src/web_server.c

   1 #ifdef HAVE_CONFIG_H
   2 #include <config.h>
   3 #endif
   4 #include <unistd.h>
   5 #include <stdlib.h>
   6 #include <sys/types.h>
   7 #include <sys/socket.h>
   8 #include <netinet/in.h>
   9 #include <arpa/inet.h>
  10 #include <errno.h>
  11 #include <pthread.h>
  12 #include <sys/stat.h>
  13 #include <fcntl.h>
  14 #include <netinet/tcp.h>
  15 #include <malloc.h>
  16 #include <poll.h>
  17
  18 #include "common.h"
  19 #include "log.h"
  20 #include "appconfig.h"
  21 #include "url.h"
  22 #include "web_buffer.h"
  23 #include "web_client.h"
  24 #include "web_server.h"
  25 #include "global_statistics.h"
  26 #include "rrd.h"
  27 #include "rrd2json.h"
  28 #include "../config.h"
  29
  30 int listen_backlog = LISTEN_BACKLOG;
  31 int listen_fd = -1;
  32 int listen_port = LISTEN_PORT;
  33 int web_server_mode = WEB_SERVER_MODE_MULTI_THREADED;
  34
  35 #ifdef NETDATA_INTERNAL_CHECKS
  36 static void log_allocations(void)
  37 {
  38         static int mem = 0;
  39
  40         struct mallinfo mi;
  41
  42         mi = mallinfo();
  43         if(mi.uordblks > mem) {
  44                 int clients = 0;
  45                 struct web_client *w;
  46                 for(w = web_clients; w ; w = w->next) clients++;
  47
  48                 info("Allocated memory increased from %d to %d (increased by %d bytes). There are %d web clients connected.", mem, mi.uordblks, mi.uordblks - mem, clients);
  49                 mem = mi.uordblks;
  50         }
  51 }
  52 #endif
  53
  54 static int is_ip_anything(const char *ip)
  55 {
  56         if(!ip || !*ip
  57                         || !strcmp(ip, "any")
  58                         || !strcmp(ip, "all")
  59                         || !strcmp(ip, "*")
  60                         || !strcmp(ip, "::")
  61                         || !strcmp(ip, "0.0.0.0")
  62                         ) return 1;
  63
  64         return 0;
  65 }
  66
  67 int create_listen_socket4(const char *ip, int port, int listen_backlog)
  68 {
  69         int sock;
  70         int sockopt = 1;
  71
  72         debug(D_LISTENER, "IPv4 creating new listening socket on port %d", port);
  73
  74         sock = socket(AF_INET, SOCK_STREAM, 0);
  75         if(sock < 0) {
  76                 error("IPv4 socket() failed.");
  77                 return -1;
  78         }
  79
  80         /* avoid "address already in use" */
  81         setsockopt(sock, SOL_SOCKET, SO_REUSEADDR, (void*)&sockopt, sizeof(sockopt));
  82
  83         struct sockaddr_in name;
  84         memset(&name, 0, sizeof(struct sockaddr_in));
  85         name.sin_family = AF_INET;
  86         name.sin_port = htons (port);
  87
  88         if(is_ip_anything(ip)) {
  89                 name.sin_addr.s_addr = htonl(INADDR_ANY);
  90                 // info("Listening on any IPs (IPv4).");
  91         }
  92         else {
  93                 int ret = inet_pton(AF_INET, ip, (void *)&name.sin_addr.s_addr);
  94                 if(ret != 1) {
  95                         error("Failed to convert IP '%s' to a valid IPv4 address.", ip);
  96                         close(sock);
  97                         return -1;
  98                 }
  99                 // info("Listening on IP '%s' (IPv4).", ip);
 100         }
 101
 102         if(bind (sock, (struct sockaddr *) &name, sizeof (name)) < 0) {
 103                 close(sock);
 104                 error("IPv4 bind() failed.");
 105                 return -1;
 106         }
 107
 108         if(listen(sock, listen_backlog) < 0) {
 109                 close(sock);
 110                 fatal("IPv4 listen() failed.");
 111                 return -1;
 112         }
 113
 114         debug(D_LISTENER, "IPv4 listening port %d created", port);
 115         return sock;
 116 }
 117
 118 int create_listen_socket6(const char *ip, int port, int listen_backlog)
 119 {
 120         int sock = -1;
 121         int sockopt = 1;
 122
 123         debug(D_LISTENER, "IPv6 creating new listening socket on port %d", port);
 124
 125         sock = socket(AF_INET6, SOCK_STREAM, 0);
 126         if (sock < 0) {
 127                 error("IPv6 socket() failed. Disabling IPv6.");
 128                 return -1;
 129         }
 130
 131         /* avoid "address already in use" */
 132         setsockopt(sock, SOL_SOCKET, SO_REUSEADDR, (void*)&sockopt, sizeof(sockopt));
 133
 134         struct sockaddr_in6 name;
 135         memset(&name, 0, sizeof(struct sockaddr_in6));
 136         name.sin6_family = AF_INET6;
 137         name.sin6_port = htons ((uint16_t) port);
 138
 139         if(is_ip_anything(ip)) {
 140                 name.sin6_addr = in6addr_any;
 141                 // info("Listening on all IPs (IPv6 and IPv4)");
 142         }
 143         else {
 144                 int ret = inet_pton(AF_INET6, ip, (void *)&name.sin6_addr.s6_addr);
 145                 if(ret != 1) {
 146                         error("Failed to convert IP '%s' to a valid IPv6 address. Disabling IPv6.", ip);
 147                         close(sock);
 148                         return -1;
 149                 }
 150                 // info("Listening on IP '%s' (IPv6)", ip);
 151         }
 152
 153         name.sin6_scope_id = 0;
 154
 155         if (bind (sock, (struct sockaddr *) &name, sizeof (name)) < 0) {
 156                 close(sock);
 157                 error("IPv6 bind() failed. Disabling IPv6.");
 158                 return -1;
 159         }
 160
 161         if (listen(sock, listen_backlog) < 0) {
 162                 close(sock);
 163                 error("IPv6 listen() failed. Disabling IPv6.");
 164                 return -1;
 165         }
 166
 167         debug(D_LISTENER, "IPv6 listening port %d created", port);
 168         return sock;
 169 }
 170
 171
 172 int create_listen_socket(void) {
 173         listen_backlog = (int) config_get_number("global", "http port listen backlog", LISTEN_BACKLOG);
 174
 175         listen_port = (int) config_get_number("global", "port", LISTEN_PORT);
 176         if(listen_port < 1 || listen_port > 65535) {
 177                 error("Invalid listen port %d given. Defaulting to %d.", listen_port, LISTEN_PORT);
 178                 listen_port = LISTEN_PORT;
 179         }
 180         else debug(D_OPTIONS, "Listen port set to %d.", listen_port);
 181
 182         int ip = 0;
 183         char *ipv = config_get("global", "ip version", "any");
 184         if(!strcmp(ipv, "any") || !strcmp(ipv, "both") || !strcmp(ipv, "all")) ip = 0;
 185         else if(!strcmp(ipv, "ipv4") || !strcmp(ipv, "IPV4") || !strcmp(ipv, "IPv4") || !strcmp(ipv, "4")) ip = 4;
 186         else if(!strcmp(ipv, "ipv6") || !strcmp(ipv, "IPV6") || !strcmp(ipv, "IPv6") || !strcmp(ipv, "6")) ip = 6;
 187         else error("Cannot understand ip version '%s'. Assuming 'any'.", ipv);
 188
 189         if(ip == 0 || ip == 6) listen_fd = create_listen_socket6(config_get("global", "bind socket to IP", "*"), listen_port, listen_backlog);
 190         if(listen_fd < 0) {
 191                 listen_fd = create_listen_socket4(config_get("global", "bind socket to IP", "*"), listen_port, listen_backlog);
 192                 // if(listen_fd >= 0 && ip != 4) info("Managed to open an IPv4 socket on port %d.", listen_port);
 193         }
 194
 195         return listen_fd;
 196 }
 197
 198 // --------------------------------------------------------------------------------------
 199 // the main socket listener
 200
 201 // 1. it accepts new incoming requests on our port
 202 // 2. creates a new web_client for each connection received
 203 // 3. spawns a new pthread to serve the client (this is optimal for keep-alive clients)
 204 // 4. cleans up old web_clients that their pthreads have been exited
 205
 206 #define CLEANUP_EVERY_EVENTS 1000
 207
 208 void *socket_listen_main_multi_threaded(void *ptr) {
 209         (void)ptr;
 210
 211         web_server_mode = WEB_SERVER_MODE_MULTI_THREADED;
 212         info("Multi-threaded WEB SERVER thread created with task id %d", gettid());
 213
 214         struct web_client *w;
 215         int retval, failures = 0, counter = 0;
 216
 217         if(pthread_setcanceltype(PTHREAD_CANCEL_DEFERRED, NULL) != 0)
 218                 error("Cannot set pthread cancel type to DEFERRED.");
 219
 220         if(pthread_setcancelstate(PTHREAD_CANCEL_ENABLE, NULL) != 0)
 221                 error("Cannot set pthread cancel state to ENABLE.");
 222
 223         if(listen_fd < 0)
 224                 fatal("LISTENER: Listen socket %d is not ready, or invalid.", listen_fd);
 225
 226         for(;;) {
 227                 struct pollfd fd = { .fd = listen_fd, .events = POLLIN, .revents = 0 };
 228                 int timeout = 10 * 1000;
 229
 230                 // debug(D_WEB_CLIENT, "LISTENER: Waiting...");
 231                 retval = poll(&fd, 1, timeout);
 232
 233                 if(unlikely(retval == -1)) {
 234                         error("LISTENER: poll() failed.");
 235                         failures++;
 236
 237                         if(failures > 10) {
 238                                 error("LISTENER: our listen port %d seems dead. Re-opening it.", listen_fd);
 239                                 close(listen_fd);
 240                                 listen_fd = -1;
 241                                 sleep(5);
 242                                 create_listen_socket();
 243                                 if(listen_fd < 0)
 244                                         fatal("Cannot listen for web clients (connected clients %llu).", global_statistics.connected_clients);
 245
 246                                 failures = 0;
 247                         }
 248
 249                         continue;
 250                 }
 251                 else if(likely(retval)) {
 252                         // check for new incoming connections
 253                         if(fd.revents & POLLIN || fd.revents & POLLPRI) {
 254                                 w = web_client_create(listen_fd);
 255                                 if(unlikely(!w)) {
 256                                         // no need for error log - web_client_create already logged the error
 257                                         continue;
 258                                 }
 259
 260                                 if(pthread_create(&w->thread, NULL, web_client_main, w) != 0) {
 261                                         error("%llu: failed to create new thread for web client.");
 262                                         w->obsolete = 1;
 263                                 }
 264                                 else if(pthread_detach(w->thread) != 0) {
 265                                         error("%llu: Cannot request detach of newly created web client thread.", w->id);
 266                                         w->obsolete = 1;
 267                                 }
 268                         }
 269                         else {
 270                                 failures++;
 271                                 debug(D_WEB_CLIENT, "LISTENER: select() didn't do anything.");
 272                                 continue;
 273                         }
 274                 }
 275                 else {
 276                         debug(D_WEB_CLIENT, "LISTENER: select() timeout.");
 277                         counter = CLEANUP_EVERY_EVENTS;
 278                 }
 279
 280                 // cleanup unused clients
 281                 counter++;
 282                 if(counter > CLEANUP_EVERY_EVENTS) {
 283                         counter = 0;
 284                         for (w = web_clients; w;) {
 285                                 if (w->obsolete) {
 286                                         debug(D_WEB_CLIENT, "%llu: Removing client.", w->id);
 287                                         // pthread_cancel(w->thread);
 288                                         // pthread_join(w->thread, NULL);
 289                                         w = web_client_free(w);
 290 #ifdef NETDATA_INTERNAL_CHECKS
 291                                         log_allocations();
 292 #endif
 293                                 }
 294                                 else w = w->next;
 295                         }
 296                 }
 297
 298                 failures = 0;
 299         }
 300
 301         error("LISTENER: exit!");
 302         close(listen_fd);
 303         listen_fd = -1;
 304         return NULL;
 305 }
 306
 307 struct web_client *single_threaded_clients[FD_SETSIZE];
 308
 309 static inline int single_threaded_link_client(struct web_client *w, fd_set *ifds, fd_set *ofds, fd_set *efds, int *max) {
 310         if(unlikely(w->obsolete || w->dead || (!w->wait_receive && !w->wait_send)))
 311                 return 1;
 312
 313         if(unlikely(w->ifd < 0 || w->ifd >= FD_SETSIZE || w->ofd < 0 || w->ofd >= FD_SETSIZE)) {
 314                 error("%llu: invalid file descriptor, ifd = %d, ofd = %d (required 0 <= fd < FD_SETSIZE (%d)", w->id, w->ifd, w->ofd, FD_SETSIZE);
 315                 return 1;
 316         }
 317
 318         FD_SET(w->ifd, efds);
 319         if(unlikely(*max < w->ifd)) *max = w->ifd;
 320
 321         if(unlikely(w->ifd != w->ofd)) {
 322                 if(*max < w->ofd) *max = w->ofd;
 323                 FD_SET(w->ofd, efds);
 324         }
 325
 326         if(w->wait_receive) FD_SET(w->ifd, ifds);
 327         if(w->wait_send)    FD_SET(w->ofd, ofds);
 328
 329         single_threaded_clients[w->ifd] = w;
 330         single_threaded_clients[w->ofd] = w;
 331
 332         return 0;
 333 }
 334
 335 static inline int single_threaded_unlink_client(struct web_client *w, fd_set *ifds, fd_set *ofds, fd_set *efds) {
 336         FD_CLR(w->ifd, efds);
 337         if(unlikely(w->ifd != w->ofd)) FD_CLR(w->ofd, efds);
 338
 339         if(w->wait_receive) FD_CLR(w->ifd, ifds);
 340         if(w->wait_send)    FD_CLR(w->ofd, ofds);
 341
 342         single_threaded_clients[w->ifd] = NULL;
 343         single_threaded_clients[w->ofd] = NULL;
 344
 345         if(unlikely(w->obsolete || w->dead || (!w->wait_receive && !w->wait_send)))
 346                 return 1;
 347
 348         return 0;
 349 }
 350
 351 void *socket_listen_main_single_threaded(void *ptr) {
 352         (void)ptr;
 353
 354         web_server_mode = WEB_SERVER_MODE_SINGLE_THREADED;
 355
 356         info("Single threaded WEB SERVER thread created with task id %d", gettid());
 357
 358         struct web_client *w;
 359         int retval, failures = 0;
 360
 361         if(ptr) { ; }
 362
 363         if(pthread_setcanceltype(PTHREAD_CANCEL_DEFERRED, NULL) != 0)
 364                 error("Cannot set pthread cancel type to DEFERRED.");
 365
 366         if(pthread_setcancelstate(PTHREAD_CANCEL_ENABLE, NULL) != 0)
 367                 error("Cannot set pthread cancel state to ENABLE.");
 368
 369         if(listen_fd < 0 || listen_fd >= FD_SETSIZE)
 370                 fatal("LISTENER: Listen socket %d is not ready, or invalid.", listen_fd);
 371
 372         int i;
 373         for(i = 0; i < FD_SETSIZE ; i++)
 374                 single_threaded_clients[i] = NULL;
 375
 376         fd_set ifds, ofds, efds, rifds, rofds, refds;
 377         FD_ZERO (&ifds);
 378         FD_ZERO (&ofds);
 379         FD_ZERO (&efds);
 380         FD_SET(listen_fd, &ifds);
 381         FD_SET(listen_fd, &efds);
 382         int fdmax = listen_fd;
 383
 384         for(;;) {
 385                 debug(D_WEB_CLIENT_ACCESS, "LISTENER: single threaded web server waiting (listen fd = %d, fdmax = %d)...", listen_fd, fdmax);
 386
 387                 struct timeval tv = { .tv_sec = 1, .tv_usec = 0 };
 388                 rifds = ifds;
 389                 rofds = ofds;
 390                 refds = efds;
 391                 retval = select(fdmax+1, &rifds, &rofds, &refds, &tv);
 392
 393                 if(unlikely(retval == -1)) {
 394                         error("LISTENER: select() failed.");
 395                         failures++;
 396                         if(failures > 10) {
 397                                 if(global_statistics.connected_clients) {
 398                                         error("REMOVING ALL %lu WEB CLIENTS !", global_statistics.connected_clients);
 399                                         while (web_clients) {
 400                                                 single_threaded_unlink_client(web_clients, &ifds, &ofds, &efds);
 401                                                 web_client_free(web_clients);
 402                                         }
 403                                 }
 404
 405                                 error("LISTENER: our listen port %d seems dead. Re-opening it.", listen_fd);
 406
 407                                 close(listen_fd);
 408                                 listen_fd = -1;
 409                                 sleep(5);
 410
 411                                 create_listen_socket();
 412                                 if(listen_fd < 0 || listen_fd >= FD_SETSIZE)
 413                                         fatal("Cannot listen for web clients (connected clients %llu).", global_statistics.connected_clients);
 414
 415                                 FD_ZERO (&ifds);
 416                                 FD_ZERO (&ofds);
 417                                 FD_ZERO (&efds);
 418                                 FD_SET(listen_fd, &ifds);
 419                                 FD_SET(listen_fd, &efds);
 420                                 failures = 0;
 421                         }
 422                 }
 423                 else if(likely(retval)) {
 424                         failures = 0;
 425                         debug(D_WEB_CLIENT_ACCESS, "LISTENER: got something.");
 426
 427                         if(FD_ISSET(listen_fd, &rifds)) {
 428                                 debug(D_WEB_CLIENT_ACCESS, "LISTENER: new connection.");
 429                                 w = web_client_create(listen_fd);
 430                                 if(single_threaded_link_client(w, &ifds, &ofds, &ifds, &fdmax) != 0) {
 431                                         web_client_free(w);
 432                                 }
 433                         }
 434
 435                         for(i = 0 ; i <= fdmax ; i++) {
 436                                 if(likely(!FD_ISSET(i, &rifds) && !FD_ISSET(i, &rofds) && !FD_ISSET(i, &refds)))
 437                                         continue;
 438
 439                                 w = single_threaded_clients[i];
 440                                 if(unlikely(!w))
 441                                         continue;
 442
 443                                 if(unlikely(single_threaded_unlink_client(w, &ifds, &ofds, &efds) != 0)) {
 444                                         web_client_free(w);
 445                                         continue;
 446                                 }
 447
 448                                 if (unlikely(FD_ISSET(w->ifd, &refds) || FD_ISSET(w->ofd, &refds))) {
 449                                         web_client_free(w);
 450                                         continue;
 451                                 }
 452
 453                                 if (unlikely(w->wait_receive && FD_ISSET(w->ifd, &rifds))) {
 454                                         if (unlikely(web_client_receive(w) < 0)) {
 455                                                 web_client_free(w);
 456                                                 continue;
 457                                         }
 458
 459                                         if (w->mode != WEB_CLIENT_MODE_FILECOPY) {
 460                                                 debug(D_WEB_CLIENT, "%llu: Processing received data.", w->id);
 461                                                 web_client_process(w);
 462                                         }
 463                                 }
 464
 465                                 if (unlikely(w->wait_send && FD_ISSET(w->ofd, &rofds))) {
 466                                         if (unlikely(web_client_send(w) < 0)) {
 467                                                 debug(D_WEB_CLIENT, "%llu: Cannot send data to client. Closing client.", w->id);
 468                                                 web_client_free(w);
 469                                                 continue;
 470                                         }
 471                                 }
 472
 473                                 if(unlikely(single_threaded_link_client(w, &ifds, &ofds, &efds, &fdmax) != 0)) {
 474                                         web_client_free(w);
 475                                 }
 476                         }
 477                 }
 478                 else {
 479                         debug(D_WEB_CLIENT_ACCESS, "LISTENER: single threaded web server timeout.");
 480 #ifdef NETDATA_INTERNAL_CHECKS
 481                         log_allocations();
 482 #endif
 483                 }
 484         }
 485
 486         error("LISTENER: exit!");
 487         close(listen_fd);
 488         listen_fd = -1;
 489         return NULL;
 490 }