src/web_server.c

   1 #ifdef HAVE_CONFIG_H
   2 #include <config.h>
   3 #endif
   4 #include <unistd.h>
   5 #include <stdlib.h>
   6 #include <sys/types.h>
   7 #include <sys/socket.h>
   8 #include <netinet/in.h>
   9 #include <arpa/inet.h>
  10 #include <errno.h>
  11 #include <pthread.h>
  12 #include <sys/stat.h>
  13 #include <fcntl.h>
  14 #include <netinet/tcp.h>
  15 #include <malloc.h>
  16
  17 #include "common.h"
  18 #include "log.h"
  19 #include "appconfig.h"
  20 #include "url.h"
  21 #include "web_buffer.h"
  22 #include "web_client.h"
  23 #include "web_server.h"
  24 #include "global_statistics.h"
  25 #include "rrd.h"
  26 #include "rrd2json.h"
  27 #include "../config.h"
  28
  29 int listen_backlog = LISTEN_BACKLOG;
  30 int listen_fd = -1;
  31 int listen_port = LISTEN_PORT;
  32 int web_server_mode = WEB_SERVER_MODE_MULTI_THREADED;
  33
  34 #ifdef NETDATA_INTERNAL_CHECKS
  35 static void log_allocations(void)
  36 {
  37         static int mem = 0;
  38
  39         struct mallinfo mi;
  40
  41         mi = mallinfo();
  42         if(mi.uordblks > mem) {
  43                 int clients = 0;
  44                 struct web_client *w;
  45                 for(w = web_clients; w ; w = w->next) clients++;
  46
  47                 info("Allocated memory increased from %d to %d (increased by %d bytes). There are %d web clients connected.", mem, mi.uordblks, mi.uordblks - mem, clients);
  48                 mem = mi.uordblks;
  49         }
  50 }
  51 #endif
  52
  53 static int is_ip_anything(const char *ip)
  54 {
  55         if(!ip || !*ip
  56                         || !strcmp(ip, "any")
  57                         || !strcmp(ip, "all")
  58                         || !strcmp(ip, "*")
  59                         || !strcmp(ip, "::")
  60                         || !strcmp(ip, "0.0.0.0")
  61                         ) return 1;
  62
  63         return 0;
  64 }
  65
  66 int create_listen_socket4(const char *ip, int port, int listen_backlog)
  67 {
  68         int sock;
  69         int sockopt = 1;
  70
  71         debug(D_LISTENER, "IPv4 creating new listening socket on port %d", port);
  72
  73         sock = socket(AF_INET, SOCK_STREAM, 0);
  74         if(sock < 0) {
  75                 error("IPv4 socket() failed.");
  76                 return -1;
  77         }
  78
  79         /* avoid "address already in use" */
  80         setsockopt(sock, SOL_SOCKET, SO_REUSEADDR, (void*)&sockopt, sizeof(sockopt));
  81
  82         struct sockaddr_in name;
  83         memset(&name, 0, sizeof(struct sockaddr_in));
  84         name.sin_family = AF_INET;
  85         name.sin_port = htons (port);
  86
  87         if(is_ip_anything(ip)) {
  88                 name.sin_addr.s_addr = htonl(INADDR_ANY);
  89                 // info("Listening on any IPs (IPv4).");
  90         }
  91         else {
  92                 int ret = inet_pton(AF_INET, ip, (void *)&name.sin_addr.s_addr);
  93                 if(ret != 1) {
  94                         error("Failed to convert IP '%s' to a valid IPv4 address.", ip);
  95                         close(sock);
  96                         return -1;
  97                 }
  98                 // info("Listening on IP '%s' (IPv4).", ip);
  99         }
 100
 101         if(bind (sock, (struct sockaddr *) &name, sizeof (name)) < 0) {
 102                 close(sock);
 103                 error("IPv4 bind() failed.");
 104                 return -1;
 105         }
 106
 107         if(listen(sock, listen_backlog) < 0) {
 108                 close(sock);
 109                 fatal("IPv4 listen() failed.");
 110                 return -1;
 111         }
 112
 113         debug(D_LISTENER, "IPv4 listening port %d created", port);
 114         return sock;
 115 }
 116
 117 int create_listen_socket6(const char *ip, int port, int listen_backlog)
 118 {
 119         int sock = -1;
 120         int sockopt = 1;
 121
 122         debug(D_LISTENER, "IPv6 creating new listening socket on port %d", port);
 123
 124         sock = socket(AF_INET6, SOCK_STREAM, 0);
 125         if (sock < 0) {
 126                 error("IPv6 socket() failed. Disabling IPv6.");
 127                 return -1;
 128         }
 129
 130         /* avoid "address already in use" */
 131         setsockopt(sock, SOL_SOCKET, SO_REUSEADDR, (void*)&sockopt, sizeof(sockopt));
 132
 133         struct sockaddr_in6 name;
 134         memset(&name, 0, sizeof(struct sockaddr_in6));
 135         name.sin6_family = AF_INET6;
 136         name.sin6_port = htons ((uint16_t) port);
 137
 138         if(is_ip_anything(ip)) {
 139                 name.sin6_addr = in6addr_any;
 140                 // info("Listening on all IPs (IPv6 and IPv4)");
 141         }
 142         else {
 143                 int ret = inet_pton(AF_INET6, ip, (void *)&name.sin6_addr.s6_addr);
 144                 if(ret != 1) {
 145                         error("Failed to convert IP '%s' to a valid IPv6 address. Disabling IPv6.", ip);
 146                         close(sock);
 147                         return -1;
 148                 }
 149                 // info("Listening on IP '%s' (IPv6)", ip);
 150         }
 151
 152         name.sin6_scope_id = 0;
 153
 154         if (bind (sock, (struct sockaddr *) &name, sizeof (name)) < 0) {
 155                 close(sock);
 156                 error("IPv6 bind() failed. Disabling IPv6.");
 157                 return -1;
 158         }
 159
 160         if (listen(sock, listen_backlog) < 0) {
 161                 close(sock);
 162                 error("IPv6 listen() failed. Disabling IPv6.");
 163                 return -1;
 164         }
 165
 166         debug(D_LISTENER, "IPv6 listening port %d created", port);
 167         return sock;
 168 }
 169
 170
 171 int create_listen_socket(void) {
 172         listen_backlog = (int) config_get_number("global", "http port listen backlog", LISTEN_BACKLOG);
 173
 174         listen_port = (int) config_get_number("global", "port", LISTEN_PORT);
 175         if(listen_port < 1 || listen_port > 65535) {
 176                 error("Invalid listen port %d given. Defaulting to %d.", listen_port, LISTEN_PORT);
 177                 listen_port = LISTEN_PORT;
 178         }
 179         else debug(D_OPTIONS, "Listen port set to %d.", listen_port);
 180
 181         int ip = 0;
 182         char *ipv = config_get("global", "ip version", "any");
 183         if(!strcmp(ipv, "any") || !strcmp(ipv, "both") || !strcmp(ipv, "all")) ip = 0;
 184         else if(!strcmp(ipv, "ipv4") || !strcmp(ipv, "IPV4") || !strcmp(ipv, "IPv4") || !strcmp(ipv, "4")) ip = 4;
 185         else if(!strcmp(ipv, "ipv6") || !strcmp(ipv, "IPV6") || !strcmp(ipv, "IPv6") || !strcmp(ipv, "6")) ip = 6;
 186         else error("Cannot understand ip version '%s'. Assuming 'any'.", ipv);
 187
 188         if(ip == 0 || ip == 6) listen_fd = create_listen_socket6(config_get("global", "bind socket to IP", "*"), listen_port, listen_backlog);
 189         if(listen_fd < 0) {
 190                 listen_fd = create_listen_socket4(config_get("global", "bind socket to IP", "*"), listen_port, listen_backlog);
 191                 // if(listen_fd >= 0 && ip != 4) info("Managed to open an IPv4 socket on port %d.", listen_port);
 192         }
 193
 194         return listen_fd;
 195 }
 196
 197 // --------------------------------------------------------------------------------------
 198 // the main socket listener
 199
 200 // 1. it accepts new incoming requests on our port
 201 // 2. creates a new web_client for each connection received
 202 // 3. spawns a new pthread to serve the client (this is optimal for keep-alive clients)
 203 // 4. cleans up old web_clients that their pthreads have been exited
 204
 205 void *socket_listen_main_multi_threaded(void *ptr) {
 206         if(ptr) { ; }
 207
 208         web_server_mode = WEB_SERVER_MODE_MULTI_THREADED;
 209         info("Multi-threaded WEB SERVER thread created with task id %d", gettid());
 210
 211         struct web_client *w;
 212         struct timeval tv;
 213         int retval, failures = 0;
 214
 215         if(ptr) { ; }
 216
 217         if(pthread_setcanceltype(PTHREAD_CANCEL_DEFERRED, NULL) != 0)
 218                 error("Cannot set pthread cancel type to DEFERRED.");
 219
 220         if(pthread_setcancelstate(PTHREAD_CANCEL_ENABLE, NULL) != 0)
 221                 error("Cannot set pthread cancel state to ENABLE.");
 222
 223         web_client_timeout = (int) config_get_number("global", "disconnect idle web clients after seconds", DEFAULT_DISCONNECT_IDLE_WEB_CLIENTS_AFTER_SECONDS);
 224         web_enable_gzip = config_get_boolean("global", "enable web responses gzip compression", web_enable_gzip);
 225
 226         if(listen_fd < 0) fatal("LISTENER: Listen socket is not ready.");
 227
 228         fd_set ifds;
 229         FD_ZERO (&ifds);
 230
 231         for(;;) {
 232                 tv.tv_sec = 0;
 233                 tv.tv_usec = 200000;
 234
 235                 if(likely(listen_fd >= 0))
 236                         FD_SET(listen_fd, &ifds);
 237
 238                 // debug(D_WEB_CLIENT, "LISTENER: Waiting...");
 239                 retval = select(listen_fd + 1, &ifds, NULL, NULL, &tv);
 240
 241                 if(unlikely(retval == -1)) {
 242                         error("LISTENER: select() failed.");
 243                         failures++;
 244
 245                         if(failures > 10) {
 246                                 error("LISTENER: our listen port %d seems dead. Re-opening it.", listen_fd);
 247                                 close(listen_fd);
 248                                 listen_fd = -1;
 249                                 sleep(5);
 250                                 create_listen_socket();
 251                                 if(listen_fd < 0)
 252                                         fatal("Cannot listen for web clients (connected clients %llu).", global_statistics.connected_clients);
 253
 254                                 failures = 0;
 255                         }
 256
 257                         continue;
 258                 }
 259                 else if(likely(retval)) {
 260                         // check for new incoming connections
 261                         if(likely(FD_ISSET(listen_fd, &ifds))) {
 262                                 w = web_client_create(listen_fd);
 263                                 if(unlikely(!w)) {
 264                                         // no need for error log - web_client_create already logged the error
 265                                         continue;
 266                                 }
 267
 268                                 if(pthread_create(&w->thread, NULL, web_client_main, w) != 0) {
 269                                         error("%llu: failed to create new thread for web client.");
 270                                         w->obsolete = 1;
 271                                 }
 272                                 else if(pthread_detach(w->thread) != 0) {
 273                                         error("%llu: Cannot request detach of newly created web client thread.", w->id);
 274                                         w->obsolete = 1;
 275                                 }
 276                         }
 277                         else debug(D_WEB_CLIENT, "LISTENER: select() didn't do anything.");
 278
 279                 }
 280                 //else {
 281                 //      debug(D_WEB_CLIENT, "LISTENER: select() timeout.");
 282                 //}
 283
 284                 failures = 0;
 285
 286                 // cleanup unused clients
 287                 for (w = web_clients; w; ) {
 288                         if (w->obsolete) {
 289                                 debug(D_WEB_CLIENT, "%llu: Removing client.", w->id);
 290                                 // pthread_cancel(w->thread);
 291                                 // pthread_join(w->thread, NULL);
 292                                 w = web_client_free(w);
 293 #ifdef NETDATA_INTERNAL_CHECKS
 294                                 log_allocations();
 295 #endif
 296                         }
 297                         else w = w->next;
 298                 }
 299         }
 300
 301         error("LISTENER: exit!");
 302
 303         if(listen_fd >= 0) close(listen_fd);
 304         exit(2);
 305
 306         return NULL;
 307 }
 308
 309 void *socket_listen_main_single_threaded(void *ptr) {
 310         if(ptr) { ; }
 311
 312         web_server_mode = WEB_SERVER_MODE_SINGLE_THREADED;
 313
 314         info("Single threaded WEB SERVER thread created with task id %d", gettid());
 315
 316         struct web_client *w;
 317         int retval, failures = 0;
 318
 319         if(ptr) { ; }
 320
 321         if(pthread_setcanceltype(PTHREAD_CANCEL_DEFERRED, NULL) != 0)
 322                 error("Cannot set pthread cancel type to DEFERRED.");
 323
 324         if(pthread_setcancelstate(PTHREAD_CANCEL_ENABLE, NULL) != 0)
 325                 error("Cannot set pthread cancel state to ENABLE.");
 326
 327         web_client_timeout = (int) config_get_number("global", "disconnect idle web clients after seconds", DEFAULT_DISCONNECT_IDLE_WEB_CLIENTS_AFTER_SECONDS);
 328         web_enable_gzip = config_get_boolean("global", "enable web responses gzip compression", web_enable_gzip);
 329
 330         if(listen_fd < 0) fatal("LISTENER: Listen socket is not ready.");
 331
 332         fd_set ifds, ofds, efds;
 333         int fdmax = listen_fd;
 334
 335         for(;;) {
 336                 int has_obsolete = 0;
 337                 FD_ZERO (&ifds);
 338                 FD_ZERO (&ofds);
 339                 FD_ZERO (&efds);
 340
 341                 if(listen_fd >= 0) {
 342                         // debug(D_WEB_CLIENT_ACCESS, "LISTENER: adding listen socket %d to ifds, efds", listen_fd);
 343                         FD_SET(listen_fd, &ifds);
 344                         FD_SET(listen_fd, &efds);
 345                 }
 346
 347                 for(w = web_clients; w ; w = w->next) {
 348                         if(unlikely(w->dead)) {
 349                                 error("%llu: client is dead.", w->id);
 350                                 w->obsolete = 1;
 351                         }
 352                         else if(unlikely(!w->wait_receive && !w->wait_send)) {
 353                                 error("%llu: client is not set for neither receiving nor sending data.");
 354                                 w->obsolete = 1;
 355                         }
 356
 357                         if(w->ifd < 0 || w->ifd >= FD_SETSIZE || w->ofd < 0 || w->ofd >= FD_SETSIZE) {
 358                                 error("%llu: invalid file descriptor, ifd = %d, ofd = %d (required 0 <= fd < FD_SETSIZE (%d)", w->id, w->ifd, w->ofd, FD_SETSIZE);
 359                                 w->obsolete = 1;
 360                         }
 361
 362                         if(unlikely(w->obsolete)) {
 363                                 has_obsolete++;
 364                                 continue;
 365                         }
 366
 367                         // debug(D_WEB_CLIENT_ACCESS, "%llu: adding input socket %d to efds", w->id, w->ifd);
 368                         FD_SET(w->ifd, &efds);
 369                         if(w->ifd > fdmax) fdmax = w->ifd;
 370
 371                         if(w->ifd != w->ofd) {
 372                                 // debug(D_WEB_CLIENT_ACCESS, "%llu: adding output socket %d to efds", w->id, w->ofd);
 373                                 FD_SET(w->ofd, &efds);
 374                                 if(w->ofd > fdmax) fdmax = w->ofd;
 375                         }
 376
 377                         if (w->wait_receive) {
 378                                 // debug(D_WEB_CLIENT_ACCESS, "%llu: adding input socket %d to ifds", w->id, w->ifd);
 379                                 FD_SET(w->ifd, &ifds);
 380                                 if(w->ifd > fdmax) fdmax = w->ifd;
 381                         }
 382
 383                         if (w->wait_send) {
 384                                 // debug(D_WEB_CLIENT_ACCESS, "%llu: adding output socket %d to ofds", w->id, w->ofd);
 385                                 FD_SET(w->ofd, &ofds);
 386                                 if(w->ofd > fdmax) fdmax = w->ofd;
 387                         }
 388                 }
 389
 390                 // cleanup unused clients
 391                 if(unlikely(has_obsolete)) {
 392                         for (w = web_clients; w; ) {
 393                                 if (w->obsolete) {
 394                                         debug(D_WEB_CLIENT, "%llu: Removing client.", w->id);
 395                                         w = web_client_free(w);
 396 #ifdef NETDATA_INTERNAL_CHECKS
 397                                         log_allocations();
 398 #endif
 399                                 }
 400                                 else w = w->next;
 401                         }
 402                 }
 403
 404                 debug(D_WEB_CLIENT_ACCESS, "LISTENER: Waiting...");
 405                 struct timeval tv = { .tv_sec = 1, .tv_usec = 0 };
 406                 errno = 0;
 407                 retval = select(fdmax+1, &ifds, &ofds, &efds, &tv);
 408
 409                 if(retval == -1) {
 410                         error("LISTENER: select() failed.");
 411
 412                         if(errno != EAGAIN) {
 413                                 // debug(D_WEB_CLIENT_ACCESS, "LISTENER: select() failed.");
 414                                 error("REMOVING ALL %lu WEB CLIENTS !", global_statistics.connected_clients);
 415                                 while (web_clients) web_client_free(web_clients);
 416                         }
 417
 418                         failures++;
 419                         if(failures > 10) {
 420                                 error("LISTENER: our listen port %d seems dead. Re-opening it.", listen_fd);
 421                                 close(listen_fd);
 422                                 listen_fd = -1;
 423                                 sleep(5);
 424                                 create_listen_socket();
 425                                 if(listen_fd < 0)
 426                                         fatal("Cannot listen for web clients (connected clients %llu).", global_statistics.connected_clients);
 427
 428                                 failures = 0;
 429                         }
 430
 431                         continue;
 432                 }
 433                 else if(retval) {
 434                         for(w = web_clients; w ; w = w->next) {
 435                                 if (unlikely(w->obsolete)) continue;
 436
 437                                 if (unlikely(FD_ISSET(w->ifd, &efds))) {
 438                                         debug(D_WEB_CLIENT_ACCESS, "%llu: Received error on input socket.", w->id);
 439                                         web_client_reset(w);
 440                                         w->obsolete = 1;
 441                                         continue;
 442                                 }
 443
 444                                 if (unlikely(FD_ISSET(w->ofd, &efds))) {
 445                                         debug(D_WEB_CLIENT_ACCESS, "%llu: Received error on output socket.", w->id);
 446                                         web_client_reset(w);
 447                                         w->obsolete = 1;
 448                                         continue;
 449                                 }
 450
 451                                 if (unlikely(w->wait_receive && FD_ISSET(w->ifd, &ifds))) {
 452                                         long bytes;
 453                                         if (unlikely((bytes = web_client_receive(w)) < 0)) {
 454                                                 debug(D_WEB_CLIENT, "%llu: Cannot receive data from client. Closing client.", w->id);
 455                                                 errno = 0;
 456                                                 web_client_reset(w);
 457                                                 w->obsolete = 1;
 458                                                 continue;
 459                                         }
 460
 461                                         if (w->mode == WEB_CLIENT_MODE_NORMAL) {
 462                                                 debug(D_WEB_CLIENT, "%llu: Processing received data (%ld bytes).", w->id, bytes);
 463                                                 // info("%llu: Attempting to process received data (%ld bytes).", w->id, bytes);
 464                                                 web_client_process(w);
 465                                         }
 466                                         else {
 467                                                 debug(D_WEB_CLIENT, "%llu: NO Processing for received data (%ld bytes).", w->id, bytes);
 468                                         }
 469                                 }
 470
 471                                 if (unlikely(w->wait_send && FD_ISSET(w->ofd, &ofds))) {
 472                                         ssize_t bytes;
 473                                         if (unlikely((bytes = web_client_send(w)) < 0)) {
 474                                                 debug(D_WEB_CLIENT, "%llu: Cannot send data to client. Closing client.", w->id);
 475                                                 errno = 0;
 476                                                 web_client_reset(w);
 477                                                 w->obsolete = 1;
 478                                                 continue;
 479                                         }
 480                                 }
 481                         }
 482
 483                         // check for new incoming connections
 484                         if(FD_ISSET(listen_fd, &ifds)) {
 485                                 debug(D_WEB_CLIENT_ACCESS, "LISTENER: new connection.");
 486                                 web_client_create(listen_fd);
 487                         }
 488                 }
 489                 else {
 490                         debug(D_WEB_CLIENT_ACCESS, "LISTENER: timeout.");
 491                 }
 492
 493                 failures = 0;
 494         }
 495
 496         error("LISTENER: exit!");
 497
 498         if(listen_fd >= 0) close(listen_fd);
 499         exit(2);
 500
 501         return NULL;
 502 }