src/web_client.c

   1 #ifdef HAVE_CONFIG_H
   2 #include <config.h>
   3 #endif
   4 #include <unistd.h>
   5 #include <stdlib.h>
   6 #include <sys/types.h>
   7 #include <sys/socket.h>
   8 #include <netinet/in.h>
   9 #include <arpa/inet.h>
  10 #include <errno.h>
  11 #include <pthread.h>
  12 #include <sys/stat.h>
  13 #include <fcntl.h>
  14 #include <netinet/tcp.h>
  15 #include <malloc.h>
  16
  17 #include "common.h"
  18 #include "log.h"
  19 #include "appconfig.h"
  20 #include "url.h"
  21 #include "web_buffer.h"
  22 #include "web_server.h"
  23 #include "global_statistics.h"
  24 #include "rrd.h"
  25 #include "rrd2json.h"
  26
  27 #include "web_client.h"
  28
  29 #define INITIAL_WEB_DATA_LENGTH 65536
  30
  31 int web_client_timeout = DEFAULT_DISCONNECT_IDLE_WEB_CLIENTS_AFTER_SECONDS;
  32 int web_enable_gzip = 1;
  33
  34 struct web_client *web_clients = NULL;
  35 unsigned long long web_clients_count = 0;
  36
  37 struct web_client *web_client_create(int listener)
  38 {
  39         struct web_client *w;
  40
  41         w = calloc(1, sizeof(struct web_client));
  42         if(!w) {
  43                 error("Cannot allocate new web_client memory.");
  44                 return NULL;
  45         }
  46
  47         w->id = ++web_clients_count;
  48         w->mode = WEB_CLIENT_MODE_NORMAL;
  49
  50         {
  51                 struct sockaddr *sadr;
  52                 socklen_t addrlen;
  53
  54                 sadr = (struct sockaddr*) &w->clientaddr;
  55                 addrlen = sizeof(w->clientaddr);
  56
  57                 w->ifd = accept(listener, sadr, &addrlen);
  58                 if (w->ifd == -1) {
  59                         error("%llu: Cannot accept new incoming connection.", w->id);
  60                         free(w);
  61                         return NULL;
  62                 }
  63                 w->ofd = w->ifd;
  64
  65                 if(getnameinfo(sadr, addrlen, w->client_ip, NI_MAXHOST, w->client_port, NI_MAXSERV, NI_NUMERICHOST | NI_NUMERICSERV) != 0) {
  66                         error("Cannot getnameinfo() on received client connection.");
  67                         strncpy(w->client_ip,   "UNKNOWN", NI_MAXHOST);
  68                         strncpy(w->client_port, "UNKNOWN", NI_MAXSERV);
  69                 }
  70                 w->client_ip[NI_MAXHOST]   = '\0';
  71                 w->client_port[NI_MAXSERV] = '\0';
  72
  73                 switch(sadr->sa_family) {
  74
  75                 case AF_INET:
  76                         debug(D_WEB_CLIENT_ACCESS, "%llu: New IPv4 web client from %s port %s on socket %d.", w->id, w->client_ip, w->client_port, w->ifd);
  77                         break;
  78
  79                 case AF_INET6:
  80                         if(strncmp(w->client_ip, "::ffff:", 7) == 0) {
  81                                 strcpy(w->client_ip, &w->client_ip[7]);
  82                                 debug(D_WEB_CLIENT_ACCESS, "%llu: New IPv4 web client from %s port %s on socket %d.", w->id, w->client_ip, w->client_port, w->ifd);
  83                         }
  84                         debug(D_WEB_CLIENT_ACCESS, "%llu: New IPv6 web client from %s port %s on socket %d.", w->id, w->client_ip, w->client_port, w->ifd);
  85                         break;
  86
  87                 default:
  88                         debug(D_WEB_CLIENT_ACCESS, "%llu: New UNKNOWN web client from %s port %s on socket %d.", w->id, w->client_ip, w->client_port, w->ifd);
  89                         break;
  90                 }
  91
  92                 int flag = 1;
  93                 if(setsockopt(w->ifd, SOL_SOCKET, SO_KEEPALIVE, (char *) &flag, sizeof(int)) != 0) error("%llu: Cannot set SO_KEEPALIVE on socket.", w->id);
  94         }
  95
  96         w->data = web_buffer_create(INITIAL_WEB_DATA_LENGTH);
  97         if(unlikely(!w->data)) {
  98                 // no need for error log - web_buffer_create already logged the error
  99                 close(w->ifd);
 100                 free(w);
 101                 return NULL;
 102         }
 103
 104         w->wait_receive = 1;
 105
 106         if(web_clients) web_clients->prev = w;
 107         w->next = web_clients;
 108         web_clients = w;
 109
 110         global_statistics.connected_clients++;
 111
 112         return(w);
 113 }
 114
 115 struct web_client *web_client_free(struct web_client *w)
 116 {
 117         struct web_client *n = w->next;
 118
 119         debug(D_WEB_CLIENT_ACCESS, "%llu: Closing web client from %s port %s.", w->id, w->client_ip, w->client_port);
 120
 121         if(w->prev)     w->prev->next = w->next;
 122         if(w->next) w->next->prev = w->prev;
 123
 124         if(w == web_clients) web_clients = w->next;
 125
 126         if(w->data) web_buffer_free(w->data);
 127         close(w->ifd);
 128         if(w->ofd != w->ifd) close(w->ofd);
 129         free(w);
 130
 131         global_statistics.connected_clients--;
 132
 133         return(n);
 134 }
 135
 136 int mysendfile(struct web_client *w, char *filename)
 137 {
 138         static char *web_dir = NULL;
 139         if(!web_dir) web_dir = config_get("global", "web files directory", WEB_DIR);
 140
 141         debug(D_WEB_CLIENT, "%llu: Looking for file '%s'...", w->id, filename);
 142
 143         // skip leading slashes
 144         while (*filename == '/') filename++;
 145
 146         // if the filename contain known paths, skip them
 147                  if(strncmp(filename, WEB_PATH_DATA       "/", strlen(WEB_PATH_DATA)       + 1) == 0) filename = &filename[strlen(WEB_PATH_DATA)       + 1];
 148         else if(strncmp(filename, WEB_PATH_DATASOURCE "/", strlen(WEB_PATH_DATASOURCE) + 1) == 0) filename = &filename[strlen(WEB_PATH_DATASOURCE) + 1];
 149         else if(strncmp(filename, WEB_PATH_GRAPH      "/", strlen(WEB_PATH_GRAPH)      + 1) == 0) filename = &filename[strlen(WEB_PATH_GRAPH)      + 1];
 150         else if(strncmp(filename, WEB_PATH_FILE       "/", strlen(WEB_PATH_FILE)       + 1) == 0) filename = &filename[strlen(WEB_PATH_FILE)       + 1];
 151
 152         // if the filename contains a / or a .., refuse to serve it
 153         if(strchr(filename, '/') != 0 || strstr(filename, "..") != 0) {
 154                 debug(D_WEB_CLIENT_ACCESS, "%llu: File '%s' is not acceptable.", w->id, filename);
 155                 web_buffer_snprintf(w->data, FILENAME_MAX + 1024, "File '%s' cannot be served. Filenames cannot contain / or ..", filename);
 156                 return 400;
 157         }
 158
 159         // access the file
 160         char webfilename[FILENAME_MAX + 1];
 161         snprintf(webfilename, FILENAME_MAX, "%s/%s", web_dir, filename);
 162
 163         // check if the file exists
 164         struct stat stat;
 165         if(lstat(webfilename, &stat) != 0) {
 166                 error("%llu: File '%s' is not found.", w->id, webfilename);
 167                 web_buffer_snprintf(w->data, FILENAME_MAX + 1024, "File '%s' does not exist, or is not accessible.", filename);
 168                 return 404;
 169         }
 170
 171         // check if the file is owned by us
 172         if(stat.st_uid != getuid() && stat.st_uid != geteuid()) {
 173                 error("%llu: File '%s' is owned by user %d (I run as user %d). Access Denied.", w->id, webfilename, stat.st_uid, getuid());
 174                 web_buffer_snprintf(w->data, FILENAME_MAX + 1024, "Access to file '%s' is not permitted.", filename);
 175                 return 403;
 176         }
 177
 178         // open the file
 179         w->ifd = open(webfilename, O_NONBLOCK, O_RDONLY);
 180         if(w->ifd == -1) {
 181                 w->ifd = w->ofd;
 182
 183                 if(errno == EBUSY || errno == EAGAIN) {
 184                         error("%llu: File '%s' is busy, sending 307 Moved Temporarily to force retry.", w->id, webfilename);
 185                         snprintf(w->response_header, MAX_HTTP_HEADER_SIZE, "Location: /" WEB_PATH_FILE "/%s\r\n", filename);
 186                         web_buffer_snprintf(w->data, FILENAME_MAX + 1024, "The file '%s' is currently busy. Please try again later.", filename);
 187                         return 307;
 188                 }
 189                 else {
 190                         error("%llu: Cannot open file '%s'.", w->id, webfilename);
 191                         web_buffer_snprintf(w->data, FILENAME_MAX + 1024, "Cannot open file '%s'.", filename);
 192                         return 404;
 193                 }
 194         }
 195
 196         // pick a Content-Type for the file
 197                  if(strstr(filename, ".html") != NULL)  w->data->contenttype = CT_TEXT_HTML;
 198         else if(strstr(filename, ".js")   != NULL)      w->data->contenttype = CT_APPLICATION_X_JAVASCRIPT;
 199         else if(strstr(filename, ".css")  != NULL)      w->data->contenttype = CT_TEXT_CSS;
 200         else if(strstr(filename, ".xml")  != NULL)      w->data->contenttype = CT_TEXT_XML;
 201         else if(strstr(filename, ".xsl")  != NULL)      w->data->contenttype = CT_TEXT_XSL;
 202         else if(strstr(filename, ".txt")  != NULL)  w->data->contenttype = CT_TEXT_PLAIN;
 203         else if(strstr(filename, ".svg")  != NULL)  w->data->contenttype = CT_IMAGE_SVG_XML;
 204         else if(strstr(filename, ".ttf")  != NULL)  w->data->contenttype = CT_APPLICATION_X_FONT_TRUETYPE;
 205         else if(strstr(filename, ".otf")  != NULL)  w->data->contenttype = CT_APPLICATION_X_FONT_OPENTYPE;
 206         else if(strstr(filename, ".woff") != NULL)  w->data->contenttype = CT_APPLICATION_FONT_WOFF;
 207         else if(strstr(filename, ".eot")  != NULL)  w->data->contenttype = CT_APPLICATION_VND_MS_FONTOBJ;
 208         else w->data->contenttype = CT_APPLICATION_OCTET_STREAM;
 209
 210         debug(D_WEB_CLIENT_ACCESS, "%llu: Sending file '%s' (%ld bytes, ifd %d, ofd %d).", w->id, webfilename, stat.st_size, w->ifd, w->ofd);
 211
 212         w->mode = WEB_CLIENT_MODE_FILECOPY;
 213         w->wait_receive = 1;
 214         w->wait_send = 0;
 215         web_buffer_flush(w->data);
 216         w->data->rlen = stat.st_size;
 217         w->data->date = stat.st_mtim.tv_sec;
 218
 219         return 200;
 220 }
 221
 222 void web_client_reset(struct web_client *w)
 223 {
 224         struct timeval tv;
 225         gettimeofday(&tv, NULL);
 226
 227         long sent = (w->mode == WEB_CLIENT_MODE_FILECOPY)?w->data->rlen:w->data->len;
 228
 229 #ifdef NETDATA_WITH_ZLIB
 230         if(likely(w->zoutput)) sent = (long)w->zstream.total_out;
 231 #endif
 232
 233         long size = (w->mode == WEB_CLIENT_MODE_FILECOPY)?w->data->rlen:w->data->len;
 234
 235         if(likely(w->last_url[0]))
 236                 log_access("%llu: (sent/all = %ld/%ld bytes %0.0f%%, prep/sent/total = %0.2f/%0.2f/%0.2f ms) %s: '%s'",
 237                         w->id,
 238                         sent, size, -((size>0)?((float)(size-sent)/(float)size * 100.0):0.0),
 239                         (float)usecdiff(&w->tv_ready, &w->tv_in) / 1000.0,
 240                         (float)usecdiff(&tv, &w->tv_ready) / 1000.0,
 241                         (float)usecdiff(&tv, &w->tv_in) / 1000.0,
 242                         (w->mode == WEB_CLIENT_MODE_FILECOPY)?"filecopy":"data",
 243                                         w->last_url
 244                 );
 245
 246         debug(D_WEB_CLIENT, "%llu: Reseting client.", w->id);
 247
 248         if(unlikely(w->mode == WEB_CLIENT_MODE_FILECOPY)) {
 249                 debug(D_WEB_CLIENT, "%llu: Closing filecopy input file.", w->id);
 250                 close(w->ifd);
 251                 w->ifd = w->ofd;
 252         }
 253
 254         w->last_url[0] = '\0';
 255
 256         w->mode = WEB_CLIENT_MODE_NORMAL;
 257
 258         w->response_header[0] = '\0';
 259
 260         web_buffer_reset(w->data);
 261
 262         w->wait_receive = 1;
 263         w->wait_send = 0;
 264
 265         w->zoutput = 0;
 266
 267         // if we had enabled compression, release it
 268 #ifdef NETDATA_WITH_ZLIB
 269         if(w->zinitialized) {
 270                 debug(D_DEFLATE, "%llu: Reseting compression.", w->id);
 271                 deflateEnd(&w->zstream);
 272                 w->zsent = 0;
 273                 w->zhave = 0;
 274                 w->zstream.avail_in = 0;
 275                 w->zstream.avail_out = 0;
 276                 w->zstream.total_in = 0;
 277                 w->zstream.total_out = 0;
 278                 w->zinitialized = 0;
 279         }
 280 #endif // NETDATA_WITH_ZLIB
 281 }
 282
 283 #ifdef NETDATA_WITH_ZLIB
 284 void web_client_enable_deflate(struct web_client *w) {
 285         if(w->zinitialized == 1) {
 286                 error("%llu: Compression has already be initialized for this client.", w->id);
 287                 return;
 288         }
 289
 290         if(w->data->sent) {
 291                 error("%llu: Cannot enable compression in the middle of a conversation.", w->id);
 292                 return;
 293         }
 294
 295         w->zstream.zalloc = Z_NULL;
 296         w->zstream.zfree = Z_NULL;
 297         w->zstream.opaque = Z_NULL;
 298
 299         w->zstream.next_in = (Bytef *)w->data->buffer;
 300         w->zstream.avail_in = 0;
 301         w->zstream.total_in = 0;
 302
 303         w->zstream.next_out = w->zbuffer;
 304         w->zstream.avail_out = 0;
 305         w->zstream.total_out = 0;
 306
 307         w->zstream.zalloc = Z_NULL;
 308         w->zstream.zfree = Z_NULL;
 309         w->zstream.opaque = Z_NULL;
 310
 311 //      if(deflateInit(&w->zstream, Z_DEFAULT_COMPRESSION) != Z_OK) {
 312 //              error("%llu: Failed to initialize zlib. Proceeding without compression.", w->id);
 313 //              return;
 314 //      }
 315
 316         // Select GZIP compression: windowbits = 15 + 16 = 31
 317         if(deflateInit2(&w->zstream, Z_DEFAULT_COMPRESSION, Z_DEFLATED, 31, 8, Z_DEFAULT_STRATEGY) != Z_OK) {
 318                 error("%llu: Failed to initialize zlib. Proceeding without compression.", w->id);
 319                 return;
 320         }
 321
 322         w->zsent = 0;
 323         w->zoutput = 1;
 324         w->zinitialized = 1;
 325
 326         debug(D_DEFLATE, "%llu: Initialized compression.", w->id);
 327 }
 328 #endif // NETDATA_WITH_ZLIB
 329
 330 int rrdset_validate_dimensions(RRDSET *st, struct web_buffer *dimensions)
 331 {
 332         ;
 333 }
 334
 335 int web_client_api_request_v1_data_format(char *name)
 336 {
 337         if(!strcmp(name, "datatable"))
 338                 return DATASOURCE_GOOGLE_JSON;
 339
 340         else if(!strcmp(name, "datasource"))
 341                 return DATASOURCE_GOOGLE_JSONP;
 342
 343         else if(!strcmp(name, "json"))
 344                 return DATASOURCE_JSON;
 345
 346         else if(!strcmp(name, "ssv"))
 347                 return DATASOURCE_SSV;
 348
 349         else if(!strcmp(name, "csv"))
 350                 return DATASOURCE_CSV;
 351
 352         return DATASOURCE_INVALID;
 353 }
 354
 355 int web_client_api_request_v1_charts(struct web_client *w, char *url)
 356 {
 357         web_buffer_flush(w->data);
 358         rrd_stats_api_v1_charts(w->data);
 359         return 200;
 360 }
 361
 362 int web_client_api_request_v1_chart(struct web_client *w, char *url)
 363 {
 364         int ret = 400;
 365         char *chart = NULL;
 366
 367         web_buffer_flush(w->data);
 368
 369         while(url) {
 370                 char *value = mystrsep(&url, "?&[]");
 371                 if(!value || !*value) continue;
 372
 373                 char *name = mystrsep(&value, "=");
 374                 if(!name || !*name) continue;
 375                 if(!value || !*value) continue;
 376
 377                 // name and value are now the parameters
 378                 // they are not null and not empty
 379
 380                 if(!strcmp(name, "chart")) chart = value;
 381                 else {
 382                         web_buffer_snprintf(w->data, 1024, "Unknown parameter '%s' in request.", name);
 383                         goto cleanup;
 384                 }
 385         }
 386
 387         if(!chart || !*chart) {
 388                 web_buffer_snprintf(w->data, 1024, "No chart id is given at the request.");
 389                 goto cleanup;
 390         }
 391
 392         RRDSET *st = rrdset_find(chart);
 393         if(!st) st = rrdset_find_byname(chart);
 394         if(!st) {
 395                 web_buffer_snprintf(w->data, 1024, "Chart '%s' is not found.", chart);
 396                 ret = 404;
 397                 goto cleanup;
 398         }
 399
 400         w->data->contenttype = CT_APPLICATION_JSON;
 401         rrd_stats_api_v1_chart(st, w->data);
 402         return 200;
 403
 404 cleanup:
 405         return ret;
 406 }
 407
 408 int web_client_api_request_v1_data(struct web_client *w, char *url)
 409 {
 410         int ret = 400;
 411         struct web_buffer *dimensions = NULL;
 412
 413         web_buffer_flush(w->data);
 414
 415         char *chart = NULL
 416                         , *dim = NULL
 417                         , *before_str = NULL
 418                         , *after_str = NULL
 419                         , *points_str = NULL
 420                         , *group_str = NULL
 421                         , *options = NULL;
 422
 423         int format = DATASOURCE_JSON;
 424
 425         while(url) {
 426                 char *value = mystrsep(&url, "?&[]");
 427                 if(!value || !*value) continue;
 428
 429                 char *name = mystrsep(&value, "=");
 430                 if(!name || !*name) continue;
 431                 if(!value || !*value) continue;
 432
 433                 // name and value are now the parameters
 434                 // they are not null and not empty
 435
 436                 if(!strcmp(name, "chart")) chart = value;
 437                 else if(!strcmp(name, "dimension") || !strcmp(name, "dim")) {
 438                         if(!dimensions) dimensions = web_buffer_create(strlen(value));
 439                         if(dimensions) {
 440                                 web_buffer_strcat(dimensions, "|");
 441                                 web_buffer_strcat(dimensions, value);
 442                         }
 443                 }
 444                 else if(!strcmp(name, "after")) after_str = value;
 445                 else if(!strcmp(name, "before")) before_str = value;
 446                 else if(!strcmp(name, "points")) points_str = value;
 447                 else if(!strcmp(name, "group")) group_str = value;
 448                 else if(!strcmp(name, "format")) {
 449                         format = web_client_api_request_v1_data_format(value);
 450                 }
 451                 else if(!strcmp(name, "options")) {
 452                         options = value;
 453                 }
 454                 else {
 455                         web_buffer_snprintf(w->data, 1024, "Unknown parameter '%s' in request.", name);
 456                         goto cleanup;
 457                 }
 458         }
 459
 460         if(!chart || !*chart) {
 461                 web_buffer_snprintf(w->data, 1024, "No chart id is given at the request.");
 462                 goto cleanup;
 463         }
 464
 465         RRDSET *st = rrdset_find(chart);
 466         if(!st) st = rrdset_find_byname(chart);
 467         if(!st) {
 468                 web_buffer_snprintf(w->data, 1024, "Chart '%s' is not found.", chart);
 469                 ret = 404;
 470                 goto cleanup;
 471         }
 472
 473         uint32_t before = (before_str && *before_str)?atol(before_str):0;
 474         uint32_t after  = (after_str  && *after_str) ?atol(after_str):0;
 475         int      points = (points_str && *points_str)?atoi(points_str):0;
 476         int      group  = (group_str  && *group_str) ?atoi(group_str):0;
 477
 478         web_buffer_snprintf(w->data, 1024, "API command 'data' for chart '%s', dimensions '%s', after '%s', before '%s', points '%s', group '%s', format '%s', options '%s'", chart, dim, after_str, before_str, points_str, group_str, format, options);
 479
 480 cleanup:
 481         if(dimensions) web_buffer_free(dimensions);
 482         return ret;
 483 }
 484
 485 int web_client_api_request_v1(struct web_client *w, char *url)
 486 {
 487         // get the command
 488         char *tok = mystrsep(&url, "/?&");
 489         debug(D_WEB_CLIENT, "%llu: Searching for API v1 command '%s'.", w->id, tok);
 490
 491         if(strcmp(tok, "data") == 0)
 492                 return web_client_api_request_v1_data(w, url);
 493         else if(strcmp(tok, "chart") == 0)
 494                 return web_client_api_request_v1_chart(w, url);
 495         else if(strcmp(tok, "charts") == 0)
 496                 return web_client_api_request_v1_charts(w, url);
 497
 498         web_buffer_flush(w->data);
 499         web_buffer_snprintf(w->data, 1024, "Unsupported v1 API command: %s", tok);
 500         return 404;
 501 }
 502
 503 int web_client_api_request(struct web_client *w, char *url)
 504 {
 505         // get the api version
 506         char *tok = mystrsep(&url, "/?&");
 507         debug(D_WEB_CLIENT, "%llu: Searching for API version '%s'.", w->id, tok);
 508
 509         if(strcmp(tok, "v1") == 0)
 510                 return web_client_api_request_v1(w, url);
 511
 512         web_buffer_flush(w->data);
 513         web_buffer_snprintf(w->data, 1024, "Unsupported API version: %s", tok);
 514         return 404;
 515 }
 516
 517 int web_client_data_request(struct web_client *w, char *url, int datasource_type)
 518 {
 519         char *args = strchr(url, '?');
 520         if(args) {
 521                 *args='\0';
 522                 args = &args[1];
 523         }
 524
 525         // get the name of the data to show
 526         char *tok = mystrsep(&url, "/");
 527         debug(D_WEB_CLIENT, "%llu: Searching for RRD data with name '%s'.", w->id, tok);
 528
 529         // do we have such a data set?
 530         RRDSET *st = rrdset_find_byname(tok);
 531         if(!st) st = rrdset_find(tok);
 532         if(!st) {
 533                 // we don't have it
 534                 // try to send a file with that name
 535                 web_buffer_flush(w->data);
 536                 return(mysendfile(w, tok));
 537         }
 538
 539         // we have it
 540         debug(D_WEB_CLIENT, "%llu: Found RRD data with name '%s'.", w->id, tok);
 541
 542         // how many entries does the client want?
 543         long lines = rrd_default_history_entries;
 544         long group_count = 1;
 545         time_t after = 0, before = 0;
 546         int group_method = GROUP_AVERAGE;
 547         int nonzero = 0;
 548
 549         if(url) {
 550                 // parse the lines required
 551                 tok = mystrsep(&url, "/");
 552                 if(tok) lines = atoi(tok);
 553                 if(lines < 1) lines = 1;
 554         }
 555         if(url) {
 556                 // parse the group count required
 557                 tok = mystrsep(&url, "/");
 558                 if(tok) group_count = atoi(tok);
 559                 if(group_count < 1) group_count = 1;
 560                 //if(group_count > save_history / 20) group_count = save_history / 20;
 561         }
 562         if(url) {
 563                 // parse the grouping method required
 564                 tok = mystrsep(&url, "/");
 565                 if(strcmp(tok, "max") == 0) group_method = GROUP_MAX;
 566                 else if(strcmp(tok, "average") == 0) group_method = GROUP_AVERAGE;
 567                 else if(strcmp(tok, "sum") == 0) group_method = GROUP_SUM;
 568                 else debug(D_WEB_CLIENT, "%llu: Unknown group method '%s'", w->id, tok);
 569         }
 570         if(url) {
 571                 // parse after time
 572                 tok = mystrsep(&url, "/");
 573                 if(tok) after = strtoul(tok, NULL, 10);
 574                 if(after < 0) after = 0;
 575         }
 576         if(url) {
 577                 // parse before time
 578                 tok = mystrsep(&url, "/");
 579                 if(tok) before = strtoul(tok, NULL, 10);
 580                 if(before < 0) before = 0;
 581         }
 582         if(url) {
 583                 // parse nonzero
 584                 tok = mystrsep(&url, "/");
 585                 if(tok && strcmp(tok, "nonzero") == 0) nonzero = 1;
 586         }
 587
 588         w->data->contenttype = CT_APPLICATION_JSON;
 589         web_buffer_flush(w->data);
 590
 591         char *google_version = "0.6";
 592         char *google_reqId = "0";
 593         char *google_sig = "0";
 594         char *google_out = "json";
 595         char *google_responseHandler = "google.visualization.Query.setResponse";
 596         char *google_outFileName = NULL;
 597         unsigned long last_timestamp_in_data = 0;
 598         if(datasource_type == DATASOURCE_GOOGLE_JSON || datasource_type == DATASOURCE_GOOGLE_JSONP) {
 599
 600                 w->data->contenttype = CT_APPLICATION_X_JAVASCRIPT;
 601
 602                 while(args) {
 603                         tok = mystrsep(&args, "&");
 604                         if(tok) {
 605                                 char *name = mystrsep(&tok, "=");
 606                                 if(name && strcmp(name, "tqx") == 0) {
 607                                         char *key = mystrsep(&tok, ":");
 608                                         char *value = mystrsep(&tok, ";");
 609                                         if(key && value && *key && *value) {
 610                                                 if(strcmp(key, "version") == 0)
 611                                                         google_version = value;
 612
 613                                                 else if(strcmp(key, "reqId") == 0)
 614                                                         google_reqId = value;
 615
 616                                                 else if(strcmp(key, "sig") == 0)
 617                                                         google_sig = value;
 618
 619                                                 else if(strcmp(key, "out") == 0)
 620                                                         google_out = value;
 621
 622                                                 else if(strcmp(key, "responseHandler") == 0)
 623                                                         google_responseHandler = value;
 624
 625                                                 else if(strcmp(key, "outFileName") == 0)
 626                                                         google_outFileName = value;
 627                                         }
 628                                 }
 629                         }
 630                 }
 631
 632                 debug(D_WEB_CLIENT_ACCESS, "%llu: GOOGLE JSONP: version = '%s', reqId = '%s', sig = '%s', out = '%s', responseHandler = '%s', outFileName = '%s'",
 633                         w->id, google_version, google_reqId, google_sig, google_out, google_responseHandler, google_outFileName
 634                         );
 635
 636                 if(datasource_type == DATASOURCE_GOOGLE_JSONP) {
 637                         last_timestamp_in_data = strtoul(google_sig, NULL, 0);
 638
 639                         // check the client wants json
 640                         if(strcmp(google_out, "json") != 0) {
 641                                 web_buffer_snprintf(w->data, 65536,
 642                                         "%s({version:'%s',reqId:'%s',status:'error',errors:[{reason:'invalid_query',message:'output format is not supported',detailed_message:'the format %s requested is not supported by netdata.'}]});",
 643                                         google_responseHandler, google_version, google_reqId, google_out);
 644                                         return 200;
 645                         }
 646                 }
 647         }
 648
 649         if(datasource_type == DATASOURCE_GOOGLE_JSONP) {
 650                 web_buffer_snprintf(w->data, 65536,
 651                         "%s({version:'%s',reqId:'%s',status:'ok',sig:'%lu',table:",
 652                         google_responseHandler, google_version, google_reqId, st->last_updated.tv_sec);
 653         }
 654
 655         debug(D_WEB_CLIENT_ACCESS, "%llu: Sending RRD data '%s' (id %s, %d lines, %d group, %d group_method, %lu after, %lu before).", w->id, st->name, st->id, lines, group_count, group_method, after, before);
 656         unsigned long timestamp_in_data = rrd_stats_json(datasource_type, st, w->data, lines, group_count, group_method, after, before, nonzero);
 657
 658         if(datasource_type == DATASOURCE_GOOGLE_JSONP) {
 659                 if(timestamp_in_data > last_timestamp_in_data)
 660                         web_buffer_strcat(w->data, "});");
 661
 662                 else {
 663                         // the client already has the latest data
 664                         web_buffer_snprintf(w->data, 65536,
 665                                 "%s({version:'%s',reqId:'%s',status:'error',errors:[{reason:'not_modified',message:'Data not modified'}]});",
 666                                 google_responseHandler, google_version, google_reqId);
 667                 }
 668         }
 669
 670         return 200;
 671 }
 672
 673 /*
 674 int web_client_parse_request(struct web_client *w) {
 675         // protocol
 676         // hostname
 677         // path
 678         // query string name-value
 679         // http version
 680         // method
 681         // http request headers name-value
 682
 683         web_client_clean_request(w);
 684
 685         debug(D_WEB_DATA, "%llu: Processing data buffer of %d bytes: '%s'.", w->id, w->data->bytes, w->data->buffer);
 686
 687         char *buf = w->data->buffer;
 688         char *line, *tok;
 689
 690         // ------------------------------------------------------------------------
 691         // the first line
 692
 693         if(buf && (line = strsep(&buf, "\r\n"))) {
 694                 // method
 695                 if(line && (tok = strsep(&line, " "))) {
 696                         w->request.protocol = strdup(tok);
 697                 }
 698                 else goto cleanup;
 699
 700                 // url
 701         }
 702         else goto cleanup;
 703
 704         // ------------------------------------------------------------------------
 705         // the rest of the lines
 706
 707         while(buf && (line = strsep(&buf, "\r\n"))) {
 708                 while(line && (tok = strsep(&line, ": "))) {
 709                 }
 710         }
 711
 712         char *url = NULL;
 713
 714
 715 cleanup:
 716         web_client_clean_request(w);
 717         return 0;
 718 }
 719 */
 720
 721 void web_client_process(struct web_client *w) {
 722         int code = 500;
 723         int bytes;
 724
 725         w->wait_receive = 0;
 726
 727         // check if we have an empty line (end of HTTP header)
 728         if(strstr(w->data->buffer, "\r\n\r\n")) {
 729                 global_statistics_lock();
 730                 global_statistics.web_requests++;
 731                 global_statistics_unlock();
 732
 733                 gettimeofday(&w->tv_in, NULL);
 734                 debug(D_WEB_DATA, "%llu: Processing data buffer of %d bytes: '%s'.", w->id, w->data->len, w->data->buffer);
 735
 736                 // check if the client requested keep-alive HTTP
 737                 if(strcasestr(w->data->buffer, "Connection: keep-alive")) w->keepalive = 1;
 738                 else w->keepalive = 0;
 739
 740 #ifdef NETDATA_WITH_ZLIB
 741                 // check if the client accepts deflate
 742                 if(web_enable_gzip && strstr(w->data->buffer, "gzip"))
 743                         web_client_enable_deflate(w);
 744 #endif // NETDATA_WITH_ZLIB
 745
 746                 int datasource_type = DATASOURCE_GOOGLE_JSONP;
 747                 //if(strstr(w->data->buffer, "X-DataSource-Auth"))
 748                 //      datasource_type = DATASOURCE_GOOGLE_JSON;
 749
 750                 char *buf = w->data->buffer;
 751                 char *tok = strsep(&buf, " \r\n");
 752                 char *url = NULL;
 753                 char *pointer_to_free = NULL; // keep url_decode() allocated buffer
 754
 755                 if(buf && strcmp(tok, "GET") == 0) {
 756                         tok = strsep(&buf, " \r\n");
 757                         pointer_to_free = url = url_decode(tok);
 758                         debug(D_WEB_CLIENT, "%llu: Processing HTTP GET on url '%s'.", w->id, url);
 759                 }
 760                 else if (buf && strcmp(tok, "POST") == 0) {
 761                         w->keepalive = 0;
 762                         tok = strsep(&buf, " \r\n");
 763                         pointer_to_free = url = url_decode(tok);
 764
 765                         debug(D_WEB_CLIENT, "%llu: I don't know how to handle POST with form data. Assuming it is a GET on url '%s'.", w->id, url);
 766                 }
 767
 768                 w->last_url[0] = '\0';
 769                 if(url) {
 770                         strncpy(w->last_url, url, URL_MAX);
 771                         w->last_url[URL_MAX] = '\0';
 772
 773                         tok = mystrsep(&url, "/?&");
 774
 775                         debug(D_WEB_CLIENT, "%llu: Processing command '%s'.", w->id, tok);
 776
 777                         if(strcmp(tok, "api") == 0) {
 778                                 // the client is requesting api access
 779                                 datasource_type = DATASOURCE_JSON;
 780                                 code = web_client_api_request(w, url);
 781                         }
 782                         else if(strcmp(tok, WEB_PATH_DATA) == 0) { // "data"
 783                                 // the client is requesting rrd data
 784                                 datasource_type = DATASOURCE_JSON;
 785                                 code = web_client_data_request(w, url, datasource_type);
 786                         }
 787                         else if(strcmp(tok, WEB_PATH_DATASOURCE) == 0) { // "datasource"
 788                                 // the client is requesting google datasource
 789                                 code = web_client_data_request(w, url, datasource_type);
 790                         }
 791                         else if(strcmp(tok, WEB_PATH_GRAPH) == 0) { // "graph"
 792                                 // the client is requesting an rrd graph
 793
 794                                 // get the name of the data to show
 795                                 tok = mystrsep(&url, "/?&");
 796                                 debug(D_WEB_CLIENT, "%llu: Searching for RRD data with name '%s'.", w->id, tok);
 797
 798                                 // do we have such a data set?
 799                                 RRDSET *st = rrdset_find_byname(tok);
 800                                 if(!st) st = rrdset_find(tok);
 801                                 if(!st) {
 802                                         // we don't have it
 803                                         // try to send a file with that name
 804                                         web_buffer_flush(w->data);
 805                                         code = mysendfile(w, tok);
 806                                 }
 807                                 else {
 808                                         code = 200;
 809                                         debug(D_WEB_CLIENT_ACCESS, "%llu: Sending %s.json of RRD_STATS...", w->id, st->name);
 810                                         w->data->contenttype = CT_APPLICATION_JSON;
 811                                         web_buffer_flush(w->data);
 812                                         rrd_stats_graph_json(st, url, w->data);
 813                                 }
 814                         }
 815                         else if(strcmp(tok, "debug") == 0) {
 816                                 web_buffer_flush(w->data);
 817
 818                                 // get the name of the data to show
 819                                 tok = mystrsep(&url, "/?&");
 820                                 debug(D_WEB_CLIENT, "%llu: Searching for RRD data with name '%s'.", w->id, tok);
 821
 822                                 // do we have such a data set?
 823                                 RRDSET *st = rrdset_find_byname(tok);
 824                                 if(!st) st = rrdset_find(tok);
 825                                 if(!st) {
 826                                         code = 404;
 827                                         web_buffer_snprintf(w->data, 1024, "Chart %s is not found.\r\n", tok);
 828                                         debug(D_WEB_CLIENT_ACCESS, "%llu: %s is not found.", w->id, tok);
 829                                 }
 830                                 else {
 831                                         code = 200;
 832                                         debug_flags |= D_RRD_STATS;
 833                                         st->debug = st->debug?0:1;
 834                                         web_buffer_snprintf(w->data, 1024, "Chart %s has now debug %s.\r\n", tok, st->debug?"enabled":"disabled");
 835                                         debug(D_WEB_CLIENT_ACCESS, "%llu: debug for %s is %s.", w->id, tok, st->debug?"enabled":"disabled");
 836                                 }
 837                         }
 838                         else if(strcmp(tok, "mirror") == 0) {
 839                                 code = 200;
 840
 841                                 debug(D_WEB_CLIENT_ACCESS, "%llu: Mirroring...", w->id);
 842
 843                                 // replace the zero bytes with spaces
 844                                 web_buffer_char_replace(w->data, '\0', ' ');
 845
 846                                 // just leave the buffer as is
 847                                 // it will be copied back to the client
 848                         }
 849                         else if(strcmp(tok, "list") == 0) {
 850                                 code = 200;
 851
 852                                 debug(D_WEB_CLIENT_ACCESS, "%llu: Sending list of RRD_STATS...", w->id);
 853
 854                                 web_buffer_flush(w->data);
 855                                 RRDSET *st = rrdset_root;
 856
 857                                 for ( ; st ; st = st->next )
 858                                         web_buffer_snprintf(w->data, 1024, "%s\n", st->name);
 859                         }
 860                         else if(strcmp(tok, "all.json") == 0) {
 861                                 code = 200;
 862                                 debug(D_WEB_CLIENT_ACCESS, "%llu: Sending JSON list of all monitors of RRD_STATS...", w->id);
 863
 864                                 w->data->contenttype = CT_APPLICATION_JSON;
 865                                 web_buffer_flush(w->data);
 866                                 rrd_stats_all_json(w->data);
 867                         }
 868                         else if(strcmp(tok, "netdata.conf") == 0) {
 869                                 code = 200;
 870                                 debug(D_WEB_CLIENT_ACCESS, "%llu: Sending netdata.conf ...", w->id);
 871
 872                                 w->data->contenttype = CT_TEXT_PLAIN;
 873                                 web_buffer_flush(w->data);
 874                                 generate_config(w->data, 0);
 875                         }
 876                         else if(strcmp(tok, WEB_PATH_FILE) == 0) { // "file"
 877                                 tok = mystrsep(&url, "/?&");
 878                                 if(tok && *tok) code = mysendfile(w, tok);
 879                                 else {
 880                                         code = 400;
 881                                         web_buffer_flush(w->data);
 882                                         web_buffer_strcat(w->data, "You have to give a filename to get.\r\n");
 883                                 }
 884                         }
 885                         else if(!tok[0]) {
 886                                 web_buffer_flush(w->data);
 887                                 code = mysendfile(w, "index.html");
 888                         }
 889                         else {
 890                                 web_buffer_flush(w->data);
 891                                 code = mysendfile(w, tok);
 892                         }
 893
 894                 }
 895                 else {
 896                         strcpy(w->last_url, "not a valid response");
 897
 898                         if(buf) debug(D_WEB_CLIENT_ACCESS, "%llu: Cannot understand '%s'.", w->id, buf);
 899
 900                         code = 500;
 901                         web_buffer_flush(w->data);
 902                         web_buffer_strcat(w->data, "I don't understand you...\r\n");
 903                 }
 904
 905                 // free url_decode() buffer
 906                 if(pointer_to_free) free(pointer_to_free);
 907         }
 908         else if(w->data->len > 8192) {
 909                 strcpy(w->last_url, "too big request");
 910
 911                 debug(D_WEB_CLIENT_ACCESS, "%llu: Received request is too big.", w->id);
 912
 913                 code = 400;
 914                 web_buffer_flush(w->data);
 915                 web_buffer_strcat(w->data, "Received request is too big.\r\n");
 916         }
 917         else {
 918                 // wait for more data
 919                 w->wait_receive = 1;
 920                 return;
 921         }
 922
 923         if(w->data->len > w->data->size) {
 924                 error("%llu: memory overflow encountered (size is %ld, written %ld).", w->data->size, w->data->len);
 925         }
 926
 927         gettimeofday(&w->tv_ready, NULL);
 928         w->data->date = time(NULL);
 929         w->data->sent = 0;
 930
 931         // prepare the HTTP response header
 932         debug(D_WEB_CLIENT, "%llu: Generating HTTP header with response %d.", w->id, code);
 933
 934         char *content_type_string = "";
 935         switch(w->data->contenttype) {
 936                 case CT_TEXT_HTML:
 937                         content_type_string = "text/html; charset=utf-8";
 938                         break;
 939
 940                 case CT_APPLICATION_XML:
 941                         content_type_string = "application/xml; charset=utf-8";
 942                         break;
 943
 944                 case CT_APPLICATION_JSON:
 945                         content_type_string = "application/json; charset=utf-8";
 946                         break;
 947
 948                 case CT_APPLICATION_X_JAVASCRIPT:
 949                         content_type_string = "application/x-javascript; charset=utf-8";
 950                         break;
 951
 952                 case CT_TEXT_CSS:
 953                         content_type_string = "text/css; charset=utf-8";
 954                         break;
 955
 956                 case CT_TEXT_XML:
 957                         content_type_string = "text/xml; charset=utf-8";
 958                         break;
 959
 960                 case CT_TEXT_XSL:
 961                         content_type_string = "text/xsl; charset=utf-8";
 962                         break;
 963
 964                 case CT_APPLICATION_OCTET_STREAM:
 965                         content_type_string = "application/octet-stream";
 966                         break;
 967
 968                 case CT_IMAGE_SVG_XML:
 969                         content_type_string = "image/svg+xml";
 970                         break;
 971
 972                 case CT_APPLICATION_X_FONT_TRUETYPE:
 973                         content_type_string = "application/x-font-truetype";
 974                         break;
 975
 976                 case CT_APPLICATION_X_FONT_OPENTYPE:
 977                         content_type_string = "application/x-font-opentype";
 978                         break;
 979
 980                 case CT_APPLICATION_FONT_WOFF:
 981                         content_type_string = "application/font-woff";
 982                         break;
 983
 984                 case CT_APPLICATION_VND_MS_FONTOBJ:
 985                         content_type_string = "application/vnd.ms-fontobject";
 986                         break;
 987
 988                 default:
 989                 case CT_TEXT_PLAIN:
 990                         content_type_string = "text/plain; charset=utf-8";
 991                         break;
 992         }
 993
 994         char *code_msg = "";
 995         switch(code) {
 996                 case 200:
 997                         code_msg = "OK";
 998                         break;
 999
1000                 case 307:
1001                         code_msg = "Temporary Redirect";
1002                         break;
1003
1004                 case 400:
1005                         code_msg = "Bad Request";
1006                         break;
1007
1008                 case 403:
1009                         code_msg = "Forbidden";
1010                         break;
1011
1012                 case 404:
1013                         code_msg = "Not Found";
1014                         break;
1015
1016                 default:
1017                         code_msg = "Internal Server Error";
1018                         break;
1019         }
1020
1021         char date[100];
1022         struct tm tm = *gmtime(&w->data->date);
1023         strftime(date, sizeof(date), "%a, %d %b %Y %H:%M:%S %Z", &tm);
1024
1025         char custom_header[MAX_HTTP_HEADER_SIZE + 1] = "";
1026         if(w->response_header[0])
1027                 strcpy(custom_header, w->response_header);
1028
1029         int headerlen = 0;
1030         headerlen += snprintf(&w->response_header[headerlen], MAX_HTTP_HEADER_SIZE - headerlen,
1031                 "HTTP/1.1 %d %s\r\n"
1032                 "Connection: %s\r\n"
1033                 "Server: NetData Embedded HTTP Server\r\n"
1034                 "Content-Type: %s\r\n"
1035                 "Access-Control-Allow-Origin: *\r\n"
1036                 "Date: %s\r\n"
1037                 , code, code_msg
1038                 , w->keepalive?"keep-alive":"close"
1039                 , content_type_string
1040                 , date
1041                 );
1042
1043         if(custom_header[0])
1044                 headerlen += snprintf(&w->response_header[headerlen], MAX_HTTP_HEADER_SIZE - headerlen, "%s", custom_header);
1045
1046         if(w->mode == WEB_CLIENT_MODE_NORMAL) {
1047                 headerlen += snprintf(&w->response_header[headerlen], MAX_HTTP_HEADER_SIZE - headerlen,
1048                         "Expires: %s\r\n"
1049                         "Cache-Control: no-cache\r\n"
1050                         , date
1051                         );
1052         }
1053         else {
1054                 headerlen += snprintf(&w->response_header[headerlen], MAX_HTTP_HEADER_SIZE - headerlen,
1055                         "Cache-Control: public\r\n"
1056                         );
1057         }
1058
1059         // if we know the content length, put it
1060         if(!w->zoutput && (w->data->len || w->data->rlen))
1061                 headerlen += snprintf(&w->response_header[headerlen], MAX_HTTP_HEADER_SIZE - headerlen,
1062                         "Content-Length: %ld\r\n"
1063                         , w->data->len?w->data->len:w->data->rlen
1064                         );
1065         else if(!w->zoutput)
1066                 w->keepalive = 0;       // content-length is required for keep-alive
1067
1068         if(w->zoutput) {
1069                 headerlen += snprintf(&w->response_header[headerlen], MAX_HTTP_HEADER_SIZE - headerlen,
1070                         "Content-Encoding: gzip\r\n"
1071                         "Transfer-Encoding: chunked\r\n"
1072                         );
1073         }
1074
1075         headerlen += snprintf(&w->response_header[headerlen], MAX_HTTP_HEADER_SIZE - headerlen, "\r\n");
1076
1077         // disable TCP_NODELAY, to buffer the header
1078         int flag = 0;
1079         if(setsockopt(w->ofd, IPPROTO_TCP, TCP_NODELAY, (char *) &flag, sizeof(int)) != 0) error("%llu: failed to disable TCP_NODELAY on socket.", w->id);
1080
1081         // sent the HTTP header
1082         debug(D_WEB_DATA, "%llu: Sending response HTTP header of size %d: '%s'", w->id, headerlen, w->response_header);
1083
1084         bytes = send(w->ofd, w->response_header, headerlen, 0);
1085         if(bytes != headerlen)
1086                 error("%llu: HTTP Header failed to be sent (I sent %d bytes but the system sent %d bytes).", w->id, headerlen, bytes);
1087         else {
1088                 global_statistics_lock();
1089                 global_statistics.bytes_sent += bytes;
1090                 global_statistics_unlock();
1091         }
1092
1093         // enable TCP_NODELAY, to send all data immediately at the next send()
1094         flag = 1;
1095         if(setsockopt(w->ofd, IPPROTO_TCP, TCP_NODELAY, (char *) &flag, sizeof(int)) != 0) error("%llu: failed to enable TCP_NODELAY on socket.", w->id);
1096
1097         // enable sending immediately if we have data
1098         if(w->data->len) w->wait_send = 1;
1099         else w->wait_send = 0;
1100
1101         // pretty logging
1102         switch(w->mode) {
1103                 case WEB_CLIENT_MODE_NORMAL:
1104                         debug(D_WEB_CLIENT, "%llu: Done preparing the response. Sending data (%d bytes) to client.", w->id, w->data->len);
1105                         break;
1106
1107                 case WEB_CLIENT_MODE_FILECOPY:
1108                         if(w->data->rlen) {
1109                                 debug(D_WEB_CLIENT, "%llu: Done preparing the response. Will be sending data file of %d bytes to client.", w->id, w->data->rlen);
1110                                 w->wait_receive = 1;
1111
1112                                 /*
1113                                 // utilize the kernel sendfile() for copying the file to the socket.
1114                                 // this block of code can be commented, without anything missing.
1115                                 // when it is commented, the program will copy the data using async I/O.
1116                                 {
1117                                         long len = sendfile(w->ofd, w->ifd, NULL, w->data->rbytes);
1118                                         if(len != w->data->rbytes) error("%llu: sendfile() should copy %ld bytes, but copied %ld. Falling back to manual copy.", w->id, w->data->rbytes, len);
1119                                         else web_client_reset(w);
1120                                 }
1121                                 */
1122                         }
1123                         else
1124                                 debug(D_WEB_CLIENT, "%llu: Done preparing the response. Will be sending an unknown amount of bytes to client.", w->id);
1125                         break;
1126
1127                 default:
1128                         fatal("%llu: Unknown client mode %d.", w->id, w->mode);
1129                         break;
1130         }
1131 }
1132
1133 long web_client_send_chunk_header(struct web_client *w, int len)
1134 {
1135         debug(D_DEFLATE, "%llu: OPEN CHUNK of %d bytes (hex: %x).", w->id, len, len);
1136         char buf[1024];
1137         sprintf(buf, "%X\r\n", len);
1138         int bytes = send(w->ofd, buf, strlen(buf), MSG_DONTWAIT);
1139
1140         if(bytes > 0) debug(D_DEFLATE, "%llu: Sent chunk header %d bytes.", w->id, bytes);
1141         else if(bytes == 0) debug(D_DEFLATE, "%llu: Did not send chunk header to the client.", w->id);
1142         else debug(D_DEFLATE, "%llu: Failed to send chunk header to client. Reason: %s", w->id, strerror(errno));
1143
1144         return bytes;
1145 }
1146
1147 long web_client_send_chunk_close(struct web_client *w)
1148 {
1149         //debug(D_DEFLATE, "%llu: CLOSE CHUNK.", w->id);
1150
1151         int bytes = send(w->ofd, "\r\n", 2, MSG_DONTWAIT);
1152
1153         if(bytes > 0) debug(D_DEFLATE, "%llu: Sent chunk suffix %d bytes.", w->id, bytes);
1154         else if(bytes == 0) debug(D_DEFLATE, "%llu: Did not send chunk suffix to the client.", w->id);
1155         else debug(D_DEFLATE, "%llu: Failed to send chunk suffix to client. Reason: %s", w->id, strerror(errno));
1156
1157         return bytes;
1158 }
1159
1160 long web_client_send_chunk_finalize(struct web_client *w)
1161 {
1162         //debug(D_DEFLATE, "%llu: FINALIZE CHUNK.", w->id);
1163
1164         int bytes = send(w->ofd, "\r\n0\r\n\r\n", 7, MSG_DONTWAIT);
1165
1166         if(bytes > 0) debug(D_DEFLATE, "%llu: Sent chunk suffix %d bytes.", w->id, bytes);
1167         else if(bytes == 0) debug(D_DEFLATE, "%llu: Did not send chunk suffix to the client.", w->id);
1168         else debug(D_DEFLATE, "%llu: Failed to send chunk suffix to client. Reason: %s", w->id, strerror(errno));
1169
1170         return bytes;
1171 }
1172
1173 #ifdef NETDATA_WITH_ZLIB
1174 long web_client_send_deflate(struct web_client *w)
1175 {
1176         long len = 0, t = 0;
1177
1178         // when using compression,
1179         // w->data->sent is the amount of bytes passed through compression
1180
1181         // debug(D_DEFLATE, "%llu: TEST w->data->bytes = %d, w->data->sent = %d, w->zhave = %d, w->zsent = %d, w->zstream.avail_in = %d, w->zstream.avail_out = %d, w->zstream.total_in = %d, w->zstream.total_out = %d.", w->id, w->data->bytes, w->data->sent, w->zhave, w->zsent, w->zstream.avail_in, w->zstream.avail_out, w->zstream.total_in, w->zstream.total_out);
1182
1183         if(w->data->len - w->data->sent == 0 && w->zstream.avail_in == 0 && w->zhave == w->zsent && w->zstream.avail_out != 0) {
1184                 // there is nothing to send
1185
1186                 debug(D_WEB_CLIENT, "%llu: Out of output data.", w->id);
1187
1188                 // finalize the chunk
1189                 if(w->data->sent != 0)
1190                         t += web_client_send_chunk_finalize(w);
1191
1192                 // there can be two cases for this
1193                 // A. we have done everything
1194                 // B. we temporarily have nothing to send, waiting for the buffer to be filled by ifd
1195
1196                 if(w->mode == WEB_CLIENT_MODE_FILECOPY && w->wait_receive && w->ifd != w->ofd && w->data->rlen && w->data->rlen > w->data->len) {
1197                         // we have to wait, more data will come
1198                         debug(D_WEB_CLIENT, "%llu: Waiting for more data to become available.", w->id);
1199                         w->wait_send = 0;
1200                         return(0);
1201                 }
1202
1203                 if(w->keepalive == 0) {
1204                         debug(D_WEB_CLIENT, "%llu: Closing (keep-alive is not enabled). %ld bytes sent.", w->id, w->data->sent);
1205                         errno = 0;
1206                         return(-1);
1207                 }
1208
1209                 // reset the client
1210                 web_client_reset(w);
1211                 debug(D_WEB_CLIENT, "%llu: Done sending all data on socket. Waiting for next request on the same socket.", w->id);
1212                 return(0);
1213         }
1214
1215         if(w->zstream.avail_out == 0 && w->zhave == w->zsent) {
1216                 // compress more input data
1217
1218                 // close the previous open chunk
1219                 if(w->data->sent != 0) t += web_client_send_chunk_close(w);
1220
1221                 debug(D_DEFLATE, "%llu: Compressing %d bytes starting from %d.", w->id, (w->data->len - w->data->sent), w->data->sent);
1222
1223                 // give the compressor all the data not passed through the compressor yet
1224                 if(w->data->len > w->data->sent) {
1225                         w->zstream.next_in = (Bytef *)&w->data->buffer[w->data->sent];
1226                         w->zstream.avail_in = (w->data->len - w->data->sent);
1227                 }
1228
1229                 // reset the compressor output buffer
1230                 w->zstream.next_out = w->zbuffer;
1231                 w->zstream.avail_out = ZLIB_CHUNK;
1232
1233                 // ask for FINISH if we have all the input
1234                 int flush = Z_SYNC_FLUSH;
1235                 if(w->mode == WEB_CLIENT_MODE_NORMAL
1236                         || (w->mode == WEB_CLIENT_MODE_FILECOPY && w->data->len == w->data->rlen)) {
1237                         flush = Z_FINISH;
1238                         debug(D_DEFLATE, "%llu: Requesting Z_FINISH.", w->id);
1239                 }
1240                 else {
1241                         debug(D_DEFLATE, "%llu: Requesting Z_SYNC_FLUSH.", w->id);
1242                 }
1243
1244                 // compress
1245                 if(deflate(&w->zstream, flush) == Z_STREAM_ERROR) {
1246                         error("%llu: Compression failed. Closing down client.", w->id);
1247                         web_client_reset(w);
1248                         return(-1);
1249                 }
1250
1251                 w->zhave = ZLIB_CHUNK - w->zstream.avail_out;
1252                 w->zsent = 0;
1253
1254                 // keep track of the bytes passed through the compressor
1255                 w->data->sent = w->data->len;
1256
1257                 debug(D_DEFLATE, "%llu: Compression produced %d bytes.", w->id, w->zhave);
1258
1259                 // open a new chunk
1260                 t += web_client_send_chunk_header(w, w->zhave);
1261         }
1262
1263         len = send(w->ofd, &w->zbuffer[w->zsent], w->zhave - w->zsent, MSG_DONTWAIT);
1264         if(len > 0) {
1265                 w->zsent += len;
1266                 if(t > 0) len += t;
1267                 debug(D_WEB_CLIENT, "%llu: Sent %d bytes.", w->id, len);
1268         }
1269         else if(len == 0) debug(D_WEB_CLIENT, "%llu: Did not send any bytes to the client.", w->id);
1270         else debug(D_WEB_CLIENT, "%llu: Failed to send data to client. Reason: %s", w->id, strerror(errno));
1271
1272         return(len);
1273 }
1274 #endif // NETDATA_WITH_ZLIB
1275
1276 long web_client_send(struct web_client *w)
1277 {
1278 #ifdef NETDATA_WITH_ZLIB
1279         if(likely(w->zoutput)) return web_client_send_deflate(w);
1280 #endif // NETDATA_WITH_ZLIB
1281
1282         long bytes;
1283
1284         if(unlikely(w->data->len - w->data->sent == 0)) {
1285                 // there is nothing to send
1286
1287                 debug(D_WEB_CLIENT, "%llu: Out of output data.", w->id);
1288
1289                 // there can be two cases for this
1290                 // A. we have done everything
1291                 // B. we temporarily have nothing to send, waiting for the buffer to be filled by ifd
1292
1293                 if(w->mode == WEB_CLIENT_MODE_FILECOPY && w->wait_receive && w->ifd != w->ofd && w->data->rlen && w->data->rlen > w->data->len) {
1294                         // we have to wait, more data will come
1295                         debug(D_WEB_CLIENT, "%llu: Waiting for more data to become available.", w->id);
1296                         w->wait_send = 0;
1297                         return(0);
1298                 }
1299
1300                 if(unlikely(w->keepalive == 0)) {
1301                         debug(D_WEB_CLIENT, "%llu: Closing (keep-alive is not enabled). %ld bytes sent.", w->id, w->data->sent);
1302                         errno = 0;
1303                         return(-1);
1304                 }
1305
1306                 web_client_reset(w);
1307                 debug(D_WEB_CLIENT, "%llu: Done sending all data on socket. Waiting for next request on the same socket.", w->id);
1308                 return(0);
1309         }
1310
1311         bytes = send(w->ofd, &w->data->buffer[w->data->sent], w->data->len - w->data->sent, MSG_DONTWAIT);
1312         if(likely(bytes > 0)) {
1313                 w->data->sent += bytes;
1314                 debug(D_WEB_CLIENT, "%llu: Sent %d bytes.", w->id, bytes);
1315         }
1316         else if(likely(bytes == 0)) debug(D_WEB_CLIENT, "%llu: Did not send any bytes to the client.", w->id);
1317         else debug(D_WEB_CLIENT, "%llu: Failed to send data to client. Reason: %s", w->id, strerror(errno));
1318
1319         return(bytes);
1320 }
1321
1322 long web_client_receive(struct web_client *w)
1323 {
1324         // do we have any space for more data?
1325         web_buffer_need_bytes(w->data, WEB_DATA_LENGTH_INCREASE_STEP);
1326
1327         long left = w->data->size - w->data->len;
1328         long bytes;
1329
1330         if(unlikely(w->mode == WEB_CLIENT_MODE_FILECOPY))
1331                 bytes = read(w->ifd, &w->data->buffer[w->data->len], (left-1));
1332         else
1333                 bytes = recv(w->ifd, &w->data->buffer[w->data->len], left-1, MSG_DONTWAIT);
1334
1335         if(likely(bytes > 0)) {
1336                 int old = w->data->len;
1337                 w->data->len += bytes;
1338                 w->data->buffer[w->data->len] = '\0';
1339
1340                 debug(D_WEB_CLIENT, "%llu: Received %d bytes.", w->id, bytes);
1341                 debug(D_WEB_DATA, "%llu: Received data: '%s'.", w->id, &w->data->buffer[old]);
1342
1343                 if(w->mode == WEB_CLIENT_MODE_FILECOPY) {
1344                         w->wait_send = 1;
1345                         if(w->data->rlen && w->data->len >= w->data->rlen) w->wait_receive = 0;
1346                 }
1347         }
1348         else if(likely(bytes == 0)) {
1349                 debug(D_WEB_CLIENT, "%llu: Out of input data.", w->id);
1350
1351                 // if we cannot read, it means we have an error on input.
1352                 // if however, we are copying a file from ifd to ofd, we should not return an error.
1353                 // in this case, the error should be generated when the file has been sent to the client.
1354
1355                 if(w->mode == WEB_CLIENT_MODE_FILECOPY) {
1356                         // we are copying data from ifd to ofd
1357                         // let it finish copying...
1358                         w->wait_receive = 0;
1359                         debug(D_WEB_CLIENT, "%llu: Disabling input.", w->id);
1360                 }
1361                 else {
1362                         bytes = -1;
1363                         errno = 0;
1364                 }
1365         }
1366
1367         return(bytes);
1368 }
1369
1370
1371 // --------------------------------------------------------------------------------------
1372 // the thread of a single client
1373
1374 // 1. waits for input and output, using async I/O
1375 // 2. it processes HTTP requests
1376 // 3. it generates HTTP responses
1377 // 4. it copies data from input to output if mode is FILECOPY
1378
1379 void *web_client_main(void *ptr)
1380 {
1381         if(pthread_setcanceltype(PTHREAD_CANCEL_DEFERRED, NULL) != 0)
1382                 error("Cannot set pthread cancel type to DEFERRED.");
1383
1384         if(pthread_setcancelstate(PTHREAD_CANCEL_ENABLE, NULL) != 0)
1385                 error("Cannot set pthread cancel state to ENABLE.");
1386
1387         struct timeval tv;
1388         struct web_client *w = ptr;
1389         int retval;
1390         fd_set ifds, ofds, efds;
1391         int fdmax = 0;
1392
1393         log_access("%llu: %s port %s connected on thread task id %d", w->id, w->client_ip, w->client_port, gettid());
1394
1395         for(;;) {
1396                 FD_ZERO (&ifds);
1397                 FD_ZERO (&ofds);
1398                 FD_ZERO (&efds);
1399
1400                 FD_SET(w->ifd, &efds);
1401
1402                 if(w->ifd != w->ofd)
1403                         FD_SET(w->ofd, &efds);
1404
1405                 if (w->wait_receive) {
1406                         FD_SET(w->ifd, &ifds);
1407                         if(w->ifd > fdmax) fdmax = w->ifd;
1408                 }
1409
1410                 if (w->wait_send) {
1411                         FD_SET(w->ofd, &ofds);
1412                         if(w->ofd > fdmax) fdmax = w->ofd;
1413                 }
1414
1415                 tv.tv_sec = web_client_timeout;
1416                 tv.tv_usec = 0;
1417
1418                 debug(D_WEB_CLIENT, "%llu: Waiting socket async I/O for %s %s", w->id, w->wait_receive?"INPUT":"", w->wait_send?"OUTPUT":"");
1419                 retval = select(fdmax+1, &ifds, &ofds, &efds, &tv);
1420
1421                 if(retval == -1) {
1422                         debug(D_WEB_CLIENT_ACCESS, "%llu: LISTENER: select() failed.", w->id);
1423                         continue;
1424                 }
1425                 else if(!retval) {
1426                         // timeout
1427                         debug(D_WEB_CLIENT_ACCESS, "%llu: LISTENER: timeout.", w->id);
1428                         break;
1429                 }
1430
1431                 if(FD_ISSET(w->ifd, &efds)) {
1432                         debug(D_WEB_CLIENT_ACCESS, "%llu: Received error on input socket (%s).", w->id, strerror(errno));
1433                         break;
1434                 }
1435
1436                 if(FD_ISSET(w->ofd, &efds)) {
1437                         debug(D_WEB_CLIENT_ACCESS, "%llu: Received error on output socket (%s).", w->id, strerror(errno));
1438                         break;
1439                 }
1440
1441                 if(w->wait_send && FD_ISSET(w->ofd, &ofds)) {
1442                         long bytes;
1443                         if((bytes = web_client_send(w)) < 0) {
1444                                 debug(D_WEB_CLIENT, "%llu: Cannot send data to client. Closing client (ouput: %s).", w->id, strerror(errno));
1445                                 errno = 0;
1446                                 break;
1447                         }
1448
1449                         global_statistics_lock();
1450                         global_statistics.bytes_sent += bytes;
1451                         global_statistics_unlock();
1452                 }
1453
1454                 if(w->wait_receive && FD_ISSET(w->ifd, &ifds)) {
1455                         long bytes;
1456                         if((bytes = web_client_receive(w)) < 0) {
1457                                 debug(D_WEB_CLIENT, "%llu: Cannot receive data from client. Closing client (input: %s).", w->id, strerror(errno));
1458                                 errno = 0;
1459                                 break;
1460                         }
1461
1462                         global_statistics_lock();
1463                         global_statistics.bytes_received += bytes;
1464                         global_statistics_unlock();
1465
1466                         if(w->mode == WEB_CLIENT_MODE_NORMAL) {
1467                                 debug(D_WEB_CLIENT, "%llu: Attempting to process received data.", w->id);
1468                                 web_client_process(w);
1469                         }
1470                 }
1471         }
1472
1473         log_access("%llu: %s port %s disconnected from thread task id %d", w->id, w->client_ip, w->client_port, gettid());
1474         debug(D_WEB_CLIENT, "%llu: done...", w->id);
1475
1476         web_client_reset(w);
1477         w->obsolete = 1;
1478
1479         return NULL;
1480 }