src/web_client.c

   1 #ifdef HAVE_CONFIG_H
   2 #include <config.h>
   3 #endif
   4 #include <unistd.h>
   5 #include <stdlib.h>
   6 #include <sys/types.h>
   7 #include <sys/socket.h>
   8 #include <netinet/in.h>
   9 #include <arpa/inet.h>
  10 #include <errno.h>
  11 #include <pthread.h>
  12 #include <sys/stat.h>
  13 #include <fcntl.h>
  14 #include <netinet/tcp.h>
  15 #include <malloc.h>
  16 #include <pwd.h>
  17 #include <grp.h>
  18 #include <ctype.h>
  19 #include <poll.h>
  20
  21 #include "common.h"
  22 #include "log.h"
  23 #include "appconfig.h"
  24 #include "url.h"
  25 #include "web_buffer.h"
  26 #include "web_server.h"
  27 #include "global_statistics.h"
  28 #include "rrd.h"
  29 #include "rrd2json.h"
  30 #include "registry.h"
  31
  32 #include "web_client.h"
  33
  34 #define INITIAL_WEB_DATA_LENGTH 16384
  35 #define WEB_REQUEST_LENGTH 16384
  36 #define TOO_BIG_REQUEST 16384
  37
  38 int web_client_timeout = DEFAULT_DISCONNECT_IDLE_WEB_CLIENTS_AFTER_SECONDS;
  39
  40 #ifdef NETDATA_WITH_ZLIB
  41 int web_enable_gzip = 1, web_gzip_level = 3, web_gzip_strategy = Z_DEFAULT_STRATEGY;
  42 #endif /* NETDATA_WITH_ZLIB */
  43
  44 extern int netdata_exit;
  45
  46 struct web_client *web_clients = NULL;
  47 unsigned long long web_clients_count = 0;
  48
  49 inline int web_client_crock_socket(struct web_client *w) {
  50 #ifdef TCP_CORK
  51         if(likely(!w->tcp_cork && w->ofd != -1)) {
  52                 w->tcp_cork = 1;
  53                 if(unlikely(setsockopt(w->ofd, IPPROTO_TCP, TCP_CORK, (char *) &w->tcp_cork, sizeof(int)) != 0)) {
  54                         error("%llu: failed to enable TCP_CORK on socket.", w->id);
  55                         w->tcp_cork = 0;
  56                         return -1;
  57                 }
  58         }
  59 #endif /* TCP_CORK */
  60
  61         return 0;
  62 }
  63
  64 inline int web_client_uncrock_socket(struct web_client *w) {
  65 #ifdef TCP_CORK
  66         if(likely(w->tcp_cork && w->ofd != -1)) {
  67                 w->tcp_cork = 0;
  68                 if(unlikely(setsockopt(w->ofd, IPPROTO_TCP, TCP_CORK, (char *) &w->tcp_cork, sizeof(int)) != 0)) {
  69                         error("%llu: failed to disable TCP_CORK on socket.", w->id);
  70                         w->tcp_cork = 1;
  71                         return -1;
  72                 }
  73         }
  74 #endif /* TCP_CORK */
  75
  76         return 0;
  77 }
  78
  79 struct web_client *web_client_create(int listener)
  80 {
  81         struct web_client *w;
  82
  83         w = calloc(1, sizeof(struct web_client));
  84         if(!w) {
  85                 error("Cannot allocate new web_client memory.");
  86                 return NULL;
  87         }
  88
  89         w->id = ++web_clients_count;
  90         w->mode = WEB_CLIENT_MODE_NORMAL;
  91
  92         {
  93                 struct sockaddr *sadr;
  94                 socklen_t addrlen;
  95
  96                 sadr = (struct sockaddr*) &w->clientaddr;
  97                 addrlen = sizeof(w->clientaddr);
  98
  99                 w->ifd = accept(listener, sadr, &addrlen);
 100                 if (w->ifd == -1) {
 101                         error("%llu: Cannot accept new incoming connection.", w->id);
 102                         free(w);
 103                         return NULL;
 104                 }
 105                 w->ofd = w->ifd;
 106
 107                 if(getnameinfo(sadr, addrlen, w->client_ip, NI_MAXHOST, w->client_port, NI_MAXSERV, NI_NUMERICHOST | NI_NUMERICSERV) != 0) {
 108                         error("Cannot getnameinfo() on received client connection.");
 109                         strncpyz(w->client_ip,   "UNKNOWN", NI_MAXHOST);
 110                         strncpyz(w->client_port, "UNKNOWN", NI_MAXSERV);
 111                 }
 112                 w->client_ip[NI_MAXHOST]   = '\0';
 113                 w->client_port[NI_MAXSERV] = '\0';
 114
 115                 switch(sadr->sa_family) {
 116                 case AF_INET:
 117                         debug(D_WEB_CLIENT_ACCESS, "%llu: New IPv4 web client from %s port %s on socket %d.", w->id, w->client_ip, w->client_port, w->ifd);
 118                         break;
 119
 120                 case AF_INET6:
 121                         if(strncmp(w->client_ip, "::ffff:", 7) == 0) {
 122                                 strcpy(w->client_ip, &w->client_ip[7]);
 123                                 debug(D_WEB_CLIENT_ACCESS, "%llu: New IPv4 web client from %s port %s on socket %d.", w->id, w->client_ip, w->client_port, w->ifd);
 124                         }
 125                         else
 126                                 debug(D_WEB_CLIENT_ACCESS, "%llu: New IPv6 web client from %s port %s on socket %d.", w->id, w->client_ip, w->client_port, w->ifd);
 127                         break;
 128
 129                 default:
 130                         debug(D_WEB_CLIENT_ACCESS, "%llu: New UNKNOWN web client from %s port %s on socket %d.", w->id, w->client_ip, w->client_port, w->ifd);
 131                         break;
 132                 }
 133
 134                 int flag = 1;
 135                 if(setsockopt(w->ofd, IPPROTO_TCP, TCP_NODELAY, (char *) &flag, sizeof(int)) != 0)
 136                         error("%llu: failed to enable TCP_NODELAY on socket.", w->id);
 137
 138                 flag = 1;
 139                 if(setsockopt(w->ifd, SOL_SOCKET, SO_KEEPALIVE, (char *) &flag, sizeof(int)) != 0)
 140                         error("%llu: Cannot set SO_KEEPALIVE on socket.", w->id);
 141
 142
 143         }
 144
 145         w->response.data = buffer_create(INITIAL_WEB_DATA_LENGTH);
 146         if(unlikely(!w->response.data)) {
 147                 // no need for error log - web_buffer_create already logged the error
 148                 close(w->ifd);
 149                 free(w);
 150                 return NULL;
 151         }
 152
 153         w->response.header = buffer_create(HTTP_RESPONSE_HEADER_SIZE);
 154         if(unlikely(!w->response.header)) {
 155                 // no need for error log - web_buffer_create already logged the error
 156                 buffer_free(w->response.data);
 157                 close(w->ifd);
 158                 free(w);
 159                 return NULL;
 160         }
 161
 162         w->response.header_output = buffer_create(HTTP_RESPONSE_HEADER_SIZE);
 163         if(unlikely(!w->response.header_output)) {
 164                 // no need for error log - web_buffer_create already logged the error
 165                 buffer_free(w->response.header);
 166                 buffer_free(w->response.data);
 167                 close(w->ifd);
 168                 free(w);
 169                 return NULL;
 170         }
 171
 172         w->origin[0] = '*';
 173         w->wait_receive = 1;
 174
 175         if(web_clients) web_clients->prev = w;
 176         w->next = web_clients;
 177         web_clients = w;
 178
 179         global_statistics.connected_clients++;
 180
 181         return(w);
 182 }
 183
 184 void web_client_reset(struct web_client *w) {
 185         web_client_uncrock_socket(w);
 186
 187         debug(D_WEB_CLIENT, "%llu: Reseting client.", w->id);
 188
 189         if(likely(w->last_url[0])) {
 190                 struct timeval tv;
 191                 gettimeofday(&tv, NULL);
 192
 193                 size_t size = (w->mode == WEB_CLIENT_MODE_FILECOPY)?w->response.rlen:w->response.data->len;
 194                 size_t sent = size;
 195 #ifdef NETDATA_WITH_ZLIB
 196                 if(likely(w->response.zoutput)) sent = (size_t)w->response.zstream.total_out;
 197 #endif
 198
 199                 // --------------------------------------------------------------------
 200                 // global statistics
 201
 202                 if(web_server_mode == WEB_SERVER_MODE_MULTI_THREADED)
 203                         global_statistics_lock();
 204
 205                 global_statistics.web_requests++;
 206                 global_statistics.web_usec += usecdiff(&tv, &w->tv_in);
 207                 global_statistics.bytes_received += w->stats_received_bytes;
 208                 global_statistics.bytes_sent += w->stats_sent_bytes;
 209                 global_statistics.content_size += size;
 210                 global_statistics.compressed_content_size += sent;
 211
 212                 if(web_server_mode == WEB_SERVER_MODE_MULTI_THREADED)
 213                         global_statistics_unlock();
 214
 215                 w->stats_received_bytes = 0;
 216                 w->stats_sent_bytes = 0;
 217
 218
 219                 // --------------------------------------------------------------------
 220                 // access log
 221
 222                 log_access("%llu: (sent/all = %zu/%zu bytes %0.0f%%, prep/sent/total = %0.2f/%0.2f/%0.2f ms) %s: %d '%s'",
 223                                    w->id,
 224                                    sent, size, -((size > 0) ? ((float) (size - sent) / (float) size * 100.0) : 0.0),
 225                                    (float) usecdiff(&w->tv_ready, &w->tv_in) / 1000.0,
 226                                    (float) usecdiff(&tv, &w->tv_ready) / 1000.0,
 227                                    (float) usecdiff(&tv, &w->tv_in) / 1000.0,
 228                                    (w->mode == WEB_CLIENT_MODE_FILECOPY) ? "filecopy" : ((w->mode == WEB_CLIENT_MODE_OPTIONS)
 229                                                                                                                                                  ? "options" : "data"),
 230                                    w->response.code,
 231                                    w->last_url
 232                 );
 233         }
 234
 235         if(unlikely(w->mode == WEB_CLIENT_MODE_FILECOPY)) {
 236                 if(w->ifd != w->ofd) {
 237                         debug(D_WEB_CLIENT, "%llu: Closing filecopy input file descriptor %d.", w->id, w->ifd);
 238                         if(w->ifd != -1) close(w->ifd);
 239                         w->ifd = w->ofd;
 240                 }
 241         }
 242
 243         w->last_url[0] = '\0';
 244         w->cookie1[0] = '\0';
 245         w->cookie2[0] = '\0';
 246         w->origin[0] = '*';
 247         w->origin[1] = '\0';
 248
 249         w->mode = WEB_CLIENT_MODE_NORMAL;
 250
 251         w->keepalive = 0;
 252         w->decoded_url[0] = '\0';
 253
 254         buffer_reset(w->response.header_output);
 255         buffer_reset(w->response.header);
 256         buffer_reset(w->response.data);
 257         w->response.rlen = 0;
 258         w->response.sent = 0;
 259         w->response.code = 0;
 260
 261         w->wait_receive = 1;
 262         w->wait_send = 0;
 263
 264         w->response.zoutput = 0;
 265
 266         // if we had enabled compression, release it
 267 #ifdef NETDATA_WITH_ZLIB
 268         if(w->response.zinitialized) {
 269                 debug(D_DEFLATE, "%llu: Freeing compression resources.", w->id);
 270                 deflateEnd(&w->response.zstream);
 271                 w->response.zsent = 0;
 272                 w->response.zhave = 0;
 273                 w->response.zstream.avail_in = 0;
 274                 w->response.zstream.avail_out = 0;
 275                 w->response.zstream.total_in = 0;
 276                 w->response.zstream.total_out = 0;
 277                 w->response.zinitialized = 0;
 278         }
 279 #endif // NETDATA_WITH_ZLIB
 280 }
 281
 282 struct web_client *web_client_free(struct web_client *w) {
 283         web_client_reset(w);
 284
 285         struct web_client *n = w->next;
 286         if(w == web_clients) web_clients = n;
 287
 288         debug(D_WEB_CLIENT_ACCESS, "%llu: Closing web client from %s port %s.", w->id, w->client_ip, w->client_port);
 289
 290         if(w->prev)     w->prev->next = w->next;
 291         if(w->next) w->next->prev = w->prev;
 292         if(w->response.header_output) buffer_free(w->response.header_output);
 293         if(w->response.header) buffer_free(w->response.header);
 294         if(w->response.data) buffer_free(w->response.data);
 295         if(w->ifd != -1) close(w->ifd);
 296         if(w->ofd != -1 && w->ofd != w->ifd) close(w->ofd);
 297         free(w);
 298
 299         global_statistics.connected_clients--;
 300
 301         return(n);
 302 }
 303
 304 uid_t web_files_uid(void) {
 305         static char *web_owner = NULL;
 306         static uid_t owner_uid = 0;
 307
 308         if(unlikely(!web_owner)) {
 309                 web_owner = config_get("global", "web files owner", config_get("global", "run as user", ""));
 310                 if(!web_owner || !*web_owner)
 311                         owner_uid = geteuid();
 312                 else {
 313                         // getpwnam() is not thread safe,
 314                         // but we have called this function once
 315                         // while single threaded
 316                         struct passwd *pw = getpwnam(web_owner);
 317                         if(!pw) {
 318                                 error("User '%s' is not present. Ignoring option.", web_owner);
 319                                 owner_uid = geteuid();
 320                         }
 321                         else {
 322                                 debug(D_WEB_CLIENT, "Web files owner set to %s.", web_owner);
 323                                 owner_uid = pw->pw_uid;
 324                         }
 325                 }
 326         }
 327
 328         return(owner_uid);
 329 }
 330
 331 gid_t web_files_gid(void) {
 332         static char *web_group = NULL;
 333         static gid_t owner_gid = 0;
 334
 335         if(unlikely(!web_group)) {
 336                 web_group = config_get("global", "web files group", config_get("global", "web files owner", ""));
 337                 if(!web_group || !*web_group)
 338                         owner_gid = getegid();
 339                 else {
 340                         // getgrnam() is not thread safe,
 341                         // but we have called this function once
 342                         // while single threaded
 343                         struct group *gr = getgrnam(web_group);
 344                         if(!gr) {
 345                                 error("Group '%s' is not present. Ignoring option.", web_group);
 346                                 owner_gid = getegid();
 347                         }
 348                         else {
 349                                 debug(D_WEB_CLIENT, "Web files group set to %s.", web_group);
 350                                 owner_gid = gr->gr_gid;
 351                         }
 352                 }
 353         }
 354
 355         return(owner_gid);
 356 }
 357
 358 int mysendfile(struct web_client *w, char *filename)
 359 {
 360         static char *web_dir = NULL;
 361
 362         // initialize our static data
 363         if(unlikely(!web_dir)) web_dir = config_get("global", "web files directory", WEB_DIR);
 364
 365         debug(D_WEB_CLIENT, "%llu: Looking for file '%s/%s'", w->id, web_dir, filename);
 366
 367         // skip leading slashes
 368         while (*filename == '/') filename++;
 369
 370         // if the filename contain known paths, skip them
 371         if(strncmp(filename, WEB_PATH_FILE "/", strlen(WEB_PATH_FILE) + 1) == 0) filename = &filename[strlen(WEB_PATH_FILE) + 1];
 372
 373         char *s;
 374         for(s = filename; *s ;s++) {
 375                 if( !isalnum(*s) && *s != '/' && *s != '.' && *s != '-' && *s != '_') {
 376                         debug(D_WEB_CLIENT_ACCESS, "%llu: File '%s' is not acceptable.", w->id, filename);
 377                         buffer_sprintf(w->response.data, "File '%s' cannot be served. Filename contains invalid character '%c'", filename, *s);
 378                         return 400;
 379                 }
 380         }
 381
 382         // if the filename contains a .. refuse to serve it
 383         if(strstr(filename, "..") != 0) {
 384                 debug(D_WEB_CLIENT_ACCESS, "%llu: File '%s' is not acceptable.", w->id, filename);
 385                 buffer_sprintf(w->response.data, "File '%s' cannot be served. Relative filenames with '..' in them are not supported.", filename);
 386                 return 400;
 387         }
 388
 389         // access the file
 390         char webfilename[FILENAME_MAX + 1];
 391         snprintfz(webfilename, FILENAME_MAX, "%s/%s", web_dir, filename);
 392
 393         // check if the file exists
 394         struct stat stat;
 395         if(lstat(webfilename, &stat) != 0) {
 396                 debug(D_WEB_CLIENT_ACCESS, "%llu: File '%s' is not found.", w->id, webfilename);
 397                 buffer_sprintf(w->response.data, "File '%s' does not exist, or is not accessible.", webfilename);
 398                 return 404;
 399         }
 400
 401         // check if the file is owned by expected user
 402         if(stat.st_uid != web_files_uid()) {
 403                 error("%llu: File '%s' is owned by user %d (expected user %d). Access Denied.", w->id, webfilename, stat.st_uid, web_files_uid());
 404                 buffer_sprintf(w->response.data, "Access to file '%s' is not permitted.", webfilename);
 405                 return 403;
 406         }
 407
 408         // check if the file is owned by expected group
 409         if(stat.st_gid != web_files_gid()) {
 410                 error("%llu: File '%s' is owned by group %d (expected group %d). Access Denied.", w->id, webfilename, stat.st_gid, web_files_gid());
 411                 buffer_sprintf(w->response.data, "Access to file '%s' is not permitted.", webfilename);
 412                 return 403;
 413         }
 414
 415         if((stat.st_mode & S_IFMT) == S_IFDIR) {
 416                 snprintfz(webfilename, FILENAME_MAX, "%s/index.html", filename);
 417                 return mysendfile(w, webfilename);
 418         }
 419
 420         if((stat.st_mode & S_IFMT) != S_IFREG) {
 421                 error("%llu: File '%s' is not a regular file. Access Denied.", w->id, webfilename);
 422                 buffer_sprintf(w->response.data, "Access to file '%s' is not permitted.", webfilename);
 423                 return 403;
 424         }
 425
 426         // open the file
 427         w->ifd = open(webfilename, O_NONBLOCK, O_RDONLY);
 428         if(w->ifd == -1) {
 429                 w->ifd = w->ofd;
 430
 431                 if(errno == EBUSY || errno == EAGAIN) {
 432                         error("%llu: File '%s' is busy, sending 307 Moved Temporarily to force retry.", w->id, webfilename);
 433                         buffer_sprintf(w->response.header, "Location: /" WEB_PATH_FILE "/%s\r\n", filename);
 434                         buffer_sprintf(w->response.data, "The file '%s' is currently busy. Please try again later.", webfilename);
 435                         return 307;
 436                 }
 437                 else {
 438                         error("%llu: Cannot open file '%s'.", w->id, webfilename);
 439                         buffer_sprintf(w->response.data, "Cannot open file '%s'.", webfilename);
 440                         return 404;
 441                 }
 442         }
 443
 444         // pick a Content-Type for the file
 445                  if(strstr(filename, ".html") != NULL)  w->response.data->contenttype = CT_TEXT_HTML;
 446         else if(strstr(filename, ".js")   != NULL)      w->response.data->contenttype = CT_APPLICATION_X_JAVASCRIPT;
 447         else if(strstr(filename, ".css")  != NULL)      w->response.data->contenttype = CT_TEXT_CSS;
 448         else if(strstr(filename, ".xml")  != NULL)      w->response.data->contenttype = CT_TEXT_XML;
 449         else if(strstr(filename, ".xsl")  != NULL)      w->response.data->contenttype = CT_TEXT_XSL;
 450         else if(strstr(filename, ".txt")  != NULL)  w->response.data->contenttype = CT_TEXT_PLAIN;
 451         else if(strstr(filename, ".svg")  != NULL)  w->response.data->contenttype = CT_IMAGE_SVG_XML;
 452         else if(strstr(filename, ".ttf")  != NULL)  w->response.data->contenttype = CT_APPLICATION_X_FONT_TRUETYPE;
 453         else if(strstr(filename, ".otf")  != NULL)  w->response.data->contenttype = CT_APPLICATION_X_FONT_OPENTYPE;
 454         else if(strstr(filename, ".woff2")!= NULL)  w->response.data->contenttype = CT_APPLICATION_FONT_WOFF2;
 455         else if(strstr(filename, ".woff") != NULL)  w->response.data->contenttype = CT_APPLICATION_FONT_WOFF;
 456         else if(strstr(filename, ".eot")  != NULL)  w->response.data->contenttype = CT_APPLICATION_VND_MS_FONTOBJ;
 457         else if(strstr(filename, ".png")  != NULL)  w->response.data->contenttype = CT_IMAGE_PNG;
 458         else if(strstr(filename, ".jpg")  != NULL)  w->response.data->contenttype = CT_IMAGE_JPG;
 459         else if(strstr(filename, ".jpeg") != NULL)  w->response.data->contenttype = CT_IMAGE_JPG;
 460         else if(strstr(filename, ".gif")  != NULL)  w->response.data->contenttype = CT_IMAGE_GIF;
 461         else if(strstr(filename, ".bmp")  != NULL)  w->response.data->contenttype = CT_IMAGE_BMP;
 462         else if(strstr(filename, ".ico")  != NULL)  w->response.data->contenttype = CT_IMAGE_XICON;
 463         else if(strstr(filename, ".icns") != NULL)  w->response.data->contenttype = CT_IMAGE_ICNS;
 464         else w->response.data->contenttype = CT_APPLICATION_OCTET_STREAM;
 465
 466         debug(D_WEB_CLIENT_ACCESS, "%llu: Sending file '%s' (%ld bytes, ifd %d, ofd %d).", w->id, webfilename, stat.st_size, w->ifd, w->ofd);
 467
 468         w->mode = WEB_CLIENT_MODE_FILECOPY;
 469         w->wait_receive = 1;
 470         w->wait_send = 0;
 471         buffer_flush(w->response.data);
 472         w->response.rlen = stat.st_size;
 473         w->response.data->date = stat.st_mtim.tv_sec;
 474
 475         return 200;
 476 }
 477
 478
 479 #ifdef NETDATA_WITH_ZLIB
 480 void web_client_enable_deflate(struct web_client *w, int gzip) {
 481         if(unlikely(w->response.zinitialized)) {
 482                 error("%llu: Compression has already be initialized for this client.", w->id);
 483                 return;
 484         }
 485
 486         if(unlikely(w->response.sent)) {
 487                 error("%llu: Cannot enable compression in the middle of a conversation.", w->id);
 488                 return;
 489         }
 490
 491         w->response.zstream.zalloc = Z_NULL;
 492         w->response.zstream.zfree = Z_NULL;
 493         w->response.zstream.opaque = Z_NULL;
 494
 495         w->response.zstream.next_in = (Bytef *)w->response.data->buffer;
 496         w->response.zstream.avail_in = 0;
 497         w->response.zstream.total_in = 0;
 498
 499         w->response.zstream.next_out = w->response.zbuffer;
 500         w->response.zstream.avail_out = 0;
 501         w->response.zstream.total_out = 0;
 502
 503         w->response.zstream.zalloc = Z_NULL;
 504         w->response.zstream.zfree = Z_NULL;
 505         w->response.zstream.opaque = Z_NULL;
 506
 507 //      if(deflateInit(&w->response.zstream, Z_DEFAULT_COMPRESSION) != Z_OK) {
 508 //              error("%llu: Failed to initialize zlib. Proceeding without compression.", w->id);
 509 //              return;
 510 //      }
 511
 512         // Select GZIP compression: windowbits = 15 + 16 = 31
 513         if(deflateInit2(&w->response.zstream, web_gzip_level, Z_DEFLATED, 15 + ((gzip)?16:0), 8, web_gzip_strategy) != Z_OK) {
 514                 error("%llu: Failed to initialize zlib. Proceeding without compression.", w->id);
 515                 return;
 516         }
 517
 518         w->response.zsent = 0;
 519         w->response.zoutput = 1;
 520         w->response.zinitialized = 1;
 521
 522         debug(D_DEFLATE, "%llu: Initialized compression.", w->id);
 523 }
 524 #endif // NETDATA_WITH_ZLIB
 525
 526 uint32_t web_client_api_request_v1_data_options(char *o)
 527 {
 528         uint32_t ret = 0x00000000;
 529         char *tok;
 530
 531         while(o && *o && (tok = mystrsep(&o, ", |"))) {
 532                 if(!*tok) continue;
 533
 534                 if(!strcmp(tok, "nonzero"))
 535                         ret |= RRDR_OPTION_NONZERO;
 536                 else if(!strcmp(tok, "flip") || !strcmp(tok, "reversed") || !strcmp(tok, "reverse"))
 537                         ret |= RRDR_OPTION_REVERSED;
 538                 else if(!strcmp(tok, "jsonwrap"))
 539                         ret |= RRDR_OPTION_JSON_WRAP;
 540                 else if(!strcmp(tok, "min2max"))
 541                         ret |= RRDR_OPTION_MIN2MAX;
 542                 else if(!strcmp(tok, "ms") || !strcmp(tok, "milliseconds"))
 543                         ret |= RRDR_OPTION_MILLISECONDS;
 544                 else if(!strcmp(tok, "abs") || !strcmp(tok, "absolute") || !strcmp(tok, "absolute_sum") || !strcmp(tok, "absolute-sum"))
 545                         ret |= RRDR_OPTION_ABSOLUTE;
 546                 else if(!strcmp(tok, "seconds"))
 547                         ret |= RRDR_OPTION_SECONDS;
 548                 else if(!strcmp(tok, "null2zero"))
 549                         ret |= RRDR_OPTION_NULL2ZERO;
 550                 else if(!strcmp(tok, "objectrows"))
 551                         ret |= RRDR_OPTION_OBJECTSROWS;
 552                 else if(!strcmp(tok, "google_json"))
 553                         ret |= RRDR_OPTION_GOOGLE_JSON;
 554                 else if(!strcmp(tok, "percentage"))
 555                         ret |= RRDR_OPTION_PERCENTAGE;
 556         }
 557
 558         return ret;
 559 }
 560
 561 uint32_t web_client_api_request_v1_data_format(char *name)
 562 {
 563         if(!strcmp(name, DATASOURCE_FORMAT_DATATABLE_JSON)) // datatable
 564                 return DATASOURCE_DATATABLE_JSON;
 565
 566         else if(!strcmp(name, DATASOURCE_FORMAT_DATATABLE_JSONP)) // datasource
 567                 return DATASOURCE_DATATABLE_JSONP;
 568
 569         else if(!strcmp(name, DATASOURCE_FORMAT_JSON)) // json
 570                 return DATASOURCE_JSON;
 571
 572         else if(!strcmp(name, DATASOURCE_FORMAT_JSONP)) // jsonp
 573                 return DATASOURCE_JSONP;
 574
 575         else if(!strcmp(name, DATASOURCE_FORMAT_SSV)) // ssv
 576                 return DATASOURCE_SSV;
 577
 578         else if(!strcmp(name, DATASOURCE_FORMAT_CSV)) // csv
 579                 return DATASOURCE_CSV;
 580
 581         else if(!strcmp(name, DATASOURCE_FORMAT_TSV) || !strcmp(name, "tsv-excel")) // tsv
 582                 return DATASOURCE_TSV;
 583
 584         else if(!strcmp(name, DATASOURCE_FORMAT_HTML)) // html
 585                 return DATASOURCE_HTML;
 586
 587         else if(!strcmp(name, DATASOURCE_FORMAT_JS_ARRAY)) // array
 588                 return DATASOURCE_JS_ARRAY;
 589
 590         else if(!strcmp(name, DATASOURCE_FORMAT_SSV_COMMA)) // ssvcomma
 591                 return DATASOURCE_SSV_COMMA;
 592
 593         else if(!strcmp(name, DATASOURCE_FORMAT_CSV_JSON_ARRAY)) // csvjsonarray
 594                 return DATASOURCE_CSV_JSON_ARRAY;
 595
 596         return DATASOURCE_JSON;
 597 }
 598
 599 uint32_t web_client_api_request_v1_data_google_format(char *name)
 600 {
 601         if(!strcmp(name, "json"))
 602                 return DATASOURCE_DATATABLE_JSONP;
 603
 604         else if(!strcmp(name, "html"))
 605                 return DATASOURCE_HTML;
 606
 607         else if(!strcmp(name, "csv"))
 608                 return DATASOURCE_CSV;
 609
 610         else if(!strcmp(name, "tsv-excel"))
 611                 return DATASOURCE_TSV;
 612
 613         return DATASOURCE_JSON;
 614 }
 615
 616 int web_client_api_request_v1_data_group(char *name)
 617 {
 618         if(!strcmp(name, "max"))
 619                 return GROUP_MAX;
 620
 621         else if(!strcmp(name, "average"))
 622                 return GROUP_AVERAGE;
 623
 624         return GROUP_MAX;
 625 }
 626
 627 int web_client_api_request_v1_charts(struct web_client *w, char *url)
 628 {
 629         if(url) { ; }
 630
 631         buffer_flush(w->response.data);
 632         w->response.data->contenttype = CT_APPLICATION_JSON;
 633         rrd_stats_api_v1_charts(w->response.data);
 634         return 200;
 635 }
 636
 637 int web_client_api_request_v1_chart(struct web_client *w, char *url)
 638 {
 639         int ret = 400;
 640         char *chart = NULL;
 641
 642         buffer_flush(w->response.data);
 643
 644         while(url) {
 645                 char *value = mystrsep(&url, "?&[]");
 646                 if(!value || !*value) continue;
 647
 648                 char *name = mystrsep(&value, "=");
 649                 if(!name || !*name) continue;
 650                 if(!value || !*value) continue;
 651
 652                 // name and value are now the parameters
 653                 // they are not null and not empty
 654
 655                 if(!strcmp(name, "chart")) chart = value;
 656                 //else {
 657                 ///     buffer_sprintf(w->response.data, "Unknown parameter '%s' in request.", name);
 658                 //      goto cleanup;
 659                 //}
 660         }
 661
 662         if(!chart || !*chart) {
 663                 buffer_sprintf(w->response.data, "No chart id is given at the request.");
 664                 goto cleanup;
 665         }
 666
 667         RRDSET *st = rrdset_find(chart);
 668         if(!st) st = rrdset_find_byname(chart);
 669         if(!st) {
 670                 buffer_sprintf(w->response.data, "Chart '%s' is not found.", chart);
 671                 ret = 404;
 672                 goto cleanup;
 673         }
 674
 675         w->response.data->contenttype = CT_APPLICATION_JSON;
 676         rrd_stats_api_v1_chart(st, w->response.data);
 677         return 200;
 678
 679 cleanup:
 680         return ret;
 681 }
 682
 683 // returns the HTTP code
 684 int web_client_api_request_v1_data(struct web_client *w, char *url)
 685 {
 686         debug(D_WEB_CLIENT, "%llu: API v1 data with URL '%s'", w->id, url);
 687
 688         int ret = 400;
 689         BUFFER *dimensions = NULL;
 690
 691         buffer_flush(w->response.data);
 692
 693         char    *google_version = "0.6",
 694                         *google_reqId = "0",
 695                         *google_sig = "0",
 696                         *google_out = "json",
 697                         *responseHandler = NULL,
 698                         *outFileName = NULL;
 699
 700         time_t last_timestamp_in_data = 0, google_timestamp = 0;
 701
 702         char *chart = NULL
 703                         , *before_str = NULL
 704                         , *after_str = NULL
 705                         , *points_str = NULL;
 706
 707         int group = GROUP_MAX;
 708         uint32_t format = DATASOURCE_JSON;
 709         uint32_t options = 0x00000000;
 710
 711         while(url) {
 712                 char *value = mystrsep(&url, "?&[]");
 713                 if(!value || !*value) continue;
 714
 715                 char *name = mystrsep(&value, "=");
 716                 if(!name || !*name) continue;
 717                 if(!value || !*value) continue;
 718
 719                 debug(D_WEB_CLIENT, "%llu: API v1 data query param '%s' with value '%s'", w->id, name, value);
 720
 721                 // name and value are now the parameters
 722                 // they are not null and not empty
 723
 724                 if(!strcmp(name, "chart")) chart = value;
 725                 else if(!strcmp(name, "dimension") || !strcmp(name, "dim") || !strcmp(name, "dimensions") || !strcmp(name, "dims")) {
 726                         if(!dimensions) dimensions = buffer_create(strlen(value));
 727                         if(dimensions) {
 728                                 buffer_strcat(dimensions, "|");
 729                                 buffer_strcat(dimensions, value);
 730                         }
 731                 }
 732                 else if(!strcmp(name, "after")) after_str = value;
 733                 else if(!strcmp(name, "before")) before_str = value;
 734                 else if(!strcmp(name, "points")) points_str = value;
 735                 else if(!strcmp(name, "group")) {
 736                         group = web_client_api_request_v1_data_group(value);
 737                 }
 738                 else if(!strcmp(name, "format")) {
 739                         format = web_client_api_request_v1_data_format(value);
 740                 }
 741                 else if(!strcmp(name, "options")) {
 742                         options |= web_client_api_request_v1_data_options(value);
 743                 }
 744                 else if(!strcmp(name, "callback")) {
 745                         responseHandler = value;
 746                 }
 747                 else if(!strcmp(name, "filename")) {
 748                         outFileName = value;
 749                 }
 750                 else if(!strcmp(name, "tqx")) {
 751                         // parse Google Visualization API options
 752                         // https://developers.google.com/chart/interactive/docs/dev/implementing_data_source
 753                         char *tqx_name, *tqx_value;
 754
 755                         while(value) {
 756                                 tqx_value = mystrsep(&value, ";");
 757                                 if(!tqx_value || !*tqx_value) continue;
 758
 759                                 tqx_name = mystrsep(&tqx_value, ":");
 760                                 if(!tqx_name || !*tqx_name) continue;
 761                                 if(!tqx_value || !*tqx_value) continue;
 762
 763                                 if(!strcmp(tqx_name, "version"))
 764                                         google_version = tqx_value;
 765                                 else if(!strcmp(tqx_name, "reqId"))
 766                                         google_reqId = tqx_value;
 767                                 else if(!strcmp(tqx_name, "sig")) {
 768                                         google_sig = tqx_value;
 769                                         google_timestamp = strtoul(google_sig, NULL, 0);
 770                                 }
 771                                 else if(!strcmp(tqx_name, "out")) {
 772                                         google_out = tqx_value;
 773                                         format = web_client_api_request_v1_data_google_format(google_out);
 774                                 }
 775                                 else if(!strcmp(tqx_name, "responseHandler"))
 776                                         responseHandler = tqx_value;
 777                                 else if(!strcmp(tqx_name, "outFileName"))
 778                                         outFileName = tqx_value;
 779                         }
 780                 }
 781         }
 782
 783         if(!chart || !*chart) {
 784                 buffer_sprintf(w->response.data, "No chart id is given at the request.");
 785                 goto cleanup;
 786         }
 787
 788         RRDSET *st = rrdset_find(chart);
 789         if(!st) st = rrdset_find_byname(chart);
 790         if(!st) {
 791                 buffer_sprintf(w->response.data, "Chart '%s' is not found.", chart);
 792                 ret = 404;
 793                 goto cleanup;
 794         }
 795
 796         long long before = (before_str && *before_str)?atol(before_str):0;
 797         long long after  = (after_str  && *after_str) ?atol(after_str):0;
 798         int       points = (points_str && *points_str)?atoi(points_str):0;
 799
 800         debug(D_WEB_CLIENT, "%llu: API command 'data' for chart '%s', dimensions '%s', after '%lld', before '%lld', points '%d', group '%u', format '%u', options '0x%08x'"
 801                         , w->id
 802                         , chart
 803                         , (dimensions)?buffer_tostring(dimensions):""
 804                         , after
 805                         , before
 806                         , points
 807                         , group
 808                         , format
 809                         , options
 810                         );
 811
 812         if(outFileName && *outFileName) {
 813                 buffer_sprintf(w->response.header, "Content-Disposition: attachment; filename=\"%s\"\r\n", outFileName);
 814                 debug(D_WEB_CLIENT, "%llu: generating outfilename header: '%s'", w->id, outFileName);
 815         }
 816
 817         if(format == DATASOURCE_DATATABLE_JSONP) {
 818                 if(responseHandler == NULL)
 819                         responseHandler = "google.visualization.Query.setResponse";
 820
 821                 debug(D_WEB_CLIENT_ACCESS, "%llu: GOOGLE JSON/JSONP: version = '%s', reqId = '%s', sig = '%s', out = '%s', responseHandler = '%s', outFileName = '%s'",
 822                                 w->id, google_version, google_reqId, google_sig, google_out, responseHandler, outFileName
 823                         );
 824
 825                 buffer_sprintf(w->response.data,
 826                         "%s({version:'%s',reqId:'%s',status:'ok',sig:'%lu',table:",
 827                         responseHandler, google_version, google_reqId, st->last_updated.tv_sec);
 828         }
 829         else if(format == DATASOURCE_JSONP) {
 830                 if(responseHandler == NULL)
 831                         responseHandler = "callback";
 832
 833                 buffer_strcat(w->response.data, responseHandler);
 834                 buffer_strcat(w->response.data, "(");
 835         }
 836
 837         ret = rrd2format(st, w->response.data, dimensions, format, points, after, before, group, options, &last_timestamp_in_data);
 838
 839         if(format == DATASOURCE_DATATABLE_JSONP) {
 840                 if(google_timestamp < last_timestamp_in_data)
 841                         buffer_strcat(w->response.data, "});");
 842
 843                 else {
 844                         // the client already has the latest data
 845                         buffer_flush(w->response.data);
 846                         buffer_sprintf(w->response.data,
 847                                 "%s({version:'%s',reqId:'%s',status:'error',errors:[{reason:'not_modified',message:'Data not modified'}]});",
 848                                 responseHandler, google_version, google_reqId);
 849                 }
 850         }
 851         else if(format == DATASOURCE_JSONP)
 852                 buffer_strcat(w->response.data, ");");
 853
 854 cleanup:
 855         if(dimensions) buffer_free(dimensions);
 856         return ret;
 857 }
 858
 859 int web_client_api_request_v1_registry(struct web_client *w, char *url)
 860 {
 861         static uint32_t hash_action = 0, hash_access = 0, hash_hello = 0, hash_delete = 0, hash_search = 0,
 862                         hash_switch = 0, hash_machine = 0, hash_url = 0, hash_name = 0, hash_delete_url = 0, hash_for = 0,
 863                         hash_to = 0;
 864
 865         if(unlikely(!hash_action)) {
 866                 hash_action = simple_hash("action");
 867                 hash_access = simple_hash("access");
 868                 hash_hello = simple_hash("hello");
 869                 hash_delete = simple_hash("delete");
 870                 hash_search = simple_hash("search");
 871                 hash_switch = simple_hash("switch");
 872                 hash_machine = simple_hash("machine");
 873                 hash_url = simple_hash("url");
 874                 hash_name = simple_hash("name");
 875                 hash_delete_url = simple_hash("delete_url");
 876                 hash_for = simple_hash("for");
 877                 hash_to = simple_hash("to");
 878         }
 879
 880         char person_guid[36 + 1] = "";
 881
 882         debug(D_WEB_CLIENT, "%llu: API v1 registry with URL '%s'", w->id, url);
 883
 884         // FIXME
 885         // The browser may send multiple cookies with our id
 886
 887         char *cookie = strstr(w->response.data->buffer, " " NETDATA_REGISTRY_COOKIE_NAME "=");
 888         if(cookie)
 889                 strncpyz(person_guid, &cookie[sizeof(NETDATA_REGISTRY_COOKIE_NAME) + 1], 36);
 890
 891         char action = '\0';
 892         char *machine_guid = NULL,
 893                         *machine_url = NULL,
 894                         *url_name = NULL,
 895                         *search_machine_guid = NULL,
 896                         *delete_url = NULL,
 897                         *to_person_guid = NULL;
 898
 899         while(url) {
 900                 char *value = mystrsep(&url, "?&[]");
 901                 if (!value || !*value) continue;
 902
 903                 char *name = mystrsep(&value, "=");
 904                 if (!name || !*name) continue;
 905                 if (!value || !*value) continue;
 906
 907                 debug(D_WEB_CLIENT, "%llu: API v1 registry query param '%s' with value '%s'", w->id, name, value);
 908
 909                 uint32_t hash = simple_hash(name);
 910
 911                 if(hash == hash_action && !strcmp(name, "action")) {
 912                         uint32_t vhash = simple_hash(value);
 913
 914                         if(vhash == hash_access && !strcmp(value, "access")) action = 'A';
 915                         else if(vhash == hash_hello && !strcmp(value, "hello")) action = 'H';
 916                         else if(vhash == hash_delete && !strcmp(value, "delete")) action = 'D';
 917                         else if(vhash == hash_search && !strcmp(value, "search")) action = 'S';
 918                         else if(vhash == hash_switch && !strcmp(value, "switch")) action = 'W';
 919                 }
 920                 else if(hash == hash_machine && !strcmp(name, "machine"))
 921                         machine_guid = value;
 922
 923                 else if(hash == hash_url && !strcmp(name, "url"))
 924                         machine_url = value;
 925
 926                 else if(action == 'A') {
 927                         if(hash == hash_name && !strcmp(name, "name"))
 928                                 url_name = value;
 929                 }
 930                 else if(action == 'D') {
 931                         if(hash == hash_delete_url && !strcmp(name, "delete_url"))
 932                                 delete_url = value;
 933                 }
 934                 else if(action == 'S') {
 935                         if(hash == hash_for && !strcmp(name, "for"))
 936                                 search_machine_guid = value;
 937                 }
 938                 else if(action == 'W') {
 939                         if(hash == hash_to && !strcmp(name, "to"))
 940                                 to_person_guid = value;
 941                 }
 942         }
 943
 944         if(action == 'A' && (!machine_guid || !machine_url || !url_name)) {
 945                 buffer_flush(w->response.data);
 946                 buffer_sprintf(w->response.data, "Invalid registry request - access requires these parameters: machine ('%s'), url ('%s'), name ('%s')",
 947                                            machine_guid?machine_guid:"UNSET", machine_url?machine_url:"UNSET", url_name?url_name:"UNSET");
 948                 return 400;
 949         }
 950         else if(action == 'D' && (!machine_guid || !machine_url || !delete_url)) {
 951                 buffer_flush(w->response.data);
 952                 buffer_sprintf(w->response.data, "Invalid registry request - delete requires these parameters: machine ('%s'), url ('%s'), delete_url ('%s')",
 953                                            machine_guid?machine_guid:"UNSET", machine_url?machine_url:"UNSET", delete_url?delete_url:"UNSET");
 954                 return 400;
 955         }
 956         else if(action == 'S' && (!machine_guid || !machine_url || !search_machine_guid)) {
 957                 buffer_flush(w->response.data);
 958                 buffer_sprintf(w->response.data, "Invalid registry request - search requires these parameters: machine ('%s'), url ('%s'), for ('%s')",
 959                                            machine_guid?machine_guid:"UNSET", machine_url?machine_url:"UNSET", search_machine_guid?search_machine_guid:"UNSET");
 960                 return 400;
 961         }
 962         else if(action == 'W' && (!machine_guid || !machine_url || !to_person_guid)) {
 963                 buffer_flush(w->response.data);
 964                 buffer_sprintf(w->response.data, "Invalid registry request - switching identity requires these parameters: machine ('%s'), url ('%s'), to ('%s')",
 965                                            machine_guid?machine_guid:"UNSET", machine_url?machine_url:"UNSET", to_person_guid?to_person_guid:"UNSET");
 966                 return 400;
 967         }
 968
 969         switch(action) {
 970                 case 'A':
 971                         return registry_request_access_json(w, person_guid, machine_guid, machine_url, url_name, time(NULL));
 972
 973                 case 'D':
 974                         return registry_request_delete_json(w, person_guid, machine_guid, machine_url, delete_url, time(NULL));
 975
 976                 case 'S':
 977                         return registry_request_search_json(w, person_guid, machine_guid, machine_url, search_machine_guid, time(NULL));
 978
 979                 case 'W':
 980                         return registry_request_switch_json(w, person_guid, machine_guid, machine_url, to_person_guid, time(NULL));
 981
 982                 case 'H':
 983                         return registry_request_hello_json(w);
 984
 985                 default:
 986                         buffer_flush(w->response.data);
 987                         buffer_sprintf(w->response.data, "Invalid registry request - you need to set an action: hello, access, delete, search");
 988                         return 400;
 989         }
 990
 991         buffer_flush(w->response.data);
 992         buffer_sprintf(w->response.data, "Invalid or no registry action.");
 993         return 400;
 994 }
 995
 996 int web_client_api_request_v1(struct web_client *w, char *url)
 997 {
 998         static uint32_t hash_data = 0, hash_chart = 0, hash_charts = 0, hash_registry = 0;
 999
1000         if(unlikely(hash_data == 0)) {
1001                 hash_data = simple_hash("data");
1002                 hash_chart = simple_hash("chart");
1003                 hash_charts = simple_hash("charts");
1004                 hash_registry = simple_hash("registry");
1005         }
1006
1007         // get the command
1008         char *tok = mystrsep(&url, "/?&");
1009         if(tok && *tok) {
1010                 debug(D_WEB_CLIENT, "%llu: Searching for API v1 command '%s'.", w->id, tok);
1011                 uint32_t hash = simple_hash(tok);
1012
1013                 if(hash == hash_data && !strcmp(tok, "data"))
1014                         return web_client_api_request_v1_data(w, url);
1015
1016                 else if(hash == hash_chart && !strcmp(tok, "chart"))
1017                         return web_client_api_request_v1_chart(w, url);
1018
1019                 else if(hash == hash_charts && !strcmp(tok, "charts"))
1020                         return web_client_api_request_v1_charts(w, url);
1021
1022                 else if(hash == hash_registry && !strcmp(tok, "registry"))
1023                         return web_client_api_request_v1_registry(w, url);
1024
1025                 else {
1026                         buffer_flush(w->response.data);
1027                         buffer_sprintf(w->response.data, "Unsupported v1 API command: %s", tok);
1028                         return 404;
1029                 }
1030         }
1031         else {
1032                 buffer_flush(w->response.data);
1033                 buffer_sprintf(w->response.data, "API v1 command?");
1034                 return 400;
1035         }
1036 }
1037
1038 int web_client_api_request(struct web_client *w, char *url)
1039 {
1040         // get the api version
1041         char *tok = mystrsep(&url, "/?&");
1042         if(tok && *tok) {
1043                 debug(D_WEB_CLIENT, "%llu: Searching for API version '%s'.", w->id, tok);
1044                 if(strcmp(tok, "v1") == 0)
1045                         return web_client_api_request_v1(w, url);
1046                 else {
1047                         buffer_flush(w->response.data);
1048                         buffer_sprintf(w->response.data, "Unsupported API version: %s", tok);
1049                         return 404;
1050                 }
1051         }
1052         else {
1053                 buffer_flush(w->response.data);
1054                 buffer_sprintf(w->response.data, "Which API version?");
1055                 return 400;
1056         }
1057 }
1058
1059 int web_client_api_old_data_request(struct web_client *w, char *url, int datasource_type)
1060 {
1061         RRDSET *st = NULL;
1062
1063         char *args = strchr(url, '?');
1064         if(args) {
1065                 *args='\0';
1066                 args = &args[1];
1067         }
1068
1069         // get the name of the data to show
1070         char *tok = mystrsep(&url, "/");
1071
1072         // do we have such a data set?
1073         if(tok && *tok) {
1074                 debug(D_WEB_CLIENT, "%llu: Searching for RRD data with name '%s'.", w->id, tok);
1075                 st = rrdset_find_byname(tok);
1076                 if(!st) st = rrdset_find(tok);
1077         }
1078
1079         if(!st) {
1080                 // we don't have it
1081                 // try to send a file with that name
1082                 buffer_flush(w->response.data);
1083                 return(mysendfile(w, tok));
1084         }
1085
1086         // we have it
1087         debug(D_WEB_CLIENT, "%llu: Found RRD data with name '%s'.", w->id, tok);
1088
1089         // how many entries does the client want?
1090         long lines = rrd_default_history_entries;
1091         long group_count = 1;
1092         time_t after = 0, before = 0;
1093         int group_method = GROUP_AVERAGE;
1094         int nonzero = 0;
1095
1096         if(url) {
1097                 // parse the lines required
1098                 tok = mystrsep(&url, "/");
1099                 if(tok) lines = atoi(tok);
1100                 if(lines < 1) lines = 1;
1101         }
1102         if(url) {
1103                 // parse the group count required
1104                 tok = mystrsep(&url, "/");
1105                 if(tok && *tok) group_count = atoi(tok);
1106                 if(group_count < 1) group_count = 1;
1107                 //if(group_count > save_history / 20) group_count = save_history / 20;
1108         }
1109         if(url) {
1110                 // parse the grouping method required
1111                 tok = mystrsep(&url, "/");
1112                 if(tok && *tok) {
1113                         if(strcmp(tok, "max") == 0) group_method = GROUP_MAX;
1114                         else if(strcmp(tok, "average") == 0) group_method = GROUP_AVERAGE;
1115                         else if(strcmp(tok, "sum") == 0) group_method = GROUP_SUM;
1116                         else debug(D_WEB_CLIENT, "%llu: Unknown group method '%s'", w->id, tok);
1117                 }
1118         }
1119         if(url) {
1120                 // parse after time
1121                 tok = mystrsep(&url, "/");
1122                 if(tok && *tok) after = strtoul(tok, NULL, 10);
1123                 if(after < 0) after = 0;
1124         }
1125         if(url) {
1126                 // parse before time
1127                 tok = mystrsep(&url, "/");
1128                 if(tok && *tok) before = strtoul(tok, NULL, 10);
1129                 if(before < 0) before = 0;
1130         }
1131         if(url) {
1132                 // parse nonzero
1133                 tok = mystrsep(&url, "/");
1134                 if(tok && *tok && strcmp(tok, "nonzero") == 0) nonzero = 1;
1135         }
1136
1137         w->response.data->contenttype = CT_APPLICATION_JSON;
1138         buffer_flush(w->response.data);
1139
1140         char *google_version = "0.6";
1141         char *google_reqId = "0";
1142         char *google_sig = "0";
1143         char *google_out = "json";
1144         char *google_responseHandler = "google.visualization.Query.setResponse";
1145         char *google_outFileName = NULL;
1146         time_t last_timestamp_in_data = 0;
1147         if(datasource_type == DATASOURCE_DATATABLE_JSON || datasource_type == DATASOURCE_DATATABLE_JSONP) {
1148
1149                 w->response.data->contenttype = CT_APPLICATION_X_JAVASCRIPT;
1150
1151                 while(args) {
1152                         tok = mystrsep(&args, "&");
1153                         if(tok && *tok) {
1154                                 char *name = mystrsep(&tok, "=");
1155                                 if(name && *name && strcmp(name, "tqx") == 0) {
1156                                         char *key = mystrsep(&tok, ":");
1157                                         char *value = mystrsep(&tok, ";");
1158                                         if(key && value && *key && *value) {
1159                                                 if(strcmp(key, "version") == 0)
1160                                                         google_version = value;
1161
1162                                                 else if(strcmp(key, "reqId") == 0)
1163                                                         google_reqId = value;
1164
1165                                                 else if(strcmp(key, "sig") == 0)
1166                                                         google_sig = value;
1167
1168                                                 else if(strcmp(key, "out") == 0)
1169                                                         google_out = value;
1170
1171                                                 else if(strcmp(key, "responseHandler") == 0)
1172                                                         google_responseHandler = value;
1173
1174                                                 else if(strcmp(key, "outFileName") == 0)
1175                                                         google_outFileName = value;
1176                                         }
1177                                 }
1178                         }
1179                 }
1180
1181                 debug(D_WEB_CLIENT_ACCESS, "%llu: GOOGLE JSONP: version = '%s', reqId = '%s', sig = '%s', out = '%s', responseHandler = '%s', outFileName = '%s'",
1182                         w->id, google_version, google_reqId, google_sig, google_out, google_responseHandler, google_outFileName
1183                         );
1184
1185                 if(datasource_type == DATASOURCE_DATATABLE_JSONP) {
1186                         last_timestamp_in_data = strtoul(google_sig, NULL, 0);
1187
1188                         // check the client wants json
1189                         if(strcmp(google_out, "json") != 0) {
1190                                 buffer_sprintf(w->response.data,
1191                                         "%s({version:'%s',reqId:'%s',status:'error',errors:[{reason:'invalid_query',message:'output format is not supported',detailed_message:'the format %s requested is not supported by netdata.'}]});",
1192                                         google_responseHandler, google_version, google_reqId, google_out);
1193                                         return 200;
1194                         }
1195                 }
1196         }
1197
1198         if(datasource_type == DATASOURCE_DATATABLE_JSONP) {
1199                 buffer_sprintf(w->response.data,
1200                         "%s({version:'%s',reqId:'%s',status:'ok',sig:'%lu',table:",
1201                         google_responseHandler, google_version, google_reqId, st->last_updated.tv_sec);
1202         }
1203
1204         debug(D_WEB_CLIENT_ACCESS, "%llu: Sending RRD data '%s' (id %s, %d lines, %d group, %d group_method, %lu after, %lu before).", w->id, st->name, st->id, lines, group_count, group_method, after, before);
1205         time_t timestamp_in_data = rrd_stats_json(datasource_type, st, w->response.data, lines, group_count, group_method, after, before, nonzero);
1206
1207         if(datasource_type == DATASOURCE_DATATABLE_JSONP) {
1208                 if(timestamp_in_data > last_timestamp_in_data)
1209                         buffer_strcat(w->response.data, "});");
1210
1211                 else {
1212                         // the client already has the latest data
1213                         buffer_flush(w->response.data);
1214                         buffer_sprintf(w->response.data,
1215                                 "%s({version:'%s',reqId:'%s',status:'error',errors:[{reason:'not_modified',message:'Data not modified'}]});",
1216                                 google_responseHandler, google_version, google_reqId);
1217                 }
1218         }
1219
1220         return 200;
1221 }
1222
1223 const char *web_content_type_to_string(uint8_t contenttype) {
1224         switch(contenttype) {
1225                 case CT_TEXT_HTML:
1226                         return "text/html; charset=utf-8";
1227
1228                 case CT_APPLICATION_XML:
1229                         return "application/xml; charset=utf-8";
1230
1231                 case CT_APPLICATION_JSON:
1232                         return "application/json; charset=utf-8";
1233
1234                 case CT_APPLICATION_X_JAVASCRIPT:
1235                         return "application/x-javascript; charset=utf-8";
1236
1237                 case CT_TEXT_CSS:
1238                         return "text/css; charset=utf-8";
1239
1240                 case CT_TEXT_XML:
1241                         return "text/xml; charset=utf-8";
1242
1243                 case CT_TEXT_XSL:
1244                         return "text/xsl; charset=utf-8";
1245
1246                 case CT_APPLICATION_OCTET_STREAM:
1247                         return "application/octet-stream";
1248
1249                 case CT_IMAGE_SVG_XML:
1250                         return "image/svg+xml";
1251
1252                 case CT_APPLICATION_X_FONT_TRUETYPE:
1253                         return "application/x-font-truetype";
1254
1255                 case CT_APPLICATION_X_FONT_OPENTYPE:
1256                         return "application/x-font-opentype";
1257
1258                 case CT_APPLICATION_FONT_WOFF:
1259                         return "application/font-woff";
1260
1261                 case CT_APPLICATION_FONT_WOFF2:
1262                         return "application/font-woff2";
1263
1264                 case CT_APPLICATION_VND_MS_FONTOBJ:
1265                         return "application/vnd.ms-fontobject";
1266
1267                 case CT_IMAGE_PNG:
1268                         return "image/png";
1269
1270                 case CT_IMAGE_JPG:
1271                         return "image/jpeg";
1272
1273                 case CT_IMAGE_GIF:
1274                         return "image/gif";
1275
1276                 case CT_IMAGE_XICON:
1277                         return "image/x-icon";
1278
1279                 case CT_IMAGE_BMP:
1280                         return "image/bmp";
1281
1282                 case CT_IMAGE_ICNS:
1283                         return "image/icns";
1284
1285                 default:
1286                 case CT_TEXT_PLAIN:
1287                         return "text/plain; charset=utf-8";
1288         }
1289 }
1290
1291
1292 const char *web_response_code_to_string(int code) {
1293         switch(code) {
1294                 case 200:
1295                         return "OK";
1296
1297                 case 307:
1298                         return "Temporary Redirect";
1299
1300                 case 400:
1301                         return "Bad Request";
1302
1303                 case 403:
1304                         return "Forbidden";
1305
1306                 case 404:
1307                         return "Not Found";
1308
1309                 case 412:
1310                         return "Preconditions Failed";
1311
1312                 default:
1313                         if(code >= 100 && code < 200)
1314                                 return "Informational";
1315
1316                         if(code >= 200 && code < 300)
1317                                 return "Successful";
1318
1319                         if(code >= 300 && code < 400)
1320                                 return "Redirection";
1321
1322                         if(code >= 400 && code < 500)
1323                                 return "Bad Request";
1324
1325                         if(code >= 500 && code < 600)
1326                                 return "Server Error";
1327
1328                         return "Undefined Error";
1329         }
1330 }
1331
1332 static inline char *http_header_parse(struct web_client *w, char *s) {
1333         static uint32_t hash_origin = 0, hash_connection = 0, hash_accept_encoding = 0;
1334
1335         if(unlikely(!hash_origin)) {
1336                 hash_origin = simple_uhash("Origin");
1337                 hash_connection = simple_uhash("Connection");
1338                 hash_accept_encoding = simple_uhash("Accept-Encoding");
1339         }
1340
1341         char *e = s;
1342
1343         // find the :
1344         while(*e && *e != ':') e++;
1345         if(!*e || e[1] != ' ') return e;
1346
1347         // get the name
1348         *e = '\0';
1349
1350         // find the value
1351         char *v, *ve;
1352         v = ve = e + 2;
1353
1354         // find the \r
1355         while(*ve && *ve != '\r') ve++;
1356         if(!*ve || ve[1] != '\n') {
1357                 *e = ':';
1358                 return ve;
1359         }
1360
1361         // terminate the value
1362         *ve = '\0';
1363
1364         // fprintf(stderr, "HEADER: '%s' = '%s'\n", s, v);
1365         uint32_t hash = simple_uhash(s);
1366
1367         if(hash == hash_origin && !strcasecmp(s, "Origin"))
1368                 strncpyz(w->origin, v, ORIGIN_MAX);
1369
1370         else if(hash == hash_connection && !strcasecmp(s, "Connection")) {
1371                 if(strcasestr(v, "keep-alive"))
1372                         w->keepalive = 1;
1373         }
1374 #ifdef NETDATA_WITH_ZLIB
1375         else if(hash == hash_accept_encoding && !strcasecmp(s, "Accept-Encoding")) {
1376                 if(web_enable_gzip) {
1377                         if(strcasestr(v, "gzip"))
1378                                 web_client_enable_deflate(w, 1);
1379                         //
1380                         // does not seem to work
1381                         // else if(strcasestr(v, "deflate"))
1382                         //      web_client_enable_deflate(w, 0);
1383                 }
1384         }
1385 #endif /* NETDATA_WITH_ZLIB */
1386
1387         *e = ':';
1388         *ve = '\r';
1389         return ve;
1390 }
1391
1392 // http_request_validate()
1393 // returns:
1394 // = 0 : all good, process the request
1395 // > 0 : request is not supported
1396 // < 0 : request is incomplete - wait for more data
1397
1398 static inline int http_request_validate(struct web_client *w) {
1399         char *s = w->response.data->buffer, *encoded_url = NULL;
1400
1401         // is is a valid request?
1402         if(!strncmp(s, "GET ", 4)) {
1403                 encoded_url = s = &s[4];
1404                 w->mode = WEB_CLIENT_MODE_NORMAL;
1405         }
1406         else if(!strncmp(s, "OPTIONS ", 8)) {
1407                 encoded_url = s = &s[8];
1408                 w->mode = WEB_CLIENT_MODE_OPTIONS;
1409         }
1410         else {
1411                 w->wait_receive = 0;
1412                 return 1;
1413         }
1414
1415         // find the SPACE + "HTTP/"
1416         while(*s) {
1417                 // find the next space
1418                 while (*s && *s != ' ') s++;
1419
1420                 // is it SPACE + "HTTP/" ?
1421                 if(*s && !strncmp(s, " HTTP/", 6)) break;
1422                 else s++;
1423         }
1424
1425         // incomplete requests
1426         if(unlikely(!*s)) {
1427                 w->wait_receive = 1;
1428                 return -2;
1429         }
1430
1431         // we have the end of encoded_url - remember it
1432         char *ue = s;
1433
1434         // make sure we have complete request
1435         // complete requests contain: \r\n\r\n
1436         while(*s) {
1437                 // find a line feed
1438                 while(*s && *s++ != '\r');
1439
1440                 // did we reach the end?
1441                 if(unlikely(!*s)) break;
1442
1443                 // is it \r\n ?
1444                 if(likely(*s++ == '\n')) {
1445
1446                         // is it again \r\n ? (header end)
1447                         if(unlikely(*s == '\r' && s[1] == '\n')) {
1448                                 // a valid complete HTTP request found
1449
1450                                 *ue = '\0';
1451                                 url_decode_r(w->decoded_url, encoded_url, URL_MAX + 1);
1452                                 *ue = ' ';
1453
1454                                 // copy the URL - we are going to overwrite parts of it
1455                                 // FIXME -- we should avoid it
1456                                 strncpyz(w->last_url, w->decoded_url, URL_MAX);
1457
1458                                 w->wait_receive = 0;
1459                                 return 0;
1460                         }
1461
1462                         // another header line
1463                         s = http_header_parse(w, s);
1464                 }
1465         }
1466
1467         // incomplete request
1468         w->wait_receive = 1;
1469         return -3;
1470 }
1471
1472 void web_client_process(struct web_client *w) {
1473         static uint32_t hash_api = 0, hash_netdata_conf = 0, hash_data = 0, hash_datasource = 0, hash_graph = 0,
1474                         hash_list = 0, hash_all_json = 0, hash_exit = 0, hash_debug = 0, hash_mirror = 0;
1475
1476         if(unlikely(!hash_api)) {
1477                 hash_api = simple_hash("api");
1478                 hash_netdata_conf = simple_hash("netdata.conf");
1479                 hash_data = simple_hash(WEB_PATH_DATA);
1480                 hash_datasource = simple_hash(WEB_PATH_DATASOURCE);
1481                 hash_graph = simple_hash(WEB_PATH_GRAPH);
1482                 hash_list = simple_hash("list");
1483                 hash_all_json = simple_hash("all.json");
1484                 hash_exit = simple_hash("exit");
1485                 hash_debug = simple_hash("debug");
1486                 hash_mirror = simple_hash("mirror");
1487         }
1488
1489         int code = 500;
1490         ssize_t bytes;
1491
1492         int what_to_do = http_request_validate(w);
1493
1494         // wait for more data
1495         if(what_to_do < 0) {
1496                 if(w->response.data->len > TOO_BIG_REQUEST) {
1497                         strcpy(w->last_url, "too big request");
1498
1499                         debug(D_WEB_CLIENT_ACCESS, "%llu: Received request is too big (%zd bytes).", w->id, w->response.data->len);
1500
1501                         code = 400;
1502                         buffer_flush(w->response.data);
1503                         buffer_sprintf(w->response.data, "Received request is too big  (%zd bytes).\r\n", w->response.data->len);
1504                 }
1505                 else {
1506                         // wait for more data
1507                         return;
1508                 }
1509         }
1510         else if(what_to_do > 0) {
1511                 strcpy(w->last_url, "not a valid request");
1512
1513                 debug(D_WEB_CLIENT_ACCESS, "%llu: Cannot understand '%s'.", w->id, w->response.data->buffer);
1514
1515                 code = 500;
1516                 buffer_flush(w->response.data);
1517                 buffer_strcat(w->response.data, "I don't understand you...\r\n");
1518         }
1519         else { // what_to_do == 0
1520                 gettimeofday(&w->tv_in, NULL);
1521
1522                 if(w->mode == WEB_CLIENT_MODE_OPTIONS) {
1523                         code = 200;
1524                         w->response.data->contenttype = CT_TEXT_PLAIN;
1525                         buffer_flush(w->response.data);
1526                         buffer_strcat(w->response.data, "OK");
1527                 }
1528                 else {
1529                         char *url = w->decoded_url;
1530                         char *tok = mystrsep(&url, "/?");
1531                         if(tok && *tok) {
1532                                 uint32_t hash = simple_hash(tok);
1533                                 debug(D_WEB_CLIENT, "%llu: Processing command '%s'.", w->id, tok);
1534
1535                                 if(hash == hash_api && strcmp(tok, "api") == 0) {
1536                                         // the client is requesting api access
1537                                         code = web_client_api_request(w, url);
1538                                 }
1539                                 else if(hash == hash_netdata_conf && strcmp(tok, "netdata.conf") == 0) {
1540                                         code = 200;
1541                                         debug(D_WEB_CLIENT_ACCESS, "%llu: Sending netdata.conf ...", w->id);
1542
1543                                         w->response.data->contenttype = CT_TEXT_PLAIN;
1544                                         buffer_flush(w->response.data);
1545                                         generate_config(w->response.data, 0);
1546                                 }
1547                                 else if(hash == hash_data && strcmp(tok, WEB_PATH_DATA) == 0) { // "data"
1548                                         // the client is requesting rrd data -- OLD API
1549                                         code = web_client_api_old_data_request(w, url, DATASOURCE_JSON);
1550                                 }
1551                                 else if(hash == hash_datasource && strcmp(tok, WEB_PATH_DATASOURCE) == 0) { // "datasource"
1552                                         // the client is requesting google datasource -- OLD API
1553                                         code = web_client_api_old_data_request(w, url, DATASOURCE_DATATABLE_JSONP);
1554                                 }
1555                                 else if(hash == hash_graph && strcmp(tok, WEB_PATH_GRAPH) == 0) { // "graph"
1556                                         // the client is requesting an rrd graph -- OLD API
1557
1558                                         // get the name of the data to show
1559                                         tok = mystrsep(&url, "/?&");
1560                                         if(tok && *tok) {
1561                                                 debug(D_WEB_CLIENT, "%llu: Searching for RRD data with name '%s'.", w->id, tok);
1562
1563                                                 // do we have such a data set?
1564                                                 RRDSET *st = rrdset_find_byname(tok);
1565                                                 if(!st) st = rrdset_find(tok);
1566                                                 if(!st) {
1567                                                         // we don't have it
1568                                                         // try to send a file with that name
1569                                                         buffer_flush(w->response.data);
1570                                                         code = mysendfile(w, tok);
1571                                                 }
1572                                                 else {
1573                                                         code = 200;
1574                                                         debug(D_WEB_CLIENT_ACCESS, "%llu: Sending %s.json of RRD_STATS...", w->id, st->name);
1575                                                         w->response.data->contenttype = CT_APPLICATION_JSON;
1576                                                         buffer_flush(w->response.data);
1577                                                         rrd_stats_graph_json(st, url, w->response.data);
1578                                                 }
1579                                         }
1580                                         else {
1581                                                 code = 400;
1582                                                 buffer_flush(w->response.data);
1583                                                 buffer_strcat(w->response.data, "Graph name?\r\n");
1584                                         }
1585                                 }
1586                                 else if(hash == hash_list && strcmp(tok, "list") == 0) {
1587                                         // OLD API
1588                                         code = 200;
1589
1590                                         debug(D_WEB_CLIENT_ACCESS, "%llu: Sending list of RRD_STATS...", w->id);
1591
1592                                         buffer_flush(w->response.data);
1593                                         RRDSET *st = rrdset_root;
1594
1595                                         for ( ; st ; st = st->next )
1596                                                 buffer_sprintf(w->response.data, "%s\n", st->name);
1597                                 }
1598                                 else if(hash == hash_all_json && strcmp(tok, "all.json") == 0) {
1599                                         // OLD API
1600                                         code = 200;
1601                                         debug(D_WEB_CLIENT_ACCESS, "%llu: Sending JSON list of all monitors of RRD_STATS...", w->id);
1602
1603                                         w->response.data->contenttype = CT_APPLICATION_JSON;
1604                                         buffer_flush(w->response.data);
1605                                         rrd_stats_all_json(w->response.data);
1606                                 }
1607 #ifdef NETDATA_INTERNAL_CHECKS
1608                                 else if(hash == hash_exit && strcmp(tok, "exit") == 0) {
1609                                         code = 200;
1610                                         w->response.data->contenttype = CT_TEXT_PLAIN;
1611                                         buffer_flush(w->response.data);
1612
1613                                         if(!netdata_exit)
1614                                                 buffer_strcat(w->response.data, "ok, will do...");
1615                                         else
1616                                                 buffer_strcat(w->response.data, "I am doing it already");
1617
1618                                         netdata_exit = 1;
1619                                 }
1620                                 else if(hash == hash_debug && strcmp(tok, "debug") == 0) {
1621                                         buffer_flush(w->response.data);
1622
1623                                         // get the name of the data to show
1624                                         tok = mystrsep(&url, "/?&");
1625                                         if(tok && *tok) {
1626                                                 debug(D_WEB_CLIENT, "%llu: Searching for RRD data with name '%s'.", w->id, tok);
1627
1628                                                 // do we have such a data set?
1629                                                 RRDSET *st = rrdset_find_byname(tok);
1630                                                 if(!st) st = rrdset_find(tok);
1631                                                 if(!st) {
1632                                                         code = 404;
1633                                                         buffer_sprintf(w->response.data, "Chart %s is not found.\r\n", tok);
1634                                                         debug(D_WEB_CLIENT_ACCESS, "%llu: %s is not found.", w->id, tok);
1635                                                 }
1636                                                 else {
1637                                                         code = 200;
1638                                                         debug_flags |= D_RRD_STATS;
1639                                                         st->debug = !st->debug;
1640                                                         buffer_sprintf(w->response.data, "Chart %s has now debug %s.\r\n", tok, st->debug?"enabled":"disabled");
1641                                                         debug(D_WEB_CLIENT_ACCESS, "%llu: debug for %s is %s.", w->id, tok, st->debug?"enabled":"disabled");
1642                                                 }
1643                                         }
1644                                         else {
1645                                                 code = 500;
1646                                                 buffer_flush(w->response.data);
1647                                                 buffer_strcat(w->response.data, "debug which chart?\r\n");
1648                                         }
1649                                 }
1650                                 else if(hash == hash_mirror && strcmp(tok, "mirror") == 0) {
1651                                         code = 200;
1652
1653                                         debug(D_WEB_CLIENT_ACCESS, "%llu: Mirroring...", w->id);
1654
1655                                         // replace the zero bytes with spaces
1656                                         buffer_char_replace(w->response.data, '\0', ' ');
1657
1658                                         // just leave the buffer as is
1659                                         // it will be copied back to the client
1660                                 }
1661 #endif  /* NETDATA_INTERNAL_CHECKS */
1662                                 else {
1663                                         char filename[FILENAME_MAX+1];
1664                                         url = filename;
1665                                         strncpyz(filename, w->last_url, FILENAME_MAX);
1666                                         tok = mystrsep(&url, "?");
1667                                         buffer_flush(w->response.data);
1668                                         code = mysendfile(w, (tok && *tok)?tok:"/");
1669                                 }
1670                         }
1671                         else {
1672                                 char filename[FILENAME_MAX+1];
1673                                 url = filename;
1674                                 strncpyz(filename, w->last_url, FILENAME_MAX);
1675                                 tok = mystrsep(&url, "?");
1676                                 buffer_flush(w->response.data);
1677                                 code = mysendfile(w, (tok && *tok)?tok:"/");
1678                         }
1679                 }
1680         }
1681
1682         gettimeofday(&w->tv_ready, NULL);
1683         w->response.data->date = time(NULL);
1684         w->response.sent = 0;
1685         w->response.code = code;
1686
1687         // prepare the HTTP response header
1688         debug(D_WEB_CLIENT, "%llu: Generating HTTP header with response %d.", w->id, code);
1689
1690         const char *content_type_string = web_content_type_to_string(w->response.data->contenttype);
1691         const char *code_msg = web_response_code_to_string(code);
1692
1693         char date[100];
1694         struct tm tmbuf, *tm = gmtime_r(&w->response.data->date, &tmbuf);
1695         strftime(date, sizeof(date), "%a, %d %b %Y %H:%M:%S %Z", tm);
1696
1697         buffer_sprintf(w->response.header_output,
1698                 "HTTP/1.1 %d %s\r\n"
1699                 "Connection: %s\r\n"
1700                 "Server: NetData Embedded HTTP Server\r\n"
1701                 "Access-Control-Allow-Origin: %s\r\n"
1702                 "Access-Control-Allow-Credentials: true\r\n"
1703                 "Content-Type: %s\r\n"
1704                 "Date: %s\r\n"
1705                 , code, code_msg
1706                 , w->keepalive?"keep-alive":"close"
1707                 , w->origin
1708                 , content_type_string
1709                 , date
1710                 );
1711
1712         if(w->cookie1[0]) {
1713                 buffer_sprintf(w->response.header_output,
1714                    "Set-Cookie: %s\r\n",
1715                    w->cookie1);
1716         }
1717
1718         if(w->cookie2[0]) {
1719                 buffer_sprintf(w->response.header_output,
1720                    "Set-Cookie: %s\r\n",
1721                    w->cookie2);
1722         }
1723
1724         if(w->mode == WEB_CLIENT_MODE_OPTIONS) {
1725                 buffer_strcat(w->response.header_output,
1726                         "Access-Control-Allow-Methods: GET, OPTIONS\r\n"
1727                         "Access-Control-Allow-Headers: accept, x-requested-with, origin, content-type, cookie\r\n"
1728                         "Access-Control-Max-Age: 1209600\r\n" // 86400 * 14
1729                         );
1730         }
1731
1732         if(buffer_strlen(w->response.header))
1733                 buffer_strcat(w->response.header_output, buffer_tostring(w->response.header));
1734
1735         if(w->mode == WEB_CLIENT_MODE_NORMAL && (w->response.data->options & WB_CONTENT_NO_CACHEABLE)) {
1736                 buffer_sprintf(w->response.header_output,
1737                         "Expires: %s\r\n"
1738                         "Cache-Control: no-cache\r\n"
1739                         , date);
1740         }
1741         else if(w->mode != WEB_CLIENT_MODE_OPTIONS) {
1742                 char edate[100];
1743                 time_t et = w->response.data->date + (86400 * 14);
1744                 struct tm etmbuf, *etm = gmtime_r(&et, &etmbuf);
1745                 strftime(edate, sizeof(edate), "%a, %d %b %Y %H:%M:%S %Z", etm);
1746
1747                 buffer_sprintf(w->response.header_output,
1748                         "Expires: %s\r\n"
1749                         "Cache-Control: public\r\n"
1750                         , edate);
1751         }
1752
1753         // if we know the content length, put it
1754         if(!w->response.zoutput && (w->response.data->len || w->response.rlen))
1755                 buffer_sprintf(w->response.header_output,
1756                         "Content-Length: %ld\r\n"
1757                         , w->response.data->len? w->response.data->len: w->response.rlen
1758                         );
1759         else if(!w->response.zoutput)
1760                 w->keepalive = 0;       // content-length is required for keep-alive
1761
1762         if(w->response.zoutput) {
1763                 buffer_strcat(w->response.header_output,
1764                         "Content-Encoding: gzip\r\n"
1765                         "Transfer-Encoding: chunked\r\n"
1766                         );
1767         }
1768
1769         buffer_strcat(w->response.header_output, "\r\n");
1770
1771         // sent the HTTP header
1772         debug(D_WEB_DATA, "%llu: Sending response HTTP header of size %d: '%s'"
1773                         , w->id
1774                         , buffer_strlen(w->response.header_output)
1775                         , buffer_tostring(w->response.header_output)
1776                         );
1777
1778         web_client_crock_socket(w);
1779
1780         bytes = send(w->ofd, buffer_tostring(w->response.header_output), buffer_strlen(w->response.header_output), 0);
1781         if(bytes != (ssize_t) buffer_strlen(w->response.header_output)) {
1782                 if(bytes > 0)
1783                         w->stats_sent_bytes += bytes;
1784
1785                 debug(D_WEB_CLIENT, "%llu: HTTP Header failed to be sent (I sent %d bytes but the system sent %d bytes). Closing web client.", w->id,
1786                           buffer_strlen(w->response.header_output), bytes);
1787
1788                 WEB_CLIENT_IS_DEAD(w);
1789                 return;
1790         }
1791         else
1792                 w->stats_sent_bytes += bytes;
1793
1794         // enable sending immediately if we have data
1795         if(w->response.data->len) w->wait_send = 1;
1796         else w->wait_send = 0;
1797
1798         // pretty logging
1799         switch(w->mode) {
1800                 case WEB_CLIENT_MODE_OPTIONS:
1801                         debug(D_WEB_CLIENT, "%llu: Done preparing the OPTIONS response. Sending data (%d bytes) to client.", w->id, w->response.data->len);
1802                         break;
1803
1804                 case WEB_CLIENT_MODE_NORMAL:
1805                         debug(D_WEB_CLIENT, "%llu: Done preparing the response. Sending data (%d bytes) to client.", w->id, w->response.data->len);
1806                         break;
1807
1808                 case WEB_CLIENT_MODE_FILECOPY:
1809                         if(w->response.rlen) {
1810                                 debug(D_WEB_CLIENT, "%llu: Done preparing the response. Will be sending data file of %d bytes to client.", w->id, w->response.rlen);
1811                                 w->wait_receive = 1;
1812
1813                                 /*
1814                                 // utilize the kernel sendfile() for copying the file to the socket.
1815                                 // this block of code can be commented, without anything missing.
1816                                 // when it is commented, the program will copy the data using async I/O.
1817                                 {
1818                                         long len = sendfile(w->ofd, w->ifd, NULL, w->response.data->rbytes);
1819                                         if(len != w->response.data->rbytes)
1820                                                 error("%llu: sendfile() should copy %ld bytes, but copied %ld. Falling back to manual copy.", w->id, w->response.data->rbytes, len);
1821                                         else
1822                                                 web_client_reset(w);
1823                                 }
1824                                 */
1825                         }
1826                         else
1827                                 debug(D_WEB_CLIENT, "%llu: Done preparing the response. Will be sending an unknown amount of bytes to client.", w->id);
1828                         break;
1829
1830                 default:
1831                         fatal("%llu: Unknown client mode %d.", w->id, w->mode);
1832                         break;
1833         }
1834 }
1835
1836 ssize_t web_client_send_chunk_header(struct web_client *w, size_t len)
1837 {
1838         debug(D_DEFLATE, "%llu: OPEN CHUNK of %d bytes (hex: %x).", w->id, len, len);
1839         char buf[1024];
1840         sprintf(buf, "%lX\r\n", len);
1841
1842         ssize_t bytes = send(w->ofd, buf, strlen(buf), 0);
1843         if(bytes > 0) {
1844                 debug(D_DEFLATE, "%llu: Sent chunk header %d bytes.", w->id, bytes);
1845                 w->stats_sent_bytes += bytes;
1846         }
1847
1848         else if(bytes == 0) {
1849                 debug(D_WEB_CLIENT, "%llu: Did not send chunk header to the client.", w->id);
1850                 WEB_CLIENT_IS_DEAD(w);
1851         }
1852         else {
1853                 debug(D_WEB_CLIENT, "%llu: Failed to send chunk header to client.", w->id);
1854                 WEB_CLIENT_IS_DEAD(w);
1855         }
1856
1857         return bytes;
1858 }
1859
1860 ssize_t web_client_send_chunk_close(struct web_client *w)
1861 {
1862         //debug(D_DEFLATE, "%llu: CLOSE CHUNK.", w->id);
1863
1864         ssize_t bytes = send(w->ofd, "\r\n", 2, 0);
1865         if(bytes > 0) {
1866                 debug(D_DEFLATE, "%llu: Sent chunk suffix %d bytes.", w->id, bytes);
1867                 w->stats_sent_bytes += bytes;
1868         }
1869
1870         else if(bytes == 0) {
1871                 debug(D_WEB_CLIENT, "%llu: Did not send chunk suffix to the client.", w->id);
1872                 WEB_CLIENT_IS_DEAD(w);
1873         }
1874         else {
1875                 debug(D_WEB_CLIENT, "%llu: Failed to send chunk suffix to client.", w->id);
1876                 WEB_CLIENT_IS_DEAD(w);
1877         }
1878
1879         return bytes;
1880 }
1881
1882 ssize_t web_client_send_chunk_finalize(struct web_client *w)
1883 {
1884         //debug(D_DEFLATE, "%llu: FINALIZE CHUNK.", w->id);
1885
1886         ssize_t bytes = send(w->ofd, "\r\n0\r\n\r\n", 7, 0);
1887         if(bytes > 0) {
1888                 debug(D_DEFLATE, "%llu: Sent chunk suffix %d bytes.", w->id, bytes);
1889                 w->stats_sent_bytes += bytes;
1890         }
1891
1892         else if(bytes == 0) {
1893                 debug(D_WEB_CLIENT, "%llu: Did not send chunk finalize suffix to the client.", w->id);
1894                 WEB_CLIENT_IS_DEAD(w);
1895         }
1896         else {
1897                 debug(D_WEB_CLIENT, "%llu: Failed to send chunk finalize suffix to client.", w->id);
1898                 WEB_CLIENT_IS_DEAD(w);
1899         }
1900
1901         return bytes;
1902 }
1903
1904 #ifdef NETDATA_WITH_ZLIB
1905 ssize_t web_client_send_deflate(struct web_client *w)
1906 {
1907         ssize_t len = 0, t = 0;
1908
1909         // when using compression,
1910         // w->response.sent is the amount of bytes passed through compression
1911
1912         debug(D_DEFLATE, "%llu: web_client_send_deflate(): w->response.data->len = %d, w->response.sent = %d, w->response.zhave = %d, w->response.zsent = %d, w->response.zstream.avail_in = %d, w->response.zstream.avail_out = %d, w->response.zstream.total_in = %d, w->response.zstream.total_out = %d.", w->id, w->response.data->len, w->response.sent, w->response.zhave, w->response.zsent, w->response.zstream.avail_in, w->response.zstream.avail_out, w->response.zstream.total_in, w->response.zstream.total_out);
1913
1914         if(w->response.data->len - w->response.sent == 0 && w->response.zstream.avail_in == 0 && w->response.zhave == w->response.zsent && w->response.zstream.avail_out != 0) {
1915                 // there is nothing to send
1916
1917                 debug(D_WEB_CLIENT, "%llu: Out of output data.", w->id);
1918
1919                 // finalize the chunk
1920                 if(w->response.sent != 0) {
1921                         t = web_client_send_chunk_finalize(w);
1922                         if(t < 0) return t;
1923                 }
1924
1925                 if(w->mode == WEB_CLIENT_MODE_FILECOPY && w->wait_receive && w->response.rlen && w->response.rlen > w->response.data->len) {
1926                         // we have to wait, more data will come
1927                         debug(D_WEB_CLIENT, "%llu: Waiting for more data to become available.", w->id);
1928                         w->wait_send = 0;
1929                         return t;
1930                 }
1931
1932                 if(unlikely(!w->keepalive)) {
1933                         debug(D_WEB_CLIENT, "%llu: Closing (keep-alive is not enabled). %ld bytes sent.", w->id, w->response.sent);
1934                         WEB_CLIENT_IS_DEAD(w);
1935                         return t;
1936                 }
1937
1938                 // reset the client
1939                 web_client_reset(w);
1940                 debug(D_WEB_CLIENT, "%llu: Done sending all data on socket.", w->id);
1941                 return t;
1942         }
1943
1944         if(w->response.zhave == w->response.zsent) {
1945                 // compress more input data
1946
1947                 // close the previous open chunk
1948                 if(w->response.sent != 0) {
1949                         t = web_client_send_chunk_close(w);
1950                         if(t < 0) return t;
1951                 }
1952
1953                 debug(D_DEFLATE, "%llu: Compressing %d new bytes starting from %d (and %d left behind).", w->id, (w->response.data->len - w->response.sent), w->response.sent, w->response.zstream.avail_in);
1954
1955                 // give the compressor all the data not passed through the compressor yet
1956                 if(w->response.data->len > w->response.sent) {
1957                         w->response.zstream.next_in = (Bytef *)&w->response.data->buffer[w->response.sent - w->response.zstream.avail_in];
1958                         w->response.zstream.avail_in += (uInt) (w->response.data->len - w->response.sent);
1959                 }
1960
1961                 // reset the compressor output buffer
1962                 w->response.zstream.next_out = w->response.zbuffer;
1963                 w->response.zstream.avail_out = ZLIB_CHUNK;
1964
1965                 // ask for FINISH if we have all the input
1966                 int flush = Z_SYNC_FLUSH;
1967                 if(w->mode == WEB_CLIENT_MODE_NORMAL
1968                         || (w->mode == WEB_CLIENT_MODE_FILECOPY && !w->wait_receive && w->response.data->len == w->response.rlen)) {
1969                         flush = Z_FINISH;
1970                         debug(D_DEFLATE, "%llu: Requesting Z_FINISH, if possible.", w->id);
1971                 }
1972                 else {
1973                         debug(D_DEFLATE, "%llu: Requesting Z_SYNC_FLUSH.", w->id);
1974                 }
1975
1976                 // compress
1977                 if(deflate(&w->response.zstream, flush) == Z_STREAM_ERROR) {
1978                         error("%llu: Compression failed. Closing down client.", w->id);
1979                         web_client_reset(w);
1980                         return(-1);
1981                 }
1982
1983                 w->response.zhave = ZLIB_CHUNK - w->response.zstream.avail_out;
1984                 w->response.zsent = 0;
1985
1986                 // keep track of the bytes passed through the compressor
1987                 w->response.sent = w->response.data->len;
1988
1989                 debug(D_DEFLATE, "%llu: Compression produced %d bytes.", w->id, w->response.zhave);
1990
1991                 // open a new chunk
1992                 ssize_t t2 = web_client_send_chunk_header(w, w->response.zhave);
1993                 if(t2 < 0) return t2;
1994                 t += t2;
1995         }
1996
1997         debug(D_WEB_CLIENT, "%llu: Sending %d bytes of data (+%d of chunk header).", w->id, w->response.zhave - w->response.zsent, t);
1998
1999         len = send(w->ofd, &w->response.zbuffer[w->response.zsent], (size_t) (w->response.zhave - w->response.zsent), MSG_DONTWAIT);
2000         if(len > 0) {
2001                 w->stats_sent_bytes += len;
2002                 w->response.zsent += len;
2003                 len += t;
2004                 debug(D_WEB_CLIENT, "%llu: Sent %d bytes.", w->id, len);
2005         }
2006         else if(len == 0) {
2007                 debug(D_WEB_CLIENT, "%llu: Did not send any bytes to the client (zhave = %ld, zsent = %ld, need to send = %ld).", w->id, w->response.zhave, w->response.zsent, w->response.zhave - w->response.zsent);
2008                 WEB_CLIENT_IS_DEAD(w);
2009         }
2010         else {
2011                 debug(D_WEB_CLIENT, "%llu: Failed to send data to client.", w->id);
2012                 WEB_CLIENT_IS_DEAD(w);
2013         }
2014
2015         return(len);
2016 }
2017 #endif // NETDATA_WITH_ZLIB
2018
2019 ssize_t web_client_send(struct web_client *w) {
2020 #ifdef NETDATA_WITH_ZLIB
2021         if(likely(w->response.zoutput)) return web_client_send_deflate(w);
2022 #endif // NETDATA_WITH_ZLIB
2023
2024         ssize_t bytes;
2025
2026         if(unlikely(w->response.data->len - w->response.sent == 0)) {
2027                 // there is nothing to send
2028
2029                 debug(D_WEB_CLIENT, "%llu: Out of output data.", w->id);
2030
2031                 // there can be two cases for this
2032                 // A. we have done everything
2033                 // B. we temporarily have nothing to send, waiting for the buffer to be filled by ifd
2034
2035                 if(w->mode == WEB_CLIENT_MODE_FILECOPY && w->wait_receive && w->response.rlen && w->response.rlen > w->response.data->len) {
2036                         // we have to wait, more data will come
2037                         debug(D_WEB_CLIENT, "%llu: Waiting for more data to become available.", w->id);
2038                         w->wait_send = 0;
2039                         return 0;
2040                 }
2041
2042                 if(unlikely(!w->keepalive)) {
2043                         debug(D_WEB_CLIENT, "%llu: Closing (keep-alive is not enabled). %ld bytes sent.", w->id, w->response.sent);
2044                         WEB_CLIENT_IS_DEAD(w);
2045                         return 0;
2046                 }
2047
2048                 web_client_reset(w);
2049                 debug(D_WEB_CLIENT, "%llu: Done sending all data on socket. Waiting for next request on the same socket.", w->id);
2050                 return 0;
2051         }
2052
2053         bytes = send(w->ofd, &w->response.data->buffer[w->response.sent], w->response.data->len - w->response.sent, MSG_DONTWAIT);
2054         if(likely(bytes > 0)) {
2055                 w->stats_sent_bytes += bytes;
2056                 w->response.sent += bytes;
2057                 debug(D_WEB_CLIENT, "%llu: Sent %d bytes.", w->id, bytes);
2058         }
2059         else if(likely(bytes == 0)) {
2060                 debug(D_WEB_CLIENT, "%llu: Did not send any bytes to the client.", w->id);
2061                 WEB_CLIENT_IS_DEAD(w);
2062         }
2063         else {
2064                 debug(D_WEB_CLIENT, "%llu: Failed to send data to client.", w->id);
2065                 WEB_CLIENT_IS_DEAD(w);
2066         }
2067
2068         return(bytes);
2069 }
2070
2071 ssize_t web_client_receive(struct web_client *w)
2072 {
2073         // do we have any space for more data?
2074         buffer_need_bytes(w->response.data, WEB_REQUEST_LENGTH);
2075
2076         ssize_t left = w->response.data->size - w->response.data->len;
2077         ssize_t bytes;
2078
2079         if(unlikely(w->mode == WEB_CLIENT_MODE_FILECOPY))
2080                 bytes = read(w->ifd, &w->response.data->buffer[w->response.data->len], (size_t) (left - 1));
2081         else
2082                 bytes = recv(w->ifd, &w->response.data->buffer[w->response.data->len], (size_t) (left - 1), MSG_DONTWAIT);
2083
2084         if(likely(bytes > 0)) {
2085                 if(w->mode != WEB_CLIENT_MODE_FILECOPY)
2086                         w->stats_received_bytes += bytes;
2087
2088                 size_t old = w->response.data->len;
2089                 w->response.data->len += bytes;
2090                 w->response.data->buffer[w->response.data->len] = '\0';
2091
2092                 debug(D_WEB_CLIENT, "%llu: Received %d bytes.", w->id, bytes);
2093                 debug(D_WEB_DATA, "%llu: Received data: '%s'.", w->id, &w->response.data->buffer[old]);
2094
2095                 if(w->mode == WEB_CLIENT_MODE_FILECOPY) {
2096                         w->wait_send = 1;
2097
2098                         if(w->response.rlen && w->response.data->len >= w->response.rlen)
2099                                 w->wait_receive = 0;
2100                 }
2101         }
2102         else if(likely(bytes == 0)) {
2103                 debug(D_WEB_CLIENT, "%llu: Out of input data.", w->id);
2104
2105                 // if we cannot read, it means we have an error on input.
2106                 // if however, we are copying a file from ifd to ofd, we should not return an error.
2107                 // in this case, the error should be generated when the file has been sent to the client.
2108
2109                 if(w->mode == WEB_CLIENT_MODE_FILECOPY) {
2110                         // we are copying data from ifd to ofd
2111                         // let it finish copying...
2112                         w->wait_receive = 0;
2113
2114                         debug(D_WEB_CLIENT, "%llu: Read the whole file.", w->id);
2115                         if(w->ifd != w->ofd) close(w->ifd);
2116                         w->ifd = w->ofd;
2117                 }
2118                 else {
2119                         debug(D_WEB_CLIENT, "%llu: failed to receive data.", w->id);
2120                         WEB_CLIENT_IS_DEAD(w);
2121                 }
2122         }
2123         else {
2124                 debug(D_WEB_CLIENT, "%llu: receive data failed.", w->id);
2125                 WEB_CLIENT_IS_DEAD(w);
2126         }
2127
2128         return(bytes);
2129 }
2130
2131
2132 // --------------------------------------------------------------------------------------
2133 // the thread of a single client
2134
2135 // 1. waits for input and output, using async I/O
2136 // 2. it processes HTTP requests
2137 // 3. it generates HTTP responses
2138 // 4. it copies data from input to output if mode is FILECOPY
2139
2140 void *web_client_main(void *ptr)
2141 {
2142         if(pthread_setcanceltype(PTHREAD_CANCEL_DEFERRED, NULL) != 0)
2143                 error("Cannot set pthread cancel type to DEFERRED.");
2144
2145         if(pthread_setcancelstate(PTHREAD_CANCEL_ENABLE, NULL) != 0)
2146                 error("Cannot set pthread cancel state to ENABLE.");
2147
2148         struct web_client *w = ptr;
2149         struct pollfd fds[2], *ifd, *ofd;
2150         int retval, fdmax = 0, timeout;
2151
2152         log_access("%llu: %s port %s connected on thread task id %d", w->id, w->client_ip, w->client_port, gettid());
2153
2154         for(;;) {
2155                 if(unlikely(w->dead)) {
2156                         debug(D_WEB_CLIENT, "%llu: client is dead.", w->id);
2157                         break;
2158                 }
2159                 else if(unlikely(!w->wait_receive && !w->wait_send)) {
2160                         debug(D_WEB_CLIENT, "%llu: client is not set for neither receiving nor sending data.");
2161                         break;
2162                 }
2163
2164                 if(unlikely(w->ifd < 0 || w->ofd < 0)) {
2165                         error("%llu: invalid file descriptor, ifd = %d, ofd = %d (required 0 <= fd", w->id, w->ifd, w->ofd);
2166                         break;
2167                 }
2168
2169                 if(w->ifd == w->ofd) {
2170                         fds[0].fd = w->ifd;
2171                         fds[0].events = 0;
2172                         fds[0].revents = 0;
2173
2174                         if(w->wait_receive) fds[0].events |= POLLIN;
2175                         if(w->wait_send)    fds[0].events |= POLLOUT;
2176
2177                         fds[1].fd = -1;
2178                         fds[1].events = 0;
2179                         fds[1].revents = 0;
2180
2181                         ifd = ofd = &fds[0];
2182
2183                         fdmax = 1;
2184                 }
2185                 else {
2186                         fds[0].fd = w->ifd;
2187                         fds[0].events = 0;
2188                         fds[0].revents = 0;
2189                         if(w->wait_receive) fds[0].events |= POLLIN;
2190                         ifd = &fds[0];
2191
2192                         fds[1].fd = w->ofd;
2193                         fds[1].events = 0;
2194                         fds[1].revents = 0;
2195                         if(w->wait_send)    fds[1].events |= POLLOUT;
2196                         ofd = &fds[1];
2197
2198                         fdmax = 2;
2199                 }
2200
2201                 debug(D_WEB_CLIENT, "%llu: Waiting socket async I/O for %s %s", w->id, w->wait_receive?"INPUT":"", w->wait_send?"OUTPUT":"");
2202                 errno = 0;
2203                 timeout = web_client_timeout * 1000;
2204                 retval = poll(fds, fdmax, timeout);
2205
2206                 if(unlikely(retval == -1)) {
2207                         if(errno == EAGAIN || errno == EINTR) {
2208                                 debug(D_WEB_CLIENT, "%llu: EAGAIN received.", w->id);
2209                                 continue;
2210                         }
2211
2212                         debug(D_WEB_CLIENT, "%llu: LISTENER: poll() failed (input fd = %d, output fd = %d). Closing client.", w->id, w->ifd, w->ofd);
2213                         break;
2214                 }
2215                 else if(unlikely(!retval)) {
2216                         debug(D_WEB_CLIENT, "%llu: Timeout while waiting socket async I/O for %s %s", w->id, w->wait_receive?"INPUT":"", w->wait_send?"OUTPUT":"");
2217                         break;
2218                 }
2219
2220                 int used = 0;
2221                 if(w->wait_send && ofd->revents & POLLOUT) {
2222                         used++;
2223                         if(web_client_send(w) < 0) {
2224                                 debug(D_WEB_CLIENT, "%llu: Cannot send data to client. Closing client.", w->id);
2225                                 break;
2226                         }
2227                 }
2228
2229                 if(w->wait_receive && (ifd->revents & POLLIN || ifd->revents & POLLPRI)) {
2230                         used++;
2231                         if(web_client_receive(w) < 0) {
2232                                 debug(D_WEB_CLIENT, "%llu: Cannot receive data from client. Closing client.", w->id);
2233                                 break;
2234                         }
2235
2236                         if(w->mode == WEB_CLIENT_MODE_NORMAL) {
2237                                 debug(D_WEB_CLIENT, "%llu: Attempting to process received data.", w->id);
2238                                 web_client_process(w);
2239                         }
2240                 }
2241
2242                 if(unlikely(!used)) {
2243                         debug(D_WEB_CLIENT_ACCESS, "%llu: Received error on socket.", w->id);
2244                         break;
2245                 }
2246         }
2247
2248         web_client_reset(w);
2249
2250         log_access("%llu: %s port %s disconnected from thread task id %d", w->id, w->client_ip, w->client_port, gettid());
2251         debug(D_WEB_CLIENT, "%llu: done...", w->id);
2252
2253         // close the sockets/files now
2254         // to free file descriptors
2255         if(w->ifd == w->ofd) {
2256                 if(w->ifd != -1) close(w->ifd);
2257         }
2258         else {
2259                 if(w->ifd != -1) close(w->ifd);
2260                 if(w->ofd != -1) close(w->ofd);
2261         }
2262         w->ifd = -1;
2263         w->ofd = -1;
2264
2265         w->obsolete = 1;
2266
2267         pthread_exit(NULL);
2268         return NULL;
2269 }