netdata.c

   1 // enable strcasestr()
   2 #define _GNU_SOURCE
   3
   4 #include <stdio.h>
   5 #include <stdlib.h>
   6 #include <string.h>
   7 #include <time.h>
   8 #include <unistd.h>
   9 #include <pwd.h>
  10 #include <sys/types.h>
  11 #include <sys/time.h>
  12 #include <sys/wait.h>
  13
  14 #include <sys/socket.h>
  15 #include <sys/select.h>
  16 #include <sys/resource.h>
  17 #include <sys/stat.h>
  18 #include <sys/sendfile.h>
  19
  20 #include <arpa/inet.h>
  21 #include <netinet/in.h>
  22 #include <netinet/tcp.h>
  23
  24 #include <errno.h>
  25 #include <stdarg.h>
  26 #include <locale.h>
  27 #include <signal.h>
  28 #include <ctype.h>
  29 #include <time.h>
  30 #include <fcntl.h>
  31 #include <syslog.h>
  32
  33 #include <pthread.h>
  34 #include <zlib.h>
  35 #include <malloc.h>
  36 #include <inttypes.h>
  37 #include <dirent.h>
  38 #include <sys/mman.h>
  39
  40 // enabling this will detach the plugins from netdata
  41 // each plugin will have its own process group
  42 // #define DETACH_PLUGINS_FROM_NETDATA
  43
  44 #define RRD_TYPE_NET                            "net"
  45 #define RRD_TYPE_NET_LEN                        strlen(RRD_TYPE_NET)
  46
  47 #define RRD_TYPE_TC                                     "tc"
  48 #define RRD_TYPE_TC_LEN                         strlen(RRD_TYPE_TC)
  49
  50 #define RRD_TYPE_DISK                           "disk"
  51 #define RRD_TYPE_DISK_LEN                       strlen(RRD_TYPE_DISK)
  52
  53 #define RRD_TYPE_NET_SNMP                       "ipv4"
  54 #define RRD_TYPE_NET_SNMP_LEN           strlen(RRD_TYPE_NET_SNMP)
  55
  56 #define RRD_TYPE_NET_STAT_CONNTRACK     "conntrack"
  57 #define RRD_TYPE_NET_STAT_CONNTRACK_LEN strlen(RRD_TYPE_NET_STAT_CONNTRACK)
  58
  59 #define RRD_TYPE_NET_IPVS                       "ipvs"
  60 #define RRD_TYPE_NET_IPVS_LEN           strlen(RRD_TYPE_NET_IPVS)
  61
  62 #define RRD_TYPE_STAT                           "cpu"
  63 #define RRD_TYPE_STAT_LEN                       strlen(RRD_TYPE_STAT)
  64
  65 #define WEB_PATH_FILE                           "file"
  66 #define WEB_PATH_DATA                           "data"
  67 #define WEB_PATH_DATASOURCE                     "datasource"
  68 #define WEB_PATH_GRAPH                          "graph"
  69
  70 // type of JSON generations
  71 #define DATASOURCE_JSON 0
  72 #define DATASOURCE_GOOGLE_JSON 1
  73 #define DATASOURCE_GOOGLE_JSONP 2
  74
  75 // internal defaults
  76 #define UPDATE_EVERY 1
  77 #define UPDATE_EVERY_MAX 3600
  78 #define LISTEN_PORT 19999
  79 #define HISTORY 3600
  80 #define HISTORY_MAX (86400*10)
  81 #define SAVE_PATH "/tmp"
  82
  83 #define D_WEB_BUFFER            0x00000001
  84 #define D_WEB_CLIENT            0x00000002
  85 #define D_LISTENER              0x00000004
  86 #define D_WEB_DATA              0x00000008
  87 #define D_OPTIONS               0x00000010
  88 #define D_PROCNETDEV_LOOP   0x00000020
  89 #define D_RRD_STATS             0x00000040
  90 #define D_WEB_CLIENT_ACCESS     0x00000080
  91 #define D_TC_LOOP           0x00000100
  92 #define D_DEFLATE           0x00000200
  93 #define D_CONFIG            0x00000400
  94 #define D_PLUGINSD          0x00000800
  95 #define D_CHILDS            0x00001000
  96 #define D_EXIT              0x00002000
  97
  98 #define CT_APPLICATION_JSON                             1
  99 #define CT_TEXT_PLAIN                                   2
 100 #define CT_TEXT_HTML                                    3
 101 #define CT_APPLICATION_X_JAVASCRIPT             4
 102 #define CT_TEXT_CSS                                             5
 103 #define CT_TEXT_XML                                             6
 104 #define CT_APPLICATION_XML                              7
 105 #define CT_TEXT_XSL                                             8
 106 #define CT_APPLICATION_OCTET_STREAM             9
 107 #define CT_APPLICATION_X_FONT_TRUETYPE  10
 108 #define CT_APPLICATION_X_FONT_OPENTYPE  11
 109 #define CT_APPLICATION_FONT_WOFF                12
 110 #define CT_APPLICATION_VND_MS_FONTOBJ   13
 111 #define CT_IMAGE_SVG_XML                                14
 112
 113
 114 #define DEBUG (D_WEB_CLIENT_ACCESS|D_LISTENER|D_RRD_STATS)
 115 //#define DEBUG 0xffffffff
 116 //#define DEBUG (0)
 117
 118 #define HOSTNAME_MAX 1024
 119 char *hostname;
 120
 121 #define EXIT_FAILURE 1
 122 #define LISTEN_BACKLOG 100
 123
 124 #define INITIAL_WEB_DATA_LENGTH 65536
 125 #define WEB_DATA_LENGTH_INCREASE_STEP 65536
 126 #define ZLIB_CHUNK      16384
 127
 128 #define MAX_HTTP_HEADER_SIZE 16384
 129
 130 #define MAX_PROC_NET_SNMP_LINE 4096
 131 #define MAX_PROC_NET_SNMP_NAME 1024
 132
 133 #define MAX_PROC_NET_STAT_CONNTRACK_LINE 4096
 134 #define MAX_PROC_NET_STAT_CONNTRACK_NAME 1024
 135
 136 #define MAX_PROC_NET_IPVS_LINE 4096
 137 #define MAX_PROC_NET_IPVS_NAME 1024
 138
 139 #define MAX_PROC_STAT_LINE 4096
 140 #define MAX_PROC_STAT_NAME 1024
 141
 142
 143 int silent = 0;
 144 int save_history = HISTORY;
 145 int update_every = UPDATE_EVERY;
 146 int listen_port = LISTEN_PORT;
 147
 148
 149 // ----------------------------------------------------------------------------
 150 // global statistics
 151
 152 struct global_statistics {
 153
 154         unsigned long long connected_clients;
 155
 156         unsigned long long web_requests;
 157
 158         unsigned long long bytes_received;
 159         unsigned long long bytes_sent;
 160
 161 } global_statistics = { 0ULL, 0ULL, 0ULL, 0ULL };
 162
 163 pthread_mutex_t global_statistics_mutex = PTHREAD_MUTEX_INITIALIZER;
 164
 165 void global_statistics_lock(void)
 166 {
 167         pthread_mutex_lock(&global_statistics_mutex);
 168 }
 169 void global_statistics_unlock(void)
 170 {
 171         pthread_mutex_unlock(&global_statistics_mutex);
 172 }
 173
 174 // ----------------------------------------------------------------------------
 175 // LOG
 176
 177 unsigned long long debug_flags = DEBUG;
 178
 179 int access_fd = -1;
 180 FILE *stdaccess = NULL;
 181
 182 int access_log_syslog = 1;
 183 int error_log_syslog = 1;
 184 int output_log_syslog = 1;      // debug log
 185
 186
 187 void log_date(FILE *out)
 188 {
 189                 char outstr[200];
 190                 time_t t;
 191                 struct tm *tmp;
 192
 193                 t = time(NULL);
 194                 tmp = localtime(&t);
 195
 196                 if (tmp == NULL) return;
 197                 if (strftime(outstr, sizeof(outstr), "%y-%m-%d %H:%M:%S", tmp) == 0) return;
 198
 199                 fprintf(out, "%s: ", outstr);
 200 }
 201
 202 int debug_variable;
 203 //#define debug(args...) debug_int(__FILE__, __FUNCTION__, __LINE__, ##args)
 204 #define debug(type, args...) do { if(!silent && debug_flags & type) debug_int(__FILE__, __FUNCTION__, __LINE__, ##args); } while(0)
 205
 206 void debug_int( const char *file, const char *function, const unsigned long line, const char *fmt, ... )
 207 {
 208         if(file) { ; }
 209         va_list args;
 210
 211         log_date(stdout);
 212         va_start( args, fmt );
 213         fprintf(stdout, "DEBUG (%04lu@%-15.15s): ", line, function);
 214         vfprintf( stdout, fmt, args );
 215         va_end( args );
 216         fprintf(stdout, "\n");
 217
 218         if(output_log_syslog) {
 219                 va_start( args, fmt );
 220                 vsyslog(LOG_ERR,  fmt, args );
 221                 va_end( args );
 222         }
 223 }
 224
 225 #define info(args...)  info_int(__FILE__, __FUNCTION__, __LINE__, ##args)
 226
 227 void info_int( const char *file, const char *function, const unsigned long line, const char *fmt, ... )
 228 {
 229         if(file) { ; }
 230         va_list args;
 231
 232         log_date(stderr);
 233
 234         va_start( args, fmt );
 235         if(debug_flags) fprintf(stderr, "INFO (%04lu@%-15.15s): ", line, function);
 236         else            fprintf(stderr, "INFO: ");
 237         vfprintf( stderr, fmt, args );
 238         va_end( args );
 239
 240         fprintf(stderr, "\n");
 241
 242         if(error_log_syslog) {
 243                 va_start( args, fmt );
 244                 vsyslog(LOG_INFO,  fmt, args );
 245                 va_end( args );
 246         }
 247 }
 248
 249 #define error(args...)  error_int(__FILE__, __FUNCTION__, __LINE__, ##args)
 250
 251 void error_int( const char *file, const char *function, const unsigned long line, const char *fmt, ... )
 252 {
 253         if(file) { ; }
 254         va_list args;
 255
 256         log_date(stderr);
 257
 258         va_start( args, fmt );
 259         if(debug_flags) fprintf(stderr, "ERROR (%04lu@%-15.15s): ", line, function);
 260         else            fprintf(stderr, "ERROR: ");
 261         vfprintf( stderr, fmt, args );
 262         va_end( args );
 263
 264         if(errno) {
 265                         fprintf(stderr, " (errno %d, %s)\n", errno, strerror(errno));
 266                         errno = 0;
 267         }
 268         else fprintf(stderr, "\n");
 269
 270         if(error_log_syslog) {
 271                 va_start( args, fmt );
 272                 vsyslog(LOG_ERR,  fmt, args );
 273                 va_end( args );
 274         }
 275 }
 276
 277 #define fatal(args...)  fatal_int(__FILE__, __FUNCTION__, __LINE__, ##args)
 278
 279 void fatal_int( const char *file, const char *function, const unsigned long line, const char *fmt, ... )
 280 {
 281         if(file) { ; }
 282         va_list args;
 283
 284         log_date(stderr);
 285
 286         va_start( args, fmt );
 287         if(debug_flags) fprintf(stderr, "FATAL (%04lu@%-15.15s): ", line, function);
 288         else            fprintf(stderr, "FATAL: ");
 289         vfprintf( stderr, fmt, args );
 290         va_end( args );
 291
 292         perror(" # ");
 293         fprintf(stderr, "\n");
 294
 295         if(error_log_syslog) {
 296                 va_start( args, fmt );
 297                 vsyslog(LOG_CRIT,  fmt, args );
 298                 va_end( args );
 299         }
 300
 301         exit(EXIT_FAILURE);
 302 }
 303
 304 void log_access( const char *fmt, ... )
 305 {
 306         va_list args;
 307
 308         if(stdaccess) {
 309                 log_date(stdaccess);
 310
 311                 va_start( args, fmt );
 312                 vfprintf( stdaccess, fmt, args );
 313                 va_end( args );
 314                 fprintf( stdaccess, "\n");
 315                 fflush( stdaccess );
 316         }
 317
 318         if(access_log_syslog) {
 319                 va_start( args, fmt );
 320                 vsyslog(LOG_INFO,  fmt, args );
 321                 va_end( args );
 322         }
 323 }
 324
 325
 326 // ----------------------------------------------------------------------------
 327 // helpers
 328
 329 unsigned long long usecdiff(struct timeval *now, struct timeval *last) {
 330                 return ((((now->tv_sec * 1000000ULL) + now->tv_usec) - ((last->tv_sec * 1000000ULL) + last->tv_usec)));
 331 }
 332
 333 unsigned long simple_hash(const char *name)
 334 {
 335         int i, len = strlen(name);
 336         unsigned long hash = 0;
 337
 338         for(i = 0; i < len ;i++) hash += (i * name[i]) + i + name[i];
 339
 340         return hash;
 341 }
 342
 343 #define PIPE_READ 0
 344 #define PIPE_WRITE 1
 345
 346 FILE *mypopen(const char *command, pid_t *pidptr)
 347 {
 348         int pipefd[2];
 349
 350         if(pipe(pipefd) == -1) return NULL;
 351
 352         int pid = fork();
 353         if(pid == -1) {
 354                 close(pipefd[PIPE_READ]);
 355                 close(pipefd[PIPE_WRITE]);
 356                 return NULL;
 357         }
 358         if(pid != 0) {
 359                 // the parent
 360                 *pidptr = pid;
 361                 close(pipefd[PIPE_WRITE]);
 362                 FILE *fp = fdopen(pipefd[PIPE_READ], "r");
 363                 return(fp);
 364         }
 365         // the child
 366
 367         // close all files
 368         int i;
 369         for(i = sysconf(_SC_OPEN_MAX); i > 0; i--)
 370                 if(i != STDIN_FILENO && i != STDERR_FILENO && i != pipefd[PIPE_WRITE]) close(i);
 371
 372         // move the pipe to stdout
 373         if(pipefd[PIPE_WRITE] != STDOUT_FILENO) {
 374                 dup2(pipefd[PIPE_WRITE], STDOUT_FILENO);
 375                 close(pipefd[PIPE_WRITE]);
 376         }
 377
 378 #ifdef DETACH_PLUGINS_FROM_NETDATA
 379         // this was an attempt to detach the child and use the suspend mode charts.d
 380         // unfortunatelly it does not work as expected.
 381
 382         // fork again to become session leader
 383         pid = fork();
 384         if(pid == -1) fprintf(stderr, "Cannot fork again on pid %d\n", getpid());
 385         if(pid != 0) {
 386                 // the parent
 387                 exit(0);
 388         }
 389
 390         // set a new process group id for just this child
 391         if( setpgid(0, 0) != 0 )
 392                 fprintf(stderr, "Cannot set a new process group for pid %d (%s)\n", getpid(), strerror(errno));
 393
 394         if( getpgid(0) != getpid() )
 395                 fprintf(stderr, "Process group set is incorrect. Expected %d, found %d\n", getpid(), getpgid(0));
 396
 397         if( setsid() != 0 )
 398                 fprintf(stderr, "Cannot set session id for pid %d (%s)\n", getpid(), strerror(errno));
 399
 400         fprintf(stdout, "MYPID %d\n", getpid());
 401         fflush(NULL);
 402 #endif
 403
 404         // ignore all signals
 405         for (i = 1 ; i < 65 ;i++) if(i != SIGSEGV) signal(i, SIG_DFL);
 406
 407         fprintf(stderr, "executing command: '%s' on pid %d.\n", command, getpid());
 408         execl("/bin/sh", "sh", "-c", command, NULL);
 409         exit(1);
 410 }
 411
 412 void mypclose(FILE *fp)
 413 {
 414         // this is a very poor implementation of pclose()
 415         // the caller should catch SIGCHLD and waitpid() on the exited child
 416         // otherwise the child will be a zombie forever
 417
 418         fclose(fp);
 419 }
 420
 421
 422 // ----------------------------------------------------------------------------
 423 // URL encode / decode
 424 // code from: http://www.geekhideout.com/urlcode.shtml
 425
 426 /* Converts a hex character to its integer value */
 427 char from_hex(char ch) {
 428         return (char)(isdigit(ch) ? ch - '0' : tolower(ch) - 'a' + 10);
 429 }
 430
 431 /* Converts an integer value to its hex character*/
 432 char to_hex(char code) {
 433         static char hex[] = "0123456789abcdef";
 434         return hex[code & 15];
 435 }
 436
 437 /* Returns a url-encoded version of str */
 438 /* IMPORTANT: be sure to free() the returned string after use */
 439 char *url_encode(char *str) {
 440         char *pstr = str,
 441                 *buf = malloc(strlen(str) * 3 + 1),
 442                 *pbuf = buf;
 443
 444         while (*pstr) {
 445                 if (isalnum(*pstr) || *pstr == '-' || *pstr == '_' || *pstr == '.' || *pstr == '~')
 446                         *pbuf++ = *pstr;
 447
 448                 else if (*pstr == ' ')
 449                         *pbuf++ = '+';
 450
 451                 else
 452                         *pbuf++ = '%', *pbuf++ = to_hex(*pstr >> 4), *pbuf++ = to_hex(*pstr & 15);
 453
 454                 pstr++;
 455         }
 456
 457         *pbuf = '\0';
 458
 459         return buf;
 460 }
 461
 462 /* Returns a url-decoded version of str */
 463 /* IMPORTANT: be sure to free() the returned string after use */
 464 char *url_decode(char *str) {
 465         char *pstr = str,
 466                 *buf = malloc(strlen(str) + 1),
 467                 *pbuf = buf;
 468
 469         if(!buf) fatal("Cannot allocate memory.");
 470
 471         while (*pstr) {
 472                 if (*pstr == '%') {
 473                         if (pstr[1] && pstr[2]) {
 474                                 *pbuf++ = from_hex(pstr[1]) << 4 | from_hex(pstr[2]);
 475                                 pstr += 2;
 476                         }
 477                 }
 478                 else if (*pstr == '+')
 479                         *pbuf++ = ' ';
 480
 481                 else
 482                         *pbuf++ = *pstr;
 483
 484                 pstr++;
 485         }
 486
 487         *pbuf = '\0';
 488
 489         return buf;
 490 }
 491
 492
 493 // ----------------------------------------------------------------------------
 494 // socket
 495
 496 int listen_fd = -1;
 497
 498 int create_listen_socket(int port)
 499 {
 500                 int sock=-1;
 501                 int sockopt=1;
 502                 struct sockaddr_in name;
 503
 504                 debug(D_LISTENER, "Creating new listening socket on port %d", port);
 505
 506                 sock = socket(AF_INET, SOCK_STREAM, 0);
 507                 if (sock < 0)
 508                                 fatal("socket() failed, errno=%d", errno);
 509
 510                 /* avoid "address already in use" */
 511                 setsockopt(sock, SOL_SOCKET, SO_REUSEADDR, (void*)&sockopt, sizeof(sockopt));
 512
 513                 memset(&name, 0, sizeof(struct sockaddr_in));
 514                 name.sin_family = AF_INET;
 515                 name.sin_port = htons (port);
 516                 name.sin_addr.s_addr = htonl (INADDR_ANY);
 517                 if (bind (sock, (struct sockaddr *) &name, sizeof (name)) < 0)
 518                         fatal("bind() failed, errno=%d", errno);
 519
 520                 if (listen(sock, LISTEN_BACKLOG) < 0)
 521                         fatal("listen() failed, errno=%d", errno);
 522
 523                 debug(D_LISTENER, "Listening Port %d created", port);
 524                 return sock;
 525 }
 526
 527
 528 // ----------------------------------------------------------------------------
 529 // CONFIG
 530
 531 #define CONFIG_MAX_NAME 100
 532 #define CONFIG_MAX_VALUE 1024
 533 #define CONFIG_FILENAME "netdata.conf"
 534 #define CONFIG_FILE_LINE_MAX 4096
 535
 536 pthread_rwlock_t config_rwlock = PTHREAD_RWLOCK_INITIALIZER;
 537
 538 struct config_value {
 539         char name[CONFIG_MAX_NAME + 1];
 540         char value[CONFIG_MAX_VALUE + 1];
 541
 542         unsigned long hash;             // a simple hash to speed up searching
 543                                                         // we first compare hashes, and only if the hashes are equal we do string comparisons
 544
 545         int loaded;                             // loaded from the user config
 546         int used;                               // has been accessed from the program
 547         int changed;                    // changed from the internal default
 548
 549         struct config_value *next;
 550 };
 551
 552 struct config {
 553         char name[CONFIG_MAX_NAME + 1];
 554
 555         unsigned long hash;             // a simple hash to speed up searching
 556                                                         // we first compare hashes, and only if the hashes are equal we do string comparisons
 557
 558         struct config_value *values;
 559
 560         struct config *next;
 561 } *config_root = NULL;
 562
 563 struct config_value *config_value_create(struct config *co, const char *name, const char *value)
 564 {
 565         debug(D_CONFIG, "Creating config entry for name '%s', value '%s', in section '%s'.", name, value, co->name);
 566
 567         struct config_value *cv = calloc(1, sizeof(struct config_value));
 568         if(!cv) fatal("Cannot allocate config_value");
 569
 570         strncpy(cv->name,  name,  CONFIG_MAX_NAME);
 571         strncpy(cv->value, value, CONFIG_MAX_VALUE);
 572         cv->hash = simple_hash(cv->name);
 573
 574         // no need for string termination, due to calloc()
 575
 576         struct config_value *cv2 = co->values;
 577         if(cv2) {
 578                 while (cv2->next) cv2 = cv2->next;
 579                 cv2->next = cv;
 580         }
 581         else co->values = cv;
 582
 583         return cv;
 584 }
 585
 586 struct config *config_create(const char *section)
 587 {
 588         debug(D_CONFIG, "Creating section '%s'.", section);
 589
 590         struct config *co = calloc(1, sizeof(struct config));
 591         if(!co) fatal("Cannot allocate config");
 592
 593         strncpy(co->name, section, CONFIG_MAX_NAME);
 594         co->hash = simple_hash(co->name);
 595
 596         // no need for string termination, due to calloc()
 597
 598         struct config *co2 = config_root;
 599         if(co2) {
 600                 while (co2->next) co2 = co2->next;
 601                 co2->next = co;
 602         }
 603         else config_root = co;
 604
 605         return co;
 606 }
 607
 608 struct config *config_find_section(const char *section)
 609 {
 610         struct config *co;
 611         unsigned long hash = simple_hash(section);
 612
 613         for(co = config_root; co ; co = co->next)
 614                 if(hash == co->hash)
 615                         if(strcmp(co->name, section) == 0)
 616                                 break;
 617
 618         return co;
 619 }
 620
 621 char *trim(char *s)
 622 {
 623         // skip leading spaces
 624         while(*s && isspace(*s)) s++;
 625         if(!*s || *s == '#') return NULL;
 626
 627         // skip tailing spaces
 628         int c = strlen(s) - 1;
 629         while(c >= 0 && isspace(s[c])) {
 630                 s[c] = '\0';
 631                 c--;
 632         }
 633         if(c < 0) return NULL;
 634         if(!*s) return NULL;
 635         return s;
 636 }
 637
 638 int load_config(char *filename, int overwrite_used)
 639 {
 640         int line = 0;
 641         struct config *co = NULL;
 642
 643         pthread_rwlock_wrlock(&config_rwlock);
 644
 645         char buffer[CONFIG_FILE_LINE_MAX + 1], *s;
 646
 647         if(!filename) filename = CONFIG_DIR "/" CONFIG_FILENAME;
 648         FILE *fp = fopen(filename, "r");
 649         if(!fp) {
 650                 error("Cannot open file '%s'", CONFIG_DIR "/" CONFIG_FILENAME);
 651                 pthread_rwlock_unlock(&config_rwlock);
 652                 return 0;
 653         }
 654
 655         while(fgets(buffer, CONFIG_FILE_LINE_MAX, fp) != NULL) {
 656                 buffer[CONFIG_FILE_LINE_MAX] = '\0';
 657                 line++;
 658
 659                 s = trim(buffer);
 660                 if(!s) {
 661                         debug(D_CONFIG, "Ignoring line %d, it is empty.", line);
 662                         continue;
 663                 }
 664
 665                 int len = strlen(s);
 666                 if(*s == '[' && s[len - 1] == ']') {
 667                         // new section
 668                         s[len - 1] = '\0';
 669                         s++;
 670
 671                         co = config_find_section(s);
 672                         if(!co) co = config_create(s);
 673
 674                         continue;
 675                 }
 676
 677                 if(!co) {
 678                         // line outside a section
 679                         error("Ignoring line %d ('%s'), it is outsize all sections.", line, s);
 680                         continue;
 681                 }
 682
 683                 char *name = s;
 684                 char *value = strchr(s, '=');
 685                 if(!value) {
 686                         error("Ignoring line %d ('%s'), there is no = in it.", line, s);
 687                         continue;
 688                 }
 689                 *value = '\0';
 690                 value++;
 691
 692                 name = trim(name);
 693                 value = trim(value);
 694
 695                 if(!name) {
 696                         error("Ignoring line %d, name is empty.", line);
 697                         continue;
 698                 }
 699                 if(!value) {
 700                         debug(D_CONFIG, "Ignoring line %d, value is empty.", line);
 701                         continue;
 702                 }
 703
 704                 struct config_value *cv;
 705                 for(cv = co->values; cv ; cv = cv->next)
 706                         if(strcmp(cv->name, name) == 0) break;
 707
 708                 if(!cv) cv = config_value_create(co, name, value);
 709                 else {
 710                         if((cv->used && overwrite_used) || !cv->used) {
 711                                 debug(D_CONFIG, "Overwriting '%s/%s'.", line, co->name, cv->name);
 712                                 strncpy(cv->value, value, CONFIG_MAX_VALUE);
 713                                 // termination is already there
 714                         }
 715                         else
 716                                 debug(D_CONFIG, "Ignoring line %d, '%s/%s' is already present and used.", line, co->name, cv->name);
 717                 }
 718                 cv->loaded = 1;
 719         }
 720
 721         fclose(fp);
 722
 723         pthread_rwlock_unlock(&config_rwlock);
 724         return 1;
 725 }
 726
 727 char *config_get(const char *section, const char *name, const char *default_value)
 728 {
 729         struct config_value *cv;
 730
 731         debug(D_CONFIG, "request to get config in section '%s', name '%s', default_value '%s'", section, name, default_value);
 732
 733         pthread_rwlock_rdlock(&config_rwlock);
 734
 735         struct config *co = config_find_section(section);
 736         if(!co) co = config_create(section);
 737
 738         unsigned long hash = simple_hash(name);
 739         for(cv = co->values; cv ; cv = cv->next)
 740                 if(hash == cv->hash)
 741                         if(strcmp(cv->name, name) == 0)
 742                                 break;
 743
 744         if(!cv) cv = config_value_create(co, name, default_value);
 745         cv->used = 1;
 746
 747         if(cv->loaded || cv->changed) {
 748                 // this is a loaded value from the config file
 749                 // if it is different that the default, mark it
 750                 if(strcmp(cv->value, default_value) != 0) cv->changed = 1;
 751         }
 752         else {
 753                 // this is not loaded from the config
 754                 // copy the default value to it
 755                 strncpy(cv->value, default_value, CONFIG_MAX_VALUE);
 756         }
 757
 758         pthread_rwlock_unlock(&config_rwlock);
 759         return(cv->value);
 760 }
 761
 762 long long config_get_number(const char *section, const char *name, long long value)
 763 {
 764         char buffer[100], *s;
 765         sprintf(buffer, "%lld", value);
 766
 767         s = config_get(section, name, buffer);
 768         return strtoll(s, NULL, 0);
 769 }
 770
 771 int config_get_boolean(const char *section, const char *name, int value)
 772 {
 773         char *s;
 774         if(value) s = "yes";
 775         else s = "no";
 776
 777         s = config_get(section, name, s);
 778
 779         if(strcmp(s, "yes") == 0 || strcmp(s, "true") == 0 || strcmp(s, "1") == 0) {
 780                 strcpy(s, "yes");
 781                 return 1;
 782         }
 783         else {
 784                 strcpy(s, "no");
 785                 return 0;
 786         }
 787 }
 788
 789 const char *config_set(const char *section, const char *name, const char *value)
 790 {
 791         struct config_value *cv;
 792
 793         debug(D_CONFIG, "request to set config in section '%s', name '%s', value '%s'", section, name, value);
 794
 795         pthread_rwlock_wrlock(&config_rwlock);
 796
 797         struct config *co = config_find_section(section);
 798         if(!co) co = config_create(section);
 799
 800         unsigned long hash = simple_hash(name);
 801         for(cv = co->values; cv ; cv = cv->next)
 802                 if(hash == cv->hash)
 803                         if(strcmp(cv->name, name) == 0)
 804                                 break;
 805
 806         if(!cv) cv = config_value_create(co, name, value);
 807         cv->used = 1;
 808
 809         if(strcmp(cv->value, value) != 0) cv->changed = 1;
 810
 811         strncpy(cv->value, value, CONFIG_MAX_VALUE);
 812         // termination is already there
 813
 814         pthread_rwlock_unlock(&config_rwlock);
 815
 816         return value;
 817 }
 818
 819 long long config_set_number(const char *section, const char *name, long long value)
 820 {
 821         char buffer[100];
 822         sprintf(buffer, "%lld", value);
 823
 824         config_set(section, name, buffer);
 825
 826         return value;
 827 }
 828
 829 int config_set_boolean(const char *section, const char *name, int value)
 830 {
 831         char *s;
 832         if(value) s = "yes";
 833         else s = "no";
 834
 835         config_set(section, name, s);
 836
 837         return value;
 838 }
 839
 840
 841 // ----------------------------------------------------------------------------
 842 // chart types
 843
 844 #define CHART_TYPE_LINE_NAME "line"
 845 #define CHART_TYPE_AREA_NAME "area"
 846 #define CHART_TYPE_STACKED_NAME "stacked"
 847
 848
 849 #define CHART_TYPE_LINE 0
 850 #define CHART_TYPE_AREA 1
 851 #define CHART_TYPE_STACKED 2
 852
 853 int chart_type_id(const char *name)
 854 {
 855         if(strcmp(name, CHART_TYPE_AREA_NAME) == 0) return CHART_TYPE_AREA;
 856         if(strcmp(name, CHART_TYPE_STACKED_NAME) == 0) return CHART_TYPE_STACKED;
 857         if(strcmp(name, CHART_TYPE_LINE_NAME) == 0) return CHART_TYPE_LINE;
 858         return CHART_TYPE_LINE;
 859 }
 860
 861 const char *chart_type_name(int chart_type)
 862 {
 863         static char line[] = CHART_TYPE_LINE_NAME;
 864         static char area[] = CHART_TYPE_AREA_NAME;
 865         static char stacked[] = CHART_TYPE_STACKED_NAME;
 866
 867         switch(chart_type) {
 868                 case CHART_TYPE_LINE:
 869                         return line;
 870
 871                 case CHART_TYPE_AREA:
 872                         return area;
 873
 874                 case CHART_TYPE_STACKED:
 875                         return stacked;
 876         }
 877         return line;
 878 }
 879
 880
 881 // ----------------------------------------------------------------------------
 882 // algorithms types
 883
 884 #define RRD_DIMENSION_ABSOLUTE_NAME                                     "absolute"
 885 #define RRD_DIMENSION_INCREMENTAL_NAME                                  "incremental"
 886 #define RRD_DIMENSION_PCENT_OVER_DIFF_TOTAL_NAME                "percentage-of-incremental-row"
 887 #define RRD_DIMENSION_PCENT_OVER_ROW_TOTAL_NAME                 "percentage-of-absolute-row"
 888 #define RRD_DIMENSION_INCREMENTAL_NO_INTERPOLATION_NAME "incremental-no-interpolation"
 889 #define RRD_DIMENSION_ABSOLUTE_NO_INTERPOLATION_NAME    "absolute-no-interpolation"
 890
 891
 892 #define RRD_DIMENSION_ABSOLUTE                                          0
 893 #define RRD_DIMENSION_INCREMENTAL                                       1
 894 #define RRD_DIMENSION_PCENT_OVER_DIFF_TOTAL             2
 895 #define RRD_DIMENSION_PCENT_OVER_ROW_TOTAL                      3
 896 #define RRD_DIMENSION_INCREMENTAL_NO_INTERPOLATION      4
 897 #define RRD_DIMENSION_ABSOLUTE_NO_INTERPOLATION         5
 898
 899 int algorithm_id(const char *name)
 900 {
 901         if(strcmp(name, RRD_DIMENSION_ABSOLUTE_NAME) == 0)                                              return RRD_DIMENSION_ABSOLUTE;
 902         if(strcmp(name, RRD_DIMENSION_ABSOLUTE_NO_INTERPOLATION_NAME) == 0)     return RRD_DIMENSION_ABSOLUTE_NO_INTERPOLATION;
 903         if(strcmp(name, RRD_DIMENSION_INCREMENTAL_NAME) == 0)                                   return RRD_DIMENSION_INCREMENTAL;
 904         if(strcmp(name, RRD_DIMENSION_INCREMENTAL_NO_INTERPOLATION_NAME) == 0)  return RRD_DIMENSION_INCREMENTAL_NO_INTERPOLATION;
 905         if(strcmp(name, RRD_DIMENSION_PCENT_OVER_ROW_TOTAL_NAME) == 0)                  return RRD_DIMENSION_PCENT_OVER_ROW_TOTAL;
 906         if(strcmp(name, RRD_DIMENSION_PCENT_OVER_DIFF_TOTAL_NAME) == 0)                 return RRD_DIMENSION_PCENT_OVER_DIFF_TOTAL;
 907         return RRD_DIMENSION_ABSOLUTE;
 908 }
 909
 910 const char *algorithm_name(int chart_type)
 911 {
 912         static char absolute[] = RRD_DIMENSION_ABSOLUTE_NAME;
 913         static char incremental[] = RRD_DIMENSION_INCREMENTAL_NAME;
 914         static char percentage_of_absolute_row[] = RRD_DIMENSION_PCENT_OVER_ROW_TOTAL_NAME;
 915         static char percentage_of_incremental_row[] = RRD_DIMENSION_PCENT_OVER_DIFF_TOTAL_NAME;
 916         static char incremental_last_value[] = RRD_DIMENSION_INCREMENTAL_NO_INTERPOLATION_NAME;
 917         static char absolute_last_value[] = RRD_DIMENSION_ABSOLUTE_NO_INTERPOLATION_NAME;
 918
 919         switch(chart_type) {
 920                 case RRD_DIMENSION_ABSOLUTE:
 921                         return absolute;
 922
 923                 case RRD_DIMENSION_ABSOLUTE_NO_INTERPOLATION:
 924                         return absolute_last_value;
 925
 926                 case RRD_DIMENSION_INCREMENTAL:
 927                         return incremental;
 928
 929                 case RRD_DIMENSION_INCREMENTAL_NO_INTERPOLATION:
 930                         return incremental_last_value;
 931
 932                 case RRD_DIMENSION_PCENT_OVER_ROW_TOTAL:
 933                         return percentage_of_absolute_row;
 934
 935                 case RRD_DIMENSION_PCENT_OVER_DIFF_TOTAL:
 936                         return percentage_of_incremental_row;
 937         }
 938         return absolute;
 939 }
 940
 941
 942 // ----------------------------------------------------------------------------
 943 // FAST NUMBER TO STRING
 944
 945 static void strreverse(char* begin, char* end)
 946 {
 947     char aux;
 948     while (end > begin)
 949         aux = *end, *end-- = *begin, *begin++ = aux;
 950 }
 951
 952
 953 // ----------------------------------------------------------------------------
 954 // mmap() wrapper
 955
 956 #define NETDATA_MEMORY_MODE_RAM_NAME "ram"
 957 #define NETDATA_MEMORY_MODE_MAP_NAME "map"
 958 #define NETDATA_MEMORY_MODE_SAVE_NAME "save"
 959
 960 #define NETDATA_MEMORY_MODE_RAM 0
 961 #define NETDATA_MEMORY_MODE_MAP 1
 962 #define NETDATA_MEMORY_MODE_SAVE 2
 963
 964 int memory_mode = NETDATA_MEMORY_MODE_SAVE;
 965
 966 const char *memory_mode_name(int id)
 967 {
 968         static const char ram[] = NETDATA_MEMORY_MODE_RAM_NAME;
 969         static const char map[] = NETDATA_MEMORY_MODE_MAP_NAME;
 970         static const char save[] = NETDATA_MEMORY_MODE_SAVE_NAME;
 971
 972         switch(id) {
 973                 case NETDATA_MEMORY_MODE_RAM:
 974                         return ram;
 975
 976                 case NETDATA_MEMORY_MODE_MAP:
 977                         return map;
 978
 979                 case NETDATA_MEMORY_MODE_SAVE:
 980                 default:
 981                         return save;
 982         }
 983
 984         return save;
 985 }
 986
 987 int memory_mode_id(const char *name)
 988 {
 989         if(!strcmp(name, NETDATA_MEMORY_MODE_RAM_NAME))
 990                 return NETDATA_MEMORY_MODE_RAM;
 991         else if(!strcmp(name, NETDATA_MEMORY_MODE_MAP_NAME))
 992                 return NETDATA_MEMORY_MODE_MAP;
 993
 994         return NETDATA_MEMORY_MODE_SAVE;
 995 }
 996
 997 void *mymmap(const char *filename, unsigned long size)
 998 {
 999         if(memory_mode == NETDATA_MEMORY_MODE_RAM) return NULL;
1000
1001         int fd, flags;
1002         void *mem = NULL;
1003
1004         if(memory_mode == NETDATA_MEMORY_MODE_MAP)
1005                 flags = MAP_SHARED;
1006         else
1007                 flags = MAP_PRIVATE;
1008
1009         errno = 0;
1010         fd = open(filename, O_RDWR|O_CREAT|O_NOATIME, 0664);
1011         if(fd != -1) {
1012                 if(lseek(fd, size, SEEK_SET) == (long)size) {
1013                         if(write(fd, "", 1) == 1) {
1014
1015                                 if(ftruncate(fd, size))
1016                                         error("Cannot truncate file '%s' to size %ld. Will use the larger file.", filename, size);
1017
1018                                 mem = mmap(NULL, size, PROT_READ|PROT_WRITE, flags, fd, 0);
1019                                 if(mem) {
1020                                         if(madvise(mem, size, MADV_SEQUENTIAL|MADV_DONTFORK|MADV_WILLNEED) != 0)
1021                                                 error("Cannot advise the kernel about the memory usage of file '%s'.", filename);
1022                                 }
1023                         }
1024                         else error("Cannot write to file '%s' at position %ld.", filename, size);
1025                 }
1026                 else error("Cannot seek file '%s' to size %ld.", filename, size);
1027
1028                 close(fd);
1029         }
1030         else error("Cannot create/open file '%s'.", filename);
1031
1032         return mem;
1033 }
1034
1035 int savememory(const char *filename, void *mem, unsigned long size)
1036 {
1037         char tmpfilename[FILENAME_MAX + 1];
1038
1039         snprintf(tmpfilename, FILENAME_MAX, "%s.%ld.tmp", filename, (long)getpid());
1040
1041         int fd = open(tmpfilename, O_RDWR|O_CREAT|O_NOATIME, 0664);
1042         if(fd < 0) {
1043                 error("Cannot create/open file '%s'.", filename);
1044                 return -1;
1045         }
1046
1047         if(write(fd, mem, size) != (long)size) {
1048                 error("Cannot write to file '%s' %ld bytes.", filename, (long)size);
1049                 close(fd);
1050                 return -1;
1051         }
1052
1053         close(fd);
1054
1055         int ret = 0;
1056         if(rename(tmpfilename, filename)) {
1057                 error("Cannot rename '%s' to '%s'", tmpfilename, filename);
1058                 ret = -1;
1059         }
1060
1061         return ret;
1062 }
1063
1064
1065 // ----------------------------------------------------------------------------
1066 // RRD STATS
1067
1068 #define RRD_STATS_NAME_MAX 1024
1069
1070 typedef long double calculated_number;
1071 #define CALCULATED_NUMBER_FORMAT "%0.3Lf"
1072 //typedef long long calculated_number;
1073 //#define CALCULATED_NUMBER_FORMAT "%lld"
1074
1075 typedef long long collected_number;
1076 #define COLLECTED_NUMBER_FORMAT "%lld"
1077
1078 typedef long long total_number;
1079 #define TOTAL_NUMBER_FORMAT "%lld"
1080
1081 typedef int32_t storage_number;
1082 typedef uint32_t ustorage_number;
1083 #define STORAGE_NUMBER_FORMAT "%d"
1084
1085 #define RRD_STATS_MAGIC     "NETDATA CACHE STATS FILE V007"
1086 #define RRD_DIMENSION_MAGIC "NETDATA CACHE DIMENSION FILE V005"
1087
1088 struct rrd_dimension {
1089         char magic[sizeof(RRD_DIMENSION_MAGIC) + 1];// our magic
1090         char id[RRD_STATS_NAME_MAX + 1];                        // the id of this dimension (for internal identification)
1091         char *name;                                                                     // the name of this dimension (as presented to user)
1092         char cache_file[FILENAME_MAX+1];
1093
1094         unsigned long hash;                                                     // a simple hash on the id, to speed up searching
1095                                                                                                 // we first compare hashes, and only if the hashes are equal we do string comparisons
1096
1097         long entries;                                                           // how many entries this dimension has
1098                                                                                                 // this should be the same to the entries of the data set
1099         long current_entry;                                                     // the entry that is currently being updated
1100
1101         int hidden;                                                                     // if set to non zero, this dimension will not be sent to the client
1102         int mapped;                                                                     // 1 if the file is mapped
1103         unsigned long memsize;                                          // the memory allocated for this dimension
1104
1105         int algorithm;
1106         long multiplier;
1107         long divisor;
1108
1109         struct timeval last_collected;                          // when was this dimension last updated
1110
1111         calculated_number calculated_value;
1112         calculated_number last_calculated_value;
1113
1114         collected_number collected_value;
1115         collected_number last_collected_value;
1116
1117         struct rrd_dimension *next;                                     // linking of dimensions within the same data set
1118
1119         storage_number values[];                                        // the array of values - THIS HAS TO BE THE LAST MEMBER
1120 };
1121 typedef struct rrd_dimension RRD_DIMENSION;
1122
1123 struct rrd_stats {
1124         char magic[sizeof(RRD_STATS_MAGIC) + 1];// our magic
1125
1126         char id[RRD_STATS_NAME_MAX + 1];                        // id of the data set
1127         char *name;                                                                     // name of the data set
1128         char *cache_dir;                                                        // the directory to store dimension maps
1129         char cache_file[FILENAME_MAX+1];
1130
1131         char *type;                                                                     // the type of graph RRD_TYPE_* (a category, for determining graphing options)
1132         char *family;                                                           // the family of this data set (for grouping them together)
1133         char *title;                                                            // title shown to user
1134         char *units;                                                            // units of measurement
1135
1136         pthread_rwlock_t rwlock;
1137         unsigned long counter;                                          // the number of times we added values to this rrd
1138         unsigned long counter_done;                                     // the number of times we added values to this rrd
1139
1140         int mapped;                                                                     // if set to 1, this is memory mapped
1141         unsigned long memsize;                                          // how much mem we have allocated for this (without dimensions)
1142
1143         unsigned long hash_name;                                        // a simple hash on the name
1144         unsigned long hash;                                                     // a simple hash on the id, to speed up searching
1145                                                                                                 // we first compare hashes, and only if the hashes are equal we do string comparisons
1146
1147         long priority;
1148
1149         long entries;                                                           // total number of entries in the data set
1150         long current_entry;                                                     // the entry that is currently being updated
1151                                                                                                 // it goes around in a round-robin fashion
1152
1153         int update_every;                                                       // every how many seconds is this updated?
1154         unsigned long long first_entry_t;                       // the timestamp (in microseconds) of the oldest entry in the db
1155         struct timeval last_updated;                            // when this data set was last updated
1156         struct timeval last_collected;
1157         unsigned long long usec_since_last_update;
1158
1159         total_number absolute_total;
1160         total_number last_absolute_total;
1161
1162         int chart_type;
1163         int debug;
1164         int enabled;
1165         int isdetail;                                                           // if set, the data set should be considered as a detail of another
1166                                                                                                 // (the master data set should be the one that has the same family and is not detail)
1167
1168         RRD_DIMENSION *dimensions;                                      // the actual data for every dimension
1169
1170         struct rrd_stats *next;                                         // linking of rrd stats
1171 };
1172 typedef struct rrd_stats RRD_STATS;
1173
1174 RRD_STATS *root = NULL;
1175 pthread_rwlock_t root_rwlock = PTHREAD_RWLOCK_INITIALIZER;
1176
1177 char *rrd_stats_strncpy_name(char *to, const char *from, int length)
1178 {
1179         int i;
1180         for(i = 0; i < length && from[i] ;i++) {
1181                 if(from[i] == '.' || isalpha(from[i]) || isdigit(from[i])) to[i] = from[i];
1182                 else to[i] = '_';
1183         }
1184         if(i < length) to[i] = '\0';
1185         to[length - 1] = '\0';
1186
1187         return to;
1188 }
1189
1190 void rrd_stats_set_name(RRD_STATS *st, const char *name)
1191 {
1192         char b[CONFIG_MAX_VALUE + 1];
1193         char n[RRD_STATS_NAME_MAX + 1];
1194
1195         snprintf(n, RRD_STATS_NAME_MAX, "%s.%s", st->type, name);
1196         rrd_stats_strncpy_name(b, n, CONFIG_MAX_VALUE);
1197         st->name = config_get(st->id, "name", b);
1198         st->hash_name = simple_hash(st->name);
1199 }
1200
1201 char *rrd_stats_cache_dir(const char *id)
1202 {
1203         char *ret = NULL;
1204
1205         static char *cache_dir = NULL;
1206         if(!cache_dir) cache_dir = config_get("global", "database directory", "cache");
1207
1208         char b[FILENAME_MAX + 1];
1209         char n[FILENAME_MAX + 1];
1210         rrd_stats_strncpy_name(b, id, FILENAME_MAX);
1211
1212         snprintf(n, FILENAME_MAX, "%s/%s", cache_dir, b);
1213         ret = config_get(id, "database directory", n);
1214
1215         if(memory_mode == NETDATA_MEMORY_MODE_MAP || memory_mode == NETDATA_MEMORY_MODE_SAVE) {
1216                 int r = mkdir(ret, 0775);
1217                 if(r != 0 && errno != EEXIST)
1218                         error("Cannot create directory '%s'", ret);
1219         }
1220
1221         return ret;
1222 }
1223
1224 RRD_STATS *rrd_stats_create(const char *type, const char *id, const char *name, const char *family, const char *title, const char *units, long priority, int update_every, int chart_type)
1225 {
1226         if(!id || !id[0]) {
1227                 fatal("Cannot create rrd stats without an id.");
1228                 return NULL;
1229         }
1230
1231         char fullid[RRD_STATS_NAME_MAX + 1];
1232         char fullfilename[FILENAME_MAX + 1];
1233         RRD_STATS *st;
1234
1235         snprintf(fullid, RRD_STATS_NAME_MAX, "%s.%s", type, id);
1236
1237         long entries = config_get_number(fullid, "history", save_history);
1238         if(entries < 5) entries = config_set_number(fullid, "history", 5);
1239         if(entries > HISTORY_MAX) entries = config_set_number(fullid, "history", HISTORY_MAX);
1240
1241         int enabled = config_get_boolean(fullid, "enabled", 1);
1242         if(!enabled) entries = 5;
1243
1244         unsigned long size = sizeof(RRD_STATS);
1245         char *cache_dir = rrd_stats_cache_dir(fullid);
1246
1247         debug(D_RRD_STATS, "Creating RRD_STATS for '%s.%s'.", type, id);
1248
1249         snprintf(fullfilename, FILENAME_MAX, "%s/main.db", cache_dir);
1250         st = (RRD_STATS *)mymmap(fullfilename, size);
1251         if(st) {
1252                 if(strcmp(st->magic, RRD_STATS_MAGIC) != 0) {
1253                         errno = 0;
1254                         error("File %s does not have our version. Clearing it.", fullfilename);
1255                         bzero(st, size);
1256                 }
1257                 else if(strcmp(st->id, fullid) != 0) {
1258                         errno = 0;
1259                         error("File %s does not have our id. Unmapping it.", fullfilename);
1260                         munmap(st, size);
1261                         st = NULL;
1262                 }
1263                 else if(st->memsize != size || st->entries != entries) {
1264                         errno = 0;
1265                         error("File %s does not have the desired size. Clearing it.", fullfilename);
1266                         bzero(st, size);
1267                 }
1268                 else if(st->update_every != update_every) {
1269                         errno = 0;
1270                         error("File %s does not have the desired update frequency. Clearing it.", fullfilename);
1271                         bzero(st, size);
1272                 }
1273         }
1274
1275         if(st) {
1276                 st->name = NULL;
1277                 st->type = NULL;
1278                 st->family = NULL;
1279                 st->title = NULL;
1280                 st->units = NULL;
1281                 st->dimensions = NULL;
1282                 st->next = NULL;
1283                 st->mapped = memory_mode;
1284         }
1285         else {
1286                 st = calloc(1, size);
1287                 if(!st) {
1288                         fatal("Cannot allocate memory for RRD_STATS %s.%s", type, id);
1289                         return NULL;
1290                 }
1291                 st->mapped = NETDATA_MEMORY_MODE_RAM;
1292         }
1293         st->memsize = size;
1294         st->entries = entries;
1295         st->update_every = update_every;
1296
1297         strcpy(st->cache_file, fullfilename);
1298         strcpy(st->magic, RRD_STATS_MAGIC);
1299
1300         strcpy(st->id, fullid);
1301         st->hash = simple_hash(st->id);
1302
1303         st->cache_dir = cache_dir;
1304
1305         st->family     = config_get(st->id, "family", family?family:st->id);
1306         st->units      = config_get(st->id, "units", units?units:"");
1307         st->type       = config_get(st->id, "type", type);
1308         st->chart_type = chart_type_id(config_get(st->id, "chart type", chart_type_name(chart_type)));
1309
1310         if(name && *name) rrd_stats_set_name(st, name);
1311         else rrd_stats_set_name(st, id);
1312
1313         {
1314                 char varvalue[CONFIG_MAX_VALUE + 1];
1315                 snprintf(varvalue, CONFIG_MAX_VALUE, "%s (%s)", title?title:"", st->name);
1316                 st->title = config_get(st->id, "title", varvalue);
1317         }
1318
1319         st->priority = config_get_number(st->id, "priority", priority);
1320         st->enabled = enabled;
1321
1322         st->debug = 0;
1323
1324         st->last_collected.tv_sec = 0;
1325         st->last_collected.tv_usec = 0;
1326
1327         pthread_rwlock_init(&st->rwlock, NULL);
1328         pthread_rwlock_wrlock(&root_rwlock);
1329
1330         st->next = root;
1331         root = st;
1332
1333         pthread_rwlock_unlock(&root_rwlock);
1334
1335         return(st);
1336 }
1337
1338 RRD_DIMENSION *rrd_stats_dimension_add(RRD_STATS *st, const char *id, const char *name, long multiplier, long divisor, int algorithm)
1339 {
1340         char filename[FILENAME_MAX + 1];
1341         char fullfilename[FILENAME_MAX + 1];
1342
1343         char varname[CONFIG_MAX_NAME + 1];
1344         RRD_DIMENSION *rd;
1345         unsigned long size = sizeof(RRD_DIMENSION) + (st->entries * sizeof(storage_number));
1346
1347         debug(D_RRD_STATS, "Adding dimension '%s/%s'.", st->id, id);
1348
1349         rrd_stats_strncpy_name(filename, id, FILENAME_MAX);
1350         snprintf(fullfilename, FILENAME_MAX, "%s/%s.db", st->cache_dir, filename);
1351         rd = (RRD_DIMENSION *)mymmap(fullfilename, size);
1352         if(rd) {
1353                 if(strcmp(rd->magic, RRD_DIMENSION_MAGIC) != 0) {
1354                         errno = 0;
1355                         error("File %s does not have our version. Clearing it.", fullfilename);
1356                         bzero(rd, size);
1357                 }
1358                 else if(rd->memsize != size) {
1359                         errno = 0;
1360                         error("File %s does not have the desired size. Clearing it.", fullfilename);
1361                         bzero(rd, size);
1362                 }
1363                 else if(rd->multiplier != multiplier) {
1364                         errno = 0;
1365                         error("File %s does not have the same multiplier. Clearing it.", fullfilename);
1366                         bzero(rd, size);
1367                 }
1368                 else if(rd->divisor != divisor) {
1369                         errno = 0;
1370                         error("File %s does not have the same divisor. Clearing it.", fullfilename);
1371                         bzero(rd, size);
1372                 }
1373                 else if(rd->algorithm != algorithm) {
1374                         errno = 0;
1375                         error("File %s does not have the same algorithm. Clearing it.", fullfilename);
1376                         bzero(rd, size);
1377                 }
1378                 else if(strcmp(rd->id, id) != 0) {
1379                         errno = 0;
1380                         error("File %s does not have our dimension id. Unmapping it.", fullfilename);
1381                         munmap(rd, size);
1382                         rd = NULL;
1383                 }
1384         }
1385
1386         if(rd) {
1387                 // we have a file mapped for rd
1388                 rd->mapped = memory_mode;
1389                 rd->hidden = 0;
1390                 rd->next = NULL;
1391                 rd->name = NULL;
1392         }
1393         else {
1394                 // if we didn't manage to get a mmap'd dimension, just create one
1395
1396                 rd = calloc(1, size);
1397                 if(!rd) {
1398                         fatal("Cannot allocate RRD_DIMENSION %s/%s.", st->id, id);
1399                         return NULL;
1400                 }
1401
1402                 rd->mapped = NETDATA_MEMORY_MODE_RAM;
1403         }
1404         rd->memsize = size;
1405
1406         strcpy(rd->magic, RRD_DIMENSION_MAGIC);
1407         strcpy(rd->cache_file, fullfilename);
1408         strncpy(rd->id, id, RRD_STATS_NAME_MAX);
1409         rd->hash = simple_hash(rd->id);
1410
1411         snprintf(varname, CONFIG_MAX_NAME, "dim %s name", rd->id);
1412         rd->name = config_get(st->id, varname, (name && *name)?name:rd->id);
1413
1414         snprintf(varname, CONFIG_MAX_NAME, "dim %s algorithm", rd->id);
1415         rd->algorithm = algorithm_id(config_get(st->id, varname, algorithm_name(algorithm)));
1416
1417         snprintf(varname, CONFIG_MAX_NAME, "dim %s multiplier", rd->id);
1418         rd->multiplier = config_get_number(st->id, varname, multiplier);
1419
1420         snprintf(varname, CONFIG_MAX_NAME, "dim %s divisor", rd->id);
1421         rd->divisor = config_get_number(st->id, varname, divisor);
1422         if(!rd->divisor) rd->divisor = 1;
1423
1424         rd->entries = st->entries;
1425
1426         // append this dimension
1427         if(!st->dimensions)
1428                 st->dimensions = rd;
1429         else {
1430                 RRD_DIMENSION *td = st->dimensions;
1431                 for(; td->next; td = td->next) ;
1432                 td->next = rd;
1433         }
1434
1435         return(rd);
1436 }
1437
1438 void rrd_stats_dimension_set_name(RRD_STATS *st, RRD_DIMENSION *rd, const char *name)
1439 {
1440         char varname[CONFIG_MAX_NAME + 1];
1441         snprintf(varname, CONFIG_MAX_NAME, "dim %s name", rd->id);
1442         config_get(st->id, varname, name);
1443 }
1444
1445 void rrd_stats_dimension_free(RRD_DIMENSION *rd)
1446 {
1447         if(rd->next) rrd_stats_dimension_free(rd->next);
1448         // free(rd->annotations);
1449         if(rd->mapped == NETDATA_MEMORY_MODE_SAVE) {
1450                 debug(D_RRD_STATS, "Saving dimension '%s' to '%s'.", rd->name, rd->cache_file);
1451                 savememory(rd->cache_file, rd, rd->memsize);
1452
1453                 debug(D_RRD_STATS, "Unmapping dimension '%s'.", rd->name);
1454                 munmap(rd, rd->memsize);
1455         }
1456         else if(rd->mapped == NETDATA_MEMORY_MODE_MAP) {
1457                 debug(D_RRD_STATS, "Unmapping dimension '%s'.", rd->name);
1458                 munmap(rd, rd->memsize);
1459         }
1460         else {
1461                 debug(D_RRD_STATS, "Removing dimension '%s'.", rd->name);
1462                 free(rd);
1463         }
1464 }
1465
1466 void rrd_stats_free_all(void)
1467 {
1468         info("Freeing all memory...");
1469
1470         RRD_STATS *st;
1471         for(st = root; st ;) {
1472                 RRD_STATS *next = st->next;
1473
1474                 if(st->dimensions) rrd_stats_dimension_free(st->dimensions);
1475                 st->dimensions = NULL;
1476
1477                 if(st->mapped == NETDATA_MEMORY_MODE_SAVE) {
1478                         debug(D_RRD_STATS, "Saving stats '%s' to '%s'.", st->name, st->cache_file);
1479                         savememory(st->cache_file, st, st->memsize);
1480
1481                         debug(D_RRD_STATS, "Unmapping stats '%s'.", st->name);
1482                         munmap(st, st->memsize);
1483                 }
1484                 else if(st->mapped == NETDATA_MEMORY_MODE_MAP) {
1485                         debug(D_RRD_STATS, "Unmapping stats '%s'.", st->name);
1486                         munmap(st, st->memsize);
1487                 }
1488                 else
1489                         free(st);
1490
1491                 st = next;
1492         }
1493         root = NULL;
1494
1495         info("Memory cleanup completed...");
1496 }
1497
1498 void rrd_stats_save_all(void)
1499 {
1500         RRD_STATS *st;
1501         RRD_DIMENSION *rd;
1502
1503         pthread_rwlock_wrlock(&root_rwlock);
1504         for(st = root; st ; st = st->next) {
1505                 pthread_rwlock_wrlock(&st->rwlock);
1506
1507                 if(st->mapped == NETDATA_MEMORY_MODE_SAVE) {
1508                         debug(D_RRD_STATS, "Saving stats '%s' to '%s'.", st->name, st->cache_file);
1509                         savememory(st->cache_file, st, st->memsize);
1510                 }
1511
1512                 for(rd = st->dimensions; rd ; rd = rd->next) {
1513                         if(rd->mapped == NETDATA_MEMORY_MODE_SAVE) {
1514                                 debug(D_RRD_STATS, "Saving dimension '%s' to '%s'.", rd->name, rd->cache_file);
1515                                 savememory(rd->cache_file, rd, rd->memsize);
1516                         }
1517                 }
1518
1519                 pthread_rwlock_unlock(&st->rwlock);
1520         }
1521         pthread_rwlock_unlock(&root_rwlock);
1522 }
1523
1524
1525 RRD_STATS *rrd_stats_find(const char *id)
1526 {
1527         unsigned long hash = simple_hash(id);
1528
1529         pthread_rwlock_rdlock(&root_rwlock);
1530         RRD_STATS *st = root;
1531         for ( ; st ; st = st->next )
1532                 if(hash == st->hash)
1533                         if(strcmp(st->id, id) == 0)
1534                                 break;
1535         pthread_rwlock_unlock(&root_rwlock);
1536
1537         return(st);
1538 }
1539
1540 RRD_STATS *rrd_stats_find_bytype(const char *type, const char *id)
1541 {
1542         char buf[RRD_STATS_NAME_MAX + 1];
1543
1544         strncpy(buf, type, RRD_STATS_NAME_MAX - 1);
1545         buf[RRD_STATS_NAME_MAX - 1] = '\0';
1546         strcat(buf, ".");
1547         int len = strlen(buf);
1548         strncpy(&buf[len], id, RRD_STATS_NAME_MAX - len);
1549         buf[RRD_STATS_NAME_MAX] = '\0';
1550
1551         return(rrd_stats_find(buf));
1552 }
1553
1554 RRD_STATS *rrd_stats_find_byname(const char *name)
1555 {
1556         char b[CONFIG_MAX_VALUE + 1];
1557
1558         rrd_stats_strncpy_name(b, name, CONFIG_MAX_VALUE);
1559         unsigned long hash = simple_hash(b);
1560
1561         pthread_rwlock_rdlock(&root_rwlock);
1562         RRD_STATS *st = root;
1563         for ( ; st ; st = st->next ) {
1564                 if(hash == st->hash_name && strcmp(st->name, b) == 0) break;
1565         }
1566         pthread_rwlock_unlock(&root_rwlock);
1567
1568         return(st);
1569 }
1570
1571 RRD_DIMENSION *rrd_stats_dimension_find(RRD_STATS *st, const char *id)
1572 {
1573         unsigned long hash = simple_hash(id);
1574
1575         RRD_DIMENSION *rd = st->dimensions;
1576
1577         for ( ; rd ; rd = rd->next )
1578                 if(hash == rd->hash)
1579                         if(strcmp(rd->id, id) == 0)
1580                                 break;
1581
1582         return(rd);
1583 }
1584
1585 int rrd_stats_dimension_hide(RRD_STATS *st, const char *id)
1586 {
1587         RRD_DIMENSION *rd = rrd_stats_dimension_find(st, id);
1588         if(!rd) {
1589                 error("Cannot find dimension with id '%s' on stats '%s' (%s).", id, st->name, st->id);
1590                 return 1;
1591         }
1592
1593         rd->hidden = 1;
1594         return 0;
1595 }
1596
1597 void rrd_stats_dimension_set_by_pointer(RRD_STATS *st, RRD_DIMENSION *rd, collected_number value)
1598 {
1599         if(!st->last_collected.tv_sec) gettimeofday(&st->last_collected, NULL);
1600
1601         rd->last_collected.tv_sec = st->last_collected.tv_sec;
1602         rd->last_collected.tv_usec = st->last_collected.tv_usec;
1603
1604         rd->collected_value = value;
1605 }
1606
1607 int rrd_stats_dimension_set(RRD_STATS *st, char *id, collected_number value)
1608 {
1609         RRD_DIMENSION *rd = rrd_stats_dimension_find(st, id);
1610         if(!rd) {
1611                 error("Cannot find dimension with id '%s' on stats '%s' (%s).", id, st->name, st->id);
1612                 return 1;
1613         }
1614
1615         rrd_stats_dimension_set_by_pointer(st, rd, value);
1616         return 0;
1617 }
1618
1619 void rrd_stats_next_internal(RRD_STATS *st)
1620 {
1621         // a read lock is OK here
1622         pthread_rwlock_rdlock(&st->rwlock);
1623
1624         RRD_DIMENSION *rd;
1625         for( rd = st->dimensions; rd ; rd = rd->next ) {
1626                 rd->last_collected_value = rd->collected_value;
1627                 rd->collected_value = 0;
1628         }
1629
1630         pthread_rwlock_unlock(&st->rwlock);
1631 }
1632
1633 void rrd_stats_next_timeval(RRD_STATS *st, struct timeval *now)
1634 {
1635         if(!st->last_collected.tv_sec) {
1636                 gettimeofday(&st->last_collected, NULL);
1637                 unsigned long long ut = st->last_collected.tv_sec * 1000000ULL + st->last_collected.tv_usec - st->update_every * 1000000ULL;
1638                 st->last_collected.tv_sec = ut / 1000000ULL;
1639                 st->last_collected.tv_usec = ut % 1000000ULL;
1640         }
1641
1642         st->usec_since_last_update = usecdiff(now, &st->last_collected);
1643         st->last_collected.tv_sec = now->tv_sec;
1644         st->last_collected.tv_usec = now->tv_usec;
1645
1646         rrd_stats_next_internal(st);
1647 }
1648
1649 void rrd_stats_next_usec(RRD_STATS *st, unsigned long long microseconds)
1650 {
1651         if(!st->last_collected.tv_sec) {
1652                 gettimeofday(&st->last_collected, NULL);
1653                 unsigned long long ut = st->last_collected.tv_sec * 1000000ULL + st->last_collected.tv_usec;
1654                 st->last_collected.tv_sec = ut / 1000000ULL;
1655                 st->last_collected.tv_usec = ut % 1000000ULL;
1656         }
1657         else {
1658                 unsigned long long ut = st->last_collected.tv_sec * 1000000ULL + st->last_collected.tv_usec + microseconds;
1659                 st->last_collected.tv_sec = ut / 1000000ULL;
1660                 st->last_collected.tv_usec = ut % 1000000ULL;
1661         }
1662         st->usec_since_last_update = microseconds;
1663
1664         rrd_stats_next_internal(st);
1665 }
1666
1667 void rrd_stats_next(RRD_STATS *st)
1668 {
1669         if(st->last_collected.tv_sec) {
1670                 struct timeval now;
1671                 gettimeofday(&now, NULL);
1672
1673                 rrd_stats_next_timeval(st, &now);
1674         }
1675         else
1676                 rrd_stats_next_usec(st, st->update_every * 1000000ULL);
1677 }
1678
1679 void rrd_stats_next_plugins(RRD_STATS *st)
1680 {
1681         rrd_stats_next_usec(st, st->update_every * 1000000ULL);
1682 }
1683
1684 unsigned long long rrd_stats_done(RRD_STATS *st)
1685 {
1686         RRD_DIMENSION *rd, *last;
1687         int oldstate;
1688
1689         if(pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, &oldstate) != 0)
1690                 error("Cannot set pthread cancel state to DISABLE.");
1691
1692         // a read lock is OK here
1693         pthread_rwlock_rdlock(&st->rwlock);
1694
1695         if(!st->last_updated.tv_sec) {
1696                 unsigned long long ut = st->last_collected.tv_sec * 1000000ULL + st->last_collected.tv_usec - st->usec_since_last_update;
1697                 st->last_updated.tv_sec = ut / 1000000ULL;
1698                 st->last_updated.tv_usec = ut % 1000000ULL;
1699         }
1700
1701         unsigned long long last_ut = st->last_updated.tv_sec * 1000000ULL + st->last_updated.tv_usec;
1702         unsigned long long now_ut = st->last_collected.tv_sec * 1000000ULL + st->last_collected.tv_usec + st->usec_since_last_update;
1703         unsigned long long next_ut = (st->last_updated.tv_sec + st->update_every) * 1000000ULL;
1704
1705         st->counter_done++;
1706         if(st->counter_done == 1 || now_ut < next_ut) {
1707                 if(st->debug) debug(D_RRD_STATS, "%s: Skipping collected values (usec since last update = %llu, counter_done = %lu)", st->name, st->usec_since_last_update, st->counter_done);
1708                 // we don't have any usable data yet
1709                 pthread_rwlock_unlock(&st->rwlock);
1710
1711                 if(pthread_setcancelstate(oldstate, NULL) != 0)
1712                         error("Cannot set pthread cancel state to RESTORE (%d).", oldstate);
1713
1714                 return(st->usec_since_last_update);
1715         }
1716
1717         if(st->debug) debug(D_RRD_STATS, "microseconds since last update: %llu", st->usec_since_last_update);
1718
1719         // calculate totals and count the dimensions
1720         int dimensions;
1721         st->last_absolute_total  = st->absolute_total;
1722         st->absolute_total = 0;
1723         for( rd = st->dimensions, dimensions = 0 ; rd ; rd = rd->next, dimensions++ )
1724                 st->absolute_total += rd->collected_value;
1725
1726         // process all dimensions to calculate its values
1727         for( rd = st->dimensions ; rd ; rd = rd->next ) {
1728                 switch(rd->algorithm) {
1729                         case RRD_DIMENSION_PCENT_OVER_DIFF_TOTAL:
1730                                 // the percentage of the current increment
1731                                 // over the increment of all dimensions together
1732                                 if(st->absolute_total == st->last_absolute_total) rd->calculated_value = 0;
1733                                 else rd->calculated_value =
1734                                           (calculated_number)100
1735                                         * (calculated_number)(rd->collected_value - rd->last_collected_value)
1736                                         / (calculated_number)(st->absolute_total  - st->last_absolute_total);
1737
1738                                 if(st->debug)
1739                                         debug(D_RRD_STATS, "%s/%s: CALC "
1740                                                 CALCULATED_NUMBER_FORMAT " = 100"
1741                                                 " * (" COLLECTED_NUMBER_FORMAT " - " COLLECTED_NUMBER_FORMAT ")"
1742                                                 " / (" COLLECTED_NUMBER_FORMAT " - " COLLECTED_NUMBER_FORMAT ")"
1743                                                 , st->id, rd->name
1744                                                 , rd->calculated_value
1745                                                 , rd->collected_value, rd->last_collected_value
1746                                                 , st->absolute_total, st->last_absolute_total
1747                                                 );
1748                                 break;
1749
1750                         case RRD_DIMENSION_PCENT_OVER_ROW_TOTAL:
1751                                 if(!st->absolute_total) rd->calculated_value = 0;
1752                                 else
1753                                 // the percentage of the current value
1754                                 // over the total of all dimensions
1755                                 rd->calculated_value =
1756                                           (calculated_number)100
1757                                         * (calculated_number)rd->collected_value
1758                                         / (calculated_number)st->absolute_total;
1759
1760                                 if(st->debug)
1761                                         debug(D_RRD_STATS, "%s/%s: CALC "
1762                                                 CALCULATED_NUMBER_FORMAT " = 100"
1763                                                 " * " COLLECTED_NUMBER_FORMAT
1764                                                 " / " COLLECTED_NUMBER_FORMAT
1765                                                 , st->id, rd->name
1766                                                 , rd->calculated_value
1767                                                 , rd->collected_value
1768                                                 , st->absolute_total
1769                                                 );
1770                                 break;
1771
1772                         case RRD_DIMENSION_INCREMENTAL:
1773                                 // we need the incremental calculation to produce per second results
1774                                 // so, we multiply with 1.000.000 and divide by the microseconds passed since
1775                                 // the last entry
1776
1777                                 // if the new is smaller than the old (an overflow, or reset), set the old equal to the new
1778                                 // to reset the calculation (it will give zero as the calculation for this second)
1779                                 if(rd->last_collected_value > rd->collected_value) rd->last_collected_value = rd->collected_value;
1780
1781                                 rd->calculated_value =
1782                                           (calculated_number)1000000
1783                                         * (calculated_number)(rd->collected_value - rd->last_collected_value)
1784                                         / (calculated_number)st->usec_since_last_update;
1785
1786                                 if(st->debug)
1787                                         debug(D_RRD_STATS, "%s/%s: CALC "
1788                                                 CALCULATED_NUMBER_FORMAT " = 1000000"
1789                                                 " * (" COLLECTED_NUMBER_FORMAT " - " COLLECTED_NUMBER_FORMAT ")"
1790                                                 " / %llu"
1791                                                 , st->id, rd->name
1792                                                 , rd->calculated_value
1793                                                 , rd->collected_value, rd->last_collected_value
1794                                                 , st->usec_since_last_update
1795                                                 );
1796                                 break;
1797
1798                         case RRD_DIMENSION_INCREMENTAL_NO_INTERPOLATION:
1799                                 // we need the incremental calculation to produce per second results
1800                                 // so, we multiply with 1.000.000 and divide by the microseconds passed since
1801                                 // the last entry
1802
1803                                 // if the new is smaller than the old (an overflow, or reset), set the old equal to the new
1804                                 // to reset the calculation (it will give zero as the calculation for this second)
1805                                 if(rd->last_collected_value > rd->collected_value) rd->last_collected_value = rd->collected_value;
1806
1807                                 rd->calculated_value = (calculated_number)(rd->collected_value - rd->last_collected_value);
1808
1809                                 if(st->debug)
1810                                         debug(D_RRD_STATS, "%s/%s: CALC "
1811                                                 CALCULATED_NUMBER_FORMAT " = "
1812                                                 COLLECTED_NUMBER_FORMAT " - " COLLECTED_NUMBER_FORMAT
1813                                                 , st->id, rd->name
1814                                                 , rd->calculated_value
1815                                                 , rd->collected_value, rd->last_collected_value
1816                                                 );
1817                                 break;
1818
1819                         case RRD_DIMENSION_ABSOLUTE:
1820                         case RRD_DIMENSION_ABSOLUTE_NO_INTERPOLATION:
1821                                 rd->calculated_value = (calculated_number)rd->collected_value;
1822
1823                                 if(st->debug)
1824                                         debug(D_RRD_STATS, "%s/%s: CALC "
1825                                                 CALCULATED_NUMBER_FORMAT " = "
1826                                                 COLLECTED_NUMBER_FORMAT
1827                                                 , st->id, rd->name
1828                                                 , rd->calculated_value
1829                                                 , rd->collected_value
1830                                                 );
1831                                 break;
1832
1833                         default:
1834                                 // make the default zero, to make sure
1835                                 // it gets noticed when we add new types
1836                                 rd->calculated_value = 0;
1837
1838                                 if(st->debug)
1839                                         debug(D_RRD_STATS, "%s/%s: CALC "
1840                                                 CALCULATED_NUMBER_FORMAT " = 0"
1841                                                 , st->id, rd->name
1842                                                 , rd->calculated_value
1843                                                 );
1844                                 break;
1845                 }
1846         }
1847
1848         for( ; next_ut < now_ut ; next_ut += st->update_every * 1000000ULL ) {
1849                 unsigned long long np = next_ut - last_ut;
1850
1851                 st->last_updated.tv_sec = next_ut / 1000000ULL;
1852                 st->last_updated.tv_usec = 0;
1853
1854                 for( rd = st->dimensions ; rd ; rd = rd->next ) {
1855                         calculated_number new_value;
1856
1857                         switch(rd->algorithm) {
1858                                 case RRD_DIMENSION_ABSOLUTE_NO_INTERPOLATION:
1859                                 case RRD_DIMENSION_INCREMENTAL_NO_INTERPOLATION:
1860                                         if(next_ut + st->update_every * 1000000ULL < now_ut)
1861                                                 new_value = rd->last_calculated_value;
1862                                         else
1863                                                 new_value = rd->calculated_value;
1864
1865                                         if(st->debug)
1866                                                 debug(D_RRD_STATS, "%s/%s: CALC2 "
1867                                                         CALCULATED_NUMBER_FORMAT
1868                                                         , st->id, rd->name
1869                                                         , new_value
1870                                                         );
1871                                         break;
1872
1873                                 default:
1874                                         new_value = (calculated_number)
1875                                                 (       (         (rd->calculated_value - rd->last_calculated_value)
1876                                                                 * (calculated_number)np
1877                                                                 / (calculated_number)(now_ut - last_ut)
1878                                                         )
1879                                                         +  rd->last_calculated_value
1880                                                 );
1881
1882                                         if(st->debug)
1883                                                 debug(D_RRD_STATS, "%s/%s: CALC2 "
1884                                                         CALCULATED_NUMBER_FORMAT " = ((("
1885                                                         "(" CALCULATED_NUMBER_FORMAT " - " CALCULATED_NUMBER_FORMAT ")"
1886                                                         " * %llu"
1887                                                         " / %llu) + " CALCULATED_NUMBER_FORMAT
1888                                                         , st->id, rd->name
1889                                                         , new_value
1890                                                         , rd->calculated_value, rd->last_calculated_value
1891                                                         , np
1892                                                         , (now_ut - last_ut), rd->last_calculated_value
1893                                                         );
1894                                         break;
1895                         }
1896
1897                         rd->values[st->current_entry] = (storage_number)
1898                                 (         new_value
1899                                         * (calculated_number)10
1900                                         * (calculated_number)rd->multiplier
1901                                         / (calculated_number)rd->divisor
1902                                 );
1903
1904                         if(st->debug)
1905                                 debug(D_RRD_STATS, "%s/%s: STORE[%ld] "
1906                                         STORAGE_NUMBER_FORMAT " = " CALCULATED_NUMBER_FORMAT
1907                                         " * 10 "
1908                                         " * %ld"
1909                                         " / %ld"
1910                                         , st->id, rd->name
1911                                         , st->current_entry
1912                                         , rd->values[st->current_entry], new_value
1913                                         , rd->multiplier
1914                                         , rd->divisor
1915                                         );
1916
1917                         if((rd->algorithm == RRD_DIMENSION_ABSOLUTE_NO_INTERPOLATION || rd->algorithm == RRD_DIMENSION_ABSOLUTE_NO_INTERPOLATION)
1918                                 && (next_ut + st->update_every * 1000000ULL < now_ut)) {
1919                                 // there is another iteration
1920                                 // do not change the anything
1921                                 ;
1922                         }
1923                         else
1924                                 rd->last_calculated_value = rd->calculated_value = new_value;
1925                 }
1926
1927                 if(st->first_entry_t && st->counter >= (unsigned long long)st->entries) {
1928                         // the db is overwriting values
1929                         // add the value we will overwrite
1930                         st->first_entry_t += st->update_every * 1000000ULL;
1931                 }
1932
1933                 st->counter++;
1934                 st->current_entry = ((st->current_entry + 1) >= st->entries) ? 0 : st->current_entry + 1;
1935                 if(!st->first_entry_t) st->first_entry_t = next_ut;
1936                 last_ut = next_ut;
1937         }
1938
1939         // ALL DONE ABOUT THE DATA UPDATE
1940         // --------------------------------------------------------------------
1941
1942
1943         // find if there are any obsolete dimensions (not updated recently)
1944         for( rd = st->dimensions; rd ; rd = rd->next )
1945                 if((rd->last_collected.tv_sec + (10 * st->update_every)) < st->last_collected.tv_sec)
1946                         break;
1947
1948         if(rd) {
1949                 // there is dimension to free
1950                 // upgrade our read lock to a write lock
1951                 pthread_rwlock_unlock(&st->rwlock);
1952                 pthread_rwlock_wrlock(&st->rwlock);
1953
1954                 for( rd = st->dimensions, last = NULL ; rd ; ) {
1955                         if((rd->last_collected.tv_sec + (10 * st->update_every)) < st->last_collected.tv_sec) { // remove it only it is not updated in 10 seconds
1956                                 debug(D_RRD_STATS, "Removing obsolete dimension '%s' (%s) of '%s' (%s).", rd->name, rd->id, st->name, st->id);
1957
1958                                 if(!last) {
1959                                         st->dimensions = rd->next;
1960                                         rd->next = NULL;
1961                                         rrd_stats_dimension_free(rd);
1962                                         rd = st->dimensions;
1963                                         continue;
1964                                 }
1965                                 else {
1966                                         last->next = rd->next;
1967                                         rd->next = NULL;
1968                                         rrd_stats_dimension_free(rd);
1969                                         rd = last->next;
1970                                         continue;
1971                                 }
1972                         }
1973
1974                         last = rd;
1975                         rd = rd->next;
1976                 }
1977
1978                 if(!st->dimensions) st->enabled = 0;
1979         }
1980
1981         pthread_rwlock_unlock(&st->rwlock);
1982
1983         if(pthread_setcancelstate(oldstate, NULL) != 0)
1984                 error("Cannot set pthread cancel state to RESTORE (%d).", oldstate);
1985
1986         return(st->usec_since_last_update);
1987 }
1988
1989
1990
1991 // ----------------------------------------------------------------------------
1992 // web buffer
1993
1994 struct web_buffer {
1995         long size;      // allocation size of buffer
1996         long bytes;     // current data length in buffer
1997         long sent;      // current data length sent to output
1998         char *buffer;   // the buffer
1999         int contenttype;
2000         long rbytes;    // if non-zero, the excepted size of ifd
2001         time_t date;    // the date this content has been generated
2002 };
2003
2004 #define web_buffer_printf(wb, args...) wb->bytes += snprintf(&wb->buffer[wb->bytes], (wb->size - wb->bytes), ##args)
2005 #define web_buffer_reset(wb) wb->buffer[wb->bytes = 0] = '\0'
2006
2007 void web_buffer_strcpy(struct web_buffer *wb, const char *txt)
2008 {
2009         char *buffer = wb->buffer;
2010         long bytes = wb->bytes, size = wb->size, i = 0;
2011
2012         while(txt[i] && bytes < size)
2013                 buffer[bytes++] = txt[i++];
2014
2015         wb->bytes = bytes;
2016 }
2017
2018 void web_buffer_rrd_value(struct web_buffer *wb, storage_number value)
2019 {
2020         if(wb->size - wb->bytes < 11) return;
2021
2022         char *str = &wb->buffer[wb->bytes];
2023         char *wstr = str;
2024
2025         // make sure it is unsigned
2026         ustorage_number uvalue = (value < 0) ? -value : value;
2027
2028         // print each digit
2029         do *wstr++ = (char)(48 + (uvalue % 10)); while(uvalue /= 10);
2030
2031         // if it is just one byte, add a zero
2032         if((wstr - str) == 1) *wstr++ = '0';
2033
2034         // put the sign back
2035         if (value < 0) *wstr++ = '-';
2036
2037         // reverse it
2038         wstr--;
2039         strreverse(str, wstr);
2040
2041         if(*wstr != '0') {
2042                 // move the last digit one byte to the right
2043                 wstr[1] = wstr[0];
2044
2045                 // put the dot in the hole
2046                 wstr[0] = '.';
2047
2048                 // terminate it
2049                 wstr += 2;
2050         }
2051         *wstr='\0';
2052
2053         // update the buffer length
2054         wb->bytes += (wstr - str);
2055 }
2056
2057 // generate a javascript date, the fastest possible way...
2058 void web_buffer_jsdate(struct web_buffer *wb, int year, int month, int day, int hours, int minutes, int seconds)
2059 {
2060         //         10        20        30      = 35
2061         // 01234567890123456789012345678901234
2062         // Date(2014, 04, 01, 03, 28, 20, 065)
2063
2064         if(wb->size - wb->bytes < 36) return;
2065
2066         char *b = &wb->buffer[wb->bytes];
2067
2068         int i = 0;
2069         b[i++]='D';
2070         b[i++]='a';
2071         b[i++]='t';
2072         b[i++]='e';
2073         b[i++]='(';
2074         b[i++]= 48 + year / 1000; year -= (year / 1000) * 1000;
2075         b[i++]= 48 + year / 100; year -= (year / 100) * 100;
2076         b[i++]= 48 + year / 10;
2077         b[i++]= 48 + year % 10;
2078         b[i++]=',';
2079         //b[i++]=' ';
2080         b[i]= 48 + month / 10; if(b[i] != '0') i++;
2081         b[i++]= 48 + month % 10;
2082         b[i++]=',';
2083         //b[i++]=' ';
2084         b[i]= 48 + day / 10; if(b[i] != '0') i++;
2085         b[i++]= 48 + day % 10;
2086         b[i++]=',';
2087         //b[i++]=' ';
2088         b[i]= 48 + hours / 10; if(b[i] != '0') i++;
2089         b[i++]= 48 + hours % 10;
2090         b[i++]=',';
2091         //b[i++]=' ';
2092         b[i]= 48 + minutes / 10; if(b[i] != '0') i++;
2093         b[i++]= 48 + minutes % 10;
2094         b[i++]=',';
2095         //b[i++]=' ';
2096         b[i]= 48 + seconds / 10; if(b[i] != '0') i++;
2097         b[i++]= 48 + seconds % 10;
2098         b[i++]=')';
2099         b[i]='\0';
2100
2101         wb->bytes += i;
2102 }
2103
2104 struct web_buffer *web_buffer_create(long size)
2105 {
2106         struct web_buffer *b;
2107
2108         debug(D_WEB_BUFFER, "Creating new web buffer of size %d.", size);
2109
2110         b = calloc(1, sizeof(struct web_buffer));
2111         if(!b) {
2112                 error("Cannot allocate a web_buffer.");
2113                 return NULL;
2114         }
2115
2116         b->buffer = malloc(size);
2117         if(!b->buffer) {
2118                 error("Cannot allocate a buffer of size %u.", size);
2119                 free(b);
2120                 return NULL;
2121         }
2122         b->buffer[0] = '\0';
2123         b->size = size;
2124         b->contenttype = CT_TEXT_PLAIN;
2125         return(b);
2126 }
2127
2128 void web_buffer_free(struct web_buffer *b)
2129 {
2130         debug(D_WEB_BUFFER, "Freeing web buffer of size %d.", b->size);
2131
2132         if(b->buffer) free(b->buffer);
2133         free(b);
2134 }
2135
2136 void web_buffer_increase(struct web_buffer *b, long free_size_required)
2137 {
2138         long left = b->size - b->bytes;
2139
2140         if(left >= free_size_required) return;
2141         long increase = free_size_required - left;
2142         if(increase < WEB_DATA_LENGTH_INCREASE_STEP) increase = WEB_DATA_LENGTH_INCREASE_STEP;
2143
2144         debug(D_WEB_BUFFER, "Increasing data buffer from size %d to %d.", b->size, b->size + increase);
2145
2146         b->buffer = realloc(b->buffer, b->size + increase);
2147         if(!b->buffer) fatal("Failed to increase data buffer from size %d to %d.", b->size, b->size + increase);
2148
2149         b->size += increase;
2150 }
2151
2152 #define WEB_CLIENT_MODE_NORMAL          0
2153 #define WEB_CLIENT_MODE_FILECOPY        1
2154
2155 #define URL_MAX 8192
2156
2157 struct web_client {
2158         unsigned long long id;
2159         char client_ip[101];
2160         char last_url[URL_MAX+1];
2161
2162         struct timeval tv_in, tv_ready;
2163
2164         int mode;
2165         int keepalive;
2166
2167         struct sockaddr_in clientaddr;
2168
2169         pthread_t thread;                               // the thread servicing this client
2170         int obsolete;                                   // if set to 1, the listener will remove this client
2171
2172         int ifd;
2173         int ofd;
2174
2175         struct web_buffer *data;
2176
2177         int zoutput;                                    // if set to 1, web_client_send() will send compressed data
2178         z_stream zstream;                               // zlib stream for sending compressed output to client
2179         Bytef zbuffer[ZLIB_CHUNK];              // temporary buffer for storing compressed output
2180         long zsent;                                     // the compressed bytes we have sent to the client
2181         long zhave;                                     // the compressed bytes that we have to send
2182         int zinitialized;
2183
2184         int wait_receive;
2185         int wait_send;
2186
2187         char response_header[MAX_HTTP_HEADER_SIZE+1];
2188
2189         struct web_client *prev;
2190         struct web_client *next;
2191 } *web_clients = NULL;
2192
2193 unsigned long long web_clients_count = 0;
2194
2195 struct web_client *web_client_create(int listener)
2196 {
2197         struct web_client *w;
2198         socklen_t addrlen;
2199
2200         w = calloc(1, sizeof(struct web_client));
2201         if(!w) {
2202                 error("Cannot allocate new web_client memory.");
2203                 return NULL;
2204         }
2205
2206         w->id = ++web_clients_count;
2207         w->mode = WEB_CLIENT_MODE_NORMAL;
2208
2209         addrlen = sizeof(w->clientaddr);
2210         w->ifd = accept(listener, (struct sockaddr *)&w->clientaddr, &addrlen);
2211         if (w->ifd == -1) {
2212                 error("%llu: Cannot accept new incoming connection.", w->id);
2213                 free(w);
2214                 return NULL;
2215         }
2216         w->ofd = w->ifd;
2217
2218         strncpy(w->client_ip, inet_ntoa(w->clientaddr.sin_addr), 100);
2219         w->client_ip[100] = '\0';
2220
2221         debug(D_WEB_CLIENT_ACCESS, "%llu: New web client from %s on socket %d.", w->id, w->client_ip, w->ifd);
2222
2223         {
2224                 int flag = 1;
2225                 if(setsockopt(w->ifd, SOL_SOCKET, SO_KEEPALIVE, (char *) &flag, sizeof(int)) != 0) error("%llu: Cannot set SO_KEEPALIVE on socket.", w->id);
2226         }
2227
2228         w->data = web_buffer_create(INITIAL_WEB_DATA_LENGTH);
2229         if(!w->data) {
2230                 close(w->ifd);
2231                 free(w);
2232                 return NULL;
2233         }
2234
2235         w->wait_receive = 1;
2236
2237         if(web_clients) web_clients->prev = w;
2238         w->next = web_clients;
2239         web_clients = w;
2240
2241         global_statistics.connected_clients++;
2242
2243         return(w);
2244 }
2245
2246 struct web_client *web_client_free(struct web_client *w)
2247 {
2248         struct web_client *n = w->next;
2249
2250         debug(D_WEB_CLIENT_ACCESS, "%llu: Closing web client from %s.", w->id, inet_ntoa(w->clientaddr.sin_addr));
2251
2252         if(w->prev)     w->prev->next = w->next;
2253         if(w->next) w->next->prev = w->prev;
2254
2255         if(w == web_clients) web_clients = w->next;
2256
2257         if(w->data) web_buffer_free(w->data);
2258         close(w->ifd);
2259         if(w->ofd != w->ifd) close(w->ofd);
2260         free(w);
2261
2262         global_statistics.connected_clients--;
2263
2264         return(n);
2265 }
2266
2267 #define GROUP_AVERAGE   0
2268 #define GROUP_MAX               1
2269
2270 // find the oldest entry in the data, skipping all empty slots
2271 time_t rrd_stats_first_entry_t(RRD_STATS *st)
2272 {
2273         if(!st->first_entry_t) return st->last_updated.tv_sec;
2274
2275         return st->first_entry_t / 1000000;
2276 }
2277
2278 unsigned long rrd_stats_one_json(RRD_STATS *st, char *options, struct web_buffer *wb)
2279 {
2280         time_t now = time(NULL);
2281         web_buffer_increase(wb, 16384);
2282
2283         pthread_rwlock_rdlock(&st->rwlock);
2284
2285         web_buffer_printf(wb,
2286                 "\t\t{\n"
2287                 "\t\t\t\"id\": \"%s\",\n"
2288                 "\t\t\t\"name\": \"%s\",\n"
2289                 "\t\t\t\"type\": \"%s\",\n"
2290                 "\t\t\t\"family\": \"%s\",\n"
2291                 "\t\t\t\"title\": \"%s\",\n"
2292                 "\t\t\t\"priority\": %ld,\n"
2293                 "\t\t\t\"enabled\": %d,\n"
2294                 "\t\t\t\"units\": \"%s\",\n"
2295                 "\t\t\t\"url\": \"/data/%s/%s\",\n"
2296                 "\t\t\t\"chart_type\": \"%s\",\n"
2297                 "\t\t\t\"counter\": %ld,\n"
2298                 "\t\t\t\"entries\": %ld,\n"
2299                 "\t\t\t\"first_entry_t\": %lu,\n"
2300                 "\t\t\t\"last_entry\": %ld,\n"
2301                 "\t\t\t\"last_entry_t\": %lu,\n"
2302                 "\t\t\t\"last_entry_secs_ago\": %lu,\n"
2303                 "\t\t\t\"update_every\": %d,\n"
2304                 "\t\t\t\"isdetail\": %d,\n"
2305                 "\t\t\t\"usec_since_last_update\": %llu,\n"
2306                 "\t\t\t\"absolute_total\": " TOTAL_NUMBER_FORMAT ",\n"
2307                 "\t\t\t\"last_absolute_total\": " TOTAL_NUMBER_FORMAT ",\n"
2308                 "\t\t\t\"dimensions\": [\n"
2309                 , st->id
2310                 , st->name
2311                 , st->type
2312                 , st->family
2313                 , st->title
2314                 , st->priority
2315                 , st->enabled
2316                 , st->units
2317                 , st->name, options?options:""
2318                 , chart_type_name(st->chart_type)
2319                 , st->counter
2320                 , st->entries
2321                 , rrd_stats_first_entry_t(st)
2322                 , st->current_entry
2323                 , st->last_updated.tv_sec
2324                 , now - (st->last_updated.tv_sec > now) ? now : st->last_updated.tv_sec
2325                 , st->update_every
2326                 , st->isdetail
2327                 , st->usec_since_last_update
2328                 , st->absolute_total
2329                 , st->last_absolute_total
2330                 );
2331
2332         unsigned long memory = st->memsize;
2333
2334         RRD_DIMENSION *rd;
2335         for(rd = st->dimensions; rd ; rd = rd->next) {
2336                 memory += rd->memsize;
2337
2338                 web_buffer_printf(wb,
2339                         "\t\t\t\t{\n"
2340                         "\t\t\t\t\t\"id\": \"%s\",\n"
2341                         "\t\t\t\t\t\"name\": \"%s\",\n"
2342                         "\t\t\t\t\t\"entries\": %ld,\n"
2343                         "\t\t\t\t\t\"isHidden\": %d,\n"
2344                         "\t\t\t\t\t\"algorithm\": \"%s\",\n"
2345                         "\t\t\t\t\t\"multiplier\": %ld,\n"
2346                         "\t\t\t\t\t\"divisor\": %ld,\n"
2347                         "\t\t\t\t\t\"last_entry_t\": %lu,\n"
2348                         "\t\t\t\t\t\"collected_value\": " COLLECTED_NUMBER_FORMAT ",\n"
2349                         "\t\t\t\t\t\"calculated_value\": " CALCULATED_NUMBER_FORMAT ",\n"
2350                         "\t\t\t\t\t\"last_collected_value\": " COLLECTED_NUMBER_FORMAT ",\n"
2351                         "\t\t\t\t\t\"last_calculated_value\": " CALCULATED_NUMBER_FORMAT ",\n"
2352                         "\t\t\t\t\t\"memory\": %lu\n"
2353                         "\t\t\t\t}%s\n"
2354                         , rd->id
2355                         , rd->name
2356                         , rd->entries
2357                         , rd->hidden
2358                         , algorithm_name(rd->algorithm)
2359                         , rd->multiplier
2360                         , rd->divisor
2361                         , rd->last_collected.tv_sec
2362                         , rd->collected_value
2363                         , rd->calculated_value
2364                         , rd->last_collected_value
2365                         , rd->last_calculated_value
2366                         , rd->memsize
2367                         , rd->next?",":""
2368                         );
2369         }
2370
2371         web_buffer_printf(wb,
2372                 "\t\t\t],\n"
2373                 "\t\t\t\"memory\" : %lu\n"
2374                 "\t\t}"
2375                 , memory
2376                 );
2377
2378         pthread_rwlock_unlock(&st->rwlock);
2379         return memory;
2380 }
2381
2382 #define RRD_GRAPH_JSON_HEADER "{\n\t\"charts\": [\n"
2383 #define RRD_GRAPH_JSON_FOOTER "\n\t]\n}\n"
2384
2385 void rrd_stats_graph_json(RRD_STATS *st, char *options, struct web_buffer *wb)
2386 {
2387         web_buffer_increase(wb, 16384);
2388
2389         web_buffer_printf(wb, RRD_GRAPH_JSON_HEADER);
2390         rrd_stats_one_json(st, options, wb);
2391         web_buffer_printf(wb, RRD_GRAPH_JSON_FOOTER);
2392 }
2393
2394 void rrd_stats_all_json(struct web_buffer *wb)
2395 {
2396         web_buffer_increase(wb, 1024);
2397
2398         unsigned long memory = 0;
2399         long c;
2400         RRD_STATS *st;
2401
2402         web_buffer_printf(wb, RRD_GRAPH_JSON_HEADER);
2403
2404         pthread_rwlock_rdlock(&root_rwlock);
2405         for(st = root, c = 0; st ; st = st->next) {
2406                 if(st->enabled) {
2407                         if(c) web_buffer_printf(wb, "%s", ",\n");
2408                         memory += rrd_stats_one_json(st, NULL, wb);
2409                         c++;
2410                 }
2411         }
2412         pthread_rwlock_unlock(&root_rwlock);
2413
2414         web_buffer_printf(wb, "\n\t],\n"
2415                 "\t\"hostname\": \"%s\",\n"
2416                 "\t\"update_every\": %d,\n"
2417                 "\t\"history\": %d,\n"
2418                 "\t\"memory\": %lu\n"
2419                 "}\n"
2420                 , hostname
2421                 , update_every
2422                 , save_history
2423                 , memory
2424                 );
2425 }
2426
2427 unsigned long rrd_stats_json(int type, RRD_STATS *st, struct web_buffer *wb, int entries_to_show, int group, int group_method, time_t after, time_t before, int only_non_zero)
2428 {
2429         int c;
2430         pthread_rwlock_rdlock(&st->rwlock);
2431
2432
2433         // -------------------------------------------------------------------------
2434         // switch from JSON to google JSON
2435
2436         char kq[2] = "\"";
2437         char sq[2] = "\"";
2438         switch(type) {
2439                 case DATASOURCE_GOOGLE_JSON:
2440                 case DATASOURCE_GOOGLE_JSONP:
2441                         kq[0] = '\0';
2442                         sq[0] = '\'';
2443                         break;
2444
2445                 case DATASOURCE_JSON:
2446                 default:
2447                         break;
2448         }
2449
2450
2451         // -------------------------------------------------------------------------
2452         // validate the parameters
2453
2454         if(entries_to_show < 1) entries_to_show = 1;
2455         if(group < 1) group = 1;
2456
2457         // make sure current_entry is within limits
2458         long current_entry = (long)st->current_entry - (long)1;
2459         if(current_entry < 0) current_entry = 0;
2460         else if(current_entry >= st->entries) current_entry = st->entries - 1;
2461
2462         // find the oldest entry of the round-robin
2463         long max_entries_init = (st->counter < (unsigned long)st->entries) ? st->counter : (unsigned long)st->entries;
2464
2465         if(before == 0) before = st->last_updated.tv_sec;
2466         if(after  == 0) after = rrd_stats_first_entry_t(st);
2467
2468         time_t time_init = st->last_updated.tv_sec;
2469
2470         // ---
2471
2472         // our return value (the last timestamp printed)
2473         // this is required to detect re-transmit in google JSONP
2474         time_t last_timestamp = 0;
2475
2476
2477         // -------------------------------------------------------------------------
2478         // find how many dimensions we have
2479
2480         int dimensions = 0;
2481         RRD_DIMENSION *rd;
2482         for( rd = st->dimensions ; rd ; rd = rd->next) dimensions++;
2483         if(!dimensions) {
2484                 pthread_rwlock_unlock(&st->rwlock);
2485                 web_buffer_printf(wb, "No dimensions yet.");
2486                 return 0;
2487         }
2488
2489
2490         // -------------------------------------------------------------------------
2491         // prepare various strings, to speed up the loop
2492
2493         char overflow_annotation[201]; snprintf(overflow_annotation, 200, ",{%sv%s:%sRESET OR OVERFLOW%s},{%sv%s:%sThe counters have been wrapped.%s}", kq, kq, sq, sq, kq, kq, sq, sq);
2494         char normal_annotation[201];   snprintf(normal_annotation,   200, ",{%sv%s:null},{%sv%s:null}", kq, kq, kq, kq);
2495         char pre_date[51];             snprintf(pre_date,             50, "             {%sc%s:[{%sv%s:%s", kq, kq, kq, kq, sq);
2496         char post_date[21];            snprintf(post_date,            20, "%s}", sq);
2497         char pre_value[21];            snprintf(pre_value,            20, ",{%sv%s:", kq, kq);
2498         char post_value[21];           snprintf(post_value,           20, "}");
2499
2500
2501         // -------------------------------------------------------------------------
2502         // checks for debuging
2503
2504         if(st->debug) {
2505                 debug(D_RRD_STATS, "%s first_entry_t = %lu, last_entry_t = %lu, duration = %lu, after = %lu, before = %lu, duration = %lu, entries_to_show = %lu, group = %lu, max_entries = %ld"
2506                         , st->id
2507                         , rrd_stats_first_entry_t(st)
2508                         , st->last_updated.tv_sec
2509                         , st->last_updated.tv_sec - rrd_stats_first_entry_t(st)
2510                         , after
2511                         , before
2512                         , before - after
2513                         , entries_to_show
2514                         , group
2515                         , max_entries_init
2516                         );
2517
2518                 if(before < after)
2519                         debug(D_RRD_STATS, "WARNING: %s The newest value in the database (%lu) is earlier than the oldest (%lu)", st->name, before, after);
2520
2521                 if((before - after) > st->entries * st->update_every)
2522                         debug(D_RRD_STATS, "WARNING: %s The time difference between the oldest and the newest entries (%lu) is higher than the capacity of the database (%lu)", st->name, before - after, st->entries * st->update_every);
2523         }
2524
2525
2526         // -------------------------------------------------------------------------
2527         // temp arrays for keeping values per dimension
2528
2529         calculated_number group_values[dimensions]; // keep sums when grouping
2530         storage_number    print_values[dimensions]; // keep the final value to be printed
2531         int               print_hidden[dimensions]; // keep hidden flags
2532         int               found_non_zero[dimensions];
2533
2534         // initialize them
2535         for( rd = st->dimensions, c = 0 ; rd && c < dimensions ; rd = rd->next, c++) {
2536                 group_values[c] = print_values[c] = 0;
2537                 print_hidden[c] = rd->hidden;
2538                 found_non_zero[c] = 0;
2539         }
2540
2541         // -------------------------------------------------------------------------
2542         // remove dimensions that contain only zeros
2543
2544         int max_loop = 1;
2545         if(only_non_zero) max_loop = 2;
2546
2547         for(; max_loop ; max_loop--) {
2548
2549                 // -------------------------------------------------------------------------
2550                 // print the JSON header
2551
2552                 web_buffer_printf(wb, "{\n      %scols%s:\n     [\n", kq, kq);
2553                 web_buffer_printf(wb, "         {%sid%s:%s%s,%slabel%s:%stime%s,%spattern%s:%s%s,%stype%s:%sdatetime%s},\n", kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq);
2554                 web_buffer_printf(wb, "         {%sid%s:%s%s,%slabel%s:%s%s,%spattern%s:%s%s,%stype%s:%sstring%s,%sp%s:{%srole%s:%sannotation%s}},\n", kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, kq, kq, sq, sq);
2555                 web_buffer_printf(wb, "         {%sid%s:%s%s,%slabel%s:%s%s,%spattern%s:%s%s,%stype%s:%sstring%s,%sp%s:{%srole%s:%sannotationText%s}}", kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, kq, kq, sq, sq);
2556
2557                 // print the header for each dimension
2558                 // and update the print_hidden array for the dimensions that should be hidden
2559                 int pc = 0;
2560                 for( rd = st->dimensions, c = 0 ; rd && c < dimensions ; rd = rd->next, c++) {
2561                         if(!print_hidden[c]) {
2562                                 pc++;
2563                                 web_buffer_printf(wb, ",\n              {%sid%s:%s%s,%slabel%s:%s%s%s,%spattern%s:%s%s,%stype%s:%snumber%s}", kq, kq, sq, sq, kq, kq, sq, rd->name, sq, kq, kq, sq, sq, kq, kq, sq, sq);
2564                         }
2565                 }
2566                 if(!pc) {
2567                         web_buffer_printf(wb, ",\n              {%sid%s:%s%s,%slabel%s:%s%s%s,%spattern%s:%s%s,%stype%s:%snumber%s}", kq, kq, sq, sq, kq, kq, sq, "no data", sq, kq, kq, sq, sq, kq, kq, sq, sq);
2568                 }
2569
2570                 // print the begin of row data
2571                 web_buffer_printf(wb, "\n       ],\n    %srows%s:\n     [\n", kq, kq);
2572
2573
2574                 // -------------------------------------------------------------------------
2575                 // the main loop
2576
2577                 int annotate_reset = 0;
2578                 int annotation_count = 0;
2579
2580                 // to allow grouping on the same values, we need a pad
2581                 long pad = before % group;
2582
2583                 // the minimum line length we expect
2584                 int line_size = 4096 + (dimensions * 200);
2585
2586                 time_t now = time_init;
2587                 long max_entries = max_entries_init;
2588
2589                 long t;
2590
2591                 long count = 0, printed = 0, group_count = 0;
2592                 last_timestamp = 0;
2593                 for(t = current_entry; max_entries ; now--, t--, max_entries--) {
2594                         if(t < 0) t = st->entries - 1;
2595
2596                         int print_this = 0;
2597
2598                         if(st->debug) {
2599                                 debug(D_RRD_STATS, "%s t = %ld, count = %ld, group_count = %ld, printed = %ld, now = %lu, %s %s"
2600                                         , st->id
2601                                         , t
2602                                         , count + 1
2603                                         , group_count + 1
2604                                         , printed
2605                                         , now
2606                                         , (((count + 1 - pad) % group) == 0)?"PRINT":"  -  "
2607                                         , (now >= after && now <= before)?"RANGE":"  -  "
2608                                         );
2609                         }
2610
2611                         // make sure we return data in the proper time range
2612                         if(now < after || now > before) continue;
2613
2614                         count++;
2615                         group_count++;
2616
2617                         if(((count - pad) % group) == 0) {
2618                                 if(printed >= entries_to_show) {
2619                                         // debug(D_RRD_STATS, "Already printed all rows. Stopping.");
2620                                         break;
2621                                 }
2622
2623                                 if(group_count != group) {
2624                                         // this is an incomplete group, skip it.
2625                                         for( rd = st->dimensions, c = 0 ; rd && c < dimensions ; rd = rd->next, c++)
2626                                                 group_values[c] = 0;
2627
2628                                         group_count = 0;
2629                                         continue;
2630                                 }
2631
2632                                 // check if we may exceed the buffer provided
2633                                 web_buffer_increase(wb, line_size);
2634
2635                                 // generate the local date time
2636                                 struct tm *tm = localtime(&now);
2637                                 if(!tm) { error("localtime() failed."); continue; }
2638                                 if(now > last_timestamp) last_timestamp = now;
2639
2640                                 if(printed) web_buffer_strcpy(wb, "]},\n");
2641                                 web_buffer_strcpy(wb, pre_date);
2642                                 web_buffer_jsdate(wb, tm->tm_year + 1900, tm->tm_mon, tm->tm_mday, tm->tm_hour, tm->tm_min, tm->tm_sec);
2643                                 web_buffer_strcpy(wb, post_date);
2644
2645                                 print_this = 1;
2646                         }
2647
2648                         for(rd = st->dimensions, c = 0 ; rd && c < dimensions ; rd = rd->next, c++) {
2649                                 long value = rd->values[t];
2650
2651                                 switch(group_method) {
2652                                         case GROUP_MAX:
2653                                                 if(abs(value) > abs(group_values[c])) group_values[c] = value;
2654                                                 break;
2655
2656                                         default:
2657                                         case GROUP_AVERAGE:
2658                                                 group_values[c] += value;
2659                                                 if(print_this) group_values[c] /= group_count;
2660                                                 break;
2661                                 }
2662
2663                                 if(print_this) {
2664                                         print_values[c] = group_values[c];
2665                                         group_values[c] = 0;
2666                                 }
2667                         }
2668
2669                         if(print_this) {
2670                                 group_count = 0;
2671
2672                                 if(annotate_reset) {
2673                                         annotation_count++;
2674                                         web_buffer_strcpy(wb, overflow_annotation);
2675                                         annotate_reset = 0;
2676                                 }
2677                                 else
2678                                         web_buffer_strcpy(wb, normal_annotation);
2679
2680                                 pc = 0;
2681                                 for(c = 0 ; c < dimensions ; c++) {
2682                                         if(!print_hidden[c]) {
2683                                                 pc++;
2684                                                 web_buffer_strcpy(wb, pre_value);
2685                                                 web_buffer_rrd_value(wb, print_values[c]);
2686                                                 web_buffer_strcpy(wb, post_value);
2687
2688                                                 if(print_values[c]) found_non_zero[c]++;
2689                                         }
2690                                 }
2691                                 if(!pc) {
2692                                         web_buffer_strcpy(wb, pre_value);
2693                                         web_buffer_rrd_value(wb, (storage_number)0);
2694                                         web_buffer_strcpy(wb, post_value);
2695                                 }
2696
2697                                 printed++;
2698                         }
2699                 }
2700
2701                 if(printed) web_buffer_printf(wb, "]}");
2702                 web_buffer_printf(wb, "\n       ]\n}\n");
2703
2704                 if(only_non_zero && max_loop > 1) {
2705                         int changed = 0;
2706                         for(rd = st->dimensions, c = 0 ; rd && c < dimensions ; rd = rd->next, c++) {
2707                                 group_values[c] = 0;
2708
2709                                 if(!print_hidden[c] && !found_non_zero[c]) {
2710                                         changed = 1;
2711                                         print_hidden[c] = 1;
2712                                 }
2713                         }
2714
2715                         if(changed) web_buffer_reset(wb);
2716                         else break;
2717                 }
2718                 else break;
2719
2720         } // max_loop
2721
2722         debug(D_RRD_STATS, "RRD_STATS_JSON: %s total %ld bytes", st->name, wb->bytes);
2723
2724         pthread_rwlock_unlock(&st->rwlock);
2725         return last_timestamp;
2726 }
2727
2728 void generate_config(struct web_buffer *wb, int only_changed)
2729 {
2730         int i, pri;
2731         struct config *co;
2732         struct config_value *cv;
2733
2734         for(i = 0; i < 3 ;i++) {
2735                 web_buffer_increase(wb, 500);
2736                 switch(i) {
2737                         case 0:
2738                                 web_buffer_printf(wb,
2739                                         "# NetData Configuration\n"
2740                                         "# You can uncomment and change any of the options bellow.\n"
2741                                         "# The value shown in the commented settings, is the default value.\n"
2742                                         "\n# global netdata configuration\n");
2743                                 break;
2744
2745                         case 1:
2746                                 web_buffer_printf(wb, "\n\n# per plugin configuration\n");
2747                                 break;
2748
2749                         case 2:
2750                                 web_buffer_printf(wb, "\n\n# per chart configuration\n");
2751                                 break;
2752                 }
2753
2754                 for(co = config_root; co ; co = co->next) {
2755                         if(strcmp(co->name, "global") == 0 || strcmp(co->name, "plugins") == 0) pri = 0;
2756                         else if(strncmp(co->name, "plugin:", 7) == 0) pri = 1;
2757                         else pri = 2;
2758
2759                         if(i == pri) {
2760                                 int used = 0;
2761                                 int changed = 0;
2762                                 int count = 0;
2763                                 for(cv = co->values; cv ; cv = cv->next) {
2764                                         used += cv->used;
2765                                         changed += cv->changed;
2766                                         count++;
2767                                 }
2768
2769                                 if(!count) continue;
2770                                 if(only_changed && !changed) continue;
2771
2772                                 if(!used) {
2773                                         web_buffer_increase(wb, 500);
2774                                         web_buffer_printf(wb, "\n# node '%s' is not used.", co->name);
2775                                 }
2776
2777                                 web_buffer_increase(wb, CONFIG_MAX_NAME + 4);
2778                                 web_buffer_printf(wb, "\n[%s]\n", co->name);
2779
2780                                 for(cv = co->values; cv ; cv = cv->next) {
2781
2782                                         if(used && !cv->used) {
2783                                                 web_buffer_increase(wb, CONFIG_MAX_NAME + 200);
2784                                                 web_buffer_printf(wb, "\n\t# option '%s' is not used.\n", cv->name);
2785                                         }
2786                                         web_buffer_increase(wb, CONFIG_MAX_NAME + CONFIG_MAX_VALUE + 5);
2787                                         web_buffer_printf(wb, "\t%s%s = %s\n", (!cv->changed && cv->used)?"# ":"", cv->name, cv->value);
2788                                 }
2789                         }
2790                 }
2791         }
2792 }
2793
2794 int mysendfile(struct web_client *w, char *filename)
2795 {
2796         static char *web_dir = NULL;
2797         if(!web_dir) web_dir = config_get("global", "web files directory", "web");
2798
2799         debug(D_WEB_CLIENT, "%llu: Looking for file '%s'...", w->id, filename);
2800
2801         // skip leading slashes
2802         while (*filename == '/') filename++;
2803
2804         // if the filename contain known paths, skip them
2805                  if(strncmp(filename, WEB_PATH_DATA       "/", strlen(WEB_PATH_DATA)       + 1) == 0) filename = &filename[strlen(WEB_PATH_DATA)       + 1];
2806         else if(strncmp(filename, WEB_PATH_DATASOURCE "/", strlen(WEB_PATH_DATASOURCE) + 1) == 0) filename = &filename[strlen(WEB_PATH_DATASOURCE) + 1];
2807         else if(strncmp(filename, WEB_PATH_GRAPH      "/", strlen(WEB_PATH_GRAPH)      + 1) == 0) filename = &filename[strlen(WEB_PATH_GRAPH)      + 1];
2808         else if(strncmp(filename, WEB_PATH_FILE       "/", strlen(WEB_PATH_FILE)       + 1) == 0) filename = &filename[strlen(WEB_PATH_FILE)       + 1];
2809
2810         // if the filename contains a / or a .., refuse to serve it
2811         if(strchr(filename, '/') != 0 || strstr(filename, "..") != 0) {
2812                 debug(D_WEB_CLIENT_ACCESS, "%llu: File '%s' is not acceptable.", w->id, filename);
2813                 web_buffer_printf(w->data, "File '%s' cannot be served. Filenames cannot contain / or ..", filename);
2814                 return 400;
2815         }
2816
2817         // access the file
2818         char webfilename[FILENAME_MAX + 1];
2819         snprintf(webfilename, FILENAME_MAX, "%s/%s", web_dir, filename);
2820
2821         // check if the file exists
2822         struct stat stat;
2823         if(lstat(webfilename, &stat) != 0) {
2824                 error("%llu: File '%s' is not found.", w->id, webfilename);
2825                 web_buffer_printf(w->data, "File '%s' does not exist, or is not accessible.", filename);
2826                 return 404;
2827         }
2828
2829         // check if the file is owned by us
2830         if(stat.st_uid != getuid() && stat.st_uid != geteuid()) {
2831                 error("%llu: File '%s' is owned by user %d (I run as user %d). Access Denied.", w->id, webfilename, stat.st_uid, getuid());
2832                 web_buffer_printf(w->data, "Access to file '%s' is not permitted.", filename);
2833                 return 403;
2834         }
2835
2836         // open the file
2837         w->ifd = open(webfilename, O_NONBLOCK, O_RDONLY);
2838         if(w->ifd == -1) {
2839                 w->ifd = w->ofd;
2840
2841                 if(errno == EBUSY || errno == EAGAIN) {
2842                         error("%llu: File '%s' is busy, sending 307 Moved Temporarily to force retry.", w->id, webfilename);
2843                         snprintf(w->response_header, MAX_HTTP_HEADER_SIZE, "Location: /" WEB_PATH_FILE "/%s\r\n", filename);
2844                         web_buffer_printf(w->data, "The file '%s' is currently busy. Please try again later.", filename);
2845                         return 307;
2846                 }
2847                 else {
2848                         error("%llu: Cannot open file '%s'.", w->id, webfilename);
2849                         web_buffer_printf(w->data, "Cannot open file '%s'.", filename);
2850                         return 404;
2851                 }
2852         }
2853
2854         // pick a Content-Type for the file
2855                  if(strstr(filename, ".html") != NULL)  w->data->contenttype = CT_TEXT_HTML;
2856         else if(strstr(filename, ".js")   != NULL)      w->data->contenttype = CT_APPLICATION_X_JAVASCRIPT;
2857         else if(strstr(filename, ".css")  != NULL)      w->data->contenttype = CT_TEXT_CSS;
2858         else if(strstr(filename, ".xml")  != NULL)      w->data->contenttype = CT_TEXT_XML;
2859         else if(strstr(filename, ".xsl")  != NULL)      w->data->contenttype = CT_TEXT_XSL;
2860         else if(strstr(filename, ".txt")  != NULL)  w->data->contenttype = CT_TEXT_PLAIN;
2861         else if(strstr(filename, ".svg")  != NULL)  w->data->contenttype = CT_IMAGE_SVG_XML;
2862         else if(strstr(filename, ".ttf")  != NULL)  w->data->contenttype = CT_APPLICATION_X_FONT_TRUETYPE;
2863         else if(strstr(filename, ".otf")  != NULL)  w->data->contenttype = CT_APPLICATION_X_FONT_OPENTYPE;
2864         else if(strstr(filename, ".woff") != NULL)  w->data->contenttype = CT_APPLICATION_FONT_WOFF;
2865         else if(strstr(filename, ".eot")  != NULL)  w->data->contenttype = CT_APPLICATION_VND_MS_FONTOBJ;
2866         else w->data->contenttype = CT_APPLICATION_OCTET_STREAM;
2867
2868         debug(D_WEB_CLIENT_ACCESS, "%llu: Sending file '%s' (%ld bytes, ifd %d, ofd %d).", w->id, webfilename, stat.st_size, w->ifd, w->ofd);
2869
2870         w->mode = WEB_CLIENT_MODE_FILECOPY;
2871         w->wait_receive = 1;
2872         w->wait_send = 0;
2873         w->data->bytes = 0;
2874         w->data->buffer[0] = '\0';
2875         w->data->rbytes = stat.st_size;
2876         w->data->date = stat.st_mtim.tv_sec;
2877
2878         return 200;
2879 }
2880
2881 char *mystrsep(char **ptr, char *s)
2882 {
2883         char *p = "";
2884         while ( p && !p[0] && *ptr ) p = strsep(ptr, s);
2885         return(p);
2886 }
2887
2888 void web_client_reset(struct web_client *w)
2889 {
2890         struct timeval tv;
2891         gettimeofday(&tv, NULL);
2892
2893         long sent = w->zoutput?(long)w->zstream.total_out:((w->mode == WEB_CLIENT_MODE_FILECOPY)?w->data->rbytes:w->data->bytes);
2894         long size = (w->mode == WEB_CLIENT_MODE_FILECOPY)?w->data->rbytes:w->data->bytes;
2895
2896         if(w->last_url[0]) log_access("%llu: (sent/all = %ld/%ld bytes %0.0f%%, prep/sent/total = %0.2f/%0.2f/%0.2f ms) %s: '%s'",
2897                 w->id,
2898                 sent, size, -((size>0)?((float)(size-sent)/(float)size * 100.0):0.0),
2899                 (float)usecdiff(&w->tv_ready, &w->tv_in) / 1000.0,
2900                 (float)usecdiff(&tv, &w->tv_ready) / 1000.0,
2901                 (float)usecdiff(&tv, &w->tv_in) / 1000.0,
2902                 (w->mode == WEB_CLIENT_MODE_FILECOPY)?"filecopy":"data",
2903                 w->last_url
2904                 );
2905
2906         debug(D_WEB_CLIENT, "%llu: Reseting client.", w->id);
2907
2908         if(w->mode == WEB_CLIENT_MODE_FILECOPY) {
2909                 debug(D_WEB_CLIENT, "%llu: Closing filecopy input file.", w->id);
2910                 close(w->ifd);
2911                 w->ifd = w->ofd;
2912         }
2913
2914         w->last_url[0] = '\0';
2915
2916         w->data->contenttype = CT_TEXT_PLAIN;
2917         w->mode = WEB_CLIENT_MODE_NORMAL;
2918
2919         w->data->rbytes = 0;
2920         w->data->bytes = 0;
2921         w->data->sent = 0;
2922
2923         w->response_header[0] = '\0';
2924         w->data->buffer[0] = '\0';
2925
2926         w->wait_receive = 1;
2927         w->wait_send = 0;
2928
2929         // if we had enabled compression, release it
2930         if(w->zinitialized) {
2931                 debug(D_DEFLATE, "%llu: Reseting compression.", w->id);
2932                 deflateEnd(&w->zstream);
2933                 w->zoutput = 0;
2934                 w->zsent = 0;
2935                 w->zhave = 0;
2936                 w->zstream.avail_in = 0;
2937                 w->zstream.avail_out = 0;
2938                 w->zstream.total_in = 0;
2939                 w->zstream.total_out = 0;
2940                 w->zinitialized = 0;
2941         }
2942 }
2943
2944 void web_client_enable_deflate(struct web_client *w) {
2945         if(w->zinitialized == 1) {
2946                 error("%llu: Compression has already be initialized for this client.", w->id);
2947                 return;
2948         }
2949
2950         if(w->data->sent) {
2951                 error("%llu: Cannot enable compression in the middle of a conversation.", w->id);
2952                 return;
2953         }
2954
2955         w->zstream.zalloc = Z_NULL;
2956         w->zstream.zfree = Z_NULL;
2957         w->zstream.opaque = Z_NULL;
2958
2959         w->zstream.next_in = (Bytef *)w->data->buffer;
2960         w->zstream.avail_in = 0;
2961         w->zstream.total_in = 0;
2962
2963         w->zstream.next_out = w->zbuffer;
2964         w->zstream.avail_out = 0;
2965         w->zstream.total_out = 0;
2966
2967         w->zstream.zalloc = Z_NULL;
2968         w->zstream.zfree = Z_NULL;
2969         w->zstream.opaque = Z_NULL;
2970
2971 //      if(deflateInit(&w->zstream, Z_DEFAULT_COMPRESSION) != Z_OK) {
2972 //              error("%llu: Failed to initialize zlib. Proceeding without compression.", w->id);
2973 //              return;
2974 //      }
2975
2976         // Select GZIP compression: windowbits = 15 + 16 = 31
2977         if(deflateInit2(&w->zstream, Z_DEFAULT_COMPRESSION, Z_DEFLATED, 31, 8, Z_DEFAULT_STRATEGY) != Z_OK) {
2978                 error("%llu: Failed to initialize zlib. Proceeding without compression.", w->id);
2979                 return;
2980         }
2981
2982         w->zsent = 0;
2983         w->zoutput = 1;
2984         w->zinitialized = 1;
2985
2986         debug(D_DEFLATE, "%llu: Initialized compression.", w->id);
2987 }
2988
2989 int web_client_data_request(struct web_client *w, char *url, int datasource_type)
2990 {
2991         char *args = strchr(url, '?');
2992         if(args) {
2993                 *args='\0';
2994                 args = &args[1];
2995         }
2996
2997         // get the name of the data to show
2998         char *tok = mystrsep(&url, "/");
2999         debug(D_WEB_CLIENT, "%llu: Searching for RRD data with name '%s'.", w->id, tok);
3000
3001         // do we have such a data set?
3002         RRD_STATS *st = rrd_stats_find_byname(tok);
3003         if(!st) st = rrd_stats_find(tok);
3004         if(!st) {
3005                 // we don't have it
3006                 // try to send a file with that name
3007                 w->data->bytes = 0;
3008                 return(mysendfile(w, tok));
3009         }
3010
3011         // we have it
3012         debug(D_WEB_CLIENT, "%llu: Found RRD data with name '%s'.", w->id, tok);
3013
3014         // how many entries does the client want?
3015         long lines = save_history;
3016         long group_count = 1;
3017         time_t after = 0, before = 0;
3018         int group_method = GROUP_AVERAGE;
3019         int nonzero = 0;
3020
3021         if(url) {
3022                 // parse the lines required
3023                 tok = mystrsep(&url, "/");
3024                 if(tok) lines = atoi(tok);
3025                 if(lines < 1) lines = 1;
3026         }
3027         if(url) {
3028                 // parse the group count required
3029                 tok = mystrsep(&url, "/");
3030                 if(tok) group_count = atoi(tok);
3031                 if(group_count < 1) group_count = 1;
3032                 //if(group_count > save_history / 20) group_count = save_history / 20;
3033         }
3034         if(url) {
3035                 // parse the grouping method required
3036                 tok = mystrsep(&url, "/");
3037                 if(strcmp(tok, "max") == 0) group_method = GROUP_MAX;
3038                 else if(strcmp(tok, "average") == 0) group_method = GROUP_AVERAGE;
3039                 else debug(D_WEB_CLIENT, "%llu: Unknown group method '%s'", w->id, tok);
3040         }
3041         if(url) {
3042                 // parse after time
3043                 tok = mystrsep(&url, "/");
3044                 if(tok) after = strtoul(tok, NULL, 10);
3045                 if(after < 0) after = 0;
3046         }
3047         if(url) {
3048                 // parse before time
3049                 tok = mystrsep(&url, "/");
3050                 if(tok) before = strtoul(tok, NULL, 10);
3051                 if(before < 0) before = 0;
3052         }
3053         if(url) {
3054                 // parse nonzero
3055                 tok = mystrsep(&url, "/");
3056                 if(tok && strcmp(tok, "nonzero") == 0) nonzero = 1;
3057         }
3058
3059         w->data->contenttype = CT_APPLICATION_JSON;
3060         w->data->bytes = 0;
3061
3062         char *google_version = "0.6";
3063         char *google_reqId = "0";
3064         char *google_sig = "0";
3065         char *google_out = "json";
3066         char *google_responseHandler = "google.visualization.Query.setResponse";
3067         char *google_outFileName = NULL;
3068         unsigned long last_timestamp_in_data = 0;
3069         if(datasource_type == DATASOURCE_GOOGLE_JSON || datasource_type == DATASOURCE_GOOGLE_JSONP) {
3070
3071                 w->data->contenttype = CT_APPLICATION_X_JAVASCRIPT;
3072
3073                 while(args) {
3074                         tok = mystrsep(&args, "&");
3075                         if(tok) {
3076                                 char *name = mystrsep(&tok, "=");
3077                                 if(name && strcmp(name, "tqx") == 0) {
3078                                         char *key = mystrsep(&tok, ":");
3079                                         char *value = mystrsep(&tok, ";");
3080                                         if(key && value && *key && *value) {
3081                                                 if(strcmp(key, "version") == 0)
3082                                                         google_version = value;
3083
3084                                                 else if(strcmp(key, "reqId") == 0)
3085                                                         google_reqId = value;
3086
3087                                                 else if(strcmp(key, "sig") == 0)
3088                                                         google_sig = value;
3089
3090                                                 else if(strcmp(key, "out") == 0)
3091                                                         google_out = value;
3092
3093                                                 else if(strcmp(key, "responseHandler") == 0)
3094                                                         google_responseHandler = value;
3095
3096                                                 else if(strcmp(key, "outFileName") == 0)
3097                                                         google_outFileName = value;
3098                                         }
3099                                 }
3100                         }
3101                 }
3102
3103                 debug(D_WEB_CLIENT_ACCESS, "%llu: GOOGLE JSONP: version = '%s', reqId = '%s', sig = '%s', out = '%s', responseHandler = '%s', outFileName = '%s'",
3104                         w->id, google_version, google_reqId, google_sig, google_out, google_responseHandler, google_outFileName
3105                         );
3106
3107                 if(datasource_type == DATASOURCE_GOOGLE_JSONP) {
3108                         last_timestamp_in_data = strtoul(google_sig, NULL, 0);
3109
3110                         // check the client wants json
3111                         if(strcmp(google_out, "json") != 0) {
3112                                 w->data->bytes = snprintf(w->data->buffer, w->data->size,
3113                                         "%s({version:'%s',reqId:'%s',status:'error',errors:[{reason:'invalid_query',message:'output format is not supported',detailed_message:'the format %s requested is not supported by netdata.'}]});",
3114                                         google_responseHandler, google_version, google_reqId, google_out);
3115                                         return 200;
3116                         }
3117                 }
3118         }
3119
3120         if(datasource_type == DATASOURCE_GOOGLE_JSONP) {
3121                 w->data->bytes = snprintf(w->data->buffer, w->data->size,
3122                         "%s({version:'%s',reqId:'%s',status:'ok',sig:'%lu',table:",
3123                         google_responseHandler, google_version, google_reqId, st->last_updated.tv_sec);
3124         }
3125
3126         debug(D_WEB_CLIENT_ACCESS, "%llu: Sending RRD data '%s' (id %s, %d lines, %d group, %d group_method, %lu after, %lu before).", w->id, st->name, st->id, lines, group_count, group_method, after, before);
3127         unsigned long timestamp_in_data = rrd_stats_json(datasource_type, st, w->data, lines, group_count, group_method, after, before, nonzero);
3128
3129         if(datasource_type == DATASOURCE_GOOGLE_JSONP) {
3130                 if(timestamp_in_data > last_timestamp_in_data)
3131                         w->data->bytes += snprintf(&w->data->buffer[w->data->bytes], w->data->size - w->data->bytes, "});");
3132
3133                 else {
3134                         // the client already has the latest data
3135                         w->data->bytes = snprintf(w->data->buffer, w->data->size,
3136                                 "%s({version:'%s',reqId:'%s',status:'error',errors:[{reason:'not_modified',message:'Data not modified'}]});",
3137                                 google_responseHandler, google_version, google_reqId);
3138                 }
3139         }
3140
3141         return 200;
3142 }
3143
3144 void web_client_process(struct web_client *w)
3145 {
3146         int code = 500;
3147         int bytes;
3148
3149         w->wait_receive = 0;
3150
3151         // check if we have an empty line (end of HTTP header)
3152         if(strstr(w->data->buffer, "\r\n\r\n")) {
3153                 global_statistics_lock();
3154                 global_statistics.web_requests++;
3155                 global_statistics_unlock();
3156
3157                 gettimeofday(&w->tv_in, NULL);
3158                 debug(D_WEB_DATA, "%llu: Processing data buffer of %d bytes: '%s'.", w->id, w->data->bytes, w->data->buffer);
3159
3160                 // check if the client requested keep-alive HTTP
3161                 if(strcasestr(w->data->buffer, "Connection: keep-alive")) w->keepalive = 1;
3162                 else w->keepalive = 0;
3163
3164                 // check if the client accepts deflate
3165                 if(strstr(w->data->buffer, "gzip"))
3166                         web_client_enable_deflate(w);
3167
3168                 int datasource_type = DATASOURCE_GOOGLE_JSONP;
3169                 //if(strstr(w->data->buffer, "X-DataSource-Auth"))
3170                 //      datasource_type = DATASOURCE_GOOGLE_JSON;
3171
3172                 char *buf = w->data->buffer;
3173                 char *tok = strsep(&buf, " \r\n");
3174                 char *url = NULL;
3175                 char *pointer_to_free = NULL; // keep url_decode() allocated buffer
3176
3177                 if(buf && strcmp(tok, "GET") == 0) {
3178                         tok = strsep(&buf, " \r\n");
3179                         pointer_to_free = url = url_decode(tok);
3180                         debug(D_WEB_CLIENT, "%llu: Processing HTTP GET on url '%s'.", w->id, url);
3181                 }
3182                 else if (buf && strcmp(tok, "POST") == 0) {
3183                         w->keepalive = 0;
3184                         tok = strsep(&buf, " \r\n");
3185                         pointer_to_free = url = url_decode(tok);
3186
3187                         debug(D_WEB_CLIENT, "%llu: I don't know how to handle POST with form data. Assuming it is a GET on url '%s'.", w->id, url);
3188                 }
3189
3190                 w->last_url[0] = '\0';
3191                 if(url) {
3192                         strncpy(w->last_url, url, URL_MAX);
3193                         w->last_url[URL_MAX] = '\0';
3194
3195                         tok = mystrsep(&url, "/?&");
3196
3197                         debug(D_WEB_CLIENT, "%llu: Processing command '%s'.", w->id, tok);
3198
3199                         if(strcmp(tok, WEB_PATH_DATA) == 0) { // "data"
3200                                 // the client is requesting rrd data
3201                                 datasource_type = DATASOURCE_JSON;
3202                                 code = web_client_data_request(w, url, datasource_type);
3203                         }
3204                         else if(strcmp(tok, WEB_PATH_DATASOURCE) == 0) { // "datasource"
3205                                 // the client is requesting google datasource
3206                                 code = web_client_data_request(w, url, datasource_type);
3207                         }
3208                         else if(strcmp(tok, WEB_PATH_GRAPH) == 0) { // "graph"
3209                                 // the client is requesting an rrd graph
3210
3211                                 // get the name of the data to show
3212                                 tok = mystrsep(&url, "/?&");
3213                                 debug(D_WEB_CLIENT, "%llu: Searching for RRD data with name '%s'.", w->id, tok);
3214
3215                                 // do we have such a data set?
3216                                 RRD_STATS *st = rrd_stats_find_byname(tok);
3217                                 if(!st) {
3218                                         // we don't have it
3219                                         // try to send a file with that name
3220                                         w->data->bytes = 0;
3221                                         code = mysendfile(w, tok);
3222                                 }
3223                                 else {
3224                                         code = 200;
3225                                         debug(D_WEB_CLIENT_ACCESS, "%llu: Sending %s.json of RRD_STATS...", w->id, st->name);
3226                                         w->data->contenttype = CT_APPLICATION_JSON;
3227                                         w->data->bytes = 0;
3228                                         rrd_stats_graph_json(st, url, w->data);
3229                                 }
3230                         }
3231                         else if(strcmp(tok, "debug") == 0) {
3232                                 w->data->bytes = 0;
3233
3234                                 // get the name of the data to show
3235                                 tok = mystrsep(&url, "/?&");
3236                                 debug(D_WEB_CLIENT, "%llu: Searching for RRD data with name '%s'.", w->id, tok);
3237
3238                                 // do we have such a data set?
3239                                 RRD_STATS *st = rrd_stats_find_byname(tok);
3240                                 if(!st) {
3241                                         code = 404;
3242                                         web_buffer_printf(w->data, "Chart %s is not found.\r\n", tok);
3243                                         debug(D_WEB_CLIENT_ACCESS, "%llu: %s is not found.", w->id, tok);
3244                                 }
3245                                 else {
3246                                         code = 200;
3247                                         debug_flags |= D_RRD_STATS;
3248                                         st->debug = st->debug?0:1;
3249                                         web_buffer_printf(w->data, "Chart %s has now debug %s.\r\n", tok, st->debug?"enabled":"disabled");
3250                                         debug(D_WEB_CLIENT_ACCESS, "%llu: debug for %s is %s.", w->id, tok, st->debug?"enabled":"disabled");
3251                                 }
3252                         }
3253                         else if(strcmp(tok, "mirror") == 0) {
3254                                 code = 200;
3255
3256                                 debug(D_WEB_CLIENT_ACCESS, "%llu: Mirroring...", w->id);
3257
3258                                 // replace the zero bytes with spaces
3259                                 int i;
3260                                 for(i = 0; i < w->data->size; i++)
3261                                         if(w->data->buffer[i] == '\0') w->data->buffer[i] = ' ';
3262
3263                                 // just leave the buffer as is
3264                                 // it will be copied back to the client
3265                         }
3266                         else if(strcmp(tok, "list") == 0) {
3267                                 code = 200;
3268
3269                                 debug(D_WEB_CLIENT_ACCESS, "%llu: Sending list of RRD_STATS...", w->id);
3270
3271                                 w->data->bytes = 0;
3272                                 RRD_STATS *st = root;
3273
3274                                 for ( ; st ; st = st->next )
3275                                         web_buffer_printf(w->data, "%s\n", st->name);
3276                         }
3277                         else if(strcmp(tok, "all.json") == 0) {
3278                                 code = 200;
3279                                 debug(D_WEB_CLIENT_ACCESS, "%llu: Sending JSON list of all monitors of RRD_STATS...", w->id);
3280
3281                                 w->data->contenttype = CT_APPLICATION_JSON;
3282                                 w->data->bytes = 0;
3283                                 rrd_stats_all_json(w->data);
3284                         }
3285                         else if(strcmp(tok, "netdata.conf") == 0) {
3286                                 code = 200;
3287                                 debug(D_WEB_CLIENT_ACCESS, "%llu: Sending netdata.conf ...", w->id);
3288
3289                                 w->data->contenttype = CT_TEXT_PLAIN;
3290                                 w->data->bytes = 0;
3291                                 generate_config(w->data, 0);
3292                         }
3293                         else if(strcmp(tok, WEB_PATH_FILE) == 0) { // "file"
3294                                 tok = mystrsep(&url, "/?&");
3295                                 if(tok && *tok) code = mysendfile(w, tok);
3296                                 else {
3297                                         code = 400;
3298                                         w->data->bytes = 0;
3299                                         strcpy(w->data->buffer, "You have to give a filename to get.\r\n");
3300                                         w->data->bytes = strlen(w->data->buffer);
3301                                 }
3302                         }
3303                         else if(!tok[0]) {
3304                                 w->data->bytes = 0;
3305                                 code = mysendfile(w, "index.html");
3306                         }
3307                         else {
3308                                 w->data->bytes = 0;
3309                                 code = mysendfile(w, tok);
3310                         }
3311
3312                 }
3313                 else {
3314                         strcpy(w->last_url, "not a valid response");
3315
3316                         if(buf) debug(D_WEB_CLIENT_ACCESS, "%llu: Cannot understand '%s'.", w->id, buf);
3317
3318                         code = 500;
3319                         w->data->bytes = 0;
3320                         strcpy(w->data->buffer, "I don't understand you...\r\n");
3321                         w->data->bytes = strlen(w->data->buffer);
3322                 }
3323
3324                 // free url_decode() buffer
3325                 if(pointer_to_free) free(pointer_to_free);
3326         }
3327         else if(w->data->bytes > 8192) {
3328                 strcpy(w->last_url, "too big request");
3329
3330                 debug(D_WEB_CLIENT_ACCESS, "%llu: Received request is too big.", w->id);
3331
3332                 code = 400;
3333                 w->data->bytes = 0;
3334                 strcpy(w->data->buffer, "Received request is too big.\r\n");
3335                 w->data->bytes = strlen(w->data->buffer);
3336         }
3337         else {
3338                 // wait for more data
3339                 w->wait_receive = 1;
3340                 return;
3341         }
3342
3343         if(w->data->bytes > w->data->size) {
3344                 error("%llu: memory overflow encountered (size is %ld, written %ld).", w->data->size, w->data->bytes);
3345         }
3346
3347         gettimeofday(&w->tv_ready, NULL);
3348         w->data->date = time(NULL);
3349         w->data->sent = 0;
3350
3351         // prepare the HTTP response header
3352         debug(D_WEB_CLIENT, "%llu: Generating HTTP header with response %d.", w->id, code);
3353
3354         char *content_type_string = "";
3355         switch(w->data->contenttype) {
3356                 case CT_TEXT_HTML:
3357                         content_type_string = "text/html";
3358                         break;
3359
3360                 case CT_APPLICATION_XML:
3361                         content_type_string = "application/xml";
3362                         break;
3363
3364                 case CT_APPLICATION_JSON:
3365                         content_type_string = "application/json";
3366                         break;
3367
3368                 case CT_APPLICATION_X_JAVASCRIPT:
3369                         content_type_string = "application/x-javascript";
3370                         break;
3371
3372                 case CT_TEXT_CSS:
3373                         content_type_string = "text/css";
3374                         break;
3375
3376                 case CT_TEXT_XML:
3377                         content_type_string = "text/xml";
3378                         break;
3379
3380                 case CT_TEXT_XSL:
3381                         content_type_string = "text/xsl";
3382                         break;
3383
3384                 case CT_APPLICATION_OCTET_STREAM:
3385                         content_type_string = "application/octet-stream";
3386                         break;
3387
3388                 case CT_IMAGE_SVG_XML:
3389                         content_type_string = "image/svg+xml";
3390                         break;
3391
3392                 case CT_APPLICATION_X_FONT_TRUETYPE:
3393                         content_type_string = "application/x-font-truetype";
3394                         break;
3395
3396                 case CT_APPLICATION_X_FONT_OPENTYPE:
3397                         content_type_string = "application/x-font-opentype";
3398                         break;
3399
3400                 case CT_APPLICATION_FONT_WOFF:
3401                         content_type_string = "application/font-woff";
3402                         break;
3403
3404                 case CT_APPLICATION_VND_MS_FONTOBJ:
3405                         content_type_string = "application/vnd.ms-fontobject";
3406                         break;
3407
3408                 default:
3409                 case CT_TEXT_PLAIN:
3410                         content_type_string = "text/plain";
3411                         break;
3412         }
3413
3414         char *code_msg = "";
3415         switch(code) {
3416                 case 200:
3417                         code_msg = "OK";
3418                         break;
3419
3420                 case 307:
3421                         code_msg = "Temporary Redirect";
3422                         break;
3423
3424                 case 400:
3425                         code_msg = "Bad Request";
3426                         break;
3427
3428                 case 403:
3429                         code_msg = "Forbidden";
3430                         break;
3431
3432                 case 404:
3433                         code_msg = "Not Found";
3434                         break;
3435
3436                 default:
3437                         code_msg = "Internal Server Error";
3438                         break;
3439         }
3440
3441         char date[100];
3442         struct tm tm = *gmtime(&w->data->date);
3443         strftime(date, sizeof(date), "%a, %d %b %Y %H:%M:%S %Z", &tm);
3444
3445         char custom_header[MAX_HTTP_HEADER_SIZE + 1] = "";
3446         if(w->response_header[0])
3447                 strcpy(custom_header, w->response_header);
3448
3449         int headerlen = 0;
3450         headerlen += snprintf(&w->response_header[headerlen], MAX_HTTP_HEADER_SIZE - headerlen,
3451                 "HTTP/1.1 %d %s\r\n"
3452                 "Connection: %s\r\n"
3453                 "Server: NetData Embedded HTTP Server\r\n"
3454                 "Content-Type: %s\r\n"
3455                 "Access-Control-Allow-Origin: *\r\n"
3456                 "Date: %s\r\n"
3457                 , code, code_msg
3458                 , w->keepalive?"keep-alive":"close"
3459                 , content_type_string
3460                 , date
3461                 );
3462
3463         if(custom_header[0])
3464                 headerlen += snprintf(&w->response_header[headerlen], MAX_HTTP_HEADER_SIZE - headerlen, "%s", custom_header);
3465
3466         if(w->mode == WEB_CLIENT_MODE_NORMAL) {
3467                 headerlen += snprintf(&w->response_header[headerlen], MAX_HTTP_HEADER_SIZE - headerlen,
3468                         "Expires: %s\r\n"
3469                         "Cache-Control: no-cache\r\n"
3470                         , date
3471                         );
3472         }
3473         else {
3474                 headerlen += snprintf(&w->response_header[headerlen], MAX_HTTP_HEADER_SIZE - headerlen,
3475                         "Cache-Control: public\r\n"
3476                         );
3477         }
3478
3479         // if we know the content length, put it
3480         if(!w->zoutput && (w->data->bytes || w->data->rbytes))
3481                 headerlen += snprintf(&w->response_header[headerlen], MAX_HTTP_HEADER_SIZE - headerlen,
3482                         "Content-Length: %ld\r\n"
3483                         , w->data->bytes?w->data->bytes:w->data->rbytes
3484                         );
3485         else if(!w->zoutput)
3486                 w->keepalive = 0;       // content-length is required for keep-alive
3487
3488         if(w->zoutput) {
3489                 headerlen += snprintf(&w->response_header[headerlen], MAX_HTTP_HEADER_SIZE - headerlen,
3490                         "Content-Encoding: gzip\r\n"
3491                         "Transfer-Encoding: chunked\r\n"
3492                         );
3493         }
3494
3495         headerlen += snprintf(&w->response_header[headerlen], MAX_HTTP_HEADER_SIZE - headerlen, "\r\n");
3496
3497         // disable TCP_NODELAY, to buffer the header
3498         int flag = 0;
3499         if(setsockopt(w->ofd, IPPROTO_TCP, TCP_NODELAY, (char *) &flag, sizeof(int)) != 0) error("%llu: failed to disable TCP_NODELAY on socket.", w->id);
3500
3501         // sent the HTTP header
3502         debug(D_WEB_DATA, "%llu: Sending response HTTP header of size %d: '%s'", w->id, headerlen, w->response_header);
3503
3504         bytes = send(w->ofd, w->response_header, headerlen, 0);
3505         if(bytes != headerlen)
3506                 error("%llu: HTTP Header failed to be sent (I sent %d bytes but the system sent %d bytes).", w->id, headerlen, bytes);
3507         else {
3508                 global_statistics_lock();
3509                 global_statistics.bytes_sent += bytes;
3510                 global_statistics_unlock();
3511         }
3512
3513         // enable TCP_NODELAY, to send all data immediately at the next send()
3514         flag = 1;
3515         if(setsockopt(w->ofd, IPPROTO_TCP, TCP_NODELAY, (char *) &flag, sizeof(int)) != 0) error("%llu: failed to enable TCP_NODELAY on socket.", w->id);
3516
3517         // enable sending immediately if we have data
3518         if(w->data->bytes) w->wait_send = 1;
3519         else w->wait_send = 0;
3520
3521         // pretty logging
3522         switch(w->mode) {
3523                 case WEB_CLIENT_MODE_NORMAL:
3524                         debug(D_WEB_CLIENT, "%llu: Done preparing the response. Sending data (%d bytes) to client.", w->id, w->data->bytes);
3525                         break;
3526
3527                 case WEB_CLIENT_MODE_FILECOPY:
3528                         if(w->data->rbytes) {
3529                                 debug(D_WEB_CLIENT, "%llu: Done preparing the response. Will be sending data file of %d bytes to client.", w->id, w->data->rbytes);
3530                                 w->wait_receive = 1;
3531
3532                                 /*
3533                                 // utilize the kernel sendfile() for copying the file to the socket.
3534                                 // this block of code can be commented, without anything missing.
3535                                 // when it is commented, the program will copy the data using async I/O.
3536                                 {
3537                                         long len = sendfile(w->ofd, w->ifd, NULL, w->data->rbytes);
3538                                         if(len != w->data->rbytes) error("%llu: sendfile() should copy %ld bytes, but copied %ld. Falling back to manual copy.", w->id, w->data->rbytes, len);
3539                                         else web_client_reset(w);
3540                                 }
3541                                 */
3542                         }
3543                         else
3544                                 debug(D_WEB_CLIENT, "%llu: Done preparing the response. Will be sending an unknown amount of bytes to client.", w->id);
3545                         break;
3546
3547                 default:
3548                         fatal("%llu: Unknown client mode %d.", w->id, w->mode);
3549                         break;
3550         }
3551 }
3552
3553 long web_client_send_chunk_header(struct web_client *w, int len)
3554 {
3555         debug(D_DEFLATE, "%llu: OPEN CHUNK of %d bytes (hex: %x).", w->id, len, len);
3556         char buf[1024];
3557         sprintf(buf, "%X\r\n", len);
3558         int bytes = send(w->ofd, buf, strlen(buf), MSG_DONTWAIT);
3559
3560         if(bytes > 0) debug(D_DEFLATE, "%llu: Sent chunk header %d bytes.", w->id, bytes);
3561         else if(bytes == 0) debug(D_DEFLATE, "%llu: Did not send chunk header to the client.", w->id);
3562         else debug(D_DEFLATE, "%llu: Failed to send chunk header to client. Reason: %s", w->id, strerror(errno));
3563
3564         return bytes;
3565 }
3566
3567 long web_client_send_chunk_close(struct web_client *w)
3568 {
3569         //debug(D_DEFLATE, "%llu: CLOSE CHUNK.", w->id);
3570
3571         int bytes = send(w->ofd, "\r\n", 2, MSG_DONTWAIT);
3572
3573         if(bytes > 0) debug(D_DEFLATE, "%llu: Sent chunk suffix %d bytes.", w->id, bytes);
3574         else if(bytes == 0) debug(D_DEFLATE, "%llu: Did not send chunk suffix to the client.", w->id);
3575         else debug(D_DEFLATE, "%llu: Failed to send chunk suffix to client. Reason: %s", w->id, strerror(errno));
3576
3577         return bytes;
3578 }
3579
3580 long web_client_send_chunk_finalize(struct web_client *w)
3581 {
3582         //debug(D_DEFLATE, "%llu: FINALIZE CHUNK.", w->id);
3583
3584         int bytes = send(w->ofd, "\r\n0\r\n\r\n", 7, MSG_DONTWAIT);
3585
3586         if(bytes > 0) debug(D_DEFLATE, "%llu: Sent chunk suffix %d bytes.", w->id, bytes);
3587         else if(bytes == 0) debug(D_DEFLATE, "%llu: Did not send chunk suffix to the client.", w->id);
3588         else debug(D_DEFLATE, "%llu: Failed to send chunk suffix to client. Reason: %s", w->id, strerror(errno));
3589
3590         return bytes;
3591 }
3592
3593 long web_client_send_deflate(struct web_client *w)
3594 {
3595         long bytes = 0, t = 0;
3596
3597         // when using compression,
3598         // w->data->sent is the amount of bytes passed through compression
3599
3600         // debug(D_DEFLATE, "%llu: TEST w->data->bytes = %d, w->data->sent = %d, w->zhave = %d, w->zsent = %d, w->zstream.avail_in = %d, w->zstream.avail_out = %d, w->zstream.total_in = %d, w->zstream.total_out = %d.", w->id, w->data->bytes, w->data->sent, w->zhave, w->zsent, w->zstream.avail_in, w->zstream.avail_out, w->zstream.total_in, w->zstream.total_out);
3601
3602         if(w->data->bytes - w->data->sent == 0 && w->zstream.avail_in == 0 && w->zhave == w->zsent && w->zstream.avail_out != 0) {
3603                 // there is nothing to send
3604
3605                 debug(D_WEB_CLIENT, "%llu: Out of output data.", w->id);
3606
3607                 // finalize the chunk
3608                 if(w->data->sent != 0)
3609                         t += web_client_send_chunk_finalize(w);
3610
3611                 // there can be two cases for this
3612                 // A. we have done everything
3613                 // B. we temporarily have nothing to send, waiting for the buffer to be filled by ifd
3614
3615                 if(w->mode == WEB_CLIENT_MODE_FILECOPY && w->wait_receive && w->ifd != w->ofd && w->data->rbytes && w->data->rbytes > w->data->bytes) {
3616                         // we have to wait, more data will come
3617                         debug(D_WEB_CLIENT, "%llu: Waiting for more data to become available.", w->id);
3618                         w->wait_send = 0;
3619                         return(0);
3620                 }
3621
3622                 if(w->keepalive == 0) {
3623                         debug(D_WEB_CLIENT, "%llu: Closing (keep-alive is not enabled). %ld bytes sent.", w->id, w->data->sent);
3624                         errno = 0;
3625                         return(-1);
3626                 }
3627
3628                 // reset the client
3629                 web_client_reset(w);
3630                 debug(D_WEB_CLIENT, "%llu: Done sending all data on socket. Waiting for next request on the same socket.", w->id);
3631                 return(0);
3632         }
3633
3634         if(w->zstream.avail_out == 0 && w->zhave == w->zsent) {
3635                 // compress more input data
3636
3637                 // close the previous open chunk
3638                 if(w->data->sent != 0) t += web_client_send_chunk_close(w);
3639
3640                 debug(D_DEFLATE, "%llu: Compressing %d bytes starting from %d.", w->id, (w->data->bytes - w->data->sent), w->data->sent);
3641
3642                 // give the compressor all the data not passed through the compressor yet
3643                 if(w->data->bytes > w->data->sent) {
3644                         w->zstream.next_in = (Bytef *)&w->data->buffer[w->data->sent];
3645                         w->zstream.avail_in = (w->data->bytes - w->data->sent);
3646                 }
3647
3648                 // reset the compressor output buffer
3649                 w->zstream.next_out = w->zbuffer;
3650                 w->zstream.avail_out = ZLIB_CHUNK;
3651
3652                 // ask for FINISH if we have all the input
3653                 int flush = Z_SYNC_FLUSH;
3654                 if(w->mode == WEB_CLIENT_MODE_NORMAL
3655                         || (w->mode == WEB_CLIENT_MODE_FILECOPY && w->data->bytes == w->data->rbytes)) {
3656                         flush = Z_FINISH;
3657                         debug(D_DEFLATE, "%llu: Requesting Z_FINISH.", w->id);
3658                 }
3659                 else {
3660                         debug(D_DEFLATE, "%llu: Requesting Z_SYNC_FLUSH.", w->id);
3661                 }
3662
3663                 // compress
3664                 if(deflate(&w->zstream, flush) == Z_STREAM_ERROR) {
3665                         error("%llu: Compression failed. Closing down client.", w->id);
3666                         web_client_reset(w);
3667                         return(-1);
3668                 }
3669
3670                 w->zhave = ZLIB_CHUNK - w->zstream.avail_out;
3671                 w->zsent = 0;
3672
3673                 // keep track of the bytes passed through the compressor
3674                 w->data->sent = w->data->bytes;
3675
3676                 debug(D_DEFLATE, "%llu: Compression produced %d bytes.", w->id, w->zhave);
3677
3678                 // open a new chunk
3679                 t += web_client_send_chunk_header(w, w->zhave);
3680         }
3681
3682         bytes = send(w->ofd, &w->zbuffer[w->zsent], w->zhave - w->zsent, MSG_DONTWAIT);
3683         if(bytes > 0) {
3684                 w->zsent += bytes;
3685                 if(t > 0) bytes += t;
3686                 debug(D_WEB_CLIENT, "%llu: Sent %d bytes.", w->id, bytes);
3687         }
3688         else if(bytes == 0) debug(D_WEB_CLIENT, "%llu: Did not send any bytes to the client.", w->id);
3689         else debug(D_WEB_CLIENT, "%llu: Failed to send data to client. Reason: %s", w->id, strerror(errno));
3690
3691         return(bytes);
3692 }
3693
3694 long web_client_send(struct web_client *w)
3695 {
3696         if(w->zoutput) return web_client_send_deflate(w);
3697
3698         long bytes;
3699
3700         if(w->data->bytes - w->data->sent == 0) {
3701                 // there is nothing to send
3702
3703                 debug(D_WEB_CLIENT, "%llu: Out of output data.", w->id);
3704
3705                 // there can be two cases for this
3706                 // A. we have done everything
3707                 // B. we temporarily have nothing to send, waiting for the buffer to be filled by ifd
3708
3709                 if(w->mode == WEB_CLIENT_MODE_FILECOPY && w->wait_receive && w->ifd != w->ofd && w->data->rbytes && w->data->rbytes > w->data->bytes) {
3710                         // we have to wait, more data will come
3711                         debug(D_WEB_CLIENT, "%llu: Waiting for more data to become available.", w->id);
3712                         w->wait_send = 0;
3713                         return(0);
3714                 }
3715
3716                 if(w->keepalive == 0) {
3717                         debug(D_WEB_CLIENT, "%llu: Closing (keep-alive is not enabled). %ld bytes sent.", w->id, w->data->sent);
3718                         errno = 0;
3719                         return(-1);
3720                 }
3721
3722                 web_client_reset(w);
3723                 debug(D_WEB_CLIENT, "%llu: Done sending all data on socket. Waiting for next request on the same socket.", w->id);
3724                 return(0);
3725         }
3726
3727         bytes = send(w->ofd, &w->data->buffer[w->data->sent], w->data->bytes - w->data->sent, MSG_DONTWAIT);
3728         if(bytes > 0) {
3729                 w->data->sent += bytes;
3730                 debug(D_WEB_CLIENT, "%llu: Sent %d bytes.", w->id, bytes);
3731         }
3732         else if(bytes == 0) debug(D_WEB_CLIENT, "%llu: Did not send any bytes to the client.", w->id);
3733         else debug(D_WEB_CLIENT, "%llu: Failed to send data to client. Reason: %s", w->id, strerror(errno));
3734
3735
3736         return(bytes);
3737 }
3738
3739 long web_client_receive(struct web_client *w)
3740 {
3741         // do we have any space for more data?
3742         web_buffer_increase(w->data, WEB_DATA_LENGTH_INCREASE_STEP);
3743
3744         long left = w->data->size - w->data->bytes;
3745         long bytes;
3746
3747         if(w->mode == WEB_CLIENT_MODE_FILECOPY)
3748                 bytes = read(w->ifd, &w->data->buffer[w->data->bytes], (left-1));
3749         else
3750                 bytes = recv(w->ifd, &w->data->buffer[w->data->bytes], left-1, MSG_DONTWAIT);
3751
3752         if(bytes > 0) {
3753                 int old = w->data->bytes;
3754                 w->data->bytes += bytes;
3755                 w->data->buffer[w->data->bytes] = '\0';
3756
3757                 debug(D_WEB_CLIENT, "%llu: Received %d bytes.", w->id, bytes);
3758                 debug(D_WEB_DATA, "%llu: Received data: '%s'.", w->id, &w->data->buffer[old]);
3759
3760                 if(w->mode == WEB_CLIENT_MODE_FILECOPY) {
3761                         w->wait_send = 1;
3762                         if(w->data->rbytes && w->data->bytes >= w->data->rbytes) w->wait_receive = 0;
3763                 }
3764         }
3765         else if(bytes == 0) {
3766                 debug(D_WEB_CLIENT, "%llu: Out of input data.", w->id);
3767
3768                 // if we cannot read, it means we have an error on input.
3769                 // if however, we are copying a file from ifd to ofd, we should not return an error.
3770                 // in this case, the error should be generated when the file has been sent to the client.
3771
3772                 if(w->mode == WEB_CLIENT_MODE_FILECOPY) {
3773                         // we are copying data fron ifd to ofd
3774                         // let it finish copying...
3775                         w->wait_receive = 0;
3776                         debug(D_WEB_CLIENT, "%llu: Disabling input.", w->id);
3777                 }
3778                 else {
3779                         bytes = -1;
3780                         errno = 0;
3781                 }
3782         }
3783
3784         return(bytes);
3785 }
3786
3787
3788 // --------------------------------------------------------------------------------------
3789 // the thread of a single client
3790
3791 // 1. waits for input and output, using async I/O
3792 // 2. it processes HTTP requests
3793 // 3. it generates HTTP responses
3794 // 4. it copies data from input to output if mode is FILECOPY
3795
3796 void *new_client(void *ptr)
3797 {
3798         if(pthread_setcanceltype(PTHREAD_CANCEL_DEFERRED, NULL) != 0)
3799                 error("Cannot set pthread cancel type to DEFERRED.");
3800
3801         if(pthread_setcancelstate(PTHREAD_CANCEL_ENABLE, NULL) != 0)
3802                 error("Cannot set pthread cancel state to ENABLE.");
3803
3804         struct timeval tv;
3805         struct web_client *w = ptr;
3806         int retval;
3807         fd_set ifds, ofds, efds;
3808         int fdmax = 0;
3809
3810         for(;;) {
3811                 FD_ZERO (&ifds);
3812                 FD_ZERO (&ofds);
3813                 FD_ZERO (&efds);
3814
3815                 FD_SET(w->ifd, &efds);
3816                 if(w->ifd != w->ofd)    FD_SET(w->ofd, &efds);
3817                 if (w->wait_receive) {
3818                         FD_SET(w->ifd, &ifds);
3819                         if(w->ifd > fdmax) fdmax = w->ifd;
3820                 }
3821                 if (w->wait_send) {
3822                         FD_SET(w->ofd, &ofds);
3823                         if(w->ofd > fdmax) fdmax = w->ofd;
3824                 }
3825
3826                 tv.tv_sec = 30;
3827                 tv.tv_usec = 0;
3828
3829                 debug(D_WEB_CLIENT, "%llu: Waiting socket async I/O for %s %s", w->id, w->wait_receive?"INPUT":"", w->wait_send?"OUTPUT":"");
3830                 retval = select(fdmax+1, &ifds, &ofds, &efds, &tv);
3831
3832                 if(retval == -1) {
3833                         error("%llu: LISTENER: select() failed.", w->id);
3834                         continue;
3835                 }
3836                 else if(!retval) {
3837                         // timeout
3838                         web_client_reset(w);
3839                         w->obsolete = 1;
3840                         return NULL;
3841                 }
3842
3843                 if(FD_ISSET(w->ifd, &efds)) {
3844                         debug(D_WEB_CLIENT_ACCESS, "%llu: Received error on input socket (%s).", w->id, strerror(errno));
3845                         web_client_reset(w);
3846                         w->obsolete = 1;
3847                         return NULL;
3848                 }
3849
3850                 if(FD_ISSET(w->ofd, &efds)) {
3851                         debug(D_WEB_CLIENT_ACCESS, "%llu: Received error on output socket (%s).", w->id, strerror(errno));
3852                         web_client_reset(w);
3853                         w->obsolete = 1;
3854                         return NULL;
3855                 }
3856
3857                 if(w->wait_send && FD_ISSET(w->ofd, &ofds)) {
3858                         long bytes;
3859                         if((bytes = web_client_send(w)) < 0) {
3860                                 debug(D_WEB_CLIENT, "%llu: Closing client (input: %s).", w->id, strerror(errno));
3861                                 web_client_reset(w);
3862                                 w->obsolete = 1;
3863                                 errno = 0;
3864                                 return NULL;
3865                         }
3866                         else {
3867                                 global_statistics_lock();
3868                                 global_statistics.bytes_sent += bytes;
3869                                 global_statistics_unlock();
3870                         }
3871                 }
3872
3873                 if(w->wait_receive && FD_ISSET(w->ifd, &ifds)) {
3874                         long bytes;
3875                         if((bytes = web_client_receive(w)) < 0) {
3876                                 debug(D_WEB_CLIENT, "%llu: Closing client (output: %s).", w->id, strerror(errno));
3877                                 web_client_reset(w);
3878                                 w->obsolete = 1;
3879                                 errno = 0;
3880                                 return NULL;
3881                         }
3882                         else {
3883                                 if(w->mode != WEB_CLIENT_MODE_FILECOPY) {
3884                                         global_statistics_lock();
3885                                         global_statistics.bytes_received += bytes;
3886                                         global_statistics_unlock();
3887                                 }
3888                         }
3889
3890                         if(w->mode == WEB_CLIENT_MODE_NORMAL) web_client_process(w);
3891                 }
3892         }
3893         debug(D_WEB_CLIENT, "%llu: done...", w->id);
3894
3895         return NULL;
3896 }
3897
3898 // --------------------------------------------------------------------------------------
3899 // the main socket listener
3900
3901 // 1. it accepts new incoming requests on our port
3902 // 2. creates a new web_client for each connection received
3903 // 3. spawns a new pthread to serve the client (this is optimal for keep-alive clients)
3904 // 4. cleans up old web_clients that their pthreads have been exited
3905
3906 void log_allocations(void)
3907 {
3908         static int mem = 0;
3909
3910         struct mallinfo mi;
3911
3912         mi = mallinfo();
3913         if(mi.uordblks > mem) {
3914                 int clients = 0;
3915                 struct web_client *w;
3916                 for(w = web_clients; w ; w = w->next) clients++;
3917
3918                 info("Allocated memory increased from %d to %d (increased by %d bytes). There are %d web clients connected.", mem, mi.uordblks, mi.uordblks - mem, clients);
3919                 mem = mi.uordblks;
3920         }
3921 }
3922
3923 void *socket_listen_main(void *ptr)
3924 {
3925         if(ptr) { ; }
3926         struct web_client *w;
3927         struct timeval tv;
3928         int retval;
3929
3930         // int listener = create_listen_socket(listen_port);
3931         int listener = listen_fd;
3932         if(listener == -1) fatal("LISTENER: Cannot create listening socket on port 19999.");
3933
3934         fd_set ifds, ofds, efds;
3935         int fdmax = listener;
3936
3937         FD_ZERO (&ifds);
3938         FD_ZERO (&ofds);
3939         FD_ZERO (&efds);
3940
3941         for(;;) {
3942                 tv.tv_sec = 0;
3943                 tv.tv_usec = 200000;
3944
3945                 FD_SET(listener, &ifds);
3946                 FD_SET(listener, &efds);
3947
3948                 // debug(D_WEB_CLIENT, "LISTENER: Waiting...");
3949                 retval = select(fdmax+1, &ifds, &ofds, &efds, &tv);
3950
3951                 if(retval == -1) {
3952                         error("LISTENER: select() failed.");
3953                         continue;
3954                 }
3955                 else if(retval) {
3956                         // check for new incoming connections
3957                         if(FD_ISSET(listener, &ifds)) {
3958                                 w = web_client_create(listener);
3959
3960                                 if(pthread_create(&w->thread, NULL, new_client, w) != 0) {
3961                                         error("%llu: failed to create new thread for web client.");
3962                                         w->obsolete = 1;
3963                                 }
3964                                 else if(pthread_detach(w->thread) != 0) {
3965                                         error("%llu: Cannot request detach of newly created web client thread.", w->id);
3966                                         w->obsolete = 1;
3967                                 }
3968
3969                                 log_access("%llu: %s connected", w->id, w->client_ip);
3970                         }
3971                         else debug(D_WEB_CLIENT, "LISTENER: select() didn't do anything.");
3972
3973                 }
3974                 //else debug(D_WEB_CLIENT, "LISTENER: select() timeout.");
3975
3976                 // cleanup unused clients
3977                 for(w = web_clients; w ; w = w?w->next:NULL) {
3978                         if(w->obsolete) {
3979                                 log_access("%llu: %s disconnected", w->id, w->client_ip);
3980                                 debug(D_WEB_CLIENT, "%llu: Removing client.", w->id);
3981                                 // pthread_join(w->thread,  NULL);
3982                                 w = web_client_free(w);
3983                                 log_allocations();
3984                         }
3985                 }
3986         }
3987
3988         error("LISTENER: exit!");
3989
3990         close(listener);
3991         exit(2);
3992
3993         return NULL;
3994 }
3995
3996 // ----------------------------------------------------------------------------
3997 // /proc/net/dev processor
3998
3999 #define MAX_PROC_NET_DEV_LINE 4096
4000 #define MAX_PROC_NET_DEV_IFACE_NAME 1024
4001
4002 int do_proc_net_dev() {
4003         static int enable_new_interfaces = -1;
4004         static int do_bandwidth = -1, do_packets = -1, do_errors = -1, do_fifo = -1, do_compressed = -1;
4005
4006         if(enable_new_interfaces == -1) enable_new_interfaces = config_get_boolean("plugin:proc:/proc/net/dev", "enable new interfaces detected at runtime", 1);
4007
4008         if(do_bandwidth == -1)  do_bandwidth = config_get_boolean("plugin:proc:/proc/net/dev", "bandwidth for all interfaces", 1);
4009         if(do_packets == -1)    do_packets = config_get_boolean("plugin:proc:/proc/net/dev", "packets for all interfaces", 1);
4010         if(do_errors == -1)             do_errors = config_get_boolean("plugin:proc:/proc/net/dev", "errors for all interfaces", 1);
4011         if(do_fifo == -1)               do_fifo = config_get_boolean("plugin:proc:/proc/net/dev", "fifo for all interfaces", 1);
4012         if(do_compressed == -1) do_compressed = config_get_boolean("plugin:proc:/proc/net/dev", "compressed packets for all interfaces", 1);
4013
4014         char buffer[MAX_PROC_NET_DEV_LINE+1] = "";
4015         char iface[MAX_PROC_NET_DEV_IFACE_NAME + 1] = "";
4016         unsigned long long rbytes, rpackets, rerrors, rdrops, rfifo, rframe, rcompressed, rmulticast;
4017         unsigned long long tbytes, tpackets, terrors, tdrops, tfifo, tcollisions, tcarrier, tcompressed;
4018
4019         int r;
4020         char *p;
4021
4022         FILE *fp = fopen("/proc/net/dev", "r");
4023         if(!fp) {
4024                 error("Cannot read /proc/net/dev.");
4025                 return 1;
4026         }
4027
4028         // skip the first two lines
4029         p = fgets(buffer, MAX_PROC_NET_DEV_LINE, fp);
4030         p = fgets(buffer, MAX_PROC_NET_DEV_LINE, fp);
4031
4032         // read the rest of the lines
4033         for(;1;) {
4034                 char *c;
4035                 p = fgets(buffer, MAX_PROC_NET_DEV_LINE, fp);
4036                 if(!p) break;
4037
4038                 c = strchr(buffer, ':');
4039                 if(c) *c = '\t';
4040
4041                 r = sscanf(buffer, "%s %llu %llu %llu %llu %llu %llu %llu %llu %llu %llu %llu %llu %llu %llu %llu %llu\n",
4042                         iface,
4043                         &rbytes, &rpackets, &rerrors, &rdrops, &rfifo, &rframe, &rcompressed, &rmulticast,
4044                         &tbytes, &tpackets, &terrors, &tdrops, &tfifo, &tcollisions, &tcarrier, &tcompressed);
4045                 if(r == EOF) break;
4046                 if(r != 17) {
4047                         error("Cannot read /proc/net/dev line. Expected 17 params, read %d.", r);
4048                         continue;
4049                 }
4050
4051                 // check if it is enabled
4052                 {
4053                         char var_name[4096 + 1];
4054                         snprintf(var_name, 4096, "interface %s", iface);
4055                         if(!config_get_boolean("plugin:proc:/proc/net/dev", var_name, enable_new_interfaces)) continue;
4056                 }
4057
4058                 RRD_STATS *st;
4059
4060                 // --------------------------------------------------------------------
4061
4062                 if(do_bandwidth) {
4063                         st = rrd_stats_find_bytype(RRD_TYPE_NET, iface);
4064                         if(!st) {
4065                                 st = rrd_stats_create(RRD_TYPE_NET, iface, NULL, iface, "Bandwidth", "kilobits/s", 1000, update_every, CHART_TYPE_AREA);
4066
4067                                 rrd_stats_dimension_add(st, "received", NULL, 8, 1024, RRD_DIMENSION_INCREMENTAL);
4068                                 rrd_stats_dimension_add(st, "sent", NULL, -8, 1024, RRD_DIMENSION_INCREMENTAL);
4069                         }
4070                         else rrd_stats_next(st);
4071
4072                         rrd_stats_dimension_set(st, "received", rbytes);
4073                         rrd_stats_dimension_set(st, "sent", tbytes);
4074                         rrd_stats_done(st);
4075                 }
4076
4077                 // --------------------------------------------------------------------
4078
4079                 if(do_packets) {
4080                         st = rrd_stats_find_bytype("net_packets", iface);
4081                         if(!st) {
4082                                 st = rrd_stats_create("net_packets", iface, NULL, iface, "Packets", "packets/s", 1001, update_every, CHART_TYPE_LINE);
4083                                 st->isdetail = 1;
4084
4085                                 rrd_stats_dimension_add(st, "received", NULL, 1, 1, RRD_DIMENSION_INCREMENTAL);
4086                                 rrd_stats_dimension_add(st, "sent", NULL, -1, 1, RRD_DIMENSION_INCREMENTAL);
4087                         }
4088                         else rrd_stats_next(st);
4089
4090                         rrd_stats_dimension_set(st, "received", rpackets);
4091                         rrd_stats_dimension_set(st, "sent", tpackets);
4092                         rrd_stats_done(st);
4093                 }
4094
4095                 // --------------------------------------------------------------------
4096
4097                 if(do_errors) {
4098                         st = rrd_stats_find_bytype("net_errors", iface);
4099                         if(!st) {
4100                                 st = rrd_stats_create("net_errors", iface, NULL, iface, "Interface Errors", "errors/s", 1002, update_every, CHART_TYPE_LINE);
4101                                 st->isdetail = 1;
4102
4103                                 rrd_stats_dimension_add(st, "receive", NULL, 1, 1, RRD_DIMENSION_INCREMENTAL);
4104                                 rrd_stats_dimension_add(st, "transmit", NULL, -1, 1, RRD_DIMENSION_INCREMENTAL);
4105                         }
4106                         else rrd_stats_next(st);
4107
4108                         rrd_stats_dimension_set(st, "receive", rerrors);
4109                         rrd_stats_dimension_set(st, "transmit", terrors);
4110                         rrd_stats_done(st);
4111                 }
4112
4113                 // --------------------------------------------------------------------
4114
4115                 if(do_fifo) {
4116                         st = rrd_stats_find_bytype("net_fifo", iface);
4117                         if(!st) {
4118                                 st = rrd_stats_create("net_fifo", iface, NULL, iface, "Interface Queue", "packets", 1100, update_every, CHART_TYPE_LINE);
4119                                 st->isdetail = 1;
4120
4121                                 rrd_stats_dimension_add(st, "receive", NULL, 1, 1, RRD_DIMENSION_ABSOLUTE);
4122                                 rrd_stats_dimension_add(st, "transmit", NULL, -1, 1, RRD_DIMENSION_ABSOLUTE);
4123                         }
4124                         else rrd_stats_next(st);
4125
4126                         rrd_stats_dimension_set(st, "receive", rfifo);
4127                         rrd_stats_dimension_set(st, "transmit", tfifo);
4128                         rrd_stats_done(st);
4129                 }
4130
4131                 // --------------------------------------------------------------------
4132
4133                 if(do_compressed) {
4134                         st = rrd_stats_find_bytype("net_compressed", iface);
4135                         if(!st) {
4136                                 st = rrd_stats_create("net_compressed", iface, NULL, iface, "Compressed Packets", "packets/s", 1200, update_every, CHART_TYPE_LINE);
4137                                 st->isdetail = 1;
4138
4139                                 rrd_stats_dimension_add(st, "received", NULL, 1, 1, RRD_DIMENSION_INCREMENTAL);
4140                                 rrd_stats_dimension_add(st, "sent", NULL, -1, 1, RRD_DIMENSION_INCREMENTAL);
4141                         }
4142                         else rrd_stats_next(st);
4143
4144                         rrd_stats_dimension_set(st, "received", rcompressed);
4145                         rrd_stats_dimension_set(st, "sent", tcompressed);
4146                         rrd_stats_done(st);
4147                 }
4148         }
4149
4150         fclose(fp);
4151         return 0;
4152 }
4153
4154 // ----------------------------------------------------------------------------
4155 // /proc/diskstats processor
4156
4157 #define MAX_PROC_DISKSTATS_LINE 4096
4158 #define MAX_PROC_DISKSTATS_DISK_NAME 1024
4159
4160 int do_proc_diskstats() {
4161         static int enable_new_disks = -1;
4162         static int do_io = -1, do_ops = -1, do_merged_ops = -1, do_iotime = -1, do_cur_ops = -1;
4163
4164         if(enable_new_disks == -1)      enable_new_disks = config_get_boolean("plugin:proc:/proc/diskstats", "enable new disks detected at runtime", 1);
4165
4166         if(do_io == -1)                 do_io = config_get_boolean("plugin:proc:/proc/diskstats", "bandwidth for all disks", 1);
4167         if(do_ops == -1)                do_ops = config_get_boolean("plugin:proc:/proc/diskstats", "operations for all disks", 1);
4168         if(do_merged_ops == -1) do_merged_ops = config_get_boolean("plugin:proc:/proc/diskstats", "merged operations for all disks", 1);
4169         if(do_iotime == -1)             do_iotime = config_get_boolean("plugin:proc:/proc/diskstats", "i/o time for all disks", 1);
4170         if(do_cur_ops == -1)    do_cur_ops = config_get_boolean("plugin:proc:/proc/diskstats", "current operations for all disks", 1);
4171
4172         char buffer[MAX_PROC_DISKSTATS_LINE+1] = "";
4173         char disk[MAX_PROC_DISKSTATS_DISK_NAME + 1] = "";
4174
4175         int r;
4176         char *p;
4177
4178         FILE *fp = fopen("/proc/diskstats", "r");
4179         if(!fp) {
4180                 error("Cannot read /proc/diskstats.");
4181                 return 1;
4182         }
4183
4184         for(;1;) {
4185                 unsigned long long      major = 0, minor = 0,
4186                                                         reads = 0,  reads_merged = 0,  readsectors = 0,  readms = 0,
4187                                                         writes = 0, writes_merged = 0, writesectors = 0, writems = 0,
4188                                                         currentios = 0, iosms = 0, wiosms = 0;
4189
4190                 p = fgets(buffer, MAX_PROC_DISKSTATS_LINE, fp);
4191                 if(!p) break;
4192
4193                 r = sscanf(buffer, "%llu %llu %s %llu %llu %llu %llu %llu %llu %llu %llu %llu %llu %llu\n",
4194                         &major, &minor, disk,
4195                         &reads, &reads_merged, &readsectors, &readms, &writes, &writes_merged, &writesectors, &writems, &currentios, &iosms, &wiosms
4196                 );
4197                 if(r == EOF) break;
4198                 if(r != 14) {
4199                         error("Cannot read /proc/diskstats line. Expected 14 params, read %d.", r);
4200                         continue;
4201                 }
4202
4203                 int def_enabled = 0;
4204
4205                 switch(major) {
4206                         case 9: // MDs
4207                         case 43: // network block
4208                         case 144: // nfs
4209                         case 145: // nfs
4210                         case 146: // nfs
4211                         case 199: // veritas
4212                         case 201: // veritas
4213                         case 251: // dm
4214                                 def_enabled = enable_new_disks;
4215                                 break;
4216
4217                         case 48: // RAID
4218                         case 49: // RAID
4219                         case 50: // RAID
4220                         case 51: // RAID
4221                         case 52: // RAID
4222                         case 53: // RAID
4223                         case 54: // RAID
4224                         case 55: // RAID
4225                         case 112: // RAID
4226                         case 136: // RAID
4227                         case 137: // RAID
4228                         case 138: // RAID
4229                         case 139: // RAID
4230                         case 140: // RAID
4231                         case 141: // RAID
4232                         case 142: // RAID
4233                         case 143: // RAID
4234                         case 179: // MMC
4235                         case 180: // USB
4236                                 if(minor % 8) def_enabled = 0; // partitions
4237                                 else def_enabled = enable_new_disks;
4238                                 break;
4239
4240                         case 8: // scsi disks
4241                         case 65: // scsi disks
4242                         case 66: // scsi disks
4243                         case 67: // scsi disks
4244                         case 68: // scsi disks
4245                         case 69: // scsi disks
4246                         case 70: // scsi disks
4247                         case 71: // scsi disks
4248                         case 72: // scsi disks
4249                         case 73: // scsi disks
4250                         case 74: // scsi disks
4251                         case 75: // scsi disks
4252                         case 76: // scsi disks
4253                         case 77: // scsi disks
4254                         case 78: // scsi disks
4255                         case 79: // scsi disks
4256                         case 80: // i2o
4257                         case 81: // i2o
4258                         case 82: // i2o
4259                         case 83: // i2o
4260                         case 84: // i2o
4261                         case 85: // i2o
4262                         case 86: // i2o
4263                         case 87: // i2o
4264                         case 101: // hyperdisk
4265                         case 102: // compressed
4266                         case 104: // scsi
4267                         case 105: // scsi
4268                         case 106: // scsi
4269                         case 107: // scsi
4270                         case 108: // scsi
4271                         case 109: // scsi
4272                         case 110: // scsi
4273                         case 111: // scsi
4274                         case 114: // bios raid
4275                         case 116: // ram board
4276                         case 128: // scsi
4277                         case 129: // scsi
4278                         case 130: // scsi
4279                         case 131: // scsi
4280                         case 132: // scsi
4281                         case 133: // scsi
4282                         case 134: // scsi
4283                         case 135: // scsi
4284                         case 153: // raid
4285                         case 202: // xen
4286                         case 256: // flash
4287                         case 257: // flash
4288                                 if(minor % 16) def_enabled = 0; // partitions
4289                                 else def_enabled = enable_new_disks;
4290                                 break;
4291
4292                         case 160: // raid
4293                         case 161: // raid
4294                                 if(minor % 32) def_enabled = 0; // partitions
4295                                 else def_enabled = enable_new_disks;
4296                                 break;
4297
4298                         case 3: // ide
4299                         case 13: // 8bit ide
4300                         case 22: // ide
4301                         case 33: // ide
4302                         case 34: // ide
4303                         case 56: // ide
4304                         case 57: // ide
4305                         case 88: // ide
4306                         case 89: // ide
4307                         case 90: // ide
4308                         case 91: // ide
4309                                 if(minor % 64) def_enabled = 0; // partitions
4310                                 else def_enabled = enable_new_disks;
4311                                 break;
4312
4313                         default:
4314                                 def_enabled = 0;
4315                                 break;
4316                 }
4317
4318                 // check if it is enabled
4319                 {
4320                         char var_name[4096 + 1];
4321                         snprintf(var_name, 4096, "disk %s", disk);
4322                         if(!config_get_boolean("plugin:proc:/proc/diskstats", var_name, def_enabled)) continue;
4323                 }
4324
4325                 RRD_STATS *st;
4326
4327                 // --------------------------------------------------------------------
4328
4329                 if(do_io) {
4330                         st = rrd_stats_find_bytype(RRD_TYPE_DISK, disk);
4331                         if(!st) {
4332                                 char tf[FILENAME_MAX + 1], *t;
4333                                 char ssfilename[FILENAME_MAX + 1];
4334                                 int sector_size = 512;
4335
4336                                 strncpy(tf, disk, FILENAME_MAX);
4337                                 tf[FILENAME_MAX] = '\0';
4338
4339                                 // replace all / with !
4340                                 while((t = strchr(tf, '/'))) *t = '!';
4341
4342                                 snprintf(ssfilename, FILENAME_MAX, "/sys/block/%s/queue/hw_sector_size", tf);
4343                                 FILE *fpss = fopen(ssfilename, "r");
4344                                 if(fpss) {
4345                                         char ssbuffer[1025];
4346                                         char *tmp = fgets(ssbuffer, 1024, fpss);
4347
4348                                         if(tmp) {
4349                                                 sector_size = atoi(tmp);
4350                                                 if(sector_size <= 0) {
4351                                                         error("Invalid sector size %d for device %s in %s. Assuming 512.", sector_size, disk, ssfilename);
4352                                                         sector_size = 512;
4353                                                 }
4354                                         }
4355                                         else error("Cannot read data for sector size for device %s from %s. Assuming 512.", disk, ssfilename);
4356
4357                                         fclose(fpss);
4358                                 }
4359                                 else error("Cannot read sector size for device %s from %s. Assuming 512.", disk, ssfilename);
4360
4361                                 st = rrd_stats_create(RRD_TYPE_DISK, disk, NULL, disk, "Disk I/O", "kilobytes/s", 2000, update_every, CHART_TYPE_AREA);
4362
4363                                 rrd_stats_dimension_add(st, "reads", NULL, sector_size, 1024, RRD_DIMENSION_INCREMENTAL);
4364                                 rrd_stats_dimension_add(st, "writes", NULL, sector_size * -1, 1024, RRD_DIMENSION_INCREMENTAL);
4365                         }
4366                         else rrd_stats_next(st);
4367
4368                         rrd_stats_dimension_set(st, "reads", readsectors);
4369                         rrd_stats_dimension_set(st, "writes", writesectors);
4370                         rrd_stats_done(st);
4371                 }
4372
4373                 // --------------------------------------------------------------------
4374
4375                 if(do_ops) {
4376                         st = rrd_stats_find_bytype("disk_ops", disk);
4377                         if(!st) {
4378                                 st = rrd_stats_create("disk_ops", disk, NULL, disk, "Disk Operations", "operations/s", 2001, update_every, CHART_TYPE_LINE);
4379                                 st->isdetail = 1;
4380
4381                                 rrd_stats_dimension_add(st, "reads", NULL, 1, 1, RRD_DIMENSION_INCREMENTAL);
4382                                 rrd_stats_dimension_add(st, "writes", NULL, -1, 1, RRD_DIMENSION_INCREMENTAL);
4383                         }
4384                         else rrd_stats_next(st);
4385
4386                         rrd_stats_dimension_set(st, "reads", reads);
4387                         rrd_stats_dimension_set(st, "writes", writes);
4388                         rrd_stats_done(st);
4389                 }
4390
4391                 // --------------------------------------------------------------------
4392
4393                 if(do_merged_ops) {
4394                         st = rrd_stats_find_bytype("disk_merged_ops", disk);
4395                         if(!st) {
4396                                 st = rrd_stats_create("disk_merged_ops", disk, NULL, disk, "Merged Disk Operations", "operations/s", 2010, update_every, CHART_TYPE_LINE);
4397                                 st->isdetail = 1;
4398
4399                                 rrd_stats_dimension_add(st, "reads", NULL, 1, 1, RRD_DIMENSION_INCREMENTAL);
4400                                 rrd_stats_dimension_add(st, "writes", NULL, -1, 1, RRD_DIMENSION_INCREMENTAL);
4401                         }
4402                         else rrd_stats_next(st);
4403
4404                         rrd_stats_dimension_set(st, "reads", reads_merged);
4405                         rrd_stats_dimension_set(st, "writes", writes_merged);
4406                         rrd_stats_done(st);
4407                 }
4408
4409                 // --------------------------------------------------------------------
4410
4411                 if(do_iotime) {
4412                         st = rrd_stats_find_bytype("disk_iotime", disk);
4413                         if(!st) {
4414                                 st = rrd_stats_create("disk_iotime", disk, NULL, disk, "Disk I/O Time", "milliseconds/s", 2005, update_every, CHART_TYPE_LINE);
4415                                 st->isdetail = 1;
4416
4417                                 rrd_stats_dimension_add(st, "reads", NULL, 1, 1, RRD_DIMENSION_INCREMENTAL);
4418                                 rrd_stats_dimension_add(st, "writes", NULL, -1, 1, RRD_DIMENSION_INCREMENTAL);
4419                                 rrd_stats_dimension_add(st, "latency", NULL, 1, 1, RRD_DIMENSION_INCREMENTAL);
4420                                 rrd_stats_dimension_add(st, "weighted", NULL, 1, 1, RRD_DIMENSION_INCREMENTAL);
4421                         }
4422                         else rrd_stats_next(st);
4423
4424                         rrd_stats_dimension_set(st, "reads", readms);
4425                         rrd_stats_dimension_set(st, "writes", writems);
4426                         rrd_stats_dimension_set(st, "latency", iosms);
4427                         rrd_stats_dimension_set(st, "weighted", wiosms);
4428                         rrd_stats_done(st);
4429                 }
4430
4431                 // --------------------------------------------------------------------
4432
4433                 if(do_cur_ops) {
4434                         st = rrd_stats_find_bytype("disk_cur_ops", disk);
4435                         if(!st) {
4436                                 st = rrd_stats_create("disk_cur_ops", disk, NULL, disk, "Current Disk I/O operations", "operations", 2004, update_every, CHART_TYPE_LINE);
4437                                 st->isdetail = 1;
4438
4439                                 rrd_stats_dimension_add(st, "operations", NULL, 1, 1, RRD_DIMENSION_ABSOLUTE);
4440                         }
4441                         else rrd_stats_next(st);
4442
4443                         rrd_stats_dimension_set(st, "operations", currentios);
4444                         rrd_stats_done(st);
4445                 }
4446         }
4447
4448         fclose(fp);
4449         return 0;
4450 }
4451
4452 // ----------------------------------------------------------------------------
4453 // /proc/net/snmp processor
4454
4455 int do_proc_net_snmp() {
4456         static int do_ip_packets = -1, do_ip_fragsout = -1, do_ip_fragsin = -1, do_ip_errors = -1,
4457                 do_tcp_sockets = -1, do_tcp_packets = -1, do_tcp_errors = -1, do_tcp_handshake = -1,
4458                 do_udp_packets = -1, do_udp_errors = -1;
4459
4460         if(do_ip_packets == -1)         do_ip_packets = config_get_boolean("plugin:proc:/proc/net/snmp", "ipv4 packets", 1);
4461         if(do_ip_fragsout == -1)        do_ip_fragsout = config_get_boolean("plugin:proc:/proc/net/snmp", "ipv4 fragrments sent", 1);
4462         if(do_ip_fragsin == -1)         do_ip_fragsin = config_get_boolean("plugin:proc:/proc/net/snmp", "ipv4 fragments assembly", 1);
4463         if(do_ip_errors == -1)          do_ip_errors = config_get_boolean("plugin:proc:/proc/net/snmp", "ipv4 errors", 1);
4464         if(do_tcp_sockets == -1)        do_tcp_sockets = config_get_boolean("plugin:proc:/proc/net/snmp", "ipv4 TCP connections", 1);
4465         if(do_tcp_packets == -1)        do_tcp_packets = config_get_boolean("plugin:proc:/proc/net/snmp", "ipv4 TCP packets", 1);
4466         if(do_tcp_errors == -1)         do_tcp_errors = config_get_boolean("plugin:proc:/proc/net/snmp", "ipv4 TCP errors", 1);
4467         if(do_tcp_handshake == -1)      do_tcp_handshake = config_get_boolean("plugin:proc:/proc/net/snmp", "ipv4 TCP handshake issues", 1);
4468         if(do_udp_packets == -1)        do_udp_packets = config_get_boolean("plugin:proc:/proc/net/snmp", "ipv4 UDP packets", 1);
4469         if(do_udp_errors == -1)         do_udp_errors = config_get_boolean("plugin:proc:/proc/net/snmp", "ipv4 UDP errors", 1);
4470
4471         char buffer[MAX_PROC_NET_SNMP_LINE+1] = "";
4472
4473         FILE *fp = fopen("/proc/net/snmp", "r");
4474         if(!fp) {
4475                 error("Cannot read /proc/net/snmp.");
4476                 return 1;
4477         }
4478
4479         RRD_STATS *st;
4480
4481         for(;1;) {
4482                 char *p = fgets(buffer, MAX_PROC_NET_SNMP_LINE, fp);
4483                 if(!p) break;
4484
4485                 if(strncmp(p, "Ip: ", 4) == 0) {
4486                         // skip the header line, read the data
4487                         p = fgets(buffer, MAX_PROC_NET_SNMP_LINE, fp);
4488                         if(!p) break;
4489
4490                         if(strncmp(p, "Ip: ", 4) != 0) {
4491                                 error("Cannot read IP line from /proc/net/snmp.");
4492                                 break;
4493                         }
4494
4495                         // see also http://net-snmp.sourceforge.net/docs/mibs/ip.html
4496                         unsigned long long Forwarding, DefaultTTL, InReceives, InHdrErrors, InAddrErrors, ForwDatagrams, InUnknownProtos, InDiscards, InDelivers,
4497                                 OutRequests, OutDiscards, OutNoRoutes, ReasmTimeout, ReasmReqds, ReasmOKs, ReasmFails, FragOKs, FragFails, FragCreates;
4498
4499                         int r = sscanf(&buffer[4], "%llu %llu %llu %llu %llu %llu %llu %llu %llu %llu %llu %llu %llu %llu %llu %llu %llu %llu %llu\n",
4500                                 &Forwarding, &DefaultTTL, &InReceives, &InHdrErrors, &InAddrErrors, &ForwDatagrams, &InUnknownProtos, &InDiscards, &InDelivers,
4501                                 &OutRequests, &OutDiscards, &OutNoRoutes, &ReasmTimeout, &ReasmReqds, &ReasmOKs, &ReasmFails, &FragOKs, &FragFails, &FragCreates);
4502
4503                         if(r == EOF) break;
4504                         if(r != 19) error("Cannot read /proc/net/snmp IP line. Expected 19 params, read %d.", r);
4505
4506                         // --------------------------------------------------------------------
4507
4508                         if(do_ip_packets) {
4509                                 st = rrd_stats_find(RRD_TYPE_NET_SNMP ".packets");
4510                                 if(!st) {
4511                                         st = rrd_stats_create(RRD_TYPE_NET_SNMP, "packets", NULL, RRD_TYPE_NET_SNMP, "IPv4 Packets", "packets/s", 3000, update_every, CHART_TYPE_LINE);
4512
4513                                         rrd_stats_dimension_add(st, "received", NULL, 1, 1, RRD_DIMENSION_INCREMENTAL);
4514                                         rrd_stats_dimension_add(st, "sent", NULL, -1, 1, RRD_DIMENSION_INCREMENTAL);
4515                                         rrd_stats_dimension_add(st, "forwarded", NULL, 1, 1, RRD_DIMENSION_INCREMENTAL);
4516                                 }
4517                                 else rrd_stats_next(st);
4518
4519                                 rrd_stats_dimension_set(st, "sent", OutRequests);
4520                                 rrd_stats_dimension_set(st, "received", InReceives);
4521                                 rrd_stats_dimension_set(st, "forwarded", ForwDatagrams);
4522                                 rrd_stats_done(st);
4523                         }
4524
4525                         // --------------------------------------------------------------------
4526
4527                         if(do_ip_fragsout) {
4528                                 st = rrd_stats_find(RRD_TYPE_NET_SNMP ".fragsout");
4529                                 if(!st) {
4530                                         st = rrd_stats_create(RRD_TYPE_NET_SNMP, "fragsout", NULL, RRD_TYPE_NET_SNMP, "IPv4 Fragments Sent", "packets/s", 3010, update_every, CHART_TYPE_LINE);
4531                                         st->isdetail = 1;
4532
4533                                         rrd_stats_dimension_add(st, "ok", NULL, 1, 1, RRD_DIMENSION_INCREMENTAL);
4534                                         rrd_stats_dimension_add(st, "failed", NULL, -1, 1, RRD_DIMENSION_INCREMENTAL);
4535                                         rrd_stats_dimension_add(st, "all", NULL, 1, 1, RRD_DIMENSION_INCREMENTAL);
4536                                 }
4537                                 else rrd_stats_next(st);
4538
4539                                 rrd_stats_dimension_set(st, "ok", FragOKs);
4540                                 rrd_stats_dimension_set(st, "failed", FragFails);
4541                                 rrd_stats_dimension_set(st, "all", FragCreates);
4542                                 rrd_stats_done(st);
4543                         }
4544
4545                         // --------------------------------------------------------------------
4546
4547                         if(do_ip_fragsin) {
4548                                 st = rrd_stats_find(RRD_TYPE_NET_SNMP ".fragsin");
4549                                 if(!st) {
4550                                         st = rrd_stats_create(RRD_TYPE_NET_SNMP, "fragsin", NULL, RRD_TYPE_NET_SNMP, "IPv4 Fragments Reassembly", "packets/s", 3011, update_every, CHART_TYPE_LINE);
4551                                         st->isdetail = 1;
4552
4553                                         rrd_stats_dimension_add(st, "ok", NULL, 1, 1, RRD_DIMENSION_INCREMENTAL);
4554                                         rrd_stats_dimension_add(st, "failed", NULL, -1, 1, RRD_DIMENSION_INCREMENTAL);
4555                                         rrd_stats_dimension_add(st, "all", NULL, 1, 1, RRD_DIMENSION_INCREMENTAL);
4556                                 }
4557                                 else rrd_stats_next(st);
4558
4559                                 rrd_stats_dimension_set(st, "ok", ReasmOKs);
4560                                 rrd_stats_dimension_set(st, "failed", ReasmFails);
4561                                 rrd_stats_dimension_set(st, "all", ReasmReqds);
4562                                 rrd_stats_done(st);
4563                         }
4564
4565                         // --------------------------------------------------------------------
4566
4567                         if(do_ip_errors) {
4568                                 st = rrd_stats_find(RRD_TYPE_NET_SNMP ".errors");
4569                                 if(!st) {
4570                                         st = rrd_stats_create(RRD_TYPE_NET_SNMP, "errors", NULL, RRD_TYPE_NET_SNMP, "IPv4 Errors", "packets/s", 3002, update_every, CHART_TYPE_LINE);
4571                                         st->isdetail = 1;
4572
4573                                         rrd_stats_dimension_add(st, "InDiscards", NULL, 1, 1, RRD_DIMENSION_INCREMENTAL);
4574                                         rrd_stats_dimension_add(st, "OutDiscards", NULL, -1, 1, RRD_DIMENSION_INCREMENTAL);
4575
4576                                         rrd_stats_dimension_add(st, "InHdrErrors", NULL, 1, 1, RRD_DIMENSION_INCREMENTAL);
4577                                         rrd_stats_dimension_add(st, "InAddrErrors", NULL, 1, 1, RRD_DIMENSION_INCREMENTAL);
4578                                         rrd_stats_dimension_add(st, "InUnknownProtos", NULL, 1, 1, RRD_DIMENSION_INCREMENTAL);
4579
4580                                         rrd_stats_dimension_add(st, "OutNoRoutes", NULL, -1, 1, RRD_DIMENSION_INCREMENTAL);
4581                                 }
4582                                 else rrd_stats_next(st);
4583
4584                                 rrd_stats_dimension_set(st, "InDiscards", InDiscards);
4585                                 rrd_stats_dimension_set(st, "OutDiscards", OutDiscards);
4586                                 rrd_stats_dimension_set(st, "InHdrErrors", InHdrErrors);
4587                                 rrd_stats_dimension_set(st, "InAddrErrors", InAddrErrors);
4588                                 rrd_stats_dimension_set(st, "InUnknownProtos", InUnknownProtos);
4589                                 rrd_stats_dimension_set(st, "OutNoRoutes", OutNoRoutes);
4590                                 rrd_stats_done(st);
4591                         }
4592                 }
4593                 else if(strncmp(p, "Tcp: ", 5) == 0) {
4594                         // skip the header line, read the data
4595                         p = fgets(buffer, MAX_PROC_NET_SNMP_LINE, fp);
4596                         if(!p) break;
4597
4598                         if(strncmp(p, "Tcp: ", 5) != 0) {
4599                                 error("Cannot read TCP line from /proc/net/snmp.");
4600                                 break;
4601                         }
4602
4603                         unsigned long long RtoAlgorithm, RtoMin, RtoMax, MaxConn, ActiveOpens, PassiveOpens, AttemptFails, EstabResets,
4604                                 CurrEstab, InSegs, OutSegs, RetransSegs, InErrs, OutRsts;
4605
4606                         int r = sscanf(&buffer[5], "%llu %llu %llu %llu %llu %llu %llu %llu %llu %llu %llu %llu %llu %llu\n",
4607                                 &RtoAlgorithm, &RtoMin, &RtoMax, &MaxConn, &ActiveOpens, &PassiveOpens, &AttemptFails, &EstabResets, &CurrEstab, &InSegs, &OutSegs, &RetransSegs, &InErrs, &OutRsts);
4608
4609                         if(r == EOF) break;
4610                         if(r != 14) error("Cannot read /proc/net/snmp TCP line. Expected 14 params, read %d.", r);
4611
4612                         // --------------------------------------------------------------------
4613
4614                         // see http://net-snmp.sourceforge.net/docs/mibs/tcp.html
4615                         if(do_tcp_sockets) {
4616                                 st = rrd_stats_find(RRD_TYPE_NET_SNMP ".tcpsock");
4617                                 if(!st) {
4618                                         st = rrd_stats_create(RRD_TYPE_NET_SNMP, "tcpsock", NULL, "tcp", "IPv4 TCP Connections", "active connections", 2500, update_every, CHART_TYPE_LINE);
4619
4620                                         rrd_stats_dimension_add(st, "connections", NULL, 1, 1, RRD_DIMENSION_ABSOLUTE);
4621                                 }
4622                                 else rrd_stats_next(st);
4623
4624                                 rrd_stats_dimension_set(st, "connections", CurrEstab);
4625                                 rrd_stats_done(st);
4626                         }
4627
4628                         // --------------------------------------------------------------------
4629
4630                         if(do_tcp_packets) {
4631                                 st = rrd_stats_find(RRD_TYPE_NET_SNMP ".tcppackets");
4632                                 if(!st) {
4633                                         st = rrd_stats_create(RRD_TYPE_NET_SNMP, "tcppackets", NULL, "tcp", "IPv4 TCP Packets", "packets/s", 2600, update_every, CHART_TYPE_LINE);
4634
4635                                         rrd_stats_dimension_add(st, "received", NULL, 1, 1, RRD_DIMENSION_INCREMENTAL);
4636                                         rrd_stats_dimension_add(st, "sent", NULL, -1, 1, RRD_DIMENSION_INCREMENTAL);
4637                                 }
4638                                 else rrd_stats_next(st);
4639
4640                                 rrd_stats_dimension_set(st, "received", InSegs);
4641                                 rrd_stats_dimension_set(st, "sent", OutSegs);
4642                                 rrd_stats_done(st);
4643                         }
4644
4645                         // --------------------------------------------------------------------
4646
4647                         if(do_tcp_errors) {
4648                                 st = rrd_stats_find(RRD_TYPE_NET_SNMP ".tcperrors");
4649                                 if(!st) {
4650                                         st = rrd_stats_create(RRD_TYPE_NET_SNMP, "tcperrors", NULL, "tcp", "IPv4 TCP Errors", "packets/s", 2700, update_every, CHART_TYPE_LINE);
4651                                         st->isdetail = 1;
4652
4653                                         rrd_stats_dimension_add(st, "InErrs", NULL, 1, 1, RRD_DIMENSION_INCREMENTAL);
4654                                         rrd_stats_dimension_add(st, "RetransSegs", NULL, -1, 1, RRD_DIMENSION_INCREMENTAL);
4655                                 }
4656                                 else rrd_stats_next(st);
4657
4658                                 rrd_stats_dimension_set(st, "InErrs", InErrs);
4659                                 rrd_stats_dimension_set(st, "RetransSegs", RetransSegs);
4660                                 rrd_stats_done(st);
4661                         }
4662
4663                         // --------------------------------------------------------------------
4664
4665                         if(do_tcp_handshake) {
4666                                 st = rrd_stats_find(RRD_TYPE_NET_SNMP ".tcphandshake");
4667                                 if(!st) {
4668                                         st = rrd_stats_create(RRD_TYPE_NET_SNMP, "tcphandshake", NULL, "tcp", "IPv4 TCP Handshake Issues", "events/s", 2900, update_every, CHART_TYPE_LINE);
4669                                         st->isdetail = 1;
4670
4671                                         rrd_stats_dimension_add(st, "EstabResets", NULL, 1, 1, RRD_DIMENSION_INCREMENTAL);
4672                                         rrd_stats_dimension_add(st, "OutRsts", NULL, -1, 1, RRD_DIMENSION_INCREMENTAL);
4673                                         rrd_stats_dimension_add(st, "ActiveOpens", NULL, 1, 1, RRD_DIMENSION_INCREMENTAL);
4674                                         rrd_stats_dimension_add(st, "PassiveOpens", NULL, 1, 1, RRD_DIMENSION_INCREMENTAL);
4675                                         rrd_stats_dimension_add(st, "AttemptFails", NULL, 1, 1, RRD_DIMENSION_INCREMENTAL);
4676                                 }
4677                                 else rrd_stats_next(st);
4678
4679                                 rrd_stats_dimension_set(st, "EstabResets", EstabResets);
4680                                 rrd_stats_dimension_set(st, "OutRsts", OutRsts);
4681                                 rrd_stats_dimension_set(st, "ActiveOpens", ActiveOpens);
4682                                 rrd_stats_dimension_set(st, "PassiveOpens", PassiveOpens);
4683                                 rrd_stats_dimension_set(st, "AttemptFails", AttemptFails);
4684                                 rrd_stats_done(st);
4685                         }
4686                 }
4687                 else if(strncmp(p, "Udp: ", 5) == 0) {
4688                         // skip the header line, read the data
4689                         p = fgets(buffer, MAX_PROC_NET_SNMP_LINE, fp);
4690                         if(!p) break;
4691
4692                         if(strncmp(p, "Udp: ", 5) != 0) {
4693                                 error("Cannot read UDP line from /proc/net/snmp.");
4694                                 break;
4695                         }
4696
4697                         unsigned long long InDatagrams, NoPorts, InErrors, OutDatagrams, RcvbufErrors, SndbufErrors;
4698
4699                         int r = sscanf(&buffer[5], "%llu %llu %llu %llu %llu %llu\n",
4700                                 &InDatagrams, &NoPorts, &InErrors, &OutDatagrams, &RcvbufErrors, &SndbufErrors);
4701
4702                         if(r == EOF) break;
4703                         if(r != 6) error("Cannot read /proc/net/snmp UDP line. Expected 6 params, read %d.", r);
4704
4705                         // --------------------------------------------------------------------
4706
4707                         // see http://net-snmp.sourceforge.net/docs/mibs/udp.html
4708                         if(do_udp_packets) {
4709                                 st = rrd_stats_find(RRD_TYPE_NET_SNMP ".udppackets");
4710                                 if(!st) {
4711                                         st = rrd_stats_create(RRD_TYPE_NET_SNMP, "udppackets", NULL, "udp", "IPv4 UDP Packets", "packets/s", 2601, update_every, CHART_TYPE_LINE);
4712
4713                                         rrd_stats_dimension_add(st, "received", NULL, 1, 1, RRD_DIMENSION_INCREMENTAL);
4714                                         rrd_stats_dimension_add(st, "sent", NULL, -1, 1, RRD_DIMENSION_INCREMENTAL);
4715                                 }
4716                                 else rrd_stats_next(st);
4717
4718                                 rrd_stats_dimension_set(st, "received", InDatagrams);
4719                                 rrd_stats_dimension_set(st, "sent", OutDatagrams);
4720                                 rrd_stats_done(st);
4721                         }
4722
4723                         // --------------------------------------------------------------------
4724
4725                         if(do_udp_errors) {
4726                                 st = rrd_stats_find(RRD_TYPE_NET_SNMP ".udperrors");
4727                                 if(!st) {
4728                                         st = rrd_stats_create(RRD_TYPE_NET_SNMP, "udperrors", NULL, "udp", "IPv4 UDP Errors", "events/s", 2701, update_every, CHART_TYPE_LINE);
4729                                         st->isdetail = 1;
4730
4731                                         rrd_stats_dimension_add(st, "RcvbufErrors", NULL, 1, 1, RRD_DIMENSION_INCREMENTAL);
4732                                         rrd_stats_dimension_add(st, "SndbufErrors", NULL, -1, 1, RRD_DIMENSION_INCREMENTAL);
4733                                         rrd_stats_dimension_add(st, "InErrors", NULL, 1, 1, RRD_DIMENSION_INCREMENTAL);
4734                                         rrd_stats_dimension_add(st, "NoPorts", NULL, 1, 1, RRD_DIMENSION_INCREMENTAL);
4735                                 }
4736                                 else rrd_stats_next(st);
4737
4738                                 rrd_stats_dimension_set(st, "InErrors", InErrors);
4739                                 rrd_stats_dimension_set(st, "NoPorts", NoPorts);
4740                                 rrd_stats_dimension_set(st, "RcvbufErrors", RcvbufErrors);
4741                                 rrd_stats_dimension_set(st, "SndbufErrors", SndbufErrors);
4742                                 rrd_stats_done(st);
4743                         }
4744                 }
4745         }
4746
4747         fclose(fp);
4748         return 0;
4749 }
4750
4751 // ----------------------------------------------------------------------------
4752 // /proc/net/netstat processor
4753
4754 int do_proc_net_netstat() {
4755         static int do_bandwidth = -1, do_inerrors = -1, do_mcast = -1, do_bcast = -1, do_mcast_p = -1, do_bcast_p = -1;
4756
4757         if(do_bandwidth == -1)  do_bandwidth = config_get_boolean("plugin:proc:/proc/net/netstat", "bandwidth", 1);
4758         if(do_inerrors == -1)   do_inerrors = config_get_boolean("plugin:proc:/proc/net/netstat", "input errors", 1);
4759         if(do_mcast == -1)              do_mcast = config_get_boolean("plugin:proc:/proc/net/netstat", "multicast bandwidth", 1);
4760         if(do_bcast == -1)              do_bcast = config_get_boolean("plugin:proc:/proc/net/netstat", "broadcast bandwidth", 1);
4761         if(do_mcast_p == -1)    do_mcast_p = config_get_boolean("plugin:proc:/proc/net/netstat", "multicast packets", 1);
4762         if(do_bcast_p == -1)    do_bcast_p = config_get_boolean("plugin:proc:/proc/net/netstat", "broadcast packets", 1);
4763
4764         char buffer[MAX_PROC_NET_SNMP_LINE+1] = "";
4765
4766         FILE *fp = fopen("/proc/net/netstat", "r");
4767         if(!fp) {
4768                 error("Cannot read /proc/net/netstat.");
4769                 return 1;
4770         }
4771
4772         for(;1;) {
4773                 char *p = fgets(buffer, MAX_PROC_NET_SNMP_LINE, fp);
4774                 if(!p) break;
4775
4776                 if(strncmp(p, "IpExt: ", 7) == 0) {
4777                         // skip the header line, read the data
4778                         p = fgets(buffer, MAX_PROC_NET_SNMP_LINE, fp);
4779                         if(!p) break;
4780
4781                         if(strncmp(p, "IpExt: ", 7) != 0) {
4782                                 error("Cannot read IpExt line from /proc/net/netstat.");
4783                                 break;
4784                         }
4785
4786                         unsigned long long
4787                                 InNoRoutes = 0, InTruncatedPkts = 0,
4788                                 InOctets = 0,  InMcastPkts = 0,  InBcastPkts = 0,  InMcastOctets = 0,  InBcastOctets = 0,
4789                                 OutOctets = 0, OutMcastPkts = 0, OutBcastPkts = 0, OutMcastOctets = 0, OutBcastOctets = 0;
4790
4791                         int r = sscanf(&buffer[7], "%llu %llu %llu %llu %llu %llu %llu %llu %llu %llu %llu %llu\n",
4792                                 &InNoRoutes, &InTruncatedPkts, &InMcastPkts, &OutMcastPkts, &InBcastPkts, &OutBcastPkts,
4793                                 &InOctets, &OutOctets, &InMcastOctets, &OutMcastOctets, &InBcastOctets, &OutBcastOctets);
4794
4795                         if(r == EOF) break;
4796                         if(r != 12) {
4797                                 error("Cannot read /proc/net/netstat IpExt line. Expected 12 params, read %d.", r);
4798                                 continue;
4799                         }
4800
4801                         RRD_STATS *st;
4802
4803                         // --------------------------------------------------------------------
4804
4805                         if(do_bandwidth) {
4806                                 st = rrd_stats_find("system.ipv4");
4807                                 if(!st) {
4808                                         st = rrd_stats_create("system", "ipv4", NULL, "ipv4", "IPv4 Bandwidth", "kilobits/s", 2000, update_every, CHART_TYPE_AREA);
4809
4810                                         rrd_stats_dimension_add(st, "received", NULL, 8, 1024, RRD_DIMENSION_INCREMENTAL);
4811                                         rrd_stats_dimension_add(st, "sent", NULL, -8, 1024, RRD_DIMENSION_INCREMENTAL);
4812                                 }
4813                                 else rrd_stats_next(st);
4814
4815                                 rrd_stats_dimension_set(st, "sent", OutOctets);
4816                                 rrd_stats_dimension_set(st, "received", InOctets);
4817                                 rrd_stats_done(st);
4818                         }
4819
4820                         // --------------------------------------------------------------------
4821
4822                         if(do_inerrors) {
4823                                 st = rrd_stats_find("ipv4.inerrors");
4824                                 if(!st) {
4825                                         st = rrd_stats_create("ipv4", "inerrors", NULL, "ipv4", "IPv4 Input Errors", "packets/s", 4000, update_every, CHART_TYPE_LINE);
4826                                         st->isdetail = 1;
4827
4828                                         rrd_stats_dimension_add(st, "noroutes", NULL, 1, 1, RRD_DIMENSION_INCREMENTAL);
4829                                         rrd_stats_dimension_add(st, "trunkated", NULL, 1, 1, RRD_DIMENSION_INCREMENTAL);
4830                                 }
4831                                 else rrd_stats_next(st);
4832
4833                                 rrd_stats_dimension_set(st, "noroutes", InNoRoutes);
4834                                 rrd_stats_dimension_set(st, "trunkated", InTruncatedPkts);
4835                                 rrd_stats_done(st);
4836                         }
4837
4838                         // --------------------------------------------------------------------
4839
4840                         if(do_mcast) {
4841                                 st = rrd_stats_find("ipv4.mcast");
4842                                 if(!st) {
4843                                         st = rrd_stats_create("ipv4", "mcast", NULL, "ipv4", "IPv4 Multicast Bandwidth", "kilobits/s", 9000, update_every, CHART_TYPE_AREA);
4844                                         st->isdetail = 1;
4845
4846                                         rrd_stats_dimension_add(st, "received", NULL, 8, 1024, RRD_DIMENSION_INCREMENTAL);
4847                                         rrd_stats_dimension_add(st, "sent", NULL, -8, 1024, RRD_DIMENSION_INCREMENTAL);
4848                                 }
4849                                 else rrd_stats_next(st);
4850
4851                                 rrd_stats_dimension_set(st, "sent", OutMcastOctets);
4852                                 rrd_stats_dimension_set(st, "received", InMcastOctets);
4853                                 rrd_stats_done(st);
4854                         }
4855
4856                         // --------------------------------------------------------------------
4857
4858                         if(do_bcast) {
4859                                 st = rrd_stats_find("ipv4.bcast");
4860                                 if(!st) {
4861                                         st = rrd_stats_create("ipv4", "bcast", NULL, "ipv4", "IPv4 Broadcast Bandwidth", "kilobits/s", 8000, update_every, CHART_TYPE_AREA);
4862                                         st->isdetail = 1;
4863
4864                                         rrd_stats_dimension_add(st, "received", NULL, 8, 1024, RRD_DIMENSION_INCREMENTAL);
4865                                         rrd_stats_dimension_add(st, "sent", NULL, -8, 1024, RRD_DIMENSION_INCREMENTAL);
4866                                 }
4867                                 else rrd_stats_next(st);
4868
4869                                 rrd_stats_dimension_set(st, "sent", OutBcastOctets);
4870                                 rrd_stats_dimension_set(st, "received", InBcastOctets);
4871                                 rrd_stats_done(st);
4872                         }
4873
4874                         // --------------------------------------------------------------------
4875
4876                         if(do_mcast_p) {
4877                                 st = rrd_stats_find("ipv4.mcastpkts");
4878                                 if(!st) {
4879                                         st = rrd_stats_create("ipv4", "mcastpkts", NULL, "ipv4", "IPv4 Multicast Packets", "packets/s", 9500, update_every, CHART_TYPE_LINE);
4880                                         st->isdetail = 1;
4881
4882                                         rrd_stats_dimension_add(st, "received", NULL, 1, 1, RRD_DIMENSION_INCREMENTAL);
4883                                         rrd_stats_dimension_add(st, "sent", NULL, -1, 1, RRD_DIMENSION_INCREMENTAL);
4884                                 }
4885                                 else rrd_stats_next(st);
4886
4887                                 rrd_stats_dimension_set(st, "sent", OutMcastPkts);
4888                                 rrd_stats_dimension_set(st, "received", InMcastPkts);
4889                                 rrd_stats_done(st);
4890                         }
4891
4892                         // --------------------------------------------------------------------
4893
4894                         if(do_bcast_p) {
4895                                 st = rrd_stats_find("ipv4.bcastpkts");
4896                                 if(!st) {
4897                                         st = rrd_stats_create("ipv4", "bcastpkts", NULL, "ipv4", "IPv4 Broadcast Packets", "packets/s", 8500, update_every, CHART_TYPE_LINE);
4898                                         st->isdetail = 1;
4899
4900                                         rrd_stats_dimension_add(st, "received", NULL, 1, 1, RRD_DIMENSION_INCREMENTAL);
4901                                         rrd_stats_dimension_add(st, "sent", NULL, -1, 1, RRD_DIMENSION_INCREMENTAL);
4902                                 }
4903                                 else rrd_stats_next(st);
4904
4905                                 rrd_stats_dimension_set(st, "sent", OutBcastPkts);
4906                                 rrd_stats_dimension_set(st, "received", InBcastPkts);
4907                                 rrd_stats_done(st);
4908                         }
4909                 }
4910         }
4911
4912         fclose(fp);
4913         return 0;
4914 }
4915
4916 // ----------------------------------------------------------------------------
4917 // /proc/net/stat/nf_conntrack processor
4918
4919 int do_proc_net_stat_conntrack() {
4920         static int do_sockets = -1, do_new = -1, do_changes = -1, do_expect = -1, do_search = -1, do_errors = -1;
4921
4922         if(do_sockets == -1)    do_sockets = config_get_boolean("plugin:proc:/proc/net/stat/nf_conntrack", "netfilter connections", 1);
4923         if(do_new == -1)                do_new = config_get_boolean("plugin:proc:/proc/net/stat/nf_conntrack", "netfilter new connections", 1);
4924         if(do_changes == -1)    do_changes = config_get_boolean("plugin:proc:/proc/net/stat/nf_conntrack", "netfilter connection changes", 1);
4925         if(do_expect == -1)             do_expect = config_get_boolean("plugin:proc:/proc/net/stat/nf_conntrack", "netfilter connection expectations", 1);
4926         if(do_search == -1)             do_search = config_get_boolean("plugin:proc:/proc/net/stat/nf_conntrack", "netfilter connection searches", 1);
4927         if(do_errors == -1)             do_errors = config_get_boolean("plugin:proc:/proc/net/stat/nf_conntrack", "netfilter errors", 1);
4928
4929
4930         char buffer[MAX_PROC_NET_STAT_CONNTRACK_LINE+1] = "";
4931
4932         FILE *fp = fopen("/proc/net/stat/nf_conntrack", "r");
4933         if(!fp) {
4934                 error("Cannot read /proc/net/stat/nf_conntrack.");
4935                 return 1;
4936         }
4937
4938         // read and discard the header
4939         char *p = fgets(buffer, MAX_PROC_NET_STAT_CONNTRACK_LINE, fp);
4940
4941         unsigned long long aentries = 0, asearched = 0, afound = 0, anew = 0, ainvalid = 0, aignore = 0, adelete = 0, adelete_list = 0,
4942                 ainsert = 0, ainsert_failed = 0, adrop = 0, aearly_drop = 0, aicmp_error = 0, aexpect_new = 0, aexpect_create = 0, aexpect_delete = 0, asearch_restart = 0;
4943
4944         for(;1;) {
4945                 p = fgets(buffer, MAX_PROC_NET_STAT_CONNTRACK_LINE, fp);
4946                 if(!p) break;
4947
4948                 unsigned long long tentries = 0, tsearched = 0, tfound = 0, tnew = 0, tinvalid = 0, tignore = 0, tdelete = 0, tdelete_list = 0, tinsert = 0, tinsert_failed = 0, tdrop = 0, tearly_drop = 0, ticmp_error = 0, texpect_new = 0, texpect_create = 0, texpect_delete = 0, tsearch_restart = 0;
4949
4950                 int r = sscanf(buffer, "%llx %llx %llx %llx %llx %llx %llx %llx %llx %llx %llx %llx %llx %llx %llx %llx %llx\n",
4951                         &tentries, &tsearched, &tfound, &tnew, &tinvalid, &tignore, &tdelete, &tdelete_list, &tinsert, &tinsert_failed, &tdrop, &tearly_drop, &ticmp_error, &texpect_new, &texpect_create, &texpect_delete, &tsearch_restart);
4952
4953                 if(r == EOF) break;
4954                 if(r < 16) error("Cannot read /proc/net/stat/nf_conntrack. Expected 17 params, read %d.", r);
4955
4956                 if(!aentries) aentries =  tentries;
4957
4958                 // sum all the cpus together
4959                 asearched                       += tsearched;                   // conntrack.search
4960                 afound                          += tfound;                              // conntrack.search
4961                 anew                            += tnew;                                // conntrack.new
4962                 ainvalid                        += tinvalid;                    // conntrack.new
4963                 aignore                         += tignore;                             // conntrack.new
4964                 adelete                         += tdelete;                             // conntrack.changes
4965                 adelete_list            += tdelete_list;                // conntrack.changes
4966                 ainsert                         += tinsert;                             // conntrack.changes
4967                 ainsert_failed          += tinsert_failed;              // conntrack.errors
4968                 adrop                           += tdrop;                               // conntrack.errors
4969                 aearly_drop             += tearly_drop;                 // conntrack.errors
4970                 aicmp_error             += ticmp_error;                 // conntrack.errors
4971                 aexpect_new             += texpect_new;                 // conntrack.expect
4972                 aexpect_create          += texpect_create;              // conntrack.expect
4973                 aexpect_delete          += texpect_delete;              // conntrack.expect
4974                 asearch_restart         += tsearch_restart;             // conntrack.search
4975         }
4976         fclose(fp);
4977
4978         RRD_STATS *st;
4979
4980         // --------------------------------------------------------------------
4981
4982         if(do_sockets) {
4983                 st = rrd_stats_find(RRD_TYPE_NET_STAT_CONNTRACK ".sockets");
4984                 if(!st) {
4985                         st = rrd_stats_create(RRD_TYPE_NET_STAT_CONNTRACK, "sockets", NULL, RRD_TYPE_NET_STAT_CONNTRACK, "Netfilter Connections", "active connections", 1000, update_every, CHART_TYPE_LINE);
4986
4987                         rrd_stats_dimension_add(st, "connections", NULL, 1, 1, RRD_DIMENSION_ABSOLUTE_NO_INTERPOLATION);
4988                 }
4989                 else rrd_stats_next(st);
4990
4991                 rrd_stats_dimension_set(st, "connections", aentries);
4992                 rrd_stats_done(st);
4993         }
4994
4995         // --------------------------------------------------------------------
4996
4997         if(do_new) {
4998                 st = rrd_stats_find(RRD_TYPE_NET_STAT_CONNTRACK ".new");
4999                 if(!st) {
5000                         st = rrd_stats_create(RRD_TYPE_NET_STAT_CONNTRACK, "new", NULL, RRD_TYPE_NET_STAT_CONNTRACK, "Netfilter New Connections", "connections/s", 1001, update_every, CHART_TYPE_LINE);
5001
5002                         rrd_stats_dimension_add(st, "new", NULL, 1, 1, RRD_DIMENSION_INCREMENTAL_NO_INTERPOLATION);
5003                         rrd_stats_dimension_add(st, "ignore", NULL, -1, 1, RRD_DIMENSION_INCREMENTAL_NO_INTERPOLATION);
5004                         rrd_stats_dimension_add(st, "invalid", NULL, -1, 1, RRD_DIMENSION_INCREMENTAL_NO_INTERPOLATION);
5005                 }
5006                 else rrd_stats_next(st);
5007
5008                 rrd_stats_dimension_set(st, "new", anew);
5009                 rrd_stats_dimension_set(st, "ignore", aignore);
5010                 rrd_stats_dimension_set(st, "invalid", ainvalid);
5011                 rrd_stats_done(st);
5012         }
5013
5014         // --------------------------------------------------------------------
5015
5016         if(do_changes) {
5017                 st = rrd_stats_find(RRD_TYPE_NET_STAT_CONNTRACK ".changes");
5018                 if(!st) {
5019                         st = rrd_stats_create(RRD_TYPE_NET_STAT_CONNTRACK, "changes", NULL, RRD_TYPE_NET_STAT_CONNTRACK, "Netfilter Connection Changes", "changes/s", 1002, update_every, CHART_TYPE_LINE);
5020                         st->isdetail = 1;
5021
5022                         rrd_stats_dimension_add(st, "inserted", NULL, 1, 1, RRD_DIMENSION_INCREMENTAL_NO_INTERPOLATION);
5023                         rrd_stats_dimension_add(st, "deleted", NULL, -1, 1, RRD_DIMENSION_INCREMENTAL_NO_INTERPOLATION);
5024                         rrd_stats_dimension_add(st, "delete_list", NULL, -1, 1, RRD_DIMENSION_INCREMENTAL_NO_INTERPOLATION);
5025                 }
5026                 else rrd_stats_next(st);
5027
5028                 rrd_stats_dimension_set(st, "inserted", ainsert);
5029                 rrd_stats_dimension_set(st, "deleted", adelete);
5030                 rrd_stats_dimension_set(st, "delete_list", adelete_list);
5031                 rrd_stats_done(st);
5032         }
5033
5034         // --------------------------------------------------------------------
5035
5036         if(do_expect) {
5037                 st = rrd_stats_find(RRD_TYPE_NET_STAT_CONNTRACK ".expect");
5038                 if(!st) {
5039                         st = rrd_stats_create(RRD_TYPE_NET_STAT_CONNTRACK, "expect", NULL, RRD_TYPE_NET_STAT_CONNTRACK, "Netfilter Connection Expectations", "expectations/s", 1003, update_every, CHART_TYPE_LINE);
5040                         st->isdetail = 1;
5041
5042                         rrd_stats_dimension_add(st, "created", NULL, 1, 1, RRD_DIMENSION_INCREMENTAL_NO_INTERPOLATION);
5043                         rrd_stats_dimension_add(st, "deleted", NULL, -1, 1, RRD_DIMENSION_INCREMENTAL_NO_INTERPOLATION);
5044                         rrd_stats_dimension_add(st, "new", NULL, 1, 1, RRD_DIMENSION_INCREMENTAL_NO_INTERPOLATION);
5045                 }
5046                 else rrd_stats_next(st);
5047
5048                 rrd_stats_dimension_set(st, "created", aexpect_create);
5049                 rrd_stats_dimension_set(st, "deleted", aexpect_delete);
5050                 rrd_stats_dimension_set(st, "new", aexpect_new);
5051                 rrd_stats_done(st);
5052         }
5053
5054         // --------------------------------------------------------------------
5055
5056         if(do_search) {
5057                 st = rrd_stats_find(RRD_TYPE_NET_STAT_CONNTRACK ".search");
5058                 if(!st) {
5059                         st = rrd_stats_create(RRD_TYPE_NET_STAT_CONNTRACK, "search", NULL, RRD_TYPE_NET_STAT_CONNTRACK, "Netfilter Connection Searches", "searches/s", 1010, update_every, CHART_TYPE_LINE);
5060                         st->isdetail = 1;
5061
5062                         rrd_stats_dimension_add(st, "searched", NULL, 1, 1, RRD_DIMENSION_INCREMENTAL_NO_INTERPOLATION);
5063                         rrd_stats_dimension_add(st, "restarted", NULL, -1, 1, RRD_DIMENSION_INCREMENTAL_NO_INTERPOLATION);
5064                         rrd_stats_dimension_add(st, "found", NULL, 1, 1, RRD_DIMENSION_INCREMENTAL_NO_INTERPOLATION);
5065                 }
5066                 else rrd_stats_next(st);
5067
5068                 rrd_stats_dimension_set(st, "searched", asearched);
5069                 rrd_stats_dimension_set(st, "restarted", asearch_restart);
5070                 rrd_stats_dimension_set(st, "found", afound);
5071                 rrd_stats_done(st);
5072         }
5073
5074         // --------------------------------------------------------------------
5075
5076         if(do_errors) {
5077                 st = rrd_stats_find(RRD_TYPE_NET_STAT_CONNTRACK ".errors");
5078                 if(!st) {
5079                         st = rrd_stats_create(RRD_TYPE_NET_STAT_CONNTRACK, "errors", NULL, RRD_TYPE_NET_STAT_CONNTRACK, "Netfilter Errors", "events/s", 1005, update_every, CHART_TYPE_LINE);
5080                         st->isdetail = 1;
5081
5082                         rrd_stats_dimension_add(st, "icmp_error", NULL, 1, 1, RRD_DIMENSION_INCREMENTAL_NO_INTERPOLATION);
5083                         rrd_stats_dimension_add(st, "insert_failed", NULL, -1, 1, RRD_DIMENSION_INCREMENTAL_NO_INTERPOLATION);
5084                         rrd_stats_dimension_add(st, "drop", NULL, -1, 1, RRD_DIMENSION_INCREMENTAL_NO_INTERPOLATION);
5085                         rrd_stats_dimension_add(st, "early_drop", NULL, -1, 1, RRD_DIMENSION_INCREMENTAL_NO_INTERPOLATION);
5086                 }
5087                 else rrd_stats_next(st);
5088
5089                 rrd_stats_dimension_set(st, "icmp_error", aicmp_error);
5090                 rrd_stats_dimension_set(st, "insert_failed", ainsert_failed);
5091                 rrd_stats_dimension_set(st, "drop", adrop);
5092                 rrd_stats_dimension_set(st, "early_drop", aearly_drop);
5093                 rrd_stats_done(st);
5094         }
5095
5096         return 0;
5097 }
5098
5099 // ----------------------------------------------------------------------------
5100 // /proc/net/ip_vs_stats processor
5101
5102 int do_proc_net_ip_vs_stats() {
5103         static int do_bandwidth = -1, do_sockets = -1, do_packets = -1;
5104
5105         if(do_bandwidth == -1)  do_bandwidth = config_get_boolean("plugin:proc:/proc/net/ip_vs_stats", "IPVS bandwidth", 1);
5106         if(do_sockets == -1)    do_sockets = config_get_boolean("plugin:proc:/proc/net/ip_vs_stats", "IPVS connections", 1);
5107         if(do_packets == -1)    do_packets = config_get_boolean("plugin:proc:/proc/net/ip_vs_stats", "IPVS packets", 1);
5108
5109         char buffer[MAX_PROC_NET_IPVS_LINE+1] = "";
5110
5111         FILE *fp = fopen("/proc/net/ip_vs_stats", "r");
5112         if(!fp) {
5113                 error("Cannot read /proc/net/ip_vs_stats.");
5114                 return 1;
5115         }
5116
5117         // read the discard the 2 header lines
5118         char *p = fgets(buffer, MAX_PROC_NET_IPVS_LINE, fp);
5119         if(!p) {
5120                 error("Cannot read /proc/net/ip_vs_stats.");
5121                 return 1;
5122         }
5123
5124         p = fgets(buffer, MAX_PROC_NET_IPVS_LINE, fp);
5125         if(!p) {
5126                 error("Cannot read /proc/net/ip_vs_stats.");
5127                 return 1;
5128         }
5129
5130         p = fgets(buffer, MAX_PROC_NET_IPVS_LINE, fp);
5131         if(!p) {
5132                 error("Cannot read /proc/net/ip_vs_stats.");
5133                 return 1;
5134         }
5135
5136         unsigned long long entries, InPackets, OutPackets, InBytes, OutBytes;
5137
5138         int r = sscanf(buffer, "%llx %llx %llx %llx %llx\n", &entries, &InPackets, &OutPackets, &InBytes, &OutBytes);
5139
5140         if(r == EOF) {
5141                 error("Cannot read /proc/net/ip_vs_stats.");
5142                 return 1;
5143         }
5144         if(r != 5) error("Cannot read /proc/net/ip_vs_stats. Expected 5 params, read %d.", r);
5145
5146         fclose(fp);
5147
5148         RRD_STATS *st;
5149
5150         // --------------------------------------------------------------------
5151
5152         if(do_sockets) {
5153                 st = rrd_stats_find(RRD_TYPE_NET_IPVS ".sockets");
5154                 if(!st) {
5155                         st = rrd_stats_create(RRD_TYPE_NET_IPVS, "sockets", NULL, RRD_TYPE_NET_IPVS, "IPVS New Connections", "connections/s", 1001, update_every, CHART_TYPE_LINE);
5156
5157                         rrd_stats_dimension_add(st, "connections", NULL, 1, 1, RRD_DIMENSION_INCREMENTAL_NO_INTERPOLATION);
5158                 }
5159                 else rrd_stats_next(st);
5160
5161                 rrd_stats_dimension_set(st, "connections", entries);
5162                 rrd_stats_done(st);
5163         }
5164
5165         // --------------------------------------------------------------------
5166
5167         if(do_packets) {
5168                 st = rrd_stats_find(RRD_TYPE_NET_IPVS ".packets");
5169                 if(!st) {
5170                         st = rrd_stats_create(RRD_TYPE_NET_IPVS, "packets", NULL, RRD_TYPE_NET_IPVS, "IPVS Packets", "packets/s", 1002, update_every, CHART_TYPE_LINE);
5171
5172                         rrd_stats_dimension_add(st, "received", NULL, 1, 1, RRD_DIMENSION_INCREMENTAL_NO_INTERPOLATION);
5173                         rrd_stats_dimension_add(st, "sent", NULL, -1, 1, RRD_DIMENSION_INCREMENTAL_NO_INTERPOLATION);
5174                 }
5175                 else rrd_stats_next(st);
5176
5177                 rrd_stats_dimension_set(st, "received", InPackets);
5178                 rrd_stats_dimension_set(st, "sent", OutPackets);
5179                 rrd_stats_done(st);
5180         }
5181
5182         // --------------------------------------------------------------------
5183
5184         if(do_bandwidth) {
5185                 st = rrd_stats_find(RRD_TYPE_NET_IPVS ".net");
5186                 if(!st) {
5187                         st = rrd_stats_create(RRD_TYPE_NET_IPVS, "net", NULL, RRD_TYPE_NET_IPVS, "IPVS Bandwidth", "kilobits/s", 1000, update_every, CHART_TYPE_AREA);
5188
5189                         rrd_stats_dimension_add(st, "received", NULL, 8, 1024, RRD_DIMENSION_INCREMENTAL);
5190                         rrd_stats_dimension_add(st, "sent", NULL, -8, 1024, RRD_DIMENSION_INCREMENTAL);
5191                 }
5192                 else rrd_stats_next(st);
5193
5194                 rrd_stats_dimension_set(st, "received", InBytes);
5195                 rrd_stats_dimension_set(st, "sent", OutBytes);
5196                 rrd_stats_done(st);
5197         }
5198
5199         return 0;
5200 }
5201
5202 int do_proc_stat() {
5203         static int do_cpu = -1, do_cpu_cores = -1, do_interrupts = -1, do_context = -1, do_forks = -1, do_processes = -1;
5204
5205         if(do_cpu == -1)                do_cpu = config_get_boolean("plugin:proc:/proc/stat", "cpu utilization", 1);
5206         if(do_cpu_cores == -1)  do_cpu_cores = config_get_boolean("plugin:proc:/proc/stat", "per cpu core utilization", 1);
5207         if(do_interrupts == -1) do_interrupts = config_get_boolean("plugin:proc:/proc/stat", "cpu interrupts", 1);
5208         if(do_context == -1)    do_context = config_get_boolean("plugin:proc:/proc/stat", "context switches", 1);
5209         if(do_forks == -1)              do_forks = config_get_boolean("plugin:proc:/proc/stat", "processes started", 1);
5210         if(do_processes == -1)  do_processes = config_get_boolean("plugin:proc:/proc/stat", "processes running", 1);
5211
5212         char buffer[MAX_PROC_STAT_LINE+1] = "";
5213
5214         FILE *fp = fopen("/proc/stat", "r");
5215         if(!fp) {
5216                 error("Cannot read /proc/stat.");
5217                 return 1;
5218         }
5219
5220         unsigned long long processes = 0, running = 0 , blocked = 0;
5221         RRD_STATS *st;
5222
5223         for(;1;) {
5224                 char *p = fgets(buffer, MAX_PROC_STAT_LINE, fp);
5225                 if(!p) break;
5226
5227                 if(strncmp(p, "cpu", 3) == 0) {
5228                         char id[MAX_PROC_STAT_NAME + 1] = "";
5229                         unsigned long long user = 0, nice = 0, system = 0, idle = 0, iowait = 0, irq = 0, softirq = 0, steal = 0, guest = 0, guest_nice = 0;
5230
5231                         int r = sscanf(buffer, "%s %llu %llu %llu %llu %llu %llu %llu %llu %llu %llu\n",
5232                                 id, &user, &nice, &system, &idle, &iowait, &irq, &softirq, &steal, &guest, &guest_nice);
5233
5234                         if(r == EOF) break;
5235                         if(r < 10) error("Cannot read /proc/stat cpu line. Expected 11 params, read %d.", r);
5236
5237                         char *title = "Core utilization";
5238                         char *type = RRD_TYPE_STAT;
5239                         long priority = 1000;
5240                         int isthistotal = 0;
5241                         if(strcmp(id, "cpu") == 0) {
5242                                 isthistotal = 1;
5243                                 title = "Total CPU utilization";
5244                                 type = "system";
5245                                 priority = 100;
5246                         }
5247
5248                         if((isthistotal && do_cpu) || (!isthistotal && do_cpu_cores)) {
5249                                 st = rrd_stats_find_bytype(type, id);
5250                                 if(!st) {
5251                                         st = rrd_stats_create(type, id, NULL, "cpu", title, "percentage", priority, update_every, CHART_TYPE_STACKED);
5252
5253                                         long multiplier = 1;
5254                                         long divisor = 1; // sysconf(_SC_CLK_TCK);
5255
5256                                         rrd_stats_dimension_add(st, "guest_nice", NULL, multiplier, divisor, RRD_DIMENSION_PCENT_OVER_DIFF_TOTAL);
5257                                         rrd_stats_dimension_add(st, "guest", NULL, multiplier, divisor, RRD_DIMENSION_PCENT_OVER_DIFF_TOTAL);
5258                                         rrd_stats_dimension_add(st, "steal", NULL, multiplier, divisor, RRD_DIMENSION_PCENT_OVER_DIFF_TOTAL);
5259                                         rrd_stats_dimension_add(st, "softirq", NULL, multiplier, divisor, RRD_DIMENSION_PCENT_OVER_DIFF_TOTAL);
5260                                         rrd_stats_dimension_add(st, "irq", NULL, multiplier, divisor, RRD_DIMENSION_PCENT_OVER_DIFF_TOTAL);
5261                                         rrd_stats_dimension_add(st, "user", NULL, multiplier, divisor, RRD_DIMENSION_PCENT_OVER_DIFF_TOTAL);
5262                                         rrd_stats_dimension_add(st, "system", NULL, multiplier, divisor, RRD_DIMENSION_PCENT_OVER_DIFF_TOTAL);
5263                                         rrd_stats_dimension_add(st, "nice", NULL, multiplier, divisor, RRD_DIMENSION_PCENT_OVER_DIFF_TOTAL);
5264                                         rrd_stats_dimension_add(st, "iowait", NULL, multiplier, divisor, RRD_DIMENSION_PCENT_OVER_DIFF_TOTAL);
5265
5266                                         rrd_stats_dimension_add(st, "idle", NULL, multiplier, divisor, RRD_DIMENSION_PCENT_OVER_DIFF_TOTAL);
5267                                         rrd_stats_dimension_hide(st, "idle");
5268                                 }
5269                                 else rrd_stats_next(st);
5270
5271                                 rrd_stats_dimension_set(st, "user", user);
5272                                 rrd_stats_dimension_set(st, "nice", nice);
5273                                 rrd_stats_dimension_set(st, "system", system);
5274                                 rrd_stats_dimension_set(st, "idle", idle);
5275                                 rrd_stats_dimension_set(st, "iowait", iowait);
5276                                 rrd_stats_dimension_set(st, "irq", irq);
5277                                 rrd_stats_dimension_set(st, "softirq", softirq);
5278                                 rrd_stats_dimension_set(st, "steal", steal);
5279                                 rrd_stats_dimension_set(st, "guest", guest);
5280                                 rrd_stats_dimension_set(st, "guest_nice", guest_nice);
5281                                 rrd_stats_done(st);
5282                         }
5283                 }
5284                 else if(strncmp(p, "intr ", 5) == 0) {
5285                         char id[MAX_PROC_STAT_NAME + 1];
5286
5287                         unsigned long long value;
5288
5289                         int r = sscanf(buffer, "%s %llu ", id, &value);
5290                         if(r == EOF) break;
5291                         if(r != 2) error("Cannot read /proc/stat intr line. Expected 2 params, read %d.", r);
5292
5293                         // --------------------------------------------------------------------
5294
5295                         if(do_interrupts) {
5296                                 st = rrd_stats_find_bytype("system", id);
5297                                 if(!st) {
5298                                         st = rrd_stats_create("system", id, NULL, "cpu", "CPU Interrupts", "interrupts/s", 900, update_every, CHART_TYPE_LINE);
5299                                         st->isdetail = 1;
5300
5301                                         rrd_stats_dimension_add(st, "interrupts", NULL, 1, 1, RRD_DIMENSION_INCREMENTAL_NO_INTERPOLATION);
5302                                 }
5303                                 else rrd_stats_next(st);
5304
5305                                 rrd_stats_dimension_set(st, "interrupts", value);
5306                                 rrd_stats_done(st);
5307                         }
5308                 }
5309                 else if(strncmp(p, "ctxt ", 5) == 0) {
5310                         char id[MAX_PROC_STAT_NAME + 1] = "";
5311
5312                         unsigned long long value;
5313
5314                         int r = sscanf(buffer, "%s %llu ", id, &value);
5315                         if(r == EOF) break;
5316                         if(r != 2) error("Cannot read /proc/stat ctxt line. Expected 2 params, read %d.", r);
5317
5318                         // --------------------------------------------------------------------
5319
5320                         if(do_context) {
5321                                 st = rrd_stats_find_bytype("system", id);
5322                                 if(!st) {
5323                                         st = rrd_stats_create("system", id, NULL, "cpu", "CPU Context Switches", "context switches/s", 800, update_every, CHART_TYPE_LINE);
5324
5325                                         rrd_stats_dimension_add(st, "switches", NULL, 1, 1, RRD_DIMENSION_INCREMENTAL_NO_INTERPOLATION);
5326                                 }
5327                                 else rrd_stats_next(st);
5328
5329                                 rrd_stats_dimension_set(st, "switches", value);
5330                                 rrd_stats_done(st);
5331                         }
5332                 }
5333                 else if(strncmp(p, "processes ", 10) == 0) {
5334                         char id[MAX_PROC_STAT_NAME + 1] = "";
5335
5336                         unsigned long long value;
5337
5338                         int r = sscanf(buffer, "%s %llu ", id, &value);
5339                         if(r == EOF) break;
5340                         if(r != 2) error("Cannot read /proc/stat processes line. Expected 2 params, read %d.", r);
5341
5342                         processes = value;
5343                 }
5344                 else if(strncmp(p, "procs_running ", 14) == 0) {
5345                         char id[MAX_PROC_STAT_NAME + 1] = "";
5346
5347                         unsigned long long value;
5348
5349                         int r = sscanf(buffer, "%s %llu ", id, &value);
5350                         if(r == EOF) break;
5351                         if(r != 2) error("Cannot read /proc/stat procs_running line. Expected 2 params, read %d.", r);
5352
5353                         running = value;
5354                 }
5355                 else if(strncmp(p, "procs_blocked ", 14) == 0) {
5356                         char id[MAX_PROC_STAT_NAME + 1] = "procs_running";
5357
5358                         unsigned long long value;
5359
5360                         int r = sscanf(buffer, "%s %llu ", id, &value);
5361                         if(r == EOF) break;
5362                         if(r != 2) error("Cannot read /proc/stat procs_blocked line. Expected 2 params, read %d.", r);
5363
5364                         blocked = value;
5365                 }
5366         }
5367         fclose(fp);
5368
5369         // --------------------------------------------------------------------
5370
5371         if(do_forks) {
5372                 st = rrd_stats_find_bytype("system", "forks");
5373                 if(!st) {
5374                         st = rrd_stats_create("system", "forks", NULL, "cpu", "New Processes", "processes/s", 700, update_every, CHART_TYPE_LINE);
5375                         st->isdetail = 1;
5376
5377                         rrd_stats_dimension_add(st, "started", NULL, 1, 1, RRD_DIMENSION_INCREMENTAL_NO_INTERPOLATION);
5378                 }
5379                 else rrd_stats_next(st);
5380
5381                 rrd_stats_dimension_set(st, "started", processes);
5382                 rrd_stats_done(st);
5383         }
5384
5385         // --------------------------------------------------------------------
5386
5387         if(do_processes) {
5388                 st = rrd_stats_find_bytype("system", "processes");
5389                 if(!st) {
5390                         st = rrd_stats_create("system", "processes", NULL, "cpu", "Processes", "processes", 600, update_every, CHART_TYPE_LINE);
5391
5392                         rrd_stats_dimension_add(st, "running", NULL, 1, 1, RRD_DIMENSION_ABSOLUTE_NO_INTERPOLATION);
5393                         rrd_stats_dimension_add(st, "blocked", NULL, -1, 1, RRD_DIMENSION_ABSOLUTE_NO_INTERPOLATION);
5394                 }
5395                 else rrd_stats_next(st);
5396
5397                 rrd_stats_dimension_set(st, "running", running);
5398                 rrd_stats_dimension_set(st, "blocked", blocked);
5399                 rrd_stats_done(st);
5400         }
5401
5402         return 0;
5403 }
5404
5405 // ----------------------------------------------------------------------------
5406 // /proc/meminfo processor
5407
5408 #define MAX_PROC_MEMINFO_LINE 4096
5409 #define MAX_PROC_MEMINFO_NAME 1024
5410
5411 int do_proc_meminfo() {
5412         static int do_ram = -1, do_swap = -1, do_hwcorrupt = -1, do_committed = -1, do_writeback = -1, do_kernel = -1, do_slab = -1;
5413
5414         if(do_ram == -1)                do_ram = config_get_boolean("plugin:proc:/proc/meminfo", "system ram", 1);
5415         if(do_swap == -1)               do_swap = config_get_boolean("plugin:proc:/proc/meminfo", "system swap", 1);
5416         if(do_hwcorrupt == -1)  do_hwcorrupt = config_get_boolean("plugin:proc:/proc/meminfo", "hardware corrupted ECC", 1);
5417         if(do_committed == -1)  do_committed = config_get_boolean("plugin:proc:/proc/meminfo", "committed memory", 1);
5418         if(do_writeback == -1)  do_writeback = config_get_boolean("plugin:proc:/proc/meminfo", "writeback memory", 1);
5419         if(do_kernel == -1)             do_kernel = config_get_boolean("plugin:proc:/proc/meminfo", "kernel memory", 1);
5420         if(do_slab == -1)               do_slab = config_get_boolean("plugin:proc:/proc/meminfo", "slab memory", 1);
5421
5422         char buffer[MAX_PROC_MEMINFO_LINE+1] = "";
5423
5424         FILE *fp = fopen("/proc/meminfo", "r");
5425         if(!fp) {
5426                 error("Cannot read /proc/meminfo.");
5427                 return 1;
5428         }
5429
5430         int hwcorrupted = 0;
5431
5432         unsigned long long MemTotal = 0, MemFree = 0, Buffers = 0, Cached = 0, SwapCached = 0,
5433                 Active = 0, Inactive = 0, ActiveAnon = 0, InactiveAnon = 0, ActiveFile = 0, InactiveFile = 0,
5434                 Unevictable = 0, Mlocked = 0, SwapTotal = 0, SwapFree = 0, Dirty = 0, Writeback = 0, AnonPages = 0,
5435                 Mapped = 0, Shmem = 0, Slab = 0, SReclaimable = 0, SUnreclaim = 0, KernelStack = 0, PageTables = 0,
5436                 NFS_Unstable = 0, Bounce = 0, WritebackTmp = 0, CommitLimit = 0, Committed_AS = 0,
5437                 VmallocTotal = 0, VmallocUsed = 0, VmallocChunk = 0,
5438                 AnonHugePages = 0, HugePages_Total = 0, HugePages_Free = 0, HugePages_Rsvd = 0, HugePages_Surp = 0, Hugepagesize = 0,
5439                 DirectMap4k = 0, DirectMap2M = 0, HardwareCorrupted = 0;
5440
5441         for(;1;) {
5442                 char *p = fgets(buffer, MAX_PROC_MEMINFO_LINE, fp);
5443                 if(!p) break;
5444
5445                 // remove ':'
5446                 while((p = strchr(buffer, ':'))) *p = ' ';
5447
5448                 char name[MAX_PROC_MEMINFO_NAME + 1] = "";
5449                 unsigned long long value = 0;
5450
5451                 int r = sscanf(buffer, "%s %llu kB\n", name, &value);
5452
5453                 if(r == EOF) break;
5454                 if(r != 2) error("Cannot read /proc/meminfo line. Expected 2 params, read %d.", r);
5455
5456                      if(!MemTotal && strcmp(name, "MemTotal") == 0) MemTotal = value;
5457                 else if(!MemFree && strcmp(name, "MemFree") == 0) MemFree = value;
5458                 else if(!Buffers && strcmp(name, "Buffers") == 0) Buffers = value;
5459                 else if(!Cached && strcmp(name, "Cached") == 0) Cached = value;
5460                 else if(!SwapCached && strcmp(name, "SwapCached") == 0) SwapCached = value;
5461                 else if(!Active && strcmp(name, "Active") == 0) Active = value;
5462                 else if(!Inactive && strcmp(name, "Inactive") == 0) Inactive = value;
5463                 else if(!ActiveAnon && strcmp(name, "ActiveAnon") == 0) ActiveAnon = value;
5464                 else if(!InactiveAnon && strcmp(name, "InactiveAnon") == 0) InactiveAnon = value;
5465                 else if(!ActiveFile && strcmp(name, "ActiveFile") == 0) ActiveFile = value;
5466                 else if(!InactiveFile && strcmp(name, "InactiveFile") == 0) InactiveFile = value;
5467                 else if(!Unevictable && strcmp(name, "Unevictable") == 0) Unevictable = value;
5468                 else if(!Mlocked && strcmp(name, "Mlocked") == 0) Mlocked = value;
5469                 else if(!SwapTotal && strcmp(name, "SwapTotal") == 0) SwapTotal = value;
5470                 else if(!SwapFree && strcmp(name, "SwapFree") == 0) SwapFree = value;
5471                 else if(!Dirty && strcmp(name, "Dirty") == 0) Dirty = value;
5472                 else if(!Writeback && strcmp(name, "Writeback") == 0) Writeback = value;
5473                 else if(!AnonPages && strcmp(name, "AnonPages") == 0) AnonPages = value;
5474                 else if(!Mapped && strcmp(name, "Mapped") == 0) Mapped = value;
5475                 else if(!Shmem && strcmp(name, "Shmem") == 0) Shmem = value;
5476                 else if(!Slab && strcmp(name, "Slab") == 0) Slab = value;
5477                 else if(!SReclaimable && strcmp(name, "SReclaimable") == 0) SReclaimable = value;
5478                 else if(!SUnreclaim && strcmp(name, "SUnreclaim") == 0) SUnreclaim = value;
5479                 else if(!KernelStack && strcmp(name, "KernelStack") == 0) KernelStack = value;
5480                 else if(!PageTables && strcmp(name, "PageTables") == 0) PageTables = value;
5481                 else if(!NFS_Unstable && strcmp(name, "NFS_Unstable") == 0) NFS_Unstable = value;
5482                 else if(!Bounce && strcmp(name, "Bounce") == 0) Bounce = value;
5483                 else if(!WritebackTmp && strcmp(name, "WritebackTmp") == 0) WritebackTmp = value;
5484                 else if(!CommitLimit && strcmp(name, "CommitLimit") == 0) CommitLimit = value;
5485                 else if(!Committed_AS && strcmp(name, "Committed_AS") == 0) Committed_AS = value;
5486                 else if(!VmallocTotal && strcmp(name, "VmallocTotal") == 0) VmallocTotal = value;
5487                 else if(!VmallocUsed && strcmp(name, "VmallocUsed") == 0) VmallocUsed = value;
5488                 else if(!VmallocChunk && strcmp(name, "VmallocChunk") == 0) VmallocChunk = value;
5489                 else if(!HardwareCorrupted && strcmp(name, "HardwareCorrupted") == 0) { HardwareCorrupted = value; hwcorrupted = 1; }
5490                 else if(!AnonHugePages && strcmp(name, "AnonHugePages") == 0) AnonHugePages = value;
5491                 else if(!HugePages_Total && strcmp(name, "HugePages_Total") == 0) HugePages_Total = value;
5492                 else if(!HugePages_Free && strcmp(name, "HugePages_Free") == 0) HugePages_Free = value;
5493                 else if(!HugePages_Rsvd && strcmp(name, "HugePages_Rsvd") == 0) HugePages_Rsvd = value;
5494                 else if(!HugePages_Surp && strcmp(name, "HugePages_Surp") == 0) HugePages_Surp = value;
5495                 else if(!Hugepagesize && strcmp(name, "Hugepagesize") == 0) Hugepagesize = value;
5496                 else if(!DirectMap4k && strcmp(name, "DirectMap4k") == 0) DirectMap4k = value;
5497                 else if(!DirectMap2M && strcmp(name, "DirectMap2M") == 0) DirectMap2M = value;
5498         }
5499         fclose(fp);
5500
5501         RRD_STATS *st;
5502
5503         // --------------------------------------------------------------------
5504
5505         // http://stackoverflow.com/questions/3019748/how-to-reliably-measure-available-memory-in-linux
5506         unsigned long long MemUsed = MemTotal - MemFree - Cached - Buffers;
5507
5508         if(do_ram) {
5509                 st = rrd_stats_find("system.ram");
5510                 if(!st) {
5511                         st = rrd_stats_create("system", "ram", NULL, "mem", "System RAM", "MB", 200, update_every, CHART_TYPE_STACKED);
5512
5513                         rrd_stats_dimension_add(st, "buffers", NULL, 1, 1024, RRD_DIMENSION_ABSOLUTE);
5514                         rrd_stats_dimension_add(st, "used",    NULL, 1, 1024, RRD_DIMENSION_ABSOLUTE);
5515                         rrd_stats_dimension_add(st, "cached",  NULL, 1, 1024, RRD_DIMENSION_ABSOLUTE);
5516                         rrd_stats_dimension_add(st, "free",    NULL, 1, 1024, RRD_DIMENSION_ABSOLUTE);
5517                 }
5518                 else rrd_stats_next(st);
5519
5520                 rrd_stats_dimension_set(st, "used", MemUsed);
5521                 rrd_stats_dimension_set(st, "free", MemFree);
5522                 rrd_stats_dimension_set(st, "cached", Cached);
5523                 rrd_stats_dimension_set(st, "buffers", Buffers);
5524                 rrd_stats_done(st);
5525         }
5526
5527         // --------------------------------------------------------------------
5528
5529         unsigned long long SwapUsed = SwapTotal - SwapFree;
5530
5531         if(do_swap) {
5532                 st = rrd_stats_find("system.swap");
5533                 if(!st) {
5534                         st = rrd_stats_create("system", "swap", NULL, "mem", "System Swap", "MB", 201, update_every, CHART_TYPE_STACKED);
5535                         st->isdetail = 1;
5536
5537                         rrd_stats_dimension_add(st, "free",    NULL, 1, 1024, RRD_DIMENSION_ABSOLUTE);
5538                         rrd_stats_dimension_add(st, "used",    NULL, 1, 1024, RRD_DIMENSION_ABSOLUTE);
5539                 }
5540                 else rrd_stats_next(st);
5541
5542                 rrd_stats_dimension_set(st, "used", SwapUsed);
5543                 rrd_stats_dimension_set(st, "free", SwapFree);
5544                 rrd_stats_done(st);
5545         }
5546
5547         // --------------------------------------------------------------------
5548
5549         if(hwcorrupted && do_hwcorrupt) {
5550                 st = rrd_stats_find("mem.hwcorrupt");
5551                 if(!st) {
5552                         st = rrd_stats_create("mem", "hwcorrupt", NULL, "mem", "Hardware Corrupted ECC", "MB", 9000, update_every, CHART_TYPE_LINE);
5553                         st->isdetail = 1;
5554
5555                         rrd_stats_dimension_add(st, "HardwareCorrupted", NULL, 1, 1024, RRD_DIMENSION_ABSOLUTE);
5556                 }
5557                 else rrd_stats_next(st);
5558
5559                 rrd_stats_dimension_set(st, "HardwareCorrupted", HardwareCorrupted);
5560                 rrd_stats_done(st);
5561         }
5562
5563         // --------------------------------------------------------------------
5564
5565         if(do_committed) {
5566                 st = rrd_stats_find("mem.committed");
5567                 if(!st) {
5568                         st = rrd_stats_create("mem", "committed", NULL, "mem", "Committed (Allocated) Memory", "MB", 5000, update_every, CHART_TYPE_AREA);
5569                         st->isdetail = 1;
5570
5571                         rrd_stats_dimension_add(st, "Committed_AS", NULL, 1, 1024, RRD_DIMENSION_ABSOLUTE);
5572                 }
5573                 else rrd_stats_next(st);
5574
5575                 rrd_stats_dimension_set(st, "Committed_AS", Committed_AS);
5576                 rrd_stats_done(st);
5577         }
5578
5579         // --------------------------------------------------------------------
5580
5581         if(do_writeback) {
5582                 st = rrd_stats_find("mem.writeback");
5583                 if(!st) {
5584                         st = rrd_stats_create("mem", "writeback", NULL, "mem", "Writeback Memory", "MB", 4000, update_every, CHART_TYPE_LINE);
5585                         st->isdetail = 1;
5586
5587                         rrd_stats_dimension_add(st, "Dirty", NULL, 1, 1024, RRD_DIMENSION_ABSOLUTE);
5588                         rrd_stats_dimension_add(st, "Writeback", NULL, 1, 1024, RRD_DIMENSION_ABSOLUTE);
5589                         rrd_stats_dimension_add(st, "FuseWriteback", NULL, 1, 1024, RRD_DIMENSION_ABSOLUTE);
5590                         rrd_stats_dimension_add(st, "NfsWriteback", NULL, 1, 1024, RRD_DIMENSION_ABSOLUTE);
5591                         rrd_stats_dimension_add(st, "Bounce", NULL, 1, 1024, RRD_DIMENSION_ABSOLUTE);
5592                 }
5593                 else rrd_stats_next(st);
5594
5595                 rrd_stats_dimension_set(st, "Dirty", Dirty);
5596                 rrd_stats_dimension_set(st, "Writeback", Writeback);
5597                 rrd_stats_dimension_set(st, "FuseWriteback", WritebackTmp);
5598                 rrd_stats_dimension_set(st, "NfsWriteback", NFS_Unstable);
5599                 rrd_stats_dimension_set(st, "Bounce", Bounce);
5600                 rrd_stats_done(st);
5601         }
5602
5603         // --------------------------------------------------------------------
5604
5605         if(do_kernel) {
5606                 st = rrd_stats_find("mem.kernel");
5607                 if(!st) {
5608                         st = rrd_stats_create("mem", "kernel", NULL, "mem", "Memory Used by Kernel", "MB", 6000, update_every, CHART_TYPE_STACKED);
5609                         st->isdetail = 1;
5610
5611                         rrd_stats_dimension_add(st, "Slab", NULL, 1, 1024, RRD_DIMENSION_ABSOLUTE);
5612                         rrd_stats_dimension_add(st, "KernelStack", NULL, 1, 1024, RRD_DIMENSION_ABSOLUTE);
5613                         rrd_stats_dimension_add(st, "PageTables", NULL, 1, 1024, RRD_DIMENSION_ABSOLUTE);
5614                         rrd_stats_dimension_add(st, "VmallocUsed", NULL, 1, 1024, RRD_DIMENSION_ABSOLUTE);
5615                 }
5616                 else rrd_stats_next(st);
5617
5618                 rrd_stats_dimension_set(st, "KernelStack", KernelStack);
5619                 rrd_stats_dimension_set(st, "Slab", Slab);
5620                 rrd_stats_dimension_set(st, "PageTables", PageTables);
5621                 rrd_stats_dimension_set(st, "VmallocUsed", VmallocUsed);
5622                 rrd_stats_done(st);
5623         }
5624
5625         // --------------------------------------------------------------------
5626
5627         if(do_slab) {
5628                 st = rrd_stats_find("mem.slab");
5629                 if(!st) {
5630                         st = rrd_stats_create("mem", "slab", NULL, "mem", "Reclaimable Kernel Memory", "MB", 6500, update_every, CHART_TYPE_STACKED);
5631                         st->isdetail = 1;
5632
5633                         rrd_stats_dimension_add(st, "reclaimable", NULL, 1, 1024, RRD_DIMENSION_ABSOLUTE);
5634                         rrd_stats_dimension_add(st, "unreclaimable", NULL, 1, 1024, RRD_DIMENSION_ABSOLUTE);
5635                 }
5636                 else rrd_stats_next(st);
5637
5638                 rrd_stats_dimension_set(st, "reclaimable", SReclaimable);
5639                 rrd_stats_dimension_set(st, "unreclaimable", SUnreclaim);
5640                 rrd_stats_done(st);
5641         }
5642
5643         return 0;
5644 }
5645
5646 // ----------------------------------------------------------------------------
5647 // /proc/vmstat processor
5648
5649 #define MAX_PROC_VMSTAT_LINE 4096
5650 #define MAX_PROC_VMSTAT_NAME 1024
5651
5652 int do_proc_vmstat() {
5653         static int do_swapio = -1, do_io = -1, do_pgfaults = -1;
5654
5655         if(do_swapio == -1)             do_swapio = config_get_boolean("plugin:proc:/proc/vmstat", "swap i/o", 1);
5656         if(do_io == -1)                 do_io = config_get_boolean("plugin:proc:/proc/vmstat", "disk i/o", 1);
5657         if(do_pgfaults == -1)   do_pgfaults = config_get_boolean("plugin:proc:/proc/vmstat", "memory page faults", 1);
5658
5659         char buffer[MAX_PROC_VMSTAT_LINE+1] = "";
5660
5661         FILE *fp = fopen("/proc/vmstat", "r");
5662         if(!fp) {
5663                 error("Cannot read /proc/vmstat.");
5664                 return 1;
5665         }
5666
5667         unsigned long long nr_free_pages = 0, nr_inactive_anon = 0, nr_active_anon = 0, nr_inactive_file = 0, nr_active_file = 0, nr_unevictable = 0, nr_mlock = 0,
5668                 nr_anon_pages = 0, nr_mapped = 0, nr_file_pages = 0, nr_dirty = 0, nr_writeback = 0, nr_slab_reclaimable = 0, nr_slab_unreclaimable = 0, nr_page_table_pages = 0,
5669                 nr_kernel_stack = 0, nr_unstable = 0, nr_bounce = 0, nr_vmscan_write = 0, nr_vmscan_immediate_reclaim = 0, nr_writeback_temp = 0, nr_isolated_anon = 0, nr_isolated_file = 0,
5670                 nr_shmem = 0, nr_dirtied = 0, nr_written = 0, nr_anon_transparent_hugepages = 0, nr_dirty_threshold = 0, nr_dirty_background_threshold = 0,
5671                 pgpgin = 0, pgpgout = 0, pswpin = 0, pswpout = 0, pgalloc_dma = 0, pgalloc_dma32 = 0, pgalloc_normal = 0, pgalloc_movable = 0, pgfree = 0, pgactivate = 0, pgdeactivate = 0,
5672                 pgfault = 0, pgmajfault = 0, pgrefill_dma = 0, pgrefill_dma32 = 0, pgrefill_normal = 0, pgrefill_movable = 0, pgsteal_kswapd_dma = 0, pgsteal_kswapd_dma32 = 0,
5673                 pgsteal_kswapd_normal = 0, pgsteal_kswapd_movable = 0, pgsteal_direct_dma = 0, pgsteal_direct_dma32 = 0, pgsteal_direct_normal = 0, pgsteal_direct_movable = 0,
5674                 pgscan_kswapd_dma = 0, pgscan_kswapd_dma32 = 0, pgscan_kswapd_normal = 0, pgscan_kswapd_movable = 0, pgscan_direct_dma = 0, pgscan_direct_dma32 = 0, pgscan_direct_normal = 0,
5675                 pgscan_direct_movable = 0, pginodesteal = 0, slabs_scanned = 0, kswapd_inodesteal = 0, kswapd_low_wmark_hit_quickly = 0, kswapd_high_wmark_hit_quickly = 0,
5676                 kswapd_skip_congestion_wait = 0, pageoutrun = 0, allocstall = 0, pgrotated = 0, compact_blocks_moved = 0, compact_pages_moved = 0, compact_pagemigrate_failed = 0,
5677                 compact_stall = 0, compact_fail = 0, compact_success = 0, htlb_buddy_alloc_success = 0, htlb_buddy_alloc_fail = 0, unevictable_pgs_culled = 0, unevictable_pgs_scanned = 0,
5678                 unevictable_pgs_rescued = 0, unevictable_pgs_mlocked = 0, unevictable_pgs_munlocked = 0, unevictable_pgs_cleared = 0, unevictable_pgs_stranded = 0, unevictable_pgs_mlockfreed = 0,
5679                 thp_fault_alloc = 0, thp_fault_fallback = 0, thp_collapse_alloc = 0, thp_collapse_alloc_failed = 0, thp_split = 0;
5680
5681         for(;1;) {
5682                 char *p = fgets(buffer, MAX_PROC_VMSTAT_NAME, fp);
5683                 if(!p) break;
5684
5685                 // remove ':'
5686                 while((p = strchr(buffer, ':'))) *p = ' ';
5687
5688                 char name[MAX_PROC_MEMINFO_NAME + 1] = "";
5689                 unsigned long long value = 0;
5690
5691                 int r = sscanf(buffer, "%s %llu\n", name, &value);
5692
5693                 if(r == EOF) break;
5694                 if(r != 2) error("Cannot read /proc/meminfo line. Expected 2 params, read %d.", r);
5695
5696                      if(!nr_free_pages && strcmp(name, "nr_free_pages") == 0) nr_free_pages = value;
5697                 else if(!nr_inactive_anon && strcmp(name, "nr_inactive_anon") == 0) nr_inactive_anon = value;
5698                 else if(!nr_active_anon && strcmp(name, "nr_active_anon") == 0) nr_active_anon = value;
5699                 else if(!nr_inactive_file && strcmp(name, "nr_inactive_file") == 0) nr_inactive_file = value;
5700                 else if(!nr_active_file && strcmp(name, "nr_active_file") == 0) nr_active_file = value;
5701                 else if(!nr_unevictable && strcmp(name, "nr_unevictable") == 0) nr_unevictable = value;
5702                 else if(!nr_mlock && strcmp(name, "nr_mlock") == 0) nr_mlock = value;
5703                 else if(!nr_anon_pages && strcmp(name, "nr_anon_pages") == 0) nr_anon_pages = value;
5704                 else if(!nr_mapped && strcmp(name, "nr_mapped") == 0) nr_mapped = value;
5705                 else if(!nr_file_pages && strcmp(name, "nr_file_pages") == 0) nr_file_pages = value;
5706                 else if(!nr_dirty && strcmp(name, "nr_dirty") == 0) nr_dirty = value;
5707                 else if(!nr_writeback && strcmp(name, "nr_writeback") == 0) nr_writeback = value;
5708                 else if(!nr_slab_reclaimable && strcmp(name, "nr_slab_reclaimable") == 0) nr_slab_reclaimable = value;
5709                 else if(!nr_slab_unreclaimable && strcmp(name, "nr_slab_unreclaimable") == 0) nr_slab_unreclaimable = value;
5710                 else if(!nr_page_table_pages && strcmp(name, "nr_page_table_pages") == 0) nr_page_table_pages = value;
5711                 else if(!nr_kernel_stack && strcmp(name, "nr_kernel_stack") == 0) nr_kernel_stack = value;
5712                 else if(!nr_unstable && strcmp(name, "nr_unstable") == 0) nr_unstable = value;
5713                 else if(!nr_bounce && strcmp(name, "nr_bounce") == 0) nr_bounce = value;
5714                 else if(!nr_vmscan_write && strcmp(name, "nr_vmscan_write") == 0) nr_vmscan_write = value;
5715                 else if(!nr_vmscan_immediate_reclaim && strcmp(name, "nr_vmscan_immediate_reclaim") == 0) nr_vmscan_immediate_reclaim = value;
5716                 else if(!nr_writeback_temp && strcmp(name, "nr_writeback_temp") == 0) nr_writeback_temp = value;
5717                 else if(!nr_isolated_anon && strcmp(name, "nr_isolated_anon") == 0) nr_isolated_anon = value;
5718                 else if(!nr_isolated_file && strcmp(name, "nr_isolated_file") == 0) nr_isolated_file = value;
5719                 else if(!nr_shmem && strcmp(name, "nr_shmem") == 0) nr_shmem = value;
5720                 else if(!nr_dirtied && strcmp(name, "nr_dirtied") == 0) nr_dirtied = value;
5721                 else if(!nr_written && strcmp(name, "nr_written") == 0) nr_written = value;
5722                 else if(!nr_anon_transparent_hugepages && strcmp(name, "nr_anon_transparent_hugepages") == 0) nr_anon_transparent_hugepages = value;
5723                 else if(!nr_dirty_threshold && strcmp(name, "nr_dirty_threshold") == 0) nr_dirty_threshold = value;
5724                 else if(!nr_dirty_background_threshold && strcmp(name, "nr_dirty_background_threshold") == 0) nr_dirty_background_threshold = value;
5725                 else if(!pgpgin && strcmp(name, "pgpgin") == 0) pgpgin = value;
5726                 else if(!pgpgout && strcmp(name, "pgpgout") == 0) pgpgout = value;
5727                 else if(!pswpin && strcmp(name, "pswpin") == 0) pswpin = value;
5728                 else if(!pswpout && strcmp(name, "pswpout") == 0) pswpout = value;
5729                 else if(!pgalloc_dma && strcmp(name, "pgalloc_dma") == 0) pgalloc_dma = value;
5730                 else if(!pgalloc_dma32 && strcmp(name, "pgalloc_dma32") == 0) pgalloc_dma32 = value;
5731                 else if(!pgalloc_normal && strcmp(name, "pgalloc_normal") == 0) pgalloc_normal = value;
5732                 else if(!pgalloc_movable && strcmp(name, "pgalloc_movable") == 0) pgalloc_movable = value;
5733                 else if(!pgfree && strcmp(name, "pgfree") == 0) pgfree = value;
5734                 else if(!pgactivate && strcmp(name, "pgactivate") == 0) pgactivate = value;
5735                 else if(!pgdeactivate && strcmp(name, "pgdeactivate") == 0) pgdeactivate = value;
5736                 else if(!pgfault && strcmp(name, "pgfault") == 0) pgfault = value;
5737                 else if(!pgmajfault && strcmp(name, "pgmajfault") == 0) pgmajfault = value;
5738                 else if(!pgrefill_dma && strcmp(name, "pgrefill_dma") == 0) pgrefill_dma = value;
5739                 else if(!pgrefill_dma32 && strcmp(name, "pgrefill_dma32") == 0) pgrefill_dma32 = value;
5740                 else if(!pgrefill_normal && strcmp(name, "pgrefill_normal") == 0) pgrefill_normal = value;
5741                 else if(!pgrefill_movable && strcmp(name, "pgrefill_movable") == 0) pgrefill_movable = value;
5742                 else if(!pgsteal_kswapd_dma && strcmp(name, "pgsteal_kswapd_dma") == 0) pgsteal_kswapd_dma = value;
5743                 else if(!pgsteal_kswapd_dma32 && strcmp(name, "pgsteal_kswapd_dma32") == 0) pgsteal_kswapd_dma32 = value;
5744                 else if(!pgsteal_kswapd_normal && strcmp(name, "pgsteal_kswapd_normal") == 0) pgsteal_kswapd_normal = value;
5745                 else if(!pgsteal_kswapd_movable && strcmp(name, "pgsteal_kswapd_movable") == 0) pgsteal_kswapd_movable = value;
5746                 else if(!pgsteal_direct_dma && strcmp(name, "pgsteal_direct_dma") == 0) pgsteal_direct_dma = value;
5747                 else if(!pgsteal_direct_dma32 && strcmp(name, "pgsteal_direct_dma32") == 0) pgsteal_direct_dma32 = value;
5748                 else if(!pgsteal_direct_normal && strcmp(name, "pgsteal_direct_normal") == 0) pgsteal_direct_normal = value;
5749                 else if(!pgsteal_direct_movable && strcmp(name, "pgsteal_direct_movable") == 0) pgsteal_direct_movable = value;
5750                 else if(!pgscan_kswapd_dma && strcmp(name, "pgscan_kswapd_dma") == 0) pgscan_kswapd_dma = value;
5751                 else if(!pgscan_kswapd_dma32 && strcmp(name, "pgscan_kswapd_dma32") == 0) pgscan_kswapd_dma32 = value;
5752                 else if(!pgscan_kswapd_normal && strcmp(name, "pgscan_kswapd_normal") == 0) pgscan_kswapd_normal = value;
5753                 else if(!pgscan_kswapd_movable && strcmp(name, "pgscan_kswapd_movable") == 0) pgscan_kswapd_movable = value;
5754                 else if(!pgscan_direct_dma && strcmp(name, "pgscan_direct_dma") == 0) pgscan_direct_dma = value;
5755                 else if(!pgscan_direct_dma32 && strcmp(name, "pgscan_direct_dma32") == 0) pgscan_direct_dma32 = value;
5756                 else if(!pgscan_direct_normal && strcmp(name, "pgscan_direct_normal") == 0) pgscan_direct_normal = value;
5757                 else if(!pgscan_direct_movable && strcmp(name, "pgscan_direct_movable") == 0) pgscan_direct_movable = value;
5758                 else if(!pginodesteal && strcmp(name, "pginodesteal") == 0) pginodesteal = value;
5759                 else if(!slabs_scanned && strcmp(name, "slabs_scanned") == 0) slabs_scanned = value;
5760                 else if(!kswapd_inodesteal && strcmp(name, "kswapd_inodesteal") == 0) kswapd_inodesteal = value;
5761                 else if(!kswapd_low_wmark_hit_quickly && strcmp(name, "kswapd_low_wmark_hit_quickly") == 0) kswapd_low_wmark_hit_quickly = value;
5762                 else if(!kswapd_high_wmark_hit_quickly && strcmp(name, "kswapd_high_wmark_hit_quickly") == 0) kswapd_high_wmark_hit_quickly = value;
5763                 else if(!kswapd_skip_congestion_wait && strcmp(name, "kswapd_skip_congestion_wait") == 0) kswapd_skip_congestion_wait = value;
5764                 else if(!pageoutrun && strcmp(name, "pageoutrun") == 0) pageoutrun = value;
5765                 else if(!allocstall && strcmp(name, "allocstall") == 0) allocstall = value;
5766                 else if(!pgrotated && strcmp(name, "pgrotated") == 0) pgrotated = value;
5767                 else if(!compact_blocks_moved && strcmp(name, "compact_blocks_moved") == 0) compact_blocks_moved = value;
5768                 else if(!compact_pages_moved && strcmp(name, "compact_pages_moved") == 0) compact_pages_moved = value;
5769                 else if(!compact_pagemigrate_failed && strcmp(name, "compact_pagemigrate_failed") == 0) compact_pagemigrate_failed = value;
5770                 else if(!compact_stall && strcmp(name, "compact_stall") == 0) compact_stall = value;
5771                 else if(!compact_fail && strcmp(name, "compact_fail") == 0) compact_fail = value;
5772                 else if(!compact_success && strcmp(name, "compact_success") == 0) compact_success = value;
5773                 else if(!htlb_buddy_alloc_success && strcmp(name, "htlb_buddy_alloc_success") == 0) htlb_buddy_alloc_success = value;
5774                 else if(!htlb_buddy_alloc_fail && strcmp(name, "htlb_buddy_alloc_fail") == 0) htlb_buddy_alloc_fail = value;
5775                 else if(!unevictable_pgs_culled && strcmp(name, "unevictable_pgs_culled") == 0) unevictable_pgs_culled = value;
5776                 else if(!unevictable_pgs_scanned && strcmp(name, "unevictable_pgs_scanned") == 0) unevictable_pgs_scanned = value;
5777                 else if(!unevictable_pgs_rescued && strcmp(name, "unevictable_pgs_rescued") == 0) unevictable_pgs_rescued = value;
5778                 else if(!unevictable_pgs_mlocked && strcmp(name, "unevictable_pgs_mlocked") == 0) unevictable_pgs_mlocked = value;
5779                 else if(!unevictable_pgs_munlocked && strcmp(name, "unevictable_pgs_munlocked") == 0) unevictable_pgs_munlocked = value;
5780                 else if(!unevictable_pgs_cleared && strcmp(name, "unevictable_pgs_cleared") == 0) unevictable_pgs_cleared = value;
5781                 else if(!unevictable_pgs_stranded && strcmp(name, "unevictable_pgs_stranded") == 0) unevictable_pgs_stranded = value;
5782                 else if(!unevictable_pgs_mlockfreed && strcmp(name, "unevictable_pgs_mlockfreed") == 0) unevictable_pgs_mlockfreed = value;
5783                 else if(!thp_fault_alloc && strcmp(name, "thp_fault_alloc") == 0) thp_fault_alloc = value;
5784                 else if(!thp_fault_fallback && strcmp(name, "thp_fault_fallback") == 0) thp_fault_fallback = value;
5785                 else if(!thp_collapse_alloc && strcmp(name, "thp_collapse_alloc") == 0) thp_collapse_alloc = value;
5786                 else if(!thp_collapse_alloc_failed && strcmp(name, "thp_collapse_alloc_failed") == 0) thp_collapse_alloc_failed = value;
5787                 else if(!thp_split && strcmp(name, "thp_split") == 0) thp_split = value;
5788         }
5789         fclose(fp);
5790
5791         RRD_STATS *st;
5792
5793         // --------------------------------------------------------------------
5794
5795         if(do_swapio) {
5796                 st = rrd_stats_find("system.swapio");
5797                 if(!st) {
5798                         st = rrd_stats_create("system", "swapio", NULL, "mem", "Swap I/O", "kilobytes/s", 250, update_every, CHART_TYPE_AREA);
5799
5800                         rrd_stats_dimension_add(st, "in",  NULL, sysconf(_SC_PAGESIZE), 1024, RRD_DIMENSION_INCREMENTAL);
5801                         rrd_stats_dimension_add(st, "out", NULL, -sysconf(_SC_PAGESIZE), 1024, RRD_DIMENSION_INCREMENTAL);
5802                 }
5803                 else rrd_stats_next(st);
5804
5805                 rrd_stats_dimension_set(st, "in", pswpin);
5806                 rrd_stats_dimension_set(st, "out", pswpout);
5807                 rrd_stats_done(st);
5808         }
5809
5810         // --------------------------------------------------------------------
5811
5812         if(do_io) {
5813                 st = rrd_stats_find("system.io");
5814                 if(!st) {
5815                         st = rrd_stats_create("system", "io", NULL, "disk", "Disk I/O", "kilobytes/s", 150, update_every, CHART_TYPE_AREA);
5816
5817                         rrd_stats_dimension_add(st, "in",  NULL,  1, 1, RRD_DIMENSION_INCREMENTAL);
5818                         rrd_stats_dimension_add(st, "out", NULL, -1, 1, RRD_DIMENSION_INCREMENTAL);
5819                 }
5820                 else rrd_stats_next(st);
5821
5822                 rrd_stats_dimension_set(st, "in", pgpgin);
5823                 rrd_stats_dimension_set(st, "out", pgpgout);
5824                 rrd_stats_done(st);
5825         }
5826
5827         // --------------------------------------------------------------------
5828
5829         if(do_pgfaults) {
5830                 st = rrd_stats_find("system.pgfaults");
5831                 if(!st) {
5832                         st = rrd_stats_create("system", "pgfaults", NULL, "mem", "Memory Page Faults", "page faults/s", 500, update_every, CHART_TYPE_LINE);
5833                         st->isdetail = 1;
5834
5835                         rrd_stats_dimension_add(st, "minor",  NULL,  1, 1, RRD_DIMENSION_INCREMENTAL);
5836                         rrd_stats_dimension_add(st, "major", NULL, -1, 1, RRD_DIMENSION_INCREMENTAL);
5837                 }
5838                 else rrd_stats_next(st);
5839
5840                 rrd_stats_dimension_set(st, "minor", pgfault);
5841                 rrd_stats_dimension_set(st, "major", pgmajfault);
5842                 rrd_stats_done(st);
5843         }
5844
5845         return 0;
5846 }
5847
5848 // ----------------------------------------------------------------------------
5849 // /proc processor
5850
5851 void *proc_main(void *ptr)
5852 {
5853         if(ptr) { ; }
5854
5855         if(pthread_setcanceltype(PTHREAD_CANCEL_DEFERRED, NULL) != 0)
5856                 error("Cannot set pthread cancel type to DEFERRED.");
5857
5858         if(pthread_setcancelstate(PTHREAD_CANCEL_ENABLE, NULL) != 0)
5859                 error("Cannot set pthread cancel state to ENABLE.");
5860
5861         struct rusage me, me_last;
5862         struct timeval last, now;
5863
5864         gettimeofday(&last, NULL);
5865         last.tv_sec -= update_every;
5866
5867         // disable (by default) various interface that are not needed
5868         config_get_boolean("plugin:proc:/proc/net/dev", "interface lo", 0);
5869         config_get_boolean("plugin:proc:/proc/net/dev", "interface fireqos_monitor", 0);
5870
5871         // when ZERO, attempt to do it
5872         int     vdo_proc_net_dev = !config_get_boolean("plugin:proc", "/proc/net/dev", 1);
5873         int vdo_proc_diskstats = !config_get_boolean("plugin:proc", "/proc/diskstats", 1);
5874         int vdo_proc_net_snmp = !config_get_boolean("plugin:proc", "/proc/net/snmp", 1);
5875         int vdo_proc_net_netstat = !config_get_boolean("plugin:proc", "/proc/net/netstat", 1);
5876         int vdo_proc_net_stat_conntrack = !config_get_boolean("plugin:proc", "/proc/net/stat/conntrack", 1);
5877         int vdo_proc_net_ip_vs_stats = !config_get_boolean("plugin:proc", "/proc/net/ip_vs/stats", 1);
5878         int vdo_proc_stat = !config_get_boolean("plugin:proc", "/proc/stat", 1);
5879         int vdo_proc_meminfo = !config_get_boolean("plugin:proc", "/proc/meminfo", 1);
5880         int vdo_proc_vmstat = !config_get_boolean("plugin:proc", "/proc/vmstat", 1);
5881         int vdo_cpu_netdata = !config_get_boolean("plugin:proc", "netdata server resources", 1);
5882
5883         RRD_STATS *stcpu = NULL, *stclients = NULL, *streqs = NULL, *stbytes = NULL;
5884
5885         gettimeofday(&last, NULL);
5886         getrusage(RUSAGE_SELF, &me_last);
5887
5888         unsigned long long usec = 0, susec = 0;
5889         for(;1;) {
5890
5891                 // BEGIN -- the job to be done
5892                 if(!vdo_proc_net_dev)                           vdo_proc_net_dev                        = do_proc_net_dev(usec);
5893                 if(!vdo_proc_diskstats)                         vdo_proc_diskstats                      = do_proc_diskstats(usec);
5894                 if(!vdo_proc_net_snmp)                          vdo_proc_net_snmp                       = do_proc_net_snmp(usec);
5895                 if(!vdo_proc_net_netstat)                       vdo_proc_net_netstat            = do_proc_net_netstat(usec);
5896                 if(!vdo_proc_net_stat_conntrack)        vdo_proc_net_stat_conntrack     = do_proc_net_stat_conntrack(usec);
5897                 if(!vdo_proc_net_ip_vs_stats)           vdo_proc_net_ip_vs_stats        = do_proc_net_ip_vs_stats(usec);
5898                 if(!vdo_proc_stat)                                      vdo_proc_stat                           = do_proc_stat(usec);
5899                 if(!vdo_proc_meminfo)                           vdo_proc_meminfo                        = do_proc_meminfo(usec);
5900                 if(!vdo_proc_vmstat)                            vdo_proc_vmstat                         = do_proc_vmstat(usec);
5901                 // END -- the job is done
5902
5903                 // find the time to sleep in order to wait exactly update_every seconds
5904                 gettimeofday(&now, NULL);
5905                 usec = usecdiff(&now, &last) - susec;
5906                 debug(D_PROCNETDEV_LOOP, "PROCNETDEV: last loop took %llu usec (worked for %llu, sleeped for %llu).", usec + susec, usec, susec);
5907
5908                 if(usec < (update_every * 1000000ULL / 2ULL)) susec = (update_every * 1000000ULL) - usec;
5909                 else susec = update_every * 1000000ULL / 2ULL;
5910
5911                 // --------------------------------------------------------------------
5912
5913                 if(!vdo_cpu_netdata && getrusage(RUSAGE_SELF, &me) == 0) {
5914
5915                         unsigned long long cpuuser = me.ru_utime.tv_sec * 1000000ULL + me.ru_utime.tv_usec;
5916                         unsigned long long cpusyst = me.ru_stime.tv_sec * 1000000ULL + me.ru_stime.tv_usec;
5917
5918                         if(!stcpu) stcpu = rrd_stats_find("netdata.server_cpu");
5919                         if(!stcpu) {
5920                                 stcpu = rrd_stats_create("netdata", "server_cpu", NULL, "netdata", "NetData CPU usage", "milliseconds/s", 9999, update_every, CHART_TYPE_STACKED);
5921
5922                                 rrd_stats_dimension_add(stcpu, "user",  NULL,  1, 1000, RRD_DIMENSION_INCREMENTAL);
5923                                 rrd_stats_dimension_add(stcpu, "system", NULL, 1, 1000, RRD_DIMENSION_INCREMENTAL);
5924                         }
5925                         else rrd_stats_next(stcpu);
5926
5927                         rrd_stats_dimension_set(stcpu, "user", cpuuser);
5928                         rrd_stats_dimension_set(stcpu, "system", cpusyst);
5929                         rrd_stats_done(stcpu);
5930
5931                         bcopy(&me, &me_last, sizeof(struct rusage));
5932
5933                         // ----------------------------------------------------------------
5934
5935                         if(!stclients) stclients = rrd_stats_find("netdata.clients");
5936                         if(!stclients) {
5937                                 stclients = rrd_stats_create("netdata", "clients", NULL, "netdata", "NetData Web Clients", "connected clients", 11000, update_every, CHART_TYPE_LINE);
5938
5939                                 rrd_stats_dimension_add(stclients, "clients",  NULL,  1, 1, RRD_DIMENSION_ABSOLUTE_NO_INTERPOLATION);
5940                         }
5941                         else rrd_stats_next(stclients);
5942
5943                         rrd_stats_dimension_set(stclients, "clients", global_statistics.connected_clients);
5944                         rrd_stats_done(stclients);
5945
5946                         // ----------------------------------------------------------------
5947
5948                         if(!streqs) streqs = rrd_stats_find("netdata.requests");
5949                         if(!streqs) {
5950                                 streqs = rrd_stats_create("netdata", "requests", NULL, "netdata", "NetData Web Requests", "requests/s", 12000, update_every, CHART_TYPE_LINE);
5951
5952                                 rrd_stats_dimension_add(streqs, "requests",  NULL,  1, 1, RRD_DIMENSION_INCREMENTAL_NO_INTERPOLATION);
5953                         }
5954                         else rrd_stats_next(streqs);
5955
5956                         rrd_stats_dimension_set(streqs, "requests", global_statistics.web_requests);
5957                         rrd_stats_done(streqs);
5958
5959                         // ----------------------------------------------------------------
5960
5961                         if(!stbytes) stbytes = rrd_stats_find("netdata.net");
5962                         if(!stbytes) {
5963                                 stbytes = rrd_stats_create("netdata", "net", NULL, "netdata", "NetData Network Traffic", "kilobits/s", 13000, update_every, CHART_TYPE_AREA);
5964
5965                                 rrd_stats_dimension_add(stbytes, "in",  NULL,  8, 1024, RRD_DIMENSION_INCREMENTAL);
5966                                 rrd_stats_dimension_add(stbytes, "out",  NULL,  -8, 1024, RRD_DIMENSION_INCREMENTAL);
5967                         }
5968                         else rrd_stats_next(stbytes);
5969
5970                         rrd_stats_dimension_set(stbytes, "in", global_statistics.bytes_received);
5971                         rrd_stats_dimension_set(stbytes, "out", global_statistics.bytes_sent);
5972                         rrd_stats_done(stbytes);
5973                 }
5974
5975                 usleep(susec);
5976
5977                 // copy current to last
5978                 bcopy(&now, &last, sizeof(struct timeval));
5979         }
5980
5981         return NULL;
5982 }
5983
5984 // ----------------------------------------------------------------------------
5985 // /sbin/tc processor
5986 // this requires the script plugins.d/tc-qos-helper.sh
5987
5988 #define TC_LINE_MAX 1024
5989
5990 struct tc_class {
5991         char id[RRD_STATS_NAME_MAX + 1];
5992         char name[RRD_STATS_NAME_MAX + 1];
5993
5994         char leafid[RRD_STATS_NAME_MAX + 1];
5995         char parentid[RRD_STATS_NAME_MAX + 1];
5996
5997         int hasparent;
5998         int isleaf;
5999         unsigned long long bytes;
6000
6001         struct tc_class *next;
6002 };
6003
6004 struct tc_device {
6005         char id[RRD_STATS_NAME_MAX + 1];
6006         char name[RRD_STATS_NAME_MAX + 1];
6007         char family[RRD_STATS_NAME_MAX + 1];
6008
6009         struct tc_class *classes;
6010 };
6011
6012 void tc_device_commit(struct tc_device *d)
6013 {
6014         static int enable_new_interfaces = -1;
6015
6016         if(enable_new_interfaces == -1) enable_new_interfaces = config_get_boolean("plugin:tc", "enable new interfaces detected at runtime", 1);
6017
6018         // we only need to add leaf classes
6019         struct tc_class *c, *x;
6020
6021         for ( c = d->classes ; c ; c = c->next)
6022                 c->isleaf = 1;
6023
6024         for ( c = d->classes ; c ; c = c->next) {
6025                 for ( x = d->classes ; x ; x = x->next) {
6026                         if(x->parentid[0] && (strcmp(c->id, x->parentid) == 0 || strcmp(c->leafid, x->parentid) == 0)) {
6027                                 // debug(D_TC_LOOP, "TC: In device '%s', class '%s' (leafid: '%s') has leaf the class '%s' (parentid: '%s').", d->name, c->name, c->leafid, x->name, x->parentid);
6028                                 c->isleaf = 0;
6029                                 x->hasparent = 1;
6030                         }
6031                 }
6032         }
6033
6034         // debugging:
6035         /*
6036         for ( c = d->classes ; c ; c = c->next) {
6037                 if(c->isleaf && c->hasparent) debug(D_TC_LOOP, "TC: Device %s, class %s, OK", d->name, c->id);
6038                 else debug(D_TC_LOOP, "TC: Device %s, class %s, IGNORE (isleaf: %d, hasparent: %d, parent: %s)", d->name, c->id, c->isleaf, c->hasparent, c->parentid);
6039         }
6040         */
6041
6042         for ( c = d->classes ; c ; c = c->next) {
6043                 if(c->isleaf && c->hasparent) break;
6044         }
6045         if(!c) {
6046                 debug(D_TC_LOOP, "TC: Ignoring TC device '%s'. No leaf classes.", d->name);
6047                 return;
6048         }
6049
6050         char var_name[4096 + 1];
6051         snprintf(var_name, 4096, "qos for %s", d->id);
6052         if(config_get_boolean("plugin:tc", var_name, enable_new_interfaces)) {
6053                 RRD_STATS *st = rrd_stats_find_bytype(RRD_TYPE_TC, d->id);
6054                 if(!st) {
6055                         debug(D_TC_LOOP, "TC: Committing new TC device '%s'", d->name);
6056
6057                         st = rrd_stats_create(RRD_TYPE_TC, d->id, d->name, d->family, "Class Usage", "kilobits/s", 1000, update_every, CHART_TYPE_STACKED);
6058
6059                         for ( c = d->classes ; c ; c = c->next) {
6060                                 if(c->isleaf && c->hasparent)
6061                                         rrd_stats_dimension_add(st, c->id, c->name, 8, 1024, RRD_DIMENSION_INCREMENTAL);
6062                         }
6063                 }
6064                 else {
6065                         rrd_stats_next_plugins(st);
6066
6067                         if(strcmp(d->id, d->name) != 0) rrd_stats_set_name(st, d->name);
6068                 }
6069
6070                 for ( c = d->classes ; c ; c = c->next) {
6071                         if(c->isleaf && c->hasparent) {
6072                                 if(rrd_stats_dimension_set(st, c->id, c->bytes) != 0) {
6073
6074                                         // new class, we have to add it
6075                                         rrd_stats_dimension_add(st, c->id, c->name, 8, 1024, RRD_DIMENSION_INCREMENTAL);
6076                                         rrd_stats_dimension_set(st, c->id, c->bytes);
6077                                 }
6078
6079                                 // if it has a name, different to the id
6080                                 if(strcmp(c->id, c->name) != 0) {
6081                                         // update the rrd dimension with the new name
6082                                         RRD_DIMENSION *rd;
6083                                         for(rd = st->dimensions ; rd ; rd = rd->next) {
6084                                                 if(strcmp(rd->id, c->id) == 0) { rrd_stats_dimension_set_name(st, rd, c->name); break; }
6085                                         }
6086                                 }
6087                         }
6088                 }
6089                 rrd_stats_done(st);
6090         }
6091 }
6092
6093 void tc_device_set_class_name(struct tc_device *d, char *id, char *name)
6094 {
6095         struct tc_class *c;
6096         for ( c = d->classes ; c ; c = c->next) {
6097                 if(strcmp(c->id, id) == 0) {
6098                         strncpy(c->name, name, RRD_STATS_NAME_MAX);
6099                         // no need for null termination - it is already null
6100                         break;
6101                 }
6102         }
6103 }
6104
6105 void tc_device_set_device_name(struct tc_device *d, char *name)
6106 {
6107         strncpy(d->name, name, RRD_STATS_NAME_MAX);
6108         // no need for null termination - it is already null
6109 }
6110
6111 void tc_device_set_device_family(struct tc_device *d, char *name)
6112 {
6113         strncpy(d->family, name, RRD_STATS_NAME_MAX);
6114         // no need for null termination - it is already null
6115 }
6116
6117 struct tc_device *tc_device_create(char *name)
6118 {
6119         struct tc_device *d;
6120
6121         d = calloc(1, sizeof(struct tc_device));
6122         if(!d) {
6123                 fatal("Cannot allocate memory for tc_device %s", name);
6124                 return NULL;
6125         }
6126
6127         strncpy(d->id, name, RRD_STATS_NAME_MAX);
6128         strcpy(d->name, d->id);
6129         strcpy(d->family, d->id);
6130
6131         // no need for null termination on the strings, because of calloc()
6132
6133         return(d);
6134 }
6135
6136 struct tc_class *tc_class_add(struct tc_device *n, char *id, char *parentid, char *leafid)
6137 {
6138         struct tc_class *c;
6139
6140         c = calloc(1, sizeof(struct tc_class));
6141         if(!c) {
6142                 fatal("Cannot allocate memory for tc class");
6143                 return NULL;
6144         }
6145
6146         c->next = n->classes;
6147         n->classes = c;
6148
6149         strncpy(c->id, id, RRD_STATS_NAME_MAX);
6150         strcpy(c->name, c->id);
6151         if(parentid) strncpy(c->parentid, parentid, RRD_STATS_NAME_MAX);
6152         if(leafid) strncpy(c->leafid, leafid, RRD_STATS_NAME_MAX);
6153
6154         // no need for null termination on the strings, because of calloc()
6155
6156         return(c);
6157 }
6158
6159 void tc_class_free(struct tc_class *c)
6160 {
6161         if(c->next) tc_class_free(c->next);
6162         free(c);
6163 }
6164
6165 void tc_device_free(struct tc_device *n)
6166 {
6167         if(n->classes) tc_class_free(n->classes);
6168         free(n);
6169 }
6170
6171 pid_t tc_child_pid = 0;
6172 void *tc_main(void *ptr)
6173 {
6174         if(ptr) { ; }
6175
6176         if(pthread_setcanceltype(PTHREAD_CANCEL_DEFERRED, NULL) != 0)
6177                 error("Cannot set pthread cancel type to DEFERRED.");
6178
6179         if(pthread_setcancelstate(PTHREAD_CANCEL_ENABLE, NULL) != 0)
6180                 error("Cannot set pthread cancel state to ENABLE.");
6181
6182         char buffer[TC_LINE_MAX+1] = "";
6183
6184         for(;1;) {
6185                 FILE *fp;
6186                 struct tc_device *device = NULL;
6187                 struct tc_class *class = NULL;
6188
6189                 snprintf(buffer, TC_LINE_MAX, "exec %s %d", config_get("plugin:tc", "script to run to get tc values", PLUGINS_DIR "/tc-qos-helper.sh"), update_every);
6190                 debug(D_TC_LOOP, "executing '%s'", buffer);
6191                 // fp = popen(buffer, "r");
6192                 fp = mypopen(buffer, &tc_child_pid);
6193                 if(!fp) {
6194                         error("TC: Cannot popen(\"%s\", \"r\").", buffer);
6195                         return NULL;
6196                 }
6197
6198                 while(fgets(buffer, TC_LINE_MAX, fp) != NULL) {
6199                         buffer[TC_LINE_MAX] = '\0';
6200                         char *b = buffer, *p;
6201                         // debug(D_TC_LOOP, "TC: read '%s'", buffer);
6202
6203                         p = strsep(&b, " \n");
6204                         while (p && (*p == ' ' || *p == '\0')) p = strsep(&b, " \n");
6205                         if(!p) continue;
6206
6207                         if(strcmp(p, "END") == 0) {
6208                                 if(device) {
6209                                         if(pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, NULL) != 0)
6210                                                 error("Cannot set pthread cancel state to DISABLE.");
6211
6212                                         tc_device_commit(device);
6213                                         tc_device_free(device);
6214                                         device = NULL;
6215                                         class = NULL;
6216
6217                                         if(pthread_setcancelstate(PTHREAD_CANCEL_ENABLE, NULL) != 0)
6218                                                 error("Cannot set pthread cancel state to ENABLE.");
6219                                 }
6220                         }
6221                         else if(strcmp(p, "BEGIN") == 0) {
6222                                 if(device) {
6223                                         tc_device_free(device);
6224                                         device = NULL;
6225                                         class = NULL;
6226                                 }
6227
6228                                 p = strsep(&b, " \n");
6229                                 if(p && *p) {
6230                                         device = tc_device_create(p);
6231                                         class = NULL;
6232                                 }
6233                         }
6234                         else if(device && (strcmp(p, "class") == 0)) {
6235                                 p = strsep(&b, " \n"); // the class: htb, fq_codel, etc
6236                                 char *id       = strsep(&b, " \n"); // the class major:minor
6237                                 char *parent   = strsep(&b, " \n"); // 'parent' or 'root'
6238                                 char *parentid = strsep(&b, " \n"); // the parent's id
6239                                 char *leaf     = strsep(&b, " \n"); // 'leaf'
6240                                 char *leafid   = strsep(&b, " \n"); // leafid
6241
6242                                 if(id && *id
6243                                         && parent && *parent
6244                                         && parentid && *parentid
6245                                         && (
6246                                                 (strcmp(parent, "parent") == 0 && parentid && *parentid)
6247                                                 || strcmp(parent, "root") == 0
6248                                         )) {
6249
6250                                         if(strcmp(parent, "root") == 0) {
6251                                                 parentid = NULL;
6252                                                 leafid = NULL;
6253                                         }
6254                                         else if(!leaf || strcmp(leaf, "leaf") != 0)
6255                                                 leafid = NULL;
6256
6257                                         char leafbuf[20 + 1] = "";
6258                                         if(leafid && leafid[strlen(leafid) - 1] == ':') {
6259                                                 strncpy(leafbuf, leafid, 20 - 1);
6260                                                 strcat(leafbuf, "1");
6261                                                 leafid = leafbuf;
6262                                         }
6263
6264                                         class = tc_class_add(device, id, parentid, leafid);
6265                                 }
6266                         }
6267                         else if(device && class && (strcmp(p, "Sent") == 0)) {
6268                                 p = strsep(&b, " \n");
6269                                 if(p && *p) class->bytes = atoll(p);
6270                         }
6271                         else if(device && (strcmp(p, "SETDEVICENAME") == 0)) {
6272                                 char *name = strsep(&b, " \n");
6273                                 if(name && *name) tc_device_set_device_name(device, name);
6274                         }
6275                         else if(device && (strcmp(p, "SETDEVICEGROUP") == 0)) {
6276                                 char *name = strsep(&b, " \n");
6277                                 if(name && *name) tc_device_set_device_family(device, name);
6278                         }
6279                         else if(device && (strcmp(p, "SETCLASSNAME") == 0)) {
6280                                 char *id    = strsep(&b, " \n");
6281                                 char *path  = strsep(&b, " \n");
6282                                 if(id && *id && path && *path) tc_device_set_class_name(device, id, path);
6283                         }
6284 #ifdef DETACH_PLUGINS_FROM_NETDATA
6285                         else if((strcmp(p, "MYPID") == 0)) {
6286                                 char *id = strsep(&b, " \n");
6287                                 pid_t pid = atol(id);
6288
6289                                 if(pid) tc_child_pid = pid;
6290
6291                                 debug(D_TC_LOOP, "TC: Child PID is %d.", tc_child_pid);
6292                         }
6293 #endif
6294                 }
6295                 mypclose(fp);
6296
6297                 if(device) {
6298                         tc_device_free(device);
6299                         device = NULL;
6300                         class = NULL;
6301                 }
6302
6303                 sleep(update_every);
6304         }
6305
6306         return NULL;
6307 }
6308
6309 // ----------------------------------------------------------------------------
6310 // cpu jitter calculation
6311
6312 #define CPU_IDLEJITTER_SLEEP_TIME_MS 20
6313
6314 void *cpuidlejitter_main(void *ptr)
6315 {
6316         if(ptr) { ; }
6317
6318         if(pthread_setcanceltype(PTHREAD_CANCEL_DEFERRED, NULL) != 0)
6319                 error("Cannot set pthread cancel type to DEFERRED.");
6320
6321         if(pthread_setcancelstate(PTHREAD_CANCEL_ENABLE, NULL) != 0)
6322                 error("Cannot set pthread cancel state to ENABLE.");
6323
6324         int sleep_ms = config_get_number("plugin:idlejitter", "loop time in ms", CPU_IDLEJITTER_SLEEP_TIME_MS);
6325         if(sleep_ms <= 0) {
6326                 config_set_number("plugin:idlejitter", "loop time in ms", CPU_IDLEJITTER_SLEEP_TIME_MS);
6327                 sleep_ms = CPU_IDLEJITTER_SLEEP_TIME_MS;
6328         }
6329
6330         RRD_STATS *st = rrd_stats_find("system.idlejitter");
6331         if(!st) {
6332                 st = rrd_stats_create("system", "idlejitter", NULL, "cpu", "CPU Idle Jitter", "microseconds lost/s", 9999, update_every, CHART_TYPE_LINE);
6333                 rrd_stats_dimension_add(st, "jitter", NULL, 1, 1, RRD_DIMENSION_ABSOLUTE);
6334         }
6335
6336         struct timeval before, after;
6337         unsigned long long counter;
6338         for(counter = 0; 1 ;counter++) {
6339                 unsigned long long usec = 0, susec = 0;
6340
6341                 while(susec < (update_every * 1000000ULL)) {
6342
6343                         gettimeofday(&before, NULL);
6344                         usleep(sleep_ms * 1000);
6345                         gettimeofday(&after, NULL);
6346
6347                         // calculate the time it took for a full loop
6348                         usec = usecdiff(&after, &before);
6349                         susec += usec;
6350                 }
6351                 usec -= (sleep_ms * 1000);
6352
6353                 if(counter) rrd_stats_next_usec(st, susec);
6354                 rrd_stats_dimension_set(st, "jitter", usec);
6355                 rrd_stats_done(st);
6356         }
6357
6358         return NULL;
6359 }
6360
6361 // ----------------------------------------------------------------------------
6362 // netdata checks
6363
6364 void *checks_main(void *ptr)
6365 {
6366         if(ptr) { ; }
6367
6368         if(pthread_setcanceltype(PTHREAD_CANCEL_DEFERRED, NULL) != 0)
6369                 error("Cannot set pthread cancel type to DEFERRED.");
6370
6371         if(pthread_setcancelstate(PTHREAD_CANCEL_ENABLE, NULL) != 0)
6372                 error("Cannot set pthread cancel state to ENABLE.");
6373
6374         unsigned long long usec = 0, susec = update_every * 1000000ULL, loop_usec = 0, total_susec = 0;
6375         struct timeval now, last, loop;
6376
6377         RRD_STATS *check1, *check2, *check3, *apps_cpu = NULL;
6378
6379         check1 = rrd_stats_create("netdata", "check1", NULL, "netdata", "Caller gives microseconds", "a million !", 99999, update_every, CHART_TYPE_LINE);
6380         rrd_stats_dimension_add(check1, "absolute", NULL, -1, 1, RRD_DIMENSION_ABSOLUTE);
6381         rrd_stats_dimension_add(check1, "incremental", NULL, 1, 1, RRD_DIMENSION_INCREMENTAL);
6382
6383         check2 = rrd_stats_create("netdata", "check2", NULL, "netdata", "Netdata calcs microseconds", "a million !", 99999, update_every, CHART_TYPE_LINE);
6384         rrd_stats_dimension_add(check2, "absolute", NULL, -1, 1, RRD_DIMENSION_ABSOLUTE);
6385         rrd_stats_dimension_add(check2, "incremental", NULL, 1, 1, RRD_DIMENSION_INCREMENTAL);
6386
6387         check3 = rrd_stats_create("netdata", "checkdt", NULL, "netdata", "Clock difference", "microseconds diff", 99999, update_every, CHART_TYPE_LINE);
6388         rrd_stats_dimension_add(check3, "caller", NULL, 1, 1, RRD_DIMENSION_ABSOLUTE);
6389         rrd_stats_dimension_add(check3, "netdata", NULL, 1, 1, RRD_DIMENSION_ABSOLUTE);
6390         rrd_stats_dimension_add(check3, "apps.plugin", NULL, 1, 1, RRD_DIMENSION_ABSOLUTE);
6391
6392         gettimeofday(&last, NULL);
6393         while(1) {
6394                 usleep(susec);
6395
6396                 // find the time to sleep in order to wait exactly update_every seconds
6397                 gettimeofday(&now, NULL);
6398                 loop_usec = usecdiff(&now, &last);
6399                 usec = loop_usec - susec;
6400                 debug(D_PROCNETDEV_LOOP, "CHECK: last loop took %llu usec (worked for %llu, sleeped for %llu).", loop_usec, usec, susec);
6401
6402                 if(usec < (update_every * 1000000ULL / 2ULL)) susec = (update_every * 1000000ULL) - usec;
6403                 else susec = update_every * 1000000ULL / 2ULL;
6404
6405                 // --------------------------------------------------------------------
6406                 // Calculate loop time
6407
6408                 last.tv_sec = now.tv_sec;
6409                 last.tv_usec = now.tv_usec;
6410                 total_susec += loop_usec;
6411
6412                 // --------------------------------------------------------------------
6413                 // check chart 1
6414
6415                 if(check1->counter_done) rrd_stats_next_usec(check1, loop_usec);
6416                 rrd_stats_dimension_set(check1, "absolute", 1000000);
6417                 rrd_stats_dimension_set(check1, "incremental", total_susec);
6418                 rrd_stats_done(check1);
6419
6420                 // --------------------------------------------------------------------
6421                 // check chart 2
6422
6423                 if(check2->counter_done) rrd_stats_next(check2);
6424                 rrd_stats_dimension_set(check2, "absolute", 1000000);
6425                 rrd_stats_dimension_set(check2, "incremental", total_susec);
6426                 rrd_stats_done(check2);
6427
6428                 // --------------------------------------------------------------------
6429                 // check chart 3
6430
6431                 if(!apps_cpu) apps_cpu = rrd_stats_find("apps.cpu");
6432                 if(check3->counter_done) rrd_stats_next_usec(check3, loop_usec);
6433                 gettimeofday(&loop, NULL);
6434                 rrd_stats_dimension_set(check3, "caller", (long long)usecdiff(&loop, &check1->last_collected));
6435                 rrd_stats_dimension_set(check3, "netdata", (long long)usecdiff(&loop, &check2->last_collected));
6436                 if(apps_cpu) rrd_stats_dimension_set(check3, "apps.plugin", (long long)usecdiff(&loop, &apps_cpu->last_collected));
6437                 rrd_stats_done(check3);
6438         }
6439
6440         return NULL;
6441 }
6442
6443 // ----------------------------------------------------------------------------
6444 // plugins.d
6445
6446 #define PLUGINSD_FILE_SUFFIX ".plugin"
6447 #define PLUGINSD_FILE_SUFFIX_LEN strlen(PLUGINSD_FILE_SUFFIX)
6448 #define PLUGINSD_CMD_MAX (FILENAME_MAX*2)
6449 #define PLUGINSD_LINE_MAX 1024
6450
6451 struct plugind {
6452         char id[CONFIG_MAX_NAME+1];                     // config node id
6453
6454         char filename[FILENAME_MAX+1];          // just the filename
6455         char fullfilename[FILENAME_MAX+1];      // with path
6456         char cmd[PLUGINSD_CMD_MAX+1];           // the command that is executes
6457
6458         pid_t pid;
6459         pthread_t thread;
6460
6461         int update_every;
6462         int obsolete;
6463         int enabled;
6464
6465         time_t started_t;
6466
6467         struct plugind *next;
6468 } *pluginsd_root = NULL;
6469
6470 // like strsep() but:
6471 // it trims spaces before and after each value
6472 // it accepts quoted values in single or double quotes
6473 char *qstrsep(char **ptr)
6474 {
6475         if(!*ptr || !**ptr) return NULL;
6476
6477         char *s, *p = *ptr;
6478
6479         // skip leading spaces
6480         while(isspace(*p)) p++;
6481
6482         // if the first char is a quote, assume quoted
6483         if(*p == '"' || *p == '\'') {
6484                 char q = *p;
6485                 s = ++p;
6486                 while(*p && *p != q) p++;
6487
6488                 if(*p == q) {
6489                         *p = '\0';
6490                         p++;
6491                 }
6492
6493                 *ptr = p;
6494                 return s;
6495         }
6496
6497         s = p;
6498         while(*p && !isspace(*p)) p++;
6499         if(!*p) *ptr = NULL;
6500         else {
6501                 *p = '\0';
6502                 *ptr = ++p;
6503         }
6504
6505         return s;
6506 }
6507
6508 void *pluginsd_worker_thread(void *arg)
6509 {
6510         struct plugind *cd = (struct plugind *)arg;
6511         char line[PLUGINSD_LINE_MAX + 1];
6512
6513 #ifdef DETACH_PLUGINS_FROM_NETDATA
6514         unsigned long long usec = 0, susec = 0;
6515         struct timeval last = {0, 0} , now = {0, 0};
6516 #endif
6517
6518         while(1) {
6519                 FILE *fp = mypopen(cd->cmd, &cd->pid);
6520                 if(!fp) {
6521                         error("Cannot popen(\"%s\", \"r\").", cd->cmd);
6522                         break;
6523                 }
6524
6525                 RRD_STATS *st = NULL;
6526
6527                 unsigned long long count = 0;
6528                 while(fgets(line, PLUGINSD_LINE_MAX, fp) != NULL) {
6529                         char *p = trim(line);
6530                         debug(D_PLUGINSD, "PLUGINSD: %s: %s", cd->filename, line);
6531
6532                         char *s = qstrsep(&p);
6533
6534                         if(!s || !*s) continue;
6535                         else if(!strcmp(s, "SET")) {
6536                                 char *t;
6537                                 while((t = strchr(p, '='))) *t = ' ';
6538
6539                                 char *dimension = qstrsep(&p);
6540                                 char *value = qstrsep(&p);
6541
6542                                 if(!dimension || !*dimension || !value) {
6543                                         error("PLUGINSD: '%s' is requesting a SET on chart '%s', like this: 'SET %s = %s'. Disabling it.", cd->fullfilename, st->id, dimension?dimension:"", value?value:"");
6544                                         cd->enabled = 0;
6545                                         kill(cd->pid, SIGTERM);
6546                                         break;
6547                                 }
6548
6549                                 if(!st) {
6550                                         error("PLUGINSD: '%s' is requesting a SET on dimension %s with value %s, without a BEGIN. Disabling it.", cd->fullfilename, dimension, value);
6551                                         cd->enabled = 0;
6552                                         kill(cd->pid, SIGTERM);
6553                                         break;
6554                                 }
6555
6556                                 if(st->debug) debug(D_PLUGINSD, "PLUGINSD: '%s' is setting dimension %s/%s to %s", cd->fullfilename, st->id, dimension, value);
6557                                 rrd_stats_dimension_set(st, dimension, atoll(value));
6558
6559                                 count++;
6560                         }
6561                         else if(!strcmp(s, "BEGIN")) {
6562                                 char *id = qstrsep(&p);
6563                                 char *microseconds_txt = qstrsep(&p);
6564
6565                                 if(!id) {
6566                                         error("PLUGINSD: '%s' is requesting a BEGIN without a chart id. Disabling it.", cd->fullfilename);
6567                                         cd->enabled = 0;
6568                                         kill(cd->pid, SIGTERM);
6569                                         break;
6570                                 }
6571
6572                                 st = rrd_stats_find(id);
6573                                 if(!st) {
6574                                         error("PLUGINSD: '%s' is requesting a BEGIN on chart '%s', which does not exist. Disabling it.", cd->fullfilename, id);
6575                                         cd->enabled = 0;
6576                                         kill(cd->pid, SIGTERM);
6577                                         break;
6578                                 }
6579
6580                                 if(st->counter_done) {
6581                                         unsigned long long microseconds = 0;
6582                                         if(microseconds_txt && *microseconds_txt) microseconds = strtoull(microseconds_txt, NULL, 10);
6583                                         if(microseconds) rrd_stats_next_usec(st, microseconds);
6584                                         else rrd_stats_next_plugins(st);
6585                                 }
6586                         }
6587                         else if(!strcmp(s, "END")) {
6588                                 if(!st) {
6589                                         error("PLUGINSD: '%s' is requesting an END, without a BEGIN. Disabling it.", cd->fullfilename);
6590                                         cd->enabled = 0;
6591                                         kill(cd->pid, SIGTERM);
6592                                         break;
6593                                 }
6594
6595                                 if(st->debug) debug(D_PLUGINSD, "PLUGINSD: '%s' is requesting a END on chart %s", cd->fullfilename, st->id);
6596
6597                                 rrd_stats_done(st);
6598                                 st = NULL;
6599                         }
6600                         else if(!strcmp(s, "FLUSH")) {
6601                                 debug(D_PLUGINSD, "PLUGINSD: '%s' is requesting a FLUSH", cd->fullfilename);
6602                                 st = NULL;
6603                         }
6604                         else if(!strcmp(s, "CHART")) {
6605                                 st = NULL;
6606
6607                                 char *type = qstrsep(&p);
6608                                 char *id = NULL;
6609                                 if(type) {
6610                                         id = strchr(type, '.');
6611                                         if(id) { *id = '\0'; id++; }
6612                                 }
6613                                 char *name = qstrsep(&p);
6614                                 char *title = qstrsep(&p);
6615                                 char *units = qstrsep(&p);
6616                                 char *family = qstrsep(&p);
6617                                 char *category = qstrsep(&p);
6618                                 char *chart = qstrsep(&p);
6619                                 char *priority_s = qstrsep(&p);
6620                                 char *update_every_s = qstrsep(&p);
6621
6622                                 if(!type || !*type || !id || !*id) {
6623                                         error("PLUGINSD: '%s' is requesting a CHART, without a type.id. Disabling it.", cd->fullfilename);
6624                                         cd->enabled = 0;
6625                                         kill(cd->pid, SIGTERM);
6626                                         break;
6627                                 }
6628
6629                                 int priority = 1000;
6630                                 if(priority_s) priority = atoi(priority_s);
6631
6632                                 int update_every = cd->update_every;
6633                                 if(update_every_s) update_every = atoi(update_every_s);
6634                                 if(!update_every) update_every = cd->update_every;
6635
6636                                 int chart_type = CHART_TYPE_LINE;
6637                                 if(chart) chart_type = chart_type_id(chart);
6638
6639                                 if(!name || !*name) name = NULL;
6640                                 if(!family || !*family) family = id;
6641                                 if(!category || !*category) category = type;
6642
6643                                 st = rrd_stats_find_bytype(type, id);
6644                                 if(!st) {
6645                                         debug(D_PLUGINSD, "PLUGINSD: Creating chart type='%s', id='%s', name='%s', family='%s', category='%s', chart='%s', priority=%d, update_every=%d"
6646                                                 , type, id
6647                                                 , name?name:""
6648                                                 , family?family:""
6649                                                 , category?category:""
6650                                                 , chart_type_name(chart_type)
6651                                                 , priority
6652                                                 , update_every
6653                                                 );
6654
6655                                         st = rrd_stats_create(type, id, name, family, title, units, priority, update_every, chart_type);
6656                                         cd->update_every = update_every;
6657
6658                                         if(strcmp(category, "none") == 0) st->isdetail = 1;
6659                                 }
6660                                 else debug(D_PLUGINSD, "PLUGINSD: Chart '%s' already exists. Not adding it again.", st->id);
6661                         }
6662                         else if(!strcmp(s, "DIMENSION")) {
6663                                 char *id = qstrsep(&p);
6664                                 char *name = qstrsep(&p);
6665                                 char *algorithm = qstrsep(&p);
6666                                 char *multiplier_s = qstrsep(&p);
6667                                 char *divisor_s = qstrsep(&p);
6668                                 char *hidden = qstrsep(&p);
6669
6670                                 if(!id || !*id) {
6671                                         error("PLUGINSD: '%s' is requesting a DIMENSION, without an id. Disabling it.", cd->fullfilename);
6672                                         cd->enabled = 0;
6673                                         kill(cd->pid, SIGTERM);
6674                                         break;
6675                                 }
6676
6677                                 if(!st) {
6678                                         error("PLUGINSD: '%s' is requesting a DIMENSION, without a CHART. Disabling it.", cd->fullfilename);
6679                                         cd->enabled = 0;
6680                                         kill(cd->pid, SIGTERM);
6681                                         break;
6682                                 }
6683
6684                                 long multiplier = 1;
6685                                 if(multiplier_s && *multiplier_s) multiplier = atol(multiplier_s);
6686                                 if(!multiplier) multiplier = 1;
6687
6688                                 long divisor = 1;
6689                                 if(divisor_s && *divisor_s) divisor = atol(divisor_s);
6690                                 if(!divisor) divisor = 1;
6691
6692                                 if(!algorithm || !*algorithm) algorithm = "absolute";
6693
6694                                 if(st->debug) debug(D_PLUGINSD, "PLUGINSD: Creating dimension in chart %s, id='%s', name='%s', algorithm='%s', multiplier=%ld, divisor=%ld, hidden='%s'"
6695                                         , st->id
6696                                         , id
6697                                         , name?name:""
6698                                         , algorithm_name(algorithm_id(algorithm))
6699                                         , multiplier
6700                                         , divisor
6701                                         , hidden?hidden:""
6702                                         );
6703
6704                                 RRD_DIMENSION *rd = rrd_stats_dimension_find(st, id);
6705                                 if(!rd) {
6706                                         rd = rrd_stats_dimension_add(st, id, name, multiplier, divisor, algorithm_id(algorithm));
6707                                         if(hidden && strcmp(hidden, "hidden") == 0) rd->hidden = 1;
6708                                 }
6709                                 else if(st->debug) debug(D_PLUGINSD, "PLUGINSD: dimension %s/%s already exists. Not adding it again.", st->id, id);
6710                         }
6711                         else if(!strcmp(s, "DISABLE")) {
6712                                 error("PLUGINSD: '%s' called DISABLE. Disabling it.", cd->fullfilename);
6713                                 cd->enabled = 0;
6714                                 kill(cd->pid, SIGTERM);
6715                                 break;
6716                         }
6717 #ifdef DETACH_PLUGINS_FROM_NETDATA
6718                         else if(!strcmp(s, "MYPID")) {
6719                                 char *pid_s = qstrsep(&p);
6720                                 pid_t pid = atol(pid_s);
6721
6722                                 if(pid) cd->pid = pid;
6723                                 debug(D_PLUGINSD, "PLUGINSD: %s is on pid %d", cd->id, cd->pid);
6724                         }
6725                         else if(!strcmp(s, "STOPPING_WAKE_ME_UP_PLEASE")) {
6726                                 error("PLUGINSD: '%s' (pid %d) called STOPPING_WAKE_ME_UP_PLEASE.", cd->fullfilename, cd->pid);
6727
6728                                 gettimeofday(&now, NULL);
6729                                 if(!usec && !susec) {
6730                                         // our first run
6731                                         susec = cd->update_every * 1000000ULL;
6732                                 }
6733                                 else {
6734                                         // second+ run
6735                                         usec = usecdiff(&now, &last) - susec;
6736                                         error("PLUGINSD: %s last loop took %llu usec (worked for %llu, sleeped for %llu).\n", cd->fullfilename, usec + susec, usec, susec);
6737                                         if(usec < (update_every * 1000000ULL / 2ULL)) susec = (update_every * 1000000ULL) - usec;
6738                                         else susec = update_every * 1000000ULL / 2ULL;
6739                                 }
6740
6741                                 error("PLUGINSD: %s sleeping for %llu. Will kill with SIGCONT pid %d to wake it up.\n", cd->fullfilename, susec, cd->pid);
6742                                 usleep(susec);
6743                                 kill(cd->pid, SIGCONT);
6744                                 bcopy(&now, &last, sizeof(struct timeval));
6745                                 break;
6746                         }
6747 #endif
6748                         else {
6749                                 error("PLUGINSD: '%s' is sending command '%s' which is not known by netdata. Disabling it.", cd->fullfilename, s);
6750                                 cd->enabled = 0;
6751                                 kill(cd->pid, SIGTERM);
6752                                 break;
6753                         }
6754                 }
6755
6756                 // fgets() failed or loop broke
6757                 mypclose(fp);
6758
6759                 if(!count && cd->enabled) {
6760                         error("PLUGINSD: '%s' does not generate usefull output. Disabling it.", cd->fullfilename);
6761                         cd->enabled = 0;
6762                         kill(cd->pid, SIGTERM);
6763                 }
6764
6765                 if(cd->enabled) sleep(cd->update_every);
6766                 else break;
6767         }
6768
6769         cd->obsolete = 1;
6770         return NULL;
6771 }
6772
6773 void *pluginsd_main(void *ptr)
6774 {
6775         if(ptr) { ; }
6776
6777         if(pthread_setcanceltype(PTHREAD_CANCEL_DEFERRED, NULL) != 0)
6778                 error("Cannot set pthread cancel type to DEFERRED.");
6779
6780         if(pthread_setcancelstate(PTHREAD_CANCEL_ENABLE, NULL) != 0)
6781                 error("Cannot set pthread cancel state to ENABLE.");
6782
6783         char *dir_name = config_get("plugins", "plugins directory", PLUGINS_DIR);
6784         int automatic_run = config_get_boolean("plugins", "enable running new plugins", 0);
6785         int scan_frequency = config_get_number("plugins", "check for new plugins every", 60);
6786         DIR *dir = NULL;
6787         struct dirent *file = NULL;
6788         struct plugind *cd;
6789
6790         // enable the apps plugin by default
6791         config_get_boolean("plugins", "apps", 1);
6792
6793         if(scan_frequency < 1) scan_frequency = 1;
6794
6795         while(1) {
6796                 dir = opendir(dir_name);
6797                 if(!dir) {
6798                         error("Cannot open directory '%s'.", dir_name);
6799                         return NULL;
6800                 }
6801
6802                 while((file = readdir(dir))) {
6803                         debug(D_PLUGINSD, "PLUGINSD: Examining file '%s'", file->d_name);
6804
6805                         if(strcmp(file->d_name, ".") == 0 || strcmp(file->d_name, "..") == 0) continue;
6806
6807                         int len = strlen(file->d_name);
6808                         if(len <= (int)PLUGINSD_FILE_SUFFIX_LEN) continue;
6809                         if(strcmp(PLUGINSD_FILE_SUFFIX, &file->d_name[len - PLUGINSD_FILE_SUFFIX_LEN]) != 0) {
6810                                 debug(D_PLUGINSD, "PLUGINSD: File '%s' does not end in '%s'.", file->d_name, PLUGINSD_FILE_SUFFIX);
6811                                 continue;
6812                         }
6813
6814                         char pluginname[CONFIG_MAX_NAME + 1];
6815                         snprintf(pluginname, CONFIG_MAX_NAME, "%.*s", (int)(len - PLUGINSD_FILE_SUFFIX_LEN), file->d_name);
6816                         int enabled = config_get_boolean("plugins", pluginname, automatic_run);
6817
6818                         if(!enabled) {
6819                                 debug(D_PLUGINSD, "PLUGINSD: plugin '%s' is not enabled", file->d_name);
6820                                 continue;
6821                         }
6822
6823                         // check if it runs already
6824                         for(cd = pluginsd_root ; cd ; cd = cd->next) {
6825                                 if(strcmp(cd->filename, file->d_name) == 0) break;
6826                         }
6827                         if(cd && !cd->obsolete) {
6828                                 debug(D_PLUGINSD, "PLUGINSD: plugin '%s' is already running", cd->filename);
6829                                 continue;
6830                         }
6831
6832                         // it is not running
6833                         // allocate a new one, or use the obsolete one
6834                         if(!cd) {
6835                                 cd = calloc(sizeof(struct plugind), 1);
6836                                 if(!cd) fatal("Cannot allocate memory for plugin.");
6837
6838                                 snprintf(cd->id, CONFIG_MAX_NAME, "plugin:%s", pluginname);
6839
6840                                 strncpy(cd->filename, file->d_name, FILENAME_MAX);
6841                                 snprintf(cd->fullfilename, FILENAME_MAX, "%s/%s", dir_name, cd->filename);
6842
6843                                 cd->enabled = enabled;
6844                                 cd->update_every = config_get_number(cd->id, "update every", update_every);
6845                                 cd->started_t = time(NULL);
6846
6847                                 char *def = "";
6848                                 snprintf(cd->cmd, PLUGINSD_CMD_MAX, "exec %s %d %s", cd->fullfilename, cd->update_every, config_get(cd->id, "command options", def));
6849
6850                                 // link it
6851                                 if(pluginsd_root) cd->next = pluginsd_root;
6852                                 pluginsd_root = cd;
6853                         }
6854                         cd->obsolete = 0;
6855
6856                         if(!cd->enabled) continue;
6857
6858                         // spawn a new thread for it
6859                         if(pthread_create(&cd->thread, NULL, pluginsd_worker_thread, cd) != 0) {
6860                                 error("CHARTS.D: failed to create new thread for chart.d %s.", cd->filename);
6861                                 cd->obsolete = 1;
6862                         }
6863                         else if(pthread_detach(cd->thread) != 0)
6864                                 error("CHARTS.D: Cannot request detach of newly created thread for chart.d %s.", cd->filename);
6865                 }
6866
6867                 closedir(dir);
6868                 sleep(scan_frequency);
6869         }
6870
6871         return NULL;
6872 }
6873
6874
6875 // ----------------------------------------------------------------------------
6876 // main and related functions
6877
6878 pthread_t *p_proc = NULL, *p_tc = NULL, *p_jitter = NULL, *p_pluginsd = NULL, *p_checks = NULL;
6879
6880 void kill_childs()
6881 {
6882         siginfo_t info;
6883
6884         struct web_client *w;
6885         for(w = web_clients; w ; w = w->next) {
6886                 debug(D_EXIT, "Stopping web client %s", w->client_ip);
6887                 pthread_cancel(w->thread);
6888                 pthread_join(w->thread, NULL);
6889         }
6890
6891         if(p_proc)      {
6892                 debug(D_EXIT, "Stopping proc thread");
6893                 pthread_cancel(*p_proc);
6894                 pthread_join(*p_proc, NULL);
6895                 p_proc = NULL;
6896         }
6897
6898         if(p_jitter) {
6899                 debug(D_EXIT, "Stopping idlejitter thread");
6900                 pthread_cancel(*p_jitter);
6901                 pthread_join(*p_jitter, NULL);
6902                 p_jitter = NULL;
6903         }
6904
6905         if(p_checks) {
6906                 debug(D_EXIT, "Stopping self-checks thread");
6907                 pthread_cancel(*p_checks);
6908                 pthread_join(*p_checks, NULL);
6909                 p_checks = NULL;
6910         }
6911
6912         if(p_tc) {
6913                 if(tc_child_pid) {
6914                         debug(D_EXIT, "Killing tc-qos-helper procees");
6915                         kill(tc_child_pid, SIGTERM);
6916                         waitid(tc_child_pid, 0, &info, WEXITED);
6917                 }
6918                 tc_child_pid = 0;
6919
6920                 debug(D_EXIT, "Stopping tc plugin thread");
6921                 pthread_cancel(*p_tc);
6922                 pthread_join(*p_tc, NULL);
6923                 p_tc = NULL;
6924         }
6925
6926         if(p_pluginsd) {
6927                 struct plugind *cd;
6928                 for(cd = pluginsd_root ; cd ; cd = cd->next) {
6929                         debug(D_EXIT, "Stopping %s plugin thread", cd->id);
6930                         pthread_cancel(cd->thread);
6931                         pthread_join(cd->thread, NULL);
6932
6933                         if(cd->pid && !cd->obsolete) {
6934                                 debug(D_EXIT, "killing %s plugin process", cd->id);
6935                                 kill(cd->pid, SIGTERM);
6936                                 waitid(cd->pid, 0, &info, WEXITED);
6937                         }
6938                 }
6939
6940                 debug(D_EXIT, "Stopping plugin manager thread");
6941                 pthread_cancel(*p_pluginsd);
6942                 pthread_join(*p_pluginsd, NULL);
6943                 p_pluginsd = NULL;
6944         }
6945
6946         debug(D_EXIT, "All threads/childs stopped.");
6947 }
6948
6949 void sig_handler(int signo)
6950 {
6951         siginfo_t info;
6952
6953         switch(signo) {
6954                 case SIGTERM:
6955                 case SIGQUIT:
6956                 case SIGINT:
6957                 case SIGHUP:
6958                 case SIGFPE:
6959                 case SIGSEGV:
6960                         debug(D_EXIT, "Signaled exit (signal %d). Errno: %d (%s)", signo, errno, strerror(errno));
6961                         signal(SIGCHLD, SIG_IGN);
6962                         signal(SIGPIPE, SIG_IGN);
6963                         signal(SIGTERM, SIG_IGN);
6964                         signal(SIGQUIT, SIG_IGN);
6965                         signal(SIGHUP,  SIG_IGN);
6966                         signal(SIGINT,  SIG_IGN);
6967                         kill_childs();
6968                         rrd_stats_free_all();
6969                         debug(D_EXIT, "Waiting for any child exits...");
6970                         while(waitid(P_ALL, 0, &info, WEXITED|WNOHANG) == 0) if(!info.si_pid) break;
6971                         //unlink("/var/run/netdata.pid");
6972                         info("NetData exiting. Bye bye...");
6973                         exit(1);
6974                         break;
6975
6976                 case SIGPIPE:
6977                         info("Ignoring signal %d. Errno: %d (%s)", signo, errno, strerror(errno));
6978                         break;
6979
6980
6981                 case SIGCHLD:
6982                         info("Received SIGCHLD (signal %d).", signo);
6983                         while(waitid(P_ALL, 0, &info, WEXITED|WNOHANG) == 0) {
6984                                 if(!info.si_pid) break;
6985                                 switch(info.si_code) {
6986                                         case CLD_EXITED:
6987                                                 error("pid %d exited with code %d.", info.si_pid, info.si_status);
6988                                                 break;
6989
6990                                         case CLD_KILLED:
6991                                                 error("pid %d killed by signal %d.", info.si_pid, info.si_status);
6992                                                 break;
6993
6994                                         case CLD_DUMPED:
6995                                                 error("pid %d core dumped by signal %d.", info.si_pid, info.si_status);
6996                                                 break;
6997
6998                                         case CLD_STOPPED:
6999                                                 error("pid %d stopped by signal %d.", info.si_pid, info.si_status);
7000                                                 break;
7001
7002                                         case CLD_TRAPPED:
7003                                                 error("pid %d trapped by signal %d.", info.si_pid, info.si_status);
7004                                                 break;
7005
7006                                         case CLD_CONTINUED:
7007                                                 error("pid %d continued by signal %d.", info.si_pid, info.si_status);
7008                                                 break;
7009
7010                                         default:
7011                                                 error("pid %d gave us a SIGCHLD with code %d and status %d.", info.si_pid, info.si_code, info.si_status);
7012                                                 break;
7013                                 }
7014                         }
7015                         break;
7016
7017                 default:
7018                         info("Signal %d received. Falling back to default action for it.", signo);
7019                         signal(signo, SIG_DFL);
7020                         break;
7021         }
7022 }
7023
7024 int become_user(const char *username)
7025 {
7026         struct passwd *pw = getpwnam(username);
7027         if(!pw) {
7028                 fprintf(stderr, "User %s is not present. Error: %s\n", username, strerror(errno));
7029                 return -1;
7030         }
7031         if(setgid(pw->pw_gid) != 0) {
7032                 fprintf(stderr, "Cannot switch to user's %s group (gid: %d). Error: %s\n", username, pw->pw_gid, strerror(errno));
7033                 return -1;
7034         }
7035         if(setegid(pw->pw_gid) != 0) {
7036                 fprintf(stderr, "Cannot effectively switch to user's %s group (gid: %d). Error: %s\n", username, pw->pw_gid, strerror(errno));
7037                 return -1;
7038         }
7039         if(setuid(pw->pw_uid) != 0) {
7040                 fprintf(stderr, "Cannot switch to user %s (uid: %d). Error: %s\n", username, pw->pw_uid, strerror(errno));
7041                 return -1;
7042         }
7043         if(seteuid(pw->pw_uid) != 0) {
7044                 fprintf(stderr, "Cannot effectively switch to user %s (uid: %d). Error: %s\n", username, pw->pw_uid, strerror(errno));
7045                 return -1;
7046         }
7047
7048         return(0);
7049 }
7050
7051 int fd_is_valid(int fd)
7052 {
7053     return fcntl(fd, F_GETFD) != -1 || errno != EBADF;
7054 }
7055
7056 int become_daemon(int close_all_files, const char *input, const char *output, const char *error, const char *access, int *access_fd, FILE **access_fp)
7057 {
7058         fflush(NULL);
7059
7060         // open the files before forking
7061         int input_fd = -1, output_fd = -1, error_fd = -1, dev_null = -1;
7062
7063         if(input && *input) {
7064                 if((input_fd = open(input, O_RDONLY, 0666)) == -1) {
7065                         fprintf(stderr, "Cannot open input file '%s' (%s).", input, strerror(errno));
7066                         return -1;
7067                 }
7068         }
7069
7070         if(output && *output) {
7071                 if((output_fd = open(output, O_RDWR | O_APPEND | O_CREAT, 0666)) == -1) {
7072                         fprintf(stderr, "Cannot open output log file '%s' (%s).", output, strerror(errno));
7073                         if(input_fd != -1) close(input_fd);
7074                         return -1;
7075                 }
7076         }
7077
7078         if(error && *error) {
7079                 if((error_fd = open(error, O_RDWR | O_APPEND | O_CREAT, 0666)) == -1) {
7080                         fprintf(stderr, "Cannot open error log file '%s' (%s).", error, strerror(errno));
7081                         if(input_fd != -1) close(input_fd);
7082                         if(output_fd != -1) close(output_fd);
7083                         return -1;
7084                 }
7085         }
7086
7087         if(access && *access && access_fd) {
7088                 if((*access_fd = open(access, O_RDWR | O_APPEND | O_CREAT, 0666)) == -1) {
7089                         fprintf(stderr, "Cannot open access log file '%s' (%s).", access, strerror(errno));
7090                         if(input_fd != -1) close(input_fd);
7091                         if(output_fd != -1) close(output_fd);
7092                         if(error_fd != -1) close(error_fd);
7093                         return -1;
7094                 }
7095
7096                 if(access_fp) {
7097                         *access_fp = fdopen(*access_fd, "w");
7098                         if(!*access_fp) {
7099                                 fprintf(stderr, "Cannot migrate file's '%s' fd %d (%s).\n", access, *access_fd, strerror(errno));
7100                                 if(input_fd != -1) close(input_fd);
7101                                 if(output_fd != -1) close(output_fd);
7102                                 if(error_fd != -1) close(error_fd);
7103                                 close(*access_fd);
7104                                 *access_fd = -1;
7105                                 return -1;
7106                         }
7107                 }
7108         }
7109
7110         if((dev_null = open("/dev/null", O_RDWR, 0666)) == -1) {
7111                 perror("Cannot open /dev/null");
7112                 if(input_fd != -1) close(input_fd);
7113                 if(output_fd != -1) close(output_fd);
7114                 if(error_fd != -1) close(error_fd);
7115                 if(access && access_fd && *access_fd != -1) {
7116                         close(*access_fd);
7117                         *access_fd = -1;
7118                         if(access_fp) {
7119                                 fclose(*access_fp);
7120                                 *access_fp = NULL;
7121                         }
7122                 }
7123                 return -1;
7124         }
7125
7126         // all files opened
7127         // lets do it
7128
7129         int i = fork();
7130         if(i == -1) {
7131                 perror("cannot fork");
7132                 exit(1);
7133         }
7134         if(i != 0) {
7135                 exit(0); // the parent
7136         }
7137
7138         // become session leader
7139         if (setsid() < 0)
7140                 exit(2);
7141
7142         signal(SIGCHLD, SIG_IGN);
7143         signal(SIGHUP, SIG_IGN);
7144         signal(SIGWINCH, SIG_IGN);
7145
7146         // fork() again
7147         i = fork();
7148         if(i == -1) {
7149                 perror("cannot fork");
7150                 exit(1);
7151         }
7152         if(i != 0) {
7153                 exit(0); // the parent
7154         }
7155
7156         // Set new file permissions
7157         umask(0);
7158
7159         // close all files
7160         if(close_all_files) {
7161                 for(i = sysconf(_SC_OPEN_MAX); i > 0; i--)
7162                         if(
7163                                 ((access_fd && i != *access_fd) || !access_fd)
7164                                 && i != dev_null
7165                                 && i != input_fd
7166                                 && i != output_fd
7167                                 && i != error_fd
7168                                 && fd_is_valid(i)
7169                                 ) close(i);
7170         }
7171         else {
7172                 close(STDIN_FILENO);
7173                 close(STDOUT_FILENO);
7174                 close(STDERR_FILENO);
7175         }
7176
7177         // put the opened files
7178         // to our standard file descriptors
7179         if(input_fd != -1) {
7180                 if(input_fd != STDIN_FILENO) {
7181                         dup2(input_fd, STDIN_FILENO);
7182                         close(input_fd);
7183                 }
7184                 input_fd = -1;
7185         }
7186         else dup2(dev_null, STDIN_FILENO);
7187
7188         if(output_fd != -1) {
7189                 if(output_fd != STDOUT_FILENO) {
7190                         dup2(output_fd, STDOUT_FILENO);
7191                         close(output_fd);
7192                 }
7193                 output_fd = -1;
7194         }
7195         else dup2(dev_null, STDOUT_FILENO);
7196
7197         if(error_fd != -1) {
7198                 if(error_fd != STDERR_FILENO) {
7199                         dup2(error_fd, STDERR_FILENO);
7200                         close(error_fd);
7201                 }
7202                 error_fd = -1;
7203         }
7204         else dup2(dev_null, STDERR_FILENO);
7205
7206         // close /dev/null
7207         if(dev_null != STDIN_FILENO && dev_null != STDOUT_FILENO && dev_null != STDERR_FILENO)
7208                 close(dev_null);
7209
7210 /*      // generate our pid file
7211         {
7212                 unlink("/var/run/netdata.pid");
7213                 int fd = open("/var/run/netdata.pid", O_RDWR | O_CREAT, 0666);
7214                 if(fd >= 0) {
7215                         char b[100];
7216                         sprintf(b, "%d\n", getpid());
7217                         write(fd, b, strlen(b));
7218                         close(fd);
7219                 }
7220         }
7221 */
7222         return(0);
7223 }
7224
7225 int main(int argc, char **argv)
7226 {
7227         int i;
7228         int config_loaded = 0;
7229
7230
7231         // parse  the arguments
7232         for(i = 1; i < argc ; i++) {
7233                 if(strcmp(argv[i], "-c") == 0 && (i+1) < argc) {
7234                         if(load_config(argv[i+1], 1) != 1) {
7235                                 fprintf(stderr, "Cannot load configuration file %s. Reason: %s\n", argv[i+1], strerror(errno));
7236                                 exit(1);
7237                         }
7238                         else {
7239                                 debug(D_OPTIONS, "Configuration loaded from %s.", argv[i+1]);
7240                                 config_loaded = 1;
7241                         }
7242                         i++;
7243                 }
7244                 else if(strcmp(argv[i], "-df") == 0 && (i+1) < argc) { config_set("global", "debug flags",  argv[i+1]); i++; }
7245                 else if(strcmp(argv[i], "-p")  == 0 && (i+1) < argc) { config_set("global", "port",         argv[i+1]); i++; }
7246                 else if(strcmp(argv[i], "-u")  == 0 && (i+1) < argc) { config_set("global", "run as user",  argv[i+1]); i++; }
7247                 else if(strcmp(argv[i], "-l")  == 0 && (i+1) < argc) { config_set("global", "history",      argv[i+1]); i++; }
7248                 else if(strcmp(argv[i], "-t")  == 0 && (i+1) < argc) { config_set("global", "update every", argv[i+1]); i++; }
7249                 else {
7250                         fprintf(stderr, "Cannot understand option '%s'.\n", argv[i]);
7251                         fprintf(stderr, "\nUSAGE: %s [-d] [-l LINES_TO_SAVE] [-u UPDATE_TIMER] [-p LISTEN_PORT] [-dl debug log file] [-df debug flags].\n\n", argv[0]);
7252                         fprintf(stderr, "  -c CONFIG FILE the configuration file to load. Default: %s.\n", CONFIG_DIR "/" CONFIG_FILENAME);
7253                         fprintf(stderr, "  -l LINES_TO_SAVE can be from 5 to %d lines in JSON data. Default: %d.\n", HISTORY_MAX, HISTORY);
7254                         fprintf(stderr, "  -t UPDATE_TIMER can be from 1 to %d seconds. Default: %d.\n", UPDATE_EVERY_MAX, UPDATE_EVERY);
7255                         fprintf(stderr, "  -p LISTEN_PORT can be from 1 to %d. Default: %d.\n", 65535, LISTEN_PORT);
7256                         fprintf(stderr, "  -u USERNAME can be any system username to run as. Default: none.\n");
7257                         fprintf(stderr, "  -df FLAGS debug options. Default: 0x%8llx.\n", debug_flags);
7258                         exit(1);
7259                 }
7260         }
7261
7262         if(!config_loaded) load_config(NULL, 0);
7263
7264         char *input_log_file = NULL;
7265         char *output_log_file = NULL;
7266         char *error_log_file = NULL;
7267         char *access_log_file = NULL;
7268         {
7269                 char buffer[1024];
7270
7271                 // --------------------------------------------------------------------
7272
7273                 sprintf(buffer, "0x%08llx", 0ULL);
7274                 char *flags = config_get("global", "debug flags", buffer);
7275                 debug_flags = strtoull(flags, NULL, 0);
7276                 debug(D_OPTIONS, "Debug flags set to '0x%8llx'.", debug_flags);
7277
7278                 // --------------------------------------------------------------------
7279
7280                 output_log_file = config_get("global", "debug log", LOG_DIR "/debug.log");
7281                 if(strcmp(output_log_file, "syslog") == 0) {
7282                         output_log_syslog = 1;
7283                         output_log_file = NULL;
7284                 }
7285                 else if(strcmp(output_log_file, "none") == 0) {
7286                         output_log_syslog = 0;
7287                         output_log_file = NULL;
7288                 }
7289                 else output_log_syslog = 0;
7290
7291                 // --------------------------------------------------------------------
7292
7293                 silent = 0;
7294                 error_log_file = config_get("global", "error log", LOG_DIR "/error.log");
7295                 if(strcmp(error_log_file, "syslog") == 0) {
7296                         error_log_syslog = 1;
7297                         error_log_file = NULL;
7298                 }
7299                 else if(strcmp(error_log_file, "none") == 0) {
7300                         error_log_syslog = 0;
7301                         error_log_file = NULL;
7302                         silent = 1; // optimization - do not even generate debug log entries
7303                 }
7304                 else error_log_syslog = 0;
7305
7306                 // --------------------------------------------------------------------
7307
7308                 access_log_file = config_get("global", "access log", LOG_DIR "/access.log");
7309                 if(strcmp(access_log_file, "syslog") == 0) {
7310                         access_log_syslog = 1;
7311                         access_log_file = NULL;
7312                 }
7313                 else if(strcmp(access_log_file, "none") == 0) {
7314                         access_log_syslog = 0;
7315                         access_log_file = NULL;
7316                 }
7317                 else access_log_syslog = 0;
7318
7319                 // --------------------------------------------------------------------
7320
7321                 memory_mode = memory_mode_id(config_get("global", "memory mode", memory_mode_name(memory_mode)));
7322
7323                 // --------------------------------------------------------------------
7324
7325                 if(gethostname(buffer, HOSTNAME_MAX) == -1)
7326                         error("WARNING: Cannot get machine hostname.");
7327                 hostname = config_get("global", "hostname", buffer);
7328                 debug(D_OPTIONS, "hostname set to '%s'", hostname);
7329
7330                 // --------------------------------------------------------------------
7331
7332                 save_history = config_get_number("global", "history", HISTORY);
7333                 if(save_history < 5 || save_history > HISTORY_MAX) {
7334                         fprintf(stderr, "Invalid save lines %d given. Defaulting to %d.\n", save_history, HISTORY);
7335                         save_history = HISTORY;
7336                 }
7337                 else {
7338                         debug(D_OPTIONS, "save lines set to %d.", save_history);
7339                 }
7340
7341                 // --------------------------------------------------------------------
7342
7343                 char *user = config_get("global", "run as user", (getuid() == 0)?"nobody":"");
7344                 if(*user) {
7345                         if(become_user(user) != 0) {
7346                                 fprintf(stderr, "Cannot become user %s.\n", user);
7347                                 exit(1);
7348                         }
7349                         else debug(D_OPTIONS, "Successfully became user %s.", user);
7350                 }
7351
7352                 // --------------------------------------------------------------------
7353
7354                 update_every = config_get_number("global", "update every", UPDATE_EVERY);
7355                 if(update_every < 1 || update_every > 600) {
7356                         fprintf(stderr, "Invalid update timer %d given. Defaulting to %d.\n", update_every, UPDATE_EVERY_MAX);
7357                         update_every = UPDATE_EVERY;
7358                 }
7359                 else debug(D_OPTIONS, "update timer set to %d.", update_every);
7360
7361                 // --------------------------------------------------------------------
7362
7363                 listen_port = config_get_number("global", "port", LISTEN_PORT);
7364                 if(listen_port < 1 || listen_port > 65535) {
7365                         fprintf(stderr, "Invalid listen port %d given. Defaulting to %d.\n", listen_port, LISTEN_PORT);
7366                         listen_port = LISTEN_PORT;
7367                 }
7368                 else debug(D_OPTIONS, "listen port set to %d.", listen_port);
7369
7370                 listen_fd = create_listen_socket(listen_port);
7371         }
7372
7373         // never become a problem
7374         if(nice(20) == -1) {
7375                 fprintf(stderr, "Cannot lower my CPU priority. Error: %s.\n", strerror(errno));
7376         }
7377
7378         if(become_daemon(0, input_log_file, output_log_file, error_log_file, access_log_file, &access_fd, &stdaccess) == -1) {
7379                 fprintf(stderr, "Cannot demonize myself (%s).", strerror(errno));
7380                 exit(1);
7381         }
7382
7383
7384         if(output_log_syslog || error_log_syslog || access_log_syslog)
7385                 openlog("netdata", LOG_PID, LOG_DAEMON);
7386
7387         info("NetData started on pid %d", getpid());
7388
7389
7390         // catch all signals
7391         for (i = 1 ; i < 65 ;i++) if(i != SIGSEGV && i != SIGFPE) signal(i,  sig_handler);
7392
7393         // spawn childs to collect data
7394         if(config_get_boolean("plugins", "tc", 1)) {
7395                 p_tc = malloc(sizeof(pthread_t));
7396                 if(pthread_create(p_tc, NULL, tc_main, NULL))
7397                         error("failed to create new thread for tc.");
7398                 else if(pthread_detach(*p_tc))
7399                         error("Cannot request detach of newly created tc thread.");
7400         }
7401
7402         if(config_get_boolean("plugins", "idlejitter", 1)) {
7403                 p_jitter = malloc(sizeof(pthread_t));
7404                 if(pthread_create(p_jitter, NULL, cpuidlejitter_main, NULL))
7405                         error("failed to create new thread for idlejitter.");
7406                 else if(pthread_detach(*p_jitter))
7407                         error("Cannot request detach of newly created idlejitter thread.");
7408         }
7409
7410         if(config_get_boolean("plugins", "proc", 1)) {
7411                 p_proc = malloc(sizeof(pthread_t));
7412                 if(pthread_create(p_proc, NULL, proc_main, NULL))
7413                         error("failed to create new thread for proc.");
7414                 else if(pthread_detach(*p_proc))
7415                         error("Cannot request detach of newly created proc thread.");
7416         }
7417
7418         p_pluginsd = malloc(sizeof(pthread_t));
7419         if(pthread_create(p_pluginsd, NULL, pluginsd_main, NULL))
7420                 error("failed to create new thread for plugins.d.");
7421         else if(pthread_detach(*p_pluginsd))
7422                 error("Cannot request detach of newly created plugins.d thread.");
7423
7424         if(config_get_boolean("plugins", "checks", 1)) {
7425                 p_checks = malloc(sizeof(pthread_t));
7426                 if(pthread_create(p_checks, NULL, checks_main, NULL))
7427                         error("failed to create new thread for checks.");
7428                 else if(pthread_detach(*p_checks))
7429                         error("Cannot request detach of newly created checks thread.");
7430         }
7431
7432         // the main process - the web server listener
7433         // this never ends
7434         socket_listen_main(NULL);
7435
7436         exit(0);
7437 }