src/plugins.d/apps_plugin.c

   1 #define __STDC_FORMAT_MACROS
   2 #include <inttypes.h>
   3
   4 #include <stdio.h>
   5 #include <stdlib.h>
   6 #include <string.h>
   7 #include <time.h>
   8 #include <unistd.h>
   9 #include <sys/types.h>
  10 #include <sys/time.h>
  11 #include <sys/wait.h>
  12
  13 #include <sys/resource.h>
  14 #include <sys/stat.h>
  15
  16 #include <errno.h>
  17 #include <stdarg.h>
  18 #include <locale.h>
  19 #include <ctype.h>
  20 #include <fcntl.h>
  21
  22 #include <malloc.h>
  23 #include <inttypes.h>
  24 #include <dirent.h>
  25 #include <arpa/inet.h>
  26
  27 #define MAX_COMPARE_NAME 15
  28 #define MAX_NAME 100
  29
  30 unsigned long long Hertz = 1;
  31
  32 long processors = 1;
  33 long pid_max = 32768;
  34 int debug = 0;
  35
  36 int update_every = 1;
  37 unsigned long long file_counter = 0;
  38
  39 #define PROC_BUFFER 4096
  40
  41
  42 // ----------------------------------------------------------------------------
  43 // helper functions
  44
  45 unsigned long long usecdiff(struct timeval *now, struct timeval *last) {
  46                 return ((((now->tv_sec * 1000000ULL) + now->tv_usec) - ((last->tv_sec * 1000000ULL) + last->tv_usec)));
  47 }
  48
  49 char *trim(char *s)
  50 {
  51         // skip leading spaces
  52         while(*s && isspace(*s)) s++;
  53         if(!*s) return NULL;
  54
  55         // skip tailing spaces
  56         int c = strlen(s) - 1;
  57         while(c >= 0 && isspace(s[c])) {
  58                 s[c] = '\0';
  59                 c--;
  60         }
  61         if(c < 0) return NULL;
  62         if(!*s) return NULL;
  63         return s;
  64 }
  65
  66 unsigned long simple_hash(const char *name)
  67 {
  68         int i, len = strlen(name);
  69         unsigned long hash = 0;
  70
  71         for(i = 0; i < len ;i++) hash += (i * name[i]) + i + name[i];
  72
  73         return hash;
  74 }
  75
  76 long get_processors(void)
  77 {
  78         char buffer[1025], *s;
  79         int processors = 0;
  80
  81         FILE *fp = fopen("/proc/stat", "r");
  82         if(!fp) return 1;
  83
  84         while((s = fgets(buffer, 1024, fp))) {
  85                 if(strncmp(buffer, "cpu", 3) == 0) processors++;
  86         }
  87         fclose(fp);
  88         processors--;
  89         if(processors < 1) processors = 1;
  90         return processors;
  91 }
  92
  93 long get_pid_max(void)
  94 {
  95         char buffer[1025], *s;
  96         long mpid = 32768;
  97
  98         FILE *fp = fopen("/proc/sys/kernel/pid_max", "r");
  99         if(!fp) return 1;
 100
 101         s = fgets(buffer, 1024, fp);
 102         if(s) mpid = atol(buffer);
 103         fclose(fp);
 104         if(mpid < 32768) mpid = 32768;
 105         return mpid;
 106 }
 107
 108 unsigned long long get_hertz(void)
 109 {
 110         unsigned long long hz = 1;
 111
 112 #ifdef _SC_CLK_TCK
 113         if((hz = sysconf(_SC_CLK_TCK)) > 0) {
 114                 return hz;
 115         }
 116 #endif
 117
 118 #ifdef HZ
 119         hz = (unsigned long long)HZ;    /* <asm/param.h> */
 120 #else
 121         /* If 32-bit or big-endian (not Alpha or ia64), assume HZ is 100. */
 122         hz = (sizeof(long)==sizeof(int) || htons(999)==999) ? 100UL : 1024UL;
 123 #endif
 124
 125         fprintf(stderr, "apps.plugin: ERROR: unknown HZ value. Assuming %llu.\n", hz);
 126         return hz;
 127 }
 128
 129
 130 // ----------------------------------------------------------------------------
 131 // target
 132 // target is the point to aggregate a process tree values
 133
 134 struct target {
 135         char compare[MAX_COMPARE_NAME + 1];
 136         char id[MAX_NAME + 1];
 137         char name[MAX_NAME + 1];
 138
 139         unsigned long long minflt;
 140         unsigned long long cminflt;
 141         unsigned long long majflt;
 142         unsigned long long cmajflt;
 143         unsigned long long utime;
 144         unsigned long long stime;
 145         unsigned long long cutime;
 146         unsigned long long cstime;
 147         unsigned long long num_threads;
 148         unsigned long long rss;
 149
 150         unsigned long long fix_minflt;
 151         unsigned long long fix_cminflt;
 152         unsigned long long fix_majflt;
 153         unsigned long long fix_cmajflt;
 154         unsigned long long fix_utime;
 155         unsigned long long fix_stime;
 156         unsigned long long fix_cutime;
 157         unsigned long long fix_cstime;
 158
 159         unsigned long long statm_size;
 160         unsigned long long statm_resident;
 161         unsigned long long statm_share;
 162         unsigned long long statm_text;
 163         unsigned long long statm_lib;
 164         unsigned long long statm_data;
 165         unsigned long long statm_dirty;
 166
 167         unsigned long long io_logical_bytes_read;
 168         unsigned long long io_logical_bytes_written;
 169         unsigned long long io_read_calls;
 170         unsigned long long io_write_calls;
 171         unsigned long long io_storage_bytes_read;
 172         unsigned long long io_storage_bytes_written;
 173         unsigned long long io_cancelled_write_bytes;
 174
 175         unsigned long long fix_io_logical_bytes_read;
 176         unsigned long long fix_io_logical_bytes_written;
 177         unsigned long long fix_io_read_calls;
 178         unsigned long long fix_io_write_calls;
 179         unsigned long long fix_io_storage_bytes_read;
 180         unsigned long long fix_io_storage_bytes_written;
 181         unsigned long long fix_io_cancelled_write_bytes;
 182
 183         int *fds;
 184         unsigned long long openfiles;
 185         unsigned long long openpipes;
 186         unsigned long long opensockets;
 187         unsigned long long openinotifies;
 188         unsigned long long openeventfds;
 189         unsigned long long opentimerfds;
 190         unsigned long long opensignalfds;
 191         unsigned long long openeventpolls;
 192         unsigned long long openother;
 193
 194         unsigned long processes;        // how many processes have been merged to this
 195         int exposed;                            // if set, we have sent this to netdata
 196         int hidden;                                     // if set, we set the hidden flag on the dimension
 197         int debug;
 198
 199         struct target *target;  // the one that will be reported to netdata
 200         struct target *next;
 201 } *target_root = NULL, *default_target = NULL;
 202
 203 long targets = 0;
 204
 205 // find or create a target
 206 // there are targets that are just agregated to other target (the second argument)
 207 struct target *get_target(const char *id, struct target *target)
 208 {
 209         const char *nid = id;
 210         if(nid[0] == '-') nid++;
 211
 212         struct target *w;
 213         for(w = target_root ; w ; w = w->next)
 214                 if(strncmp(nid, w->id, MAX_NAME) == 0) return w;
 215
 216         w = calloc(sizeof(struct target), 1);
 217         if(!w) {
 218                 fprintf(stderr, "apps.plugin: cannot allocate %lu bytes of memory\n", (unsigned long)sizeof(struct target));
 219                 return NULL;
 220         }
 221
 222         strncpy(w->id, nid, MAX_NAME);
 223         strncpy(w->name, nid, MAX_NAME);
 224         strncpy(w->compare, nid, MAX_COMPARE_NAME);
 225         if(id[0] == '-') w->hidden = 1;
 226
 227         w->target = target;
 228
 229         w->next = target_root;
 230         target_root = w;
 231
 232         if(debug) fprintf(stderr, "apps.plugin: adding hook for process '%s', compare '%s' on target '%s'\n", w->id, w->compare, w->target?w->target->id:"");
 233
 234         return w;
 235 }
 236
 237 // read the process groups file
 238 int read_process_groups(const char *name)
 239 {
 240         char buffer[4096+1];
 241         char filename[FILENAME_MAX + 1];
 242
 243         snprintf(filename, FILENAME_MAX, "%s/apps_%s.conf", CONFIG_DIR, name);
 244
 245         if(debug) fprintf(stderr, "apps.plugin: process groups file: '%s'\n", filename);
 246         FILE *fp = fopen(filename, "r");
 247         if(!fp) {
 248                 fprintf(stderr, "apps.plugin: ERROR: cannot open file '%s' (%s)\n", filename, strerror(errno));
 249                 return 1;
 250         }
 251
 252         long line = 0;
 253         while(fgets(buffer, 4096, fp) != NULL) {
 254                 int whidden = 0, wdebug = 0;
 255                 line++;
 256
 257                 // if(debug) fprintf(stderr, "apps.plugin: \tread %s\n", buffer);
 258
 259                 char *s = buffer, *t, *p;
 260                 s = trim(s);
 261                 if(!s || !*s || *s == '#') continue;
 262
 263                 if(debug) fprintf(stderr, "apps.plugin: \tread %s\n", s);
 264
 265                 // the target name
 266                 t = strsep(&s, ":");
 267                 if(t) t = trim(t);
 268                 if(!t || !*t) continue;
 269
 270                 while(t[0]) {
 271                         int stop = 1;
 272
 273                         switch(t[0]) {
 274                                 case '-':
 275                                         stop = 0;
 276                                         whidden = 1;
 277                                         t++;
 278                                         break;
 279
 280                                 case '+':
 281                                         stop = 0;
 282                                         wdebug = 1;
 283                                         t++;
 284                                         break;
 285                         }
 286
 287                         if(stop) break;
 288                 }
 289
 290                 if(debug) fprintf(stderr, "apps.plugin: \t\ttarget %s\n", t);
 291
 292                 struct target *w = NULL;
 293                 long count = 0;
 294
 295                 // the process names
 296                 while((p = strsep(&s, " "))) {
 297                         p = trim(p);
 298                         if(!p || !*p) continue;
 299
 300                         struct target *n = get_target(p, w);
 301                         n->hidden = whidden;
 302                         n->debug = wdebug;
 303                         if(!w) w = n;
 304
 305                         count++;
 306                 }
 307
 308                 if(w) strncpy(w->name, t, MAX_NAME);
 309                 if(!count) fprintf(stderr, "apps.plugin: ERROR: the line %ld on file '%s', for group '%s' does not state any process names.\n", line, filename, t);
 310         }
 311         fclose(fp);
 312
 313         default_target = get_target("+p!o@w#e$i^r&7*5(-i)l-o_", NULL); // match nothing
 314         strncpy(default_target->name, "other", MAX_NAME);
 315
 316         return 0;
 317 }
 318
 319
 320 // ----------------------------------------------------------------------------
 321 // data to store for each pid
 322 // see: man proc
 323
 324 struct pid_stat {
 325         int32_t pid;
 326         char comm[MAX_COMPARE_NAME + 1];
 327         char state;
 328         int32_t ppid;
 329         int32_t pgrp;
 330         int32_t session;
 331         int32_t tty_nr;
 332         int32_t tpgid;
 333         uint64_t flags;
 334         unsigned long long minflt;
 335         unsigned long long cminflt;
 336         unsigned long long majflt;
 337         unsigned long long cmajflt;
 338         unsigned long long utime;
 339         unsigned long long stime;
 340         unsigned long long cutime;
 341         unsigned long long cstime;
 342         int64_t priority;
 343         int64_t nice;
 344         int32_t num_threads;
 345         int64_t itrealvalue;
 346         unsigned long long starttime;
 347         unsigned long long vsize;
 348         unsigned long long rss;
 349         unsigned long long rsslim;
 350         unsigned long long starcode;
 351         unsigned long long endcode;
 352         unsigned long long startstack;
 353         unsigned long long kstkesp;
 354         unsigned long long kstkeip;
 355         uint64_t signal;
 356         uint64_t blocked;
 357         uint64_t sigignore;
 358         uint64_t sigcatch;
 359         uint64_t wchan;
 360         uint64_t nswap;
 361         uint64_t cnswap;
 362         int32_t exit_signal;
 363         int32_t processor;
 364         uint32_t rt_priority;
 365         uint32_t policy;
 366         unsigned long long delayacct_blkio_ticks;
 367         uint64_t guest_time;
 368         int64_t cguest_time;
 369
 370         unsigned long long statm_size;
 371         unsigned long long statm_resident;
 372         unsigned long long statm_share;
 373         unsigned long long statm_text;
 374         unsigned long long statm_lib;
 375         unsigned long long statm_data;
 376         unsigned long long statm_dirty;
 377
 378         unsigned long long io_logical_bytes_read;
 379         unsigned long long io_logical_bytes_written;
 380         unsigned long long io_read_calls;
 381         unsigned long long io_write_calls;
 382         unsigned long long io_storage_bytes_read;
 383         unsigned long long io_storage_bytes_written;
 384         unsigned long long io_cancelled_write_bytes;
 385
 386 #ifdef INCLUDE_CHILDS
 387         unsigned long long old_utime;
 388         unsigned long long old_stime;
 389         unsigned long long old_minflt;
 390         unsigned long long old_majflt;
 391
 392         unsigned long long old_cutime;
 393         unsigned long long old_cstime;
 394         unsigned long long old_cminflt;
 395         unsigned long long old_cmajflt;
 396
 397         unsigned long long fix_cutime;
 398         unsigned long long fix_cstime;
 399         unsigned long long fix_cminflt;
 400         unsigned long long fix_cmajflt;
 401
 402         unsigned long long diff_cutime;
 403         unsigned long long diff_cstime;
 404         unsigned long long diff_cminflt;
 405         unsigned long long diff_cmajflt;
 406 #endif
 407
 408         int *fds;                                       // array of fds it uses
 409         int fds_size;                           // the size of the fds array
 410
 411         int childs;                                     // number of processes directly referencing this
 412         int updated;                            // 1 when update
 413         int merged;                                     // 1 when it has been merged to its parent
 414         int new_entry;
 415         struct target *target;
 416         struct pid_stat *parent;
 417         struct pid_stat *prev;
 418         struct pid_stat *next;
 419 } *root = NULL, **all_pids;
 420
 421 long pids = 0;
 422
 423 struct pid_stat *get_entry(pid_t pid)
 424 {
 425         if(all_pids[pid]) {
 426                 all_pids[pid]->new_entry = 0;
 427                 return all_pids[pid];
 428         }
 429
 430         all_pids[pid] = calloc(sizeof(struct pid_stat), 1);
 431         if(!all_pids[pid]) {
 432                 fprintf(stderr, "apps.plugin: ERROR: Cannot allocate %lu bytes of memory", (unsigned long)sizeof(struct pid_stat));
 433                 return NULL;
 434         }
 435
 436         all_pids[pid]->fds = calloc(sizeof(int), 100);
 437         if(!all_pids[pid]->fds)
 438                 fprintf(stderr, "apps.plugin: ERROR: Cannot allocate %ld bytes of memory\n", (unsigned long)(sizeof(int) * 100));
 439         else all_pids[pid]->fds_size = 100;
 440
 441         if(root) root->prev = all_pids[pid];
 442         all_pids[pid]->next = root;
 443         root = all_pids[pid];
 444
 445         all_pids[pid]->new_entry = 1;
 446
 447         return all_pids[pid];
 448 }
 449
 450 void del_entry(pid_t pid)
 451 {
 452         if(!all_pids[pid]) return;
 453
 454         if(debug) fprintf(stderr, "apps.plugin: process %d %s exited, deleting it.\n", pid, all_pids[pid]->comm);
 455
 456         if(root == all_pids[pid]) root = all_pids[pid]->next;
 457         if(all_pids[pid]->next) all_pids[pid]->next->prev = all_pids[pid]->prev;
 458         if(all_pids[pid]->prev) all_pids[pid]->prev->next = all_pids[pid]->next;
 459
 460         if(all_pids[pid]->fds) free(all_pids[pid]->fds);
 461         free(all_pids[pid]);
 462         all_pids[pid] = NULL;
 463 }
 464
 465 #ifdef INCLUDE_CHILDS
 466 // print a tree view of all processes
 467 int walk_down(pid_t pid, int level) {
 468         struct pid_stat *p = NULL;
 469         char b[level+3];
 470         int i, ret = 0;
 471
 472         for(i = 0; i < level; i++) b[i] = '\t';
 473         b[level] = '|';
 474         b[level+1] = '-';
 475         b[level+2] = '\0';
 476
 477         for(p = root; p ; p = p->next) {
 478                 if(p->ppid == pid) {
 479                         ret += walk_down(p->pid, level+1);
 480                 }
 481         }
 482
 483         p = all_pids[pid];
 484         if(p) {
 485                 if(!p->updated) ret += 1;
 486                 if(ret) fprintf(stderr, "%s %s %d [%s, %s] c=%d u=%llu+%llu, s=%llu+%llu, cu=%llu+%llu, cs=%llu+%llu, n=%llu+%llu, j=%llu+%llu, cn=%llu+%llu, cj=%llu+%llu\n"
 487                         , b, p->comm, p->pid, p->updated?"OK":"KILLED", p->target->name, p->childs
 488                         , p->utime, p->utime - p->old_utime
 489                         , p->stime, p->stime - p->old_stime
 490                         , p->cutime, p->cutime - p->old_cutime
 491                         , p->cstime, p->cstime - p->old_cstime
 492                         , p->minflt, p->minflt - p->old_minflt
 493                         , p->majflt, p->majflt - p->old_majflt
 494                         , p->cminflt, p->cminflt - p->old_cminflt
 495                         , p->cmajflt, p->cmajflt - p->old_cmajflt
 496                         );
 497         }
 498
 499         return ret;
 500 }
 501 #endif
 502
 503
 504 // ----------------------------------------------------------------------------
 505 // file descriptor
 506 // this is used to keep a global list of all open files of the system
 507 // it is needed in order to figure out the unique files a process tree has open
 508
 509 struct file_descriptor {
 510         unsigned long hash;
 511         char *name;
 512         int type;
 513         long count;
 514 } *all_files = NULL;
 515
 516 int all_files_len = 0;
 517 int all_files_size = 0;
 518
 519 #define FILETYPE_OTHER 0
 520 #define FILETYPE_FILE 1
 521 #define FILETYPE_PIPE 2
 522 #define FILETYPE_SOCKET 3
 523 #define FILETYPE_INOTIFY 4
 524 #define FILETYPE_EVENTFD 5
 525 #define FILETYPE_EVENTPOLL 6
 526 #define FILETYPE_TIMERFD 7
 527 #define FILETYPE_SIGNALFD 8
 528
 529 void file_descriptor_not_used(int id)
 530 {
 531         if(id > 0 && id < all_files_len) {
 532                 if(all_files[id].count > 0)
 533                         all_files[id].count--;
 534                 else
 535                         fprintf(stderr, "apps.plugin: ERROR: request to decrease counter of fd %d (%s), while the use counter is 0\n", id, all_files[id].name);
 536         }
 537         else    fprintf(stderr, "apps.plugin: ERROR: request to decrease counter of fd %d, which is outside the array size (1 to %d)\n", id, all_files_len);
 538 }
 539
 540 unsigned long file_descriptor_find_or_add(const char *name)
 541 {
 542         int type = FILETYPE_OTHER;
 543
 544         if(name[0] == '/') type = FILETYPE_FILE;
 545         else if(strncmp(name, "pipe:", 5) == 0) type = FILETYPE_PIPE;
 546         else if(strncmp(name, "socket:", 7) == 0) type = FILETYPE_SOCKET;
 547         else if(strcmp(name, "anon_inode:inotify") == 0 || strcmp(name, "inotify") == 0) type = FILETYPE_INOTIFY;
 548         else if(strcmp(name, "anon_inode:[eventfd]") == 0) type = FILETYPE_EVENTFD;
 549         else if(strcmp(name, "anon_inode:[eventpoll]") == 0) type = FILETYPE_EVENTPOLL;
 550         else if(strcmp(name, "anon_inode:[timerfd]") == 0) type = FILETYPE_TIMERFD;
 551         else if(strcmp(name, "anon_inode:[signalfd]") == 0) type = FILETYPE_SIGNALFD;
 552         else if(strncmp(name, "anon_inode:", 11) == 0) {
 553                 if(debug) fprintf(stderr, "apps.plugin: FIXME: unknown anonymous inode: %s\n", name);
 554                 type = FILETYPE_OTHER;
 555         }
 556         else {
 557                 if(debug) fprintf(stderr, "apps.plugin: FIXME: cannot understand linkname: %s\n", name);
 558                 type = FILETYPE_OTHER;
 559         }
 560
 561         // init
 562         if(!all_files) {
 563                 all_files = malloc(1024 * sizeof(struct file_descriptor));
 564                 if(!all_files) return 0;
 565
 566                 all_files_size = 1024;
 567                 all_files_len = 1; // start from 1, skip 0
 568                 if(debug) fprintf(stderr, "apps.plugin: initialized fd array to %d entries\n", all_files_size);
 569         }
 570
 571         // try to find it
 572         unsigned long hash = simple_hash(name);
 573         int c;
 574         for( c = 0 ; c < all_files_len ; c++) {
 575                 if(all_files[c].hash == hash && strcmp(all_files[c].name, name) == 0) break;
 576         }
 577
 578         // found it
 579         if(c < all_files_len) {
 580                 all_files[c].count++;
 581                 return c;
 582         }
 583
 584         // not found, search for an empty slot
 585         for(c = 0 ; c < all_files_len ; c++) {
 586                 if(!all_files[c].count) {
 587                         if(debug) fprintf(stderr, "apps.plugin: re-using fd position %d (last name: %s)\n", c, all_files[c].name);
 588                         if(all_files[c].name) free(all_files[c].name);
 589                         break;
 590                 }
 591         }
 592
 593         if(c == all_files_len) {
 594                 // not found any emtpty slot
 595                 all_files_len++;
 596
 597                 if(c >= all_files_size) {
 598                         // not enough memory - extend it
 599                         if(debug) fprintf(stderr, "apps.plugin: extending fd array to %d entries\n", all_files_size + 1024);
 600                         all_files = realloc(all_files, (all_files_size + 1024) * sizeof(struct file_descriptor));
 601                         all_files_size += 1024;
 602                 }
 603         }
 604         // else we have the slot in 'c'
 605
 606         all_files[c].name = strdup(name);
 607         all_files[c].hash = hash;
 608         all_files[c].type = type;
 609         all_files[c].count++;
 610
 611         if(debug) fprintf(stderr, "apps.plugin: using fd position %d (name: %s)\n", c, all_files[c].name);
 612
 613         return c;
 614 }
 615
 616
 617 // ----------------------------------------------------------------------------
 618 // update pids from proc
 619
 620 // 1. read all files in /proc
 621 // 2. for each numeric directory:
 622 //    i.   read /proc/pid/stat
 623 //    ii.  read /proc/pid/statm
 624 //    iii. read /proc/pid/io (requires root access)
 625 //    iii. read the entries in directory /proc/pid/fd (requires root access)
 626 //         for each entry:
 627 //         a. find or create a struct file_descriptor
 628 //         b. cleanup any old/unused file_descriptors
 629
 630 // after all these, some pids may be linked to targets, while others may not
 631
 632 // in case of errors, only 1 every 1000 errors is printed
 633 // to avoid filling up all disk space
 634 // if debug is enabled, all errors are printed
 635
 636 int update_from_proc(void)
 637 {
 638         static long count_errors = 0;
 639
 640         char buffer[PROC_BUFFER + 1];
 641         char name[PROC_BUFFER + 1];
 642         char filename[FILENAME_MAX+1];
 643         DIR *dir = opendir("/proc");
 644         if(!dir) return 0;
 645
 646         struct dirent *file = NULL;
 647         struct pid_stat *p = NULL;
 648
 649         // mark them all as un-updated
 650         pids = 0;
 651         for(p = root; p ; p = p->next) {
 652                 pids++;
 653                 p->parent = NULL;
 654                 p->updated = 0;
 655                 p->childs = 0;
 656                 p->merged = 0;
 657                 p->new_entry = 0;
 658         }
 659
 660         while((file = readdir(dir))) {
 661                 char *endptr = file->d_name;
 662                 pid_t pid = strtoul(file->d_name, &endptr, 10);
 663                 if(pid <= 0 || pid > pid_max || endptr == file->d_name || *endptr != '\0') continue;
 664
 665
 666                 // --------------------------------------------------------------------
 667                 // /proc/<pid>/stat
 668
 669                 snprintf(filename, FILENAME_MAX, "/proc/%s/stat", file->d_name);
 670                 int fd = open(filename, O_RDONLY);
 671                 if(fd == -1) {
 672                         if(errno != ENOENT && errno != ESRCH) {
 673                                 if(!count_errors++ || debug)
 674                                         fprintf(stderr, "apps.plugin: ERROR: cannot open file '%s' for reading (%d, %s).\n", filename, errno, strerror(errno));
 675                         }
 676                         continue;
 677                 }
 678                 file_counter++;
 679
 680                 int bytes = read(fd, buffer, PROC_BUFFER);
 681                 close(fd);
 682
 683                 if(bytes == -1) {
 684                         if(!count_errors++ || debug)
 685                                 fprintf(stderr, "apps.plugin: ERROR: cannot read from file '%s' (%s).\n", filename, strerror(errno));
 686                         continue;
 687                 }
 688
 689                 if(bytes < 10) continue;
 690                 buffer[bytes] = '\0';
 691                 if(debug) fprintf(stderr, "apps.plugin: READ stat: %s", buffer);
 692
 693                 p = get_entry(pid);
 694                 if(!p) continue;
 695
 696                 int parsed = sscanf(buffer,
 697                         "%d (%[^)]) %c"                                         // pid, comm, state
 698                         " %d %d %d %d %d"                                       // ppid, pgrp, session, tty_nr, tpgid
 699                         " %" PRIu64 " %llu %llu %llu %llu"      // flags, minflt, cminflt, majflt, cmajflt
 700                         " %llu %llu %llu %llu"                          // utime, stime, cutime, cstime
 701                         " %" PRId64 " %" PRId64                         // priority, nice
 702                         " %d"                                                           // num_threads
 703                         " %" PRId64                                                     // itrealvalue
 704                         " %llu"                                                         // starttime
 705                         " %llu"                                                         // vsize
 706                         " %llu"                                                         // rss
 707                         " %llu %llu %llu %llu %llu %llu"        // rsslim, starcode, endcode, startstack, kstkesp, kstkeip
 708                         " %" PRIu64 " %" PRIu64 " %" PRIu64 " %" PRIu64 // signal, blocked, sigignore, sigcatch
 709                         " %" PRIu64 " %" PRIu64 " %" PRIu64     // wchan, nswap, cnswap
 710                         " %d %d"                                                        // exit_signal, processor
 711                         " %u %u"                                                        // rt_priority, policy
 712                         " %llu %" PRIu64 " %" PRId64            // delayacct_blkio_ticks, guest_time, cguest_time
 713                         , &p->pid, name, &p->state
 714                         , &p->ppid, &p->pgrp, &p->session, &p->tty_nr, &p->tpgid
 715                         , &p->flags, &p->minflt, &p->cminflt, &p->majflt, &p->cmajflt
 716                         , &p->utime, &p->stime, &p->cutime, &p->cstime
 717                         , &p->priority, &p->nice
 718                         , &p->num_threads
 719                         , &p->itrealvalue
 720                         , &p->starttime
 721                         , &p->vsize
 722                         , &p->rss
 723                         , &p->rsslim, &p->starcode, &p->endcode, &p->startstack, &p->kstkesp, &p->kstkeip
 724                         , &p->signal, &p->blocked, &p->sigignore, &p->sigcatch
 725                         , &p->wchan, &p->nswap, &p->cnswap
 726                         , &p->exit_signal, &p->processor
 727                         , &p->rt_priority, &p->policy
 728                         , &p->delayacct_blkio_ticks, &p->guest_time, &p->cguest_time
 729                         );
 730                 strncpy(p->comm, name, MAX_COMPARE_NAME);
 731                 p->comm[MAX_COMPARE_NAME] = '\0';
 732
 733                 if(debug || (p->target && p->target->debug)) fprintf(stderr, "apps.plugin: VALUES: %s utime=%llu, stime=%llu, cutime=%llu, cstime=%llu, minflt=%llu, majflt=%llu, cminflt=%llu, cmajflt=%llu\n", p->comm, p->utime, p->stime, p->cutime, p->cstime, p->minflt, p->majflt, p->cminflt, p->cmajflt);
 734
 735                 if(parsed < 39) {
 736                         if(!count_errors++ || debug || (p->target && p->target->debug))
 737                                 fprintf(stderr, "apps.plugin: ERROR: file %s gave %d results (expected 44)\n", filename, parsed);
 738                 }
 739
 740                 // check if it is target
 741                 // we do this only once, the first time this pid is loaded
 742                 if(p->new_entry) {
 743                         if(debug) fprintf(stderr, "apps.plugin: \tJust added %s\n", p->comm);
 744
 745                         struct target *w;
 746                         for(w = target_root; w ; w = w->next) {
 747                                 // if(debug || (p->target && p->target->debug)) fprintf(stderr, "apps.plugin: \t\tcomparing '%s' with '%s'\n", w->compare, p->comm);
 748
 749                                 if(strcmp(w->compare, p->comm) == 0) {
 750                                         if(w->target) p->target = w->target;
 751                                         else p->target = w;
 752
 753                                         if(debug || (p->target && p->target->debug)) fprintf(stderr, "apps.plugin: \t\t%s linked to target %s\n", p->comm, p->target->name);
 754                                 }
 755                         }
 756                 }
 757
 758                 // just a few checks
 759                 if(p->ppid < 0 || p->ppid > pid_max) p->ppid = 0;
 760
 761
 762                 // --------------------------------------------------------------------
 763                 // /proc/<pid>/statm
 764
 765                 snprintf(filename, FILENAME_MAX, "/proc/%s/statm", file->d_name);
 766                 fd = open(filename, O_RDONLY);
 767                 if(fd == -1) {
 768                         if(errno != ENOENT && errno != ESRCH) {
 769                                 if(!count_errors++ || debug || (p->target && p->target->debug))
 770                                         fprintf(stderr, "apps.plugin: ERROR: cannot open file '%s' for reading (%d, %s).\n", filename, errno, strerror(errno));
 771                         }
 772                 }
 773                 else {
 774                         file_counter++;
 775                         bytes = read(fd, buffer, PROC_BUFFER);
 776                         close(fd);
 777
 778                         if(bytes == -1) {
 779                                 if(!count_errors++ || debug || (p->target && p->target->debug))
 780                                         fprintf(stderr, "apps.plugin: ERROR: cannot read from file '%s' (%s).\n", filename, strerror(errno));
 781                         }
 782                         else if(bytes > 10) {
 783                                 buffer[bytes] = '\0';
 784                                 if(debug || (p->target && p->target->debug))
 785                                         fprintf(stderr, "apps.plugin: READ statm: %s", buffer);
 786
 787                                 parsed = sscanf(buffer,
 788                                         "%llu %llu %llu %llu %llu %llu %llu"
 789                                         , &p->statm_size
 790                                         , &p->statm_resident
 791                                         , &p->statm_share
 792                                         , &p->statm_text
 793                                         , &p->statm_lib
 794                                         , &p->statm_data
 795                                         , &p->statm_dirty
 796                                         );
 797
 798                                 if(parsed < 7) {
 799                                         if(!count_errors++ || debug || (p->target && p->target->debug))
 800                                                 fprintf(stderr, "apps.plugin: ERROR: file %s gave %d results (expected 7)\n", filename, parsed);
 801                                 }
 802                         }
 803                 }
 804
 805                 // --------------------------------------------------------------------
 806                 // /proc/<pid>/io
 807
 808                 snprintf(filename, FILENAME_MAX, "/proc/%s/io", file->d_name);
 809                 fd = open(filename, O_RDONLY);
 810                 if(fd == -1) {
 811                         if(errno != ENOENT && errno != ESRCH) {
 812                                 if(!count_errors++ || debug || (p->target && p->target->debug))
 813                                         fprintf(stderr, "apps.plugin: ERROR: cannot open file '%s' for reading (%d, %s).\n", filename, errno, strerror(errno));
 814                         }
 815                 }
 816                 else {
 817                         file_counter++;
 818                         bytes = read(fd, buffer, PROC_BUFFER);
 819                         close(fd);
 820
 821                         if(bytes == -1) {
 822                                 if(!count_errors++ || debug || (p->target && p->target->debug))
 823                                         fprintf(stderr, "apps.plugin: ERROR: cannot read from file '%s' (%s).\n", filename, strerror(errno));
 824                         }
 825                         else if(bytes > 10) {
 826                                 buffer[bytes] = '\0';
 827                                 if(debug || (p->target && p->target->debug)) fprintf(stderr, "apps.plugin: READ io: %s", buffer);
 828
 829                                 parsed = sscanf(buffer,
 830                                         "rchar: %llu\nwchar: %llu\nsyscr: %llu\nsyscw: %llu\nread_bytes: %llu\nwrite_bytes: %llu\ncancelled_write_bytes: %llu"
 831                                         , &p->io_logical_bytes_read
 832                                         , &p->io_logical_bytes_written
 833                                         , &p->io_read_calls
 834                                         , &p->io_write_calls
 835                                         , &p->io_storage_bytes_read
 836                                         , &p->io_storage_bytes_written
 837                                         , &p->io_cancelled_write_bytes
 838                                         );
 839
 840                                 if(parsed < 7) {
 841                                         if(!count_errors++ || debug || (p->target && p->target->debug))
 842                                                 fprintf(stderr, "apps.plugin: ERROR: file %s gave %d results (expected 7)\n", filename, parsed);
 843                                 }
 844                         }
 845                 }
 846
 847                 // --------------------------------------------------------------------
 848                 // /proc/<pid>/fd
 849
 850                 snprintf(filename, FILENAME_MAX, "/proc/%s/fd", file->d_name);
 851                 DIR *fds = opendir(filename);
 852                 if(fds) {
 853                         int c;
 854                         struct dirent *de;
 855                         char fdname[FILENAME_MAX + 1];
 856                         char linkname[FILENAME_MAX + 1];
 857
 858                         // make the array negative
 859                         for(c = 0 ; c < p->fds_size ; c++) p->fds[c] = -p->fds[c];
 860
 861                         while((de = readdir(fds))) {
 862                                 if(strcmp(de->d_name, ".") == 0 || strcmp(de->d_name, "..") == 0) continue;
 863
 864                                 // check if the fds array is small
 865                                 int fdid = atol(de->d_name);
 866                                 if(fdid < 0) continue;
 867                                 if(fdid >= p->fds_size) {
 868                                         // it is small, extend it
 869                                         if(debug) fprintf(stderr, "apps.plugin: extending fd memory slots for %s from %d to %d\n", p->comm, p->fds_size, fdid + 100);
 870                                         p->fds = realloc(p->fds, (fdid + 100) * sizeof(int));
 871                                         if(!p->fds) {
 872                                                 fprintf(stderr, "apps.plugin: ERROR: cannot re-allocate fds for %s\n", p->comm);
 873                                                 break;
 874                                         }
 875
 876                                         // and initialize it
 877                                         for(c = p->fds_size ; c < (fdid + 100) ; c++) p->fds[c] = 0;
 878                                         p->fds_size = fdid + 100;
 879                                 }
 880
 881                                 if(p->fds[fdid] == 0) {
 882                                         // we don't know this fd, get it
 883
 884                                         sprintf(fdname, "/proc/%s/fd/%s", file->d_name, de->d_name);
 885                                         int l = readlink(fdname, linkname, FILENAME_MAX);
 886                                         if(l == -1) {
 887                                                 if(debug || (p->target && p->target->debug)) {
 888                                                         if(!count_errors++ || debug || (p->target && p->target->debug))
 889                                                                 fprintf(stderr, "apps.plugin: ERROR: cannot read link %s\n", fdname);
 890                                                 }
 891                                                 continue;
 892                                         }
 893                                         linkname[l] = '\0';
 894                                         file_counter++;
 895
 896                                         // if another process already has this, we will get
 897                                         // the same id
 898                                         p->fds[fdid] = file_descriptor_find_or_add(linkname);
 899                                 }
 900
 901                                 // else make it positive again, we need it
 902                                 // of course, the actual file may have changed, but we don't care so much
 903                                 // FIXME: we could compare the inode as returned by readdir direct structure
 904                                 else p->fds[fdid] = -p->fds[fdid];
 905                         }
 906                         closedir(fds);
 907
 908                         // remove all the negative file descriptors
 909                         for(c = 0 ; c < p->fds_size ; c++) if(p->fds[c] < 0) {
 910                                 file_descriptor_not_used(-p->fds[c]);
 911                                 p->fds[c] = 0;
 912                         }
 913                 }
 914
 915                 // --------------------------------------------------------------------
 916                 // done!
 917
 918                 // mark it as updated
 919                 p->updated = 1;
 920         }
 921         if(count_errors > 1000) {
 922                 fprintf(stderr, "apps.plugin: ERROR: %ld more errors encountered\n", count_errors - 1);
 923                 count_errors = 0;
 924         }
 925
 926         closedir(dir);
 927
 928         return 1;
 929 }
 930
 931
 932 // ----------------------------------------------------------------------------
 933 // update statistics on the targets
 934
 935 // 1. link all childs to their parents
 936 // 2. go from bottom to top, marking as merged all childs to their parents
 937 //    this step links all parents without a target to the child target, if any
 938 // 3. link all top level processes (the ones not merged) to the default target
 939 // 4. go from top to bottom, linking all childs without a target, to their parent target
 940 //    after this step, all processes have a target
 941 // [5. for each killed pid (updated = 0), remove its usage from its target]
 942 // 6. zero all targets
 943 // 7. concentrate all values on the targets
 944 // 8. remove all killed processes
 945 // 9. find the unique file count for each target
 946
 947 void update_statistics(void)
 948 {
 949         int c;
 950         struct pid_stat *p = NULL;
 951
 952         // link all parents and update childs count
 953         for(p = root; p ; p = p->next) {
 954                 if(p->ppid > 0 && p->ppid <= pid_max && all_pids[p->ppid]) {
 955                         if(debug || (p->target && p->target->debug)) fprintf(stderr, "apps.plugin: \tparent of %d %s is %d %s\n", p->pid, p->comm, p->ppid, all_pids[p->ppid]->comm);
 956
 957                         p->parent = all_pids[p->ppid];
 958                         p->parent->childs++;
 959                 }
 960                 else if(p->ppid != 0) fprintf(stderr, "apps.plugin: \t\tWRONG! pid %d %s states parent %d, but the later does not exist.\n", p->pid, p->comm, p->ppid);
 961         }
 962
 963         // find all the procs with 0 childs and merge them to their parents
 964         // repeat, until nothing more can be done.
 965         int found = 1;
 966         while(found) {
 967                 found = 0;
 968                 for(p = root; p ; p = p->next) {
 969                         // if this process does not have any childs, and
 970                         // is not already merged, and
 971                         // its parent has childs waiting to be merged, and
 972                         // the target of this process and its parent is the same, or the parent does not have a target, or this process does not have a parent
 973                         // and its parent is not init
 974                         // then... merge them!
 975                         if(!p->childs && !p->merged && p->parent && p->parent->childs && (p->target == p->parent->target || !p->parent->target || !p->target) && p->ppid != 1) {
 976                                 p->parent->childs--;
 977                                 p->merged = 1;
 978
 979                                 // the parent inherits the child's target, if it does not have a target itself
 980                                 if(p->target && !p->parent->target) {
 981                                         p->parent->target = p->target;
 982                                         if(debug || (p->target && p->target->debug)) fprintf(stderr, "apps.plugin: \t\ttarget %s is inherited by %d %s from its child %d %s.\n", p->target->name, p->parent->pid, p->parent->comm, p->pid, p->comm);
 983                                 }
 984
 985                                 found++;
 986                         }
 987                 }
 988                 if(debug) fprintf(stderr, "apps.plugin: merged %d processes\n", found);
 989         }
 990
 991         // give a default target on all top level processes
 992         // init goes always to default target
 993         if(all_pids[1]) all_pids[1]->target = default_target;
 994
 995         for(p = root; p ; p = p->next) {
 996                 // if the process is not merged itself
 997                 // then is is a top level process
 998                 if(!p->merged && !p->target) p->target = default_target;
 999
1000 #ifdef INCLUDE_CHILDS
1001                 // by the way, update the diffs
1002                 // will be used later for substracting killed process times
1003                 p->diff_cutime = p->utime - p->cutime;
1004                 p->diff_cstime = p->stime - p->cstime;
1005                 p->diff_cminflt = p->minflt - p->cminflt;
1006                 p->diff_cmajflt = p->majflt - p->cmajflt;
1007 #endif
1008         }
1009
1010         // give a target to all merged child processes
1011         found = 1;
1012         while(found) {
1013                 found = 0;
1014                 for(p = root; p ; p = p->next) {
1015                         if(!p->target && p->merged && p->parent && p->parent->target) {
1016                                 p->target = p->parent->target;
1017                                 found++;
1018                         }
1019                 }
1020         }
1021
1022 #ifdef INCLUDE_CHILDS
1023         // for each killed process, remove its values from the parents
1024         // sums (we had already added them in a previous loop)
1025         for(p = root; p ; p = p->next) {
1026                 if(p->updated) continue;
1027
1028                 fprintf(stderr, "apps.plugin: UNMERGING %d %s\n", p->pid, p->comm);
1029
1030                 unsigned long long diff_utime = p->utime + p->cutime + p->fix_cutime;
1031                 unsigned long long diff_stime = p->stime + p->cstime + p->fix_cstime;
1032                 unsigned long long diff_minflt = p->minflt + p->cminflt + p->fix_cminflt;
1033                 unsigned long long diff_majflt = p->majflt + p->cmajflt + p->fix_cmajflt;
1034
1035                 struct pid_stat *t = p;
1036                 while((t = t->parent)) {
1037                         if(!t->updated) continue;
1038
1039                         unsigned long long x;
1040                         if(diff_utime && t->diff_cutime) {
1041                                 x = (t->diff_cutime < diff_utime)?t->diff_cutime:diff_utime;
1042                                 diff_utime -= x;
1043                                 t->diff_cutime -= x;
1044                                 t->fix_cutime += x;
1045                                 fprintf(stderr, "apps.plugin: \t cutime %llu from %d %s %s\n", x, t->pid, t->comm, t->target->name);
1046                         }
1047                         if(diff_stime && t->diff_cstime) {
1048                                 x = (t->diff_cstime < diff_stime)?t->diff_cstime:diff_stime;
1049                                 diff_stime -= x;
1050                                 t->diff_cstime -= x;
1051                                 t->fix_cstime += x;
1052                                 fprintf(stderr, "apps.plugin: \t cstime %llu from %d %s %s\n", x, t->pid, t->comm, t->target->name);
1053                         }
1054                         if(diff_minflt && t->diff_cminflt) {
1055                                 x = (t->diff_cminflt < diff_minflt)?t->diff_cminflt:diff_minflt;
1056                                 diff_minflt -= x;
1057                                 t->diff_cminflt -= x;
1058                                 t->fix_cminflt += x;
1059                                 fprintf(stderr, "apps.plugin: \t cminflt %llu from %d %s %s\n", x, t->pid, t->comm, t->target->name);
1060                         }
1061                         if(diff_majflt && t->diff_cmajflt) {
1062                                 x = (t->diff_cmajflt < diff_majflt)?t->diff_cmajflt:diff_majflt;
1063                                 diff_majflt -= x;
1064                                 t->diff_cmajflt -= x;
1065                                 t->fix_cmajflt += x;
1066                                 fprintf(stderr, "apps.plugin: \t cmajflt %llu from %d %s %s\n", x, t->pid, t->comm, t->target->name);
1067                         }
1068                 }
1069
1070                 if(diff_utime) fprintf(stderr, "apps.plugin: \t cannot fix up utime %llu\n", diff_utime);
1071                 if(diff_stime) fprintf(stderr, "apps.plugin: \t cannot fix up stime %llu\n", diff_stime);
1072                 if(diff_minflt) fprintf(stderr, "apps.plugin: \t cannot fix up minflt %llu\n", diff_minflt);
1073                 if(diff_majflt) fprintf(stderr, "apps.plugin: \t cannot fix up majflt %llu\n", diff_majflt);
1074         }
1075 #endif
1076
1077         // zero all the targets
1078         targets = 0;
1079         struct target *w;
1080         for (w = target_root; w ; w = w->next) {
1081                 targets++;
1082
1083                 w->fds = calloc(sizeof(int), all_files_len);
1084                 if(!w->fds)
1085                         fprintf(stderr, "apps.plugin: ERROR: cannot allocate memory for fds in %s\n", w->name);
1086
1087                 w->minflt = 0;
1088                 w->majflt = 0;
1089                 w->utime = 0;
1090                 w->stime = 0;
1091                 w->cminflt = 0;
1092                 w->cmajflt = 0;
1093                 w->cutime = 0;
1094                 w->cstime = 0;
1095                 w->num_threads = 0;
1096                 w->rss = 0;
1097                 w->processes = 0;
1098
1099                 w->statm_size = 0;
1100                 w->statm_resident = 0;
1101                 w->statm_share = 0;
1102                 w->statm_text = 0;
1103                 w->statm_lib = 0;
1104                 w->statm_data = 0;
1105                 w->statm_dirty = 0;
1106
1107                 w->io_logical_bytes_read = 0;
1108                 w->io_logical_bytes_written = 0;
1109                 w->io_read_calls = 0;
1110                 w->io_write_calls = 0;
1111                 w->io_storage_bytes_read = 0;
1112                 w->io_storage_bytes_written = 0;
1113                 w->io_cancelled_write_bytes = 0;
1114         }
1115
1116 #ifdef INCLUDE_CHILDS
1117         if(debug) walk_down(0, 1);
1118 #endif
1119
1120         // concentrate everything on the targets
1121         for(p = root; p ; p = p->next) {
1122                 if(!p->target) {
1123                         fprintf(stderr, "apps.plugin: ERROR: pid %d %s was left without a target!\n", p->pid, p->comm);
1124                         continue;
1125                 }
1126
1127                 if(p->updated) {
1128                         p->target->cutime += p->cutime; // - p->fix_cutime;
1129                         p->target->cstime += p->cstime; // - p->fix_cstime;
1130                         p->target->cminflt += p->cminflt; // - p->fix_cminflt;
1131                         p->target->cmajflt += p->cmajflt; // - p->fix_cmajflt;
1132
1133                         p->target->utime += p->utime; //+ (p->pid != 1)?(p->cutime - p->fix_cutime):0;
1134                         p->target->stime += p->stime; //+ (p->pid != 1)?(p->cstime - p->fix_cstime):0;
1135                         p->target->minflt += p->minflt; //+ (p->pid != 1)?(p->cminflt - p->fix_cminflt):0;
1136                         p->target->majflt += p->majflt; //+ (p->pid != 1)?(p->cmajflt - p->fix_cmajflt):0;
1137
1138                         p->target->num_threads += p->num_threads;
1139                         p->target->rss += p->rss;
1140
1141                         p->target->statm_size += p->statm_size;
1142                         p->target->statm_resident += p->statm_resident;
1143                         p->target->statm_share += p->statm_share;
1144                         p->target->statm_text += p->statm_text;
1145                         p->target->statm_lib += p->statm_lib;
1146                         p->target->statm_data += p->statm_data;
1147                         p->target->statm_dirty += p->statm_dirty;
1148
1149                         p->target->io_logical_bytes_read += p->io_logical_bytes_read;
1150                         p->target->io_logical_bytes_written += p->io_logical_bytes_written;
1151                         p->target->io_read_calls += p->io_read_calls;
1152                         p->target->io_write_calls += p->io_write_calls;
1153                         p->target->io_storage_bytes_read += p->io_storage_bytes_read;
1154                         p->target->io_storage_bytes_written += p->io_storage_bytes_written;
1155                         p->target->io_cancelled_write_bytes += p->io_cancelled_write_bytes;
1156
1157                         p->target->processes++;
1158
1159                         for(c = 0; c < p->fds_size ;c++) {
1160                                 if(p->fds[c] == 0) continue;
1161                                 if(p->fds[c] > 0 && p->fds[c] < all_files_len) {
1162                                         if(p->target->fds) p->target->fds[p->fds[c]]++;
1163                                 }
1164                                 else
1165                                         fprintf(stderr, "apps.plugin: ERROR: invalid fd number %d\n", p->fds[c]);
1166                         }
1167
1168                         if(debug || p->target->debug) fprintf(stderr, "apps.plugin: \tAgregating %s pid %d on %s utime=%llu, stime=%llu, cutime=%llu, cstime=%llu, minflt=%llu, majflt=%llu, cminflt=%llu, cmajflt=%llu\n", p->comm, p->pid, p->target->name, p->utime, p->stime, p->cutime, p->cstime, p->minflt, p->majflt, p->cminflt, p->cmajflt);
1169
1170 /*                      if(p->utime - p->old_utime > 100) fprintf(stderr, "BIG CHANGE: %d %s utime increased by %llu from %llu to %llu\n", p->pid, p->comm, p->utime - p->old_utime, p->old_utime, p->utime);
1171                         if(p->cutime - p->old_cutime > 100) fprintf(stderr, "BIG CHANGE: %d %s cutime increased by %llu from %llu to %llu\n", p->pid, p->comm, p->cutime - p->old_cutime, p->old_cutime, p->cutime);
1172                         if(p->stime - p->old_stime > 100) fprintf(stderr, "BIG CHANGE: %d %s stime increased by %llu from %llu to %llu\n", p->pid, p->comm, p->stime - p->old_stime, p->old_stime, p->stime);
1173                         if(p->cstime - p->old_cstime > 100) fprintf(stderr, "BIG CHANGE: %d %s cstime increased by %llu from %llu to %llu\n", p->pid, p->comm, p->cstime - p->old_cstime, p->old_cstime, p->cstime);
1174                         if(p->minflt - p->old_minflt > 5000) fprintf(stderr, "BIG CHANGE: %d %s minflt increased by %llu from %llu to %llu\n", p->pid, p->comm, p->minflt - p->old_minflt, p->old_minflt, p->minflt);
1175                         if(p->majflt - p->old_majflt > 5000) fprintf(stderr, "BIG CHANGE: %d %s majflt increased by %llu from %llu to %llu\n", p->pid, p->comm, p->majflt - p->old_majflt, p->old_majflt, p->majflt);
1176                         if(p->cminflt - p->old_cminflt > 15000) fprintf(stderr, "BIG CHANGE: %d %s cminflt increased by %llu from %llu to %llu\n", p->pid, p->comm, p->cminflt - p->old_cminflt, p->old_cminflt, p->cminflt);
1177                         if(p->cmajflt - p->old_cmajflt > 15000) fprintf(stderr, "BIG CHANGE: %d %s cmajflt increased by %llu from %llu to %llu\n", p->pid, p->comm, p->cmajflt - p->old_cmajflt, p->old_cmajflt, p->cmajflt);
1178 */
1179 #ifdef INCLUDE_CHILDS
1180                         p->old_utime = p->utime;
1181                         p->old_cutime = p->cutime;
1182                         p->old_stime = p->stime;
1183                         p->old_cstime = p->cstime;
1184                         p->old_minflt = p->minflt;
1185                         p->old_majflt = p->majflt;
1186                         p->old_cminflt = p->cminflt;
1187                         p->old_cmajflt = p->cmajflt;
1188 #endif
1189                 }
1190                 else {
1191                         // since the process has exited, the user
1192                         // will see a drop in our charts, because the incremental
1193                         // values of this process will not be there
1194
1195                         // add them to the fix_* values and they will be added to
1196                         // the reported values, so that the report goes steady
1197                         p->target->fix_minflt += p->minflt;
1198                         p->target->fix_majflt += p->majflt;
1199                         p->target->fix_utime += p->utime;
1200                         p->target->fix_stime += p->stime;
1201                         p->target->fix_cminflt += p->cminflt;
1202                         p->target->fix_cmajflt += p->cmajflt;
1203                         p->target->fix_cutime += p->cutime;
1204                         p->target->fix_cstime += p->cstime;
1205
1206                         p->target->fix_io_logical_bytes_read += p->io_logical_bytes_read;
1207                         p->target->fix_io_logical_bytes_written += p->io_logical_bytes_written;
1208                         p->target->fix_io_read_calls += p->io_read_calls;
1209                         p->target->fix_io_write_calls += p->io_write_calls;
1210                         p->target->fix_io_storage_bytes_read += p->io_storage_bytes_read;
1211                         p->target->fix_io_storage_bytes_written += p->io_storage_bytes_written;
1212                         p->target->fix_io_cancelled_write_bytes += p->io_cancelled_write_bytes;
1213                 }
1214         }
1215
1216 //      fprintf(stderr, "\n");
1217         // cleanup all un-updated processed (exited, killed, etc)
1218         for(p = root; p ;) {
1219                 if(!p->updated) {
1220 //                      fprintf(stderr, "\tEXITED %d %s [parent %d %s, target %s] utime=%llu, stime=%llu, cutime=%llu, cstime=%llu, minflt=%llu, majflt=%llu, cminflt=%llu, cmajflt=%llu\n", p->pid, p->comm, p->parent->pid, p->parent->comm, p->target->name,  p->utime, p->stime, p->cutime, p->cstime, p->minflt, p->majflt, p->cminflt, p->cmajflt);
1221
1222                         for(c = 0 ; c < p->fds_size ; c++) if(p->fds[c] > 0) {
1223                                 file_descriptor_not_used(p->fds[c]);
1224                                 p->fds[c] = 0;
1225                         }
1226
1227                         pid_t r = p->pid;
1228                         p = p->next;
1229                         del_entry(r);
1230                 }
1231                 else p = p->next;
1232         }
1233
1234         for (w = target_root; w ; w = w->next) {
1235                 w->openfiles = 0;
1236                 w->openpipes = 0;
1237                 w->opensockets = 0;
1238                 w->openinotifies = 0;
1239                 w->openeventfds = 0;
1240                 w->opentimerfds = 0;
1241                 w->opensignalfds = 0;
1242                 w->openeventpolls = 0;
1243                 w->openother = 0;
1244
1245                 for(c = 1; c < all_files_len ;c++) {
1246                         if(w->fds && w->fds[c] > 0) switch(all_files[c].type) {
1247                                 case FILETYPE_FILE:
1248                                         w->openfiles++;
1249                                         break;
1250
1251                                 case FILETYPE_PIPE:
1252                                         w->openpipes++;
1253                                         break;
1254
1255                                 case FILETYPE_SOCKET:
1256                                         w->opensockets++;
1257                                         break;
1258
1259                                 case FILETYPE_INOTIFY:
1260                                         w->openinotifies++;
1261                                         break;
1262
1263                                 case FILETYPE_EVENTFD:
1264                                         w->openeventfds++;
1265                                         break;
1266
1267                                 case FILETYPE_TIMERFD:
1268                                         w->opentimerfds++;
1269                                         break;
1270
1271                                 case FILETYPE_SIGNALFD:
1272                                         w->opensignalfds++;
1273                                         break;
1274
1275                                 case FILETYPE_EVENTPOLL:
1276                                         w->openeventpolls++;
1277                                         break;
1278
1279                                 default:
1280                                         w->openother++;
1281                         }
1282                 }
1283
1284                 free(w->fds);
1285                 w->fds = NULL;
1286         }
1287 }
1288
1289 // ----------------------------------------------------------------------------
1290 // update chart dimensions
1291
1292 void show_dimensions(void)
1293 {
1294         static struct timeval last = { 0, 0 };
1295         static struct rusage me_last;
1296
1297         struct target *w;
1298         struct timeval now;
1299         struct rusage me;
1300
1301         unsigned long long usec;
1302         unsigned long long cpuuser;
1303         unsigned long long cpusyst;
1304
1305         if(!last.tv_sec) {
1306                 gettimeofday(&last, NULL);
1307                 getrusage(RUSAGE_SELF, &me_last);
1308
1309                 // the first time, give a zero to allow
1310                 // netdata calibrate to the current time
1311                 // usec = update_every * 1000000ULL;
1312                 usec = 0ULL;
1313                 cpuuser = 0;
1314                 cpusyst = 0;
1315         }
1316         else {
1317                 gettimeofday(&now, NULL);
1318                 getrusage(RUSAGE_SELF, &me);
1319
1320                 usec = usecdiff(&now, &last);
1321                 cpuuser = me.ru_utime.tv_sec * 1000000ULL + me.ru_utime.tv_usec;
1322                 cpusyst = me.ru_stime.tv_sec * 1000000ULL + me.ru_stime.tv_usec;
1323
1324                 bcopy(&now, &last, sizeof(struct timeval));
1325                 bcopy(&me, &me_last, sizeof(struct rusage));
1326         }
1327
1328         fprintf(stdout, "BEGIN apps.cpu %llu\n", usec);
1329         for (w = target_root; w ; w = w->next) {
1330                 if(w->target || (!w->processes && !w->exposed)) continue;
1331
1332                 fprintf(stdout, "SET %s = %llu\n", w->name, w->utime + w->stime + w->fix_utime + w->fix_stime);
1333         }
1334         fprintf(stdout, "END\n");
1335
1336         fprintf(stdout, "BEGIN apps.cpu_user %llu\n", usec);
1337         for (w = target_root; w ; w = w->next) {
1338                 if(w->target || (!w->processes && !w->exposed)) continue;
1339
1340                 fprintf(stdout, "SET %s = %llu\n", w->name, w->utime + w->fix_utime);
1341         }
1342         fprintf(stdout, "END\n");
1343
1344         fprintf(stdout, "BEGIN apps.cpu_system %llu\n", usec);
1345         for (w = target_root; w ; w = w->next) {
1346                 if(w->target || (!w->processes && !w->exposed)) continue;
1347
1348                 fprintf(stdout, "SET %s = %llu\n", w->name, w->stime + w->fix_stime);
1349         }
1350         fprintf(stdout, "END\n");
1351
1352         fprintf(stdout, "BEGIN apps.threads %llu\n", usec);
1353         for (w = target_root; w ; w = w->next) {
1354                 if(w->target || (!w->processes && !w->exposed)) continue;
1355
1356                 fprintf(stdout, "SET %s = %llu\n", w->name, w->num_threads);
1357         }
1358         fprintf(stdout, "END\n");
1359
1360         fprintf(stdout, "BEGIN apps.processes %llu\n", usec);
1361         for (w = target_root; w ; w = w->next) {
1362                 if(w->target || (!w->processes && !w->exposed)) continue;
1363
1364                 fprintf(stdout, "SET %s = %lu\n", w->name, w->processes);
1365         }
1366         fprintf(stdout, "END\n");
1367
1368         fprintf(stdout, "BEGIN apps.mem %llu\n", usec);
1369         for (w = target_root; w ; w = w->next) {
1370                 if(w->target || (!w->processes && !w->exposed)) continue;
1371
1372                 fprintf(stdout, "SET %s = %lld\n", w->name, (long long)w->statm_resident - (long long)w->statm_share);
1373         }
1374         fprintf(stdout, "END\n");
1375
1376         fprintf(stdout, "BEGIN apps.minor_faults %llu\n", usec);
1377         for (w = target_root; w ; w = w->next) {
1378                 if(w->target || (!w->processes && !w->exposed)) continue;
1379
1380                 fprintf(stdout, "SET %s = %llu\n", w->name, w->minflt + w->fix_minflt);
1381         }
1382         fprintf(stdout, "END\n");
1383
1384         fprintf(stdout, "BEGIN apps.major_faults %llu\n", usec);
1385         for (w = target_root; w ; w = w->next) {
1386                 if(w->target || (!w->processes && !w->exposed)) continue;
1387
1388                 fprintf(stdout, "SET %s = %llu\n", w->name, w->majflt + w->fix_majflt);
1389         }
1390         fprintf(stdout, "END\n");
1391
1392         fprintf(stdout, "BEGIN apps.lreads %llu\n", usec);
1393         for (w = target_root; w ; w = w->next) {
1394                 if(w->target || (!w->processes && !w->exposed)) continue;
1395
1396                 fprintf(stdout, "SET %s = %llu\n", w->name, w->io_logical_bytes_read);
1397         }
1398         fprintf(stdout, "END\n");
1399
1400         fprintf(stdout, "BEGIN apps.lwrites %llu\n", usec);
1401         for (w = target_root; w ; w = w->next) {
1402                 if(w->target || (!w->processes && !w->exposed)) continue;
1403
1404                 fprintf(stdout, "SET %s = %llu\n", w->name, w->io_logical_bytes_written);
1405         }
1406         fprintf(stdout, "END\n");
1407
1408         fprintf(stdout, "BEGIN apps.preads %llu\n", usec);
1409         for (w = target_root; w ; w = w->next) {
1410                 if(w->target || (!w->processes && !w->exposed)) continue;
1411
1412                 fprintf(stdout, "SET %s = %llu\n", w->name, w->io_storage_bytes_read);
1413         }
1414         fprintf(stdout, "END\n");
1415
1416         fprintf(stdout, "BEGIN apps.pwrites %llu\n", usec);
1417         for (w = target_root; w ; w = w->next) {
1418                 if(w->target || (!w->processes && !w->exposed)) continue;
1419
1420                 fprintf(stdout, "SET %s = %llu\n", w->name, w->io_storage_bytes_written);
1421         }
1422         fprintf(stdout, "END\n");
1423
1424         fprintf(stdout, "BEGIN apps.files %llu\n", usec);
1425         for (w = target_root; w ; w = w->next) {
1426                 if(w->target || (!w->processes && !w->exposed)) continue;
1427
1428                 fprintf(stdout, "SET %s = %llu\n", w->name, w->openfiles);
1429         }
1430         fprintf(stdout, "END\n");
1431
1432         fprintf(stdout, "BEGIN apps.sockets %llu\n", usec);
1433         for (w = target_root; w ; w = w->next) {
1434                 if(w->target || (!w->processes && !w->exposed)) continue;
1435
1436                 fprintf(stdout, "SET %s = %llu\n", w->name, w->opensockets);
1437         }
1438         fprintf(stdout, "END\n");
1439
1440         fprintf(stdout, "BEGIN apps.pipes %llu\n", usec);
1441         for (w = target_root; w ; w = w->next) {
1442                 if(w->target || (!w->processes && !w->exposed)) continue;
1443
1444                 fprintf(stdout, "SET %s = %llu\n", w->name, w->openpipes);
1445         }
1446         fprintf(stdout, "END\n");
1447
1448         fprintf(stdout, "BEGIN netdata.apps_cpu %llu\n", usec);
1449         fprintf(stdout, "SET user = %llu\n", cpuuser);
1450         fprintf(stdout, "SET system = %llu\n", cpusyst);
1451         fprintf(stdout, "END\n");
1452
1453         fprintf(stdout, "BEGIN netdata.apps_files %llu\n", usec);
1454         fprintf(stdout, "SET files = %llu\n", file_counter);
1455         fprintf(stdout, "SET pids = %ld\n", pids);
1456         fprintf(stdout, "SET fds = %d\n", all_files_len);
1457         fprintf(stdout, "SET targets = %ld\n", targets);
1458         fprintf(stdout, "END\n");
1459
1460         fflush(stdout);
1461 }
1462
1463
1464 // ----------------------------------------------------------------------------
1465 // generate the charts
1466
1467 void show_charts(void)
1468 {
1469         struct target *w;
1470         int newly_added = 0;
1471
1472         for(w = target_root ; w ; w = w->next)
1473                 if(!w->exposed && w->processes) {
1474                         newly_added++;
1475                         w->exposed = 1;
1476                         if(debug || w->debug) fprintf(stderr, "apps.plugin: %s just added - regenerating charts.\n", w->name);
1477                 }
1478
1479         // nothing more to show
1480         if(!newly_added) return;
1481
1482         // we have something new to show
1483         // update the charts
1484         fprintf(stdout, "CHART apps.cpu '' 'Apps CPU Time (%ld%% = %ld core%s)' 'cpu time %%' apps apps stacked 20001 %d\n", (processors * 100), processors, (processors>1)?"s":"", update_every);
1485         for (w = target_root; w ; w = w->next) {
1486                 if(w->target || (!w->processes && !w->exposed)) continue;
1487
1488                 fprintf(stdout, "DIMENSION %s '' incremental 100 %llu %s\n", w->name, (unsigned long long)(Hertz * update_every), w->hidden?"hidden":"");
1489         }
1490
1491         fprintf(stdout, "CHART apps.mem '' 'Apps Dedicated Memory (w/o shared)' 'MB' apps apps stacked 20003 %d\n", update_every);
1492         for (w = target_root; w ; w = w->next) {
1493                 if(w->target || (!w->processes && !w->exposed)) continue;
1494
1495                 fprintf(stdout, "DIMENSION %s '' absolute %ld %ld\n", w->name, sysconf(_SC_PAGESIZE), 1024L*1024L);
1496         }
1497
1498         fprintf(stdout, "CHART apps.threads '' 'Apps Threads' 'threads' apps apps stacked 20005 %d\n", update_every);
1499         for (w = target_root; w ; w = w->next) {
1500                 if(w->target || (!w->processes && !w->exposed)) continue;
1501
1502                 fprintf(stdout, "DIMENSION %s '' absolute 1 1\n", w->name);
1503         }
1504
1505         fprintf(stdout, "CHART apps.processes '' 'Apps Processes' 'processes' apps apps stacked 20004 %d\n", update_every);
1506         for (w = target_root; w ; w = w->next) {
1507                 if(w->target || (!w->processes && !w->exposed)) continue;
1508
1509                 fprintf(stdout, "DIMENSION %s '' absolute 1 1\n", w->name);
1510         }
1511
1512         fprintf(stdout, "CHART apps.cpu_user '' 'Apps CPU User Time (%ld%% = %ld core%s)' 'cpu time %%' apps none stacked 20020 %d\n", (processors * 100), processors, (processors>1)?"s":"", update_every);
1513         for (w = target_root; w ; w = w->next) {
1514                 if(w->target || (!w->processes && !w->exposed)) continue;
1515
1516                 fprintf(stdout, "DIMENSION %s '' incremental 100 %llu\n", w->name, Hertz * processors * update_every);
1517         }
1518
1519         fprintf(stdout, "CHART apps.cpu_system '' 'Apps CPU System Time (%ld%% = %ld core%s)' 'cpu time %%' apps none stacked 20021 %d\n", (processors * 100), processors, (processors>1)?"s":"", update_every);
1520         for (w = target_root; w ; w = w->next) {
1521                 if(w->target || (!w->processes && !w->exposed)) continue;
1522
1523                 fprintf(stdout, "DIMENSION %s '' incremental 100 %llu\n", w->name, Hertz * processors * update_every);
1524         }
1525
1526         fprintf(stdout, "CHART apps.major_faults '' 'Apps Major Page Faults (swaps in)' 'page faults/s' apps apps stacked 20010 %d\n", update_every);
1527         for (w = target_root; w ; w = w->next) {
1528                 if(w->target || (!w->processes && !w->exposed)) continue;
1529
1530                 fprintf(stdout, "DIMENSION %s '' incremental 1 %d\n", w->name, update_every);
1531         }
1532
1533         fprintf(stdout, "CHART apps.minor_faults '' 'Apps Minor Page Faults' 'page faults/s' apps none stacked 20011 %d\n", update_every);
1534         for (w = target_root; w ; w = w->next) {
1535                 if(w->target || (!w->processes && !w->exposed)) continue;
1536
1537                 fprintf(stdout, "DIMENSION %s '' incremental 1 %d\n", w->name, update_every);
1538         }
1539
1540         fprintf(stdout, "CHART apps.lreads '' 'Apps Disk Logical Reads' 'kilobytes/s' apps none stacked 20042 %d\n", update_every);
1541         for (w = target_root; w ; w = w->next) {
1542                 if(w->target || (!w->processes && !w->exposed)) continue;
1543
1544                 fprintf(stdout, "DIMENSION %s '' incremental 1 %d\n", w->name, 1024 * update_every);
1545         }
1546
1547         fprintf(stdout, "CHART apps.lwrites '' 'Apps I/O Logical Writes' 'kilobytes/s' apps none stacked 20042 %d\n", update_every);
1548         for (w = target_root; w ; w = w->next) {
1549                 if(w->target || (!w->processes && !w->exposed)) continue;
1550
1551                 fprintf(stdout, "DIMENSION %s '' incremental 1 %d\n", w->name, 1024 * update_every);
1552         }
1553
1554         fprintf(stdout, "CHART apps.preads '' 'Apps Disk Reads' 'kilobytes/s' apps apps stacked 20002 %d\n", update_every);
1555         for (w = target_root; w ; w = w->next) {
1556                 if(w->target || (!w->processes && !w->exposed)) continue;
1557
1558                 fprintf(stdout, "DIMENSION %s '' incremental 1 %d\n", w->name, 1024 * update_every);
1559         }
1560
1561         fprintf(stdout, "CHART apps.pwrites '' 'Apps Disk Writes' 'kilobytes/s' apps apps stacked 20002 %d\n", update_every);
1562         for (w = target_root; w ; w = w->next) {
1563                 if(w->target || (!w->processes && !w->exposed)) continue;
1564
1565                 fprintf(stdout, "DIMENSION %s '' incremental 1 %d\n", w->name, 1024 * update_every);
1566         }
1567
1568         fprintf(stdout, "CHART apps.files '' 'Apps Open Files' 'open files' apps apps stacked 20050 %d\n", update_every);
1569         for (w = target_root; w ; w = w->next) {
1570                 if(w->target || (!w->processes && !w->exposed)) continue;
1571
1572                 fprintf(stdout, "DIMENSION %s '' absolute 1 1\n", w->name);
1573         }
1574
1575         fprintf(stdout, "CHART apps.sockets '' 'Apps Open Sockets' 'open sockets' apps apps stacked 20051 %d\n", update_every);
1576         for (w = target_root; w ; w = w->next) {
1577                 if(w->target || (!w->processes && !w->exposed)) continue;
1578
1579                 fprintf(stdout, "DIMENSION %s '' absolute 1 1\n", w->name);
1580         }
1581
1582         fprintf(stdout, "CHART apps.pipes '' 'Apps Pipes' 'open pipes' apps none stacked 20053 %d\n", update_every);
1583         for (w = target_root; w ; w = w->next) {
1584                 if(w->target || (!w->processes && !w->exposed)) continue;
1585
1586                 fprintf(stdout, "DIMENSION %s '' absolute 1 1\n", w->name);
1587         }
1588
1589         fprintf(stdout, "CHART netdata.apps_cpu '' 'Apps Plugin CPU' 'milliseconds/s' netdata netdata stacked 10000 %d\n", update_every);
1590         fprintf(stdout, "DIMENSION user '' incremental 1 %d\n", 1000 * update_every);
1591         fprintf(stdout, "DIMENSION system '' incremental 1 %d\n", 1000 * update_every);
1592
1593         fprintf(stdout, "CHART netdata.apps_files '' 'Apps Plugin Files' 'files/s' netdata netdata line 10001 %d\n", update_every);
1594         fprintf(stdout, "DIMENSION files '' incremental 1 %d\n", update_every);
1595         fprintf(stdout, "DIMENSION pids '' absolute 1 1\n");
1596         fprintf(stdout, "DIMENSION fds '' absolute 1 1\n");
1597         fprintf(stdout, "DIMENSION targets '' absolute 1 1\n");
1598
1599         fflush(stdout);
1600 }
1601
1602
1603 // ----------------------------------------------------------------------------
1604 // parse command line arguments
1605
1606 void parse_args(int argc, char **argv)
1607 {
1608         int i, freq = 0;
1609         char *name = NULL;
1610
1611         for(i = 1; i < argc; i++) {
1612                 if(!freq) {
1613                         int n = atoi(argv[i]);
1614                         if(n > 0) {
1615                                 freq = n;
1616                                 continue;
1617                         }
1618                 }
1619
1620                 if(strcmp("debug", argv[i]) == 0) {
1621                         debug = 1;
1622                         continue;
1623                 }
1624
1625                 if(!name) {
1626                         name = argv[i];
1627                         continue;
1628                 }
1629
1630                 fprintf(stderr, "apps.plugin: ERROR: cannot understand option %s\n", argv[i]);
1631                 exit(1);
1632         }
1633
1634         if(freq > 0) update_every = freq;
1635         if(!name) name = "groups";
1636
1637         if(read_process_groups(name)) {
1638                 fprintf(stderr, "apps.plugin: ERROR: cannot read process groups %s\n", name);
1639                 exit(1);
1640         }
1641 }
1642
1643 int main(int argc, char **argv)
1644 {
1645         unsigned long started_t = time(NULL), current_t;
1646         Hertz = get_hertz();
1647         pid_max = get_pid_max();
1648         processors = get_processors();
1649
1650         parse_args(argc, argv);
1651
1652         all_pids = calloc(sizeof(struct pid_stat *), pid_max);
1653         if(!all_pids) {
1654                 fprintf(stderr, "apps.plugin: ERROR: cannot allocate %lu bytes of memory.\n", sizeof(struct pid_stat *) * pid_max);
1655                 printf("DISABLE\n");
1656                 exit(1);
1657         }
1658
1659         unsigned long long counter = 1;
1660         unsigned long long usec = 0, susec = 0;
1661         struct timeval last, now;
1662         gettimeofday(&last, NULL);
1663
1664         for(;1; counter++) {
1665                 if(!update_from_proc()) {
1666                         fprintf(stderr, "apps.plugin: ERROR: cannot allocate %lu bytes of memory.\n", sizeof(struct pid_stat *) * pid_max);
1667                         printf("DISABLE\n");
1668                         exit(1);
1669                 }
1670
1671                 update_statistics();
1672                 show_charts();          // this is smart enough to show only newly added apps, when needed
1673                 show_dimensions();
1674
1675                 if(debug) fprintf(stderr, "apps.plugin: done Loop No %llu\n", counter);
1676                 fflush(NULL);
1677
1678                 gettimeofday(&now, NULL);
1679                 usec = usecdiff(&now, &last) - susec;
1680                 if(debug) fprintf(stderr, "apps.plugin: last loop took %llu usec (worked for %llu, sleeped for %llu).\n", usec + susec, usec, susec);
1681
1682                 // if the last loop took less than half the time
1683                 // wait the rest of the time
1684                 if(usec < (update_every * 1000000ULL / 2)) susec = (update_every * 1000000ULL) - usec;
1685                 else susec = update_every * 1000000ULL / 2;
1686
1687                 usleep(susec);
1688                 bcopy(&now, &last, sizeof(struct timeval));
1689
1690                 current_t = time(NULL);
1691                 if(current_t - started_t > 3600) exit(0);
1692         }
1693 }