src/rrd.c

   1 #include <string.h>
   2 #include <unistd.h>
   3 #include <time.h>
   4 #include <sys/time.h>
   5 #include <sys/mman.h>
   6 #include <pthread.h>
   7 #include <errno.h>
   8 #include <ctype.h>
   9 #include <sys/stat.h>
  10 #include <sys/types.h>
  11 #include <stdlib.h>
  12
  13 #include "log.h"
  14 #include "config.h"
  15 #include "common.h"
  16
  17 #include "rrd.h"
  18
  19 int update_every = UPDATE_EVERY;
  20 int save_history = HISTORY;
  21
  22 // ----------------------------------------------------------------------------
  23 // chart types
  24
  25 int chart_type_id(const char *name)
  26 {
  27         if(strcmp(name, CHART_TYPE_AREA_NAME) == 0) return CHART_TYPE_AREA;
  28         if(strcmp(name, CHART_TYPE_STACKED_NAME) == 0) return CHART_TYPE_STACKED;
  29         if(strcmp(name, CHART_TYPE_LINE_NAME) == 0) return CHART_TYPE_LINE;
  30         return CHART_TYPE_LINE;
  31 }
  32
  33 const char *chart_type_name(int chart_type)
  34 {
  35         static char line[] = CHART_TYPE_LINE_NAME;
  36         static char area[] = CHART_TYPE_AREA_NAME;
  37         static char stacked[] = CHART_TYPE_STACKED_NAME;
  38
  39         switch(chart_type) {
  40                 case CHART_TYPE_LINE:
  41                         return line;
  42
  43                 case CHART_TYPE_AREA:
  44                         return area;
  45
  46                 case CHART_TYPE_STACKED:
  47                         return stacked;
  48         }
  49         return line;
  50 }
  51
  52 // ----------------------------------------------------------------------------
  53 // mmap() wrapper
  54
  55 int memory_mode = NETDATA_MEMORY_MODE_SAVE;
  56
  57 const char *memory_mode_name(int id)
  58 {
  59         static const char ram[] = NETDATA_MEMORY_MODE_RAM_NAME;
  60         static const char map[] = NETDATA_MEMORY_MODE_MAP_NAME;
  61         static const char save[] = NETDATA_MEMORY_MODE_SAVE_NAME;
  62
  63         switch(id) {
  64                 case NETDATA_MEMORY_MODE_RAM:
  65                         return ram;
  66
  67                 case NETDATA_MEMORY_MODE_MAP:
  68                         return map;
  69
  70                 case NETDATA_MEMORY_MODE_SAVE:
  71                 default:
  72                         return save;
  73         }
  74
  75         return save;
  76 }
  77
  78 int memory_mode_id(const char *name)
  79 {
  80         if(!strcmp(name, NETDATA_MEMORY_MODE_RAM_NAME))
  81                 return NETDATA_MEMORY_MODE_RAM;
  82         else if(!strcmp(name, NETDATA_MEMORY_MODE_MAP_NAME))
  83                 return NETDATA_MEMORY_MODE_MAP;
  84
  85         return NETDATA_MEMORY_MODE_SAVE;
  86 }
  87
  88 // ----------------------------------------------------------------------------
  89 // algorithms types
  90
  91 int algorithm_id(const char *name)
  92 {
  93         if(strcmp(name, RRD_DIMENSION_ABSOLUTE_NAME) == 0)                      return RRD_DIMENSION_ABSOLUTE;
  94         if(strcmp(name, RRD_DIMENSION_INCREMENTAL_NAME) == 0)                   return RRD_DIMENSION_INCREMENTAL;
  95         if(strcmp(name, RRD_DIMENSION_PCENT_OVER_ROW_TOTAL_NAME) == 0)          return RRD_DIMENSION_PCENT_OVER_ROW_TOTAL;
  96         if(strcmp(name, RRD_DIMENSION_PCENT_OVER_DIFF_TOTAL_NAME) == 0)         return RRD_DIMENSION_PCENT_OVER_DIFF_TOTAL;
  97         return RRD_DIMENSION_ABSOLUTE;
  98 }
  99
 100 const char *algorithm_name(int chart_type)
 101 {
 102         static char absolute[] = RRD_DIMENSION_ABSOLUTE_NAME;
 103         static char incremental[] = RRD_DIMENSION_INCREMENTAL_NAME;
 104         static char percentage_of_absolute_row[] = RRD_DIMENSION_PCENT_OVER_ROW_TOTAL_NAME;
 105         static char percentage_of_incremental_row[] = RRD_DIMENSION_PCENT_OVER_DIFF_TOTAL_NAME;
 106
 107         switch(chart_type) {
 108                 case RRD_DIMENSION_ABSOLUTE:
 109                         return absolute;
 110
 111                 case RRD_DIMENSION_INCREMENTAL:
 112                         return incremental;
 113
 114                 case RRD_DIMENSION_PCENT_OVER_ROW_TOTAL:
 115                         return percentage_of_absolute_row;
 116
 117                 case RRD_DIMENSION_PCENT_OVER_DIFF_TOTAL:
 118                         return percentage_of_incremental_row;
 119         }
 120         return absolute;
 121 }
 122
 123 RRD_STATS *root = NULL;
 124 pthread_rwlock_t root_rwlock = PTHREAD_RWLOCK_INITIALIZER;
 125
 126 char *rrd_stats_strncpy_name(char *to, const char *from, int length)
 127 {
 128         int i;
 129         for(i = 0; i < length && from[i] ;i++) {
 130                 if(from[i] == '.' || isalpha(from[i]) || isdigit(from[i])) to[i] = from[i];
 131                 else to[i] = '_';
 132         }
 133         if(i < length) to[i] = '\0';
 134         to[length - 1] = '\0';
 135
 136         return to;
 137 }
 138
 139 void rrd_stats_set_name(RRD_STATS *st, const char *name)
 140 {
 141         char b[CONFIG_MAX_VALUE + 1];
 142         char n[RRD_STATS_NAME_MAX + 1];
 143
 144         snprintf(n, RRD_STATS_NAME_MAX, "%s.%s", st->type, name);
 145         rrd_stats_strncpy_name(b, n, CONFIG_MAX_VALUE);
 146         st->name = config_get(st->id, "name", b);
 147         st->hash_name = simple_hash(st->name);
 148 }
 149
 150 char *rrd_stats_cache_dir(const char *id)
 151 {
 152         char *ret = NULL;
 153
 154         static char *cache_dir = NULL;
 155         if(!cache_dir) cache_dir = config_get("global", "database directory", "cache");
 156
 157         char b[FILENAME_MAX + 1];
 158         char n[FILENAME_MAX + 1];
 159         rrd_stats_strncpy_name(b, id, FILENAME_MAX);
 160
 161         snprintf(n, FILENAME_MAX, "%s/%s", cache_dir, b);
 162         ret = config_get(id, "database directory", n);
 163
 164         if(memory_mode == NETDATA_MEMORY_MODE_MAP || memory_mode == NETDATA_MEMORY_MODE_SAVE) {
 165                 int r = mkdir(ret, 0775);
 166                 if(r != 0 && errno != EEXIST)
 167                         error("Cannot create directory '%s'", ret);
 168         }
 169
 170         return ret;
 171 }
 172
 173 void rrd_stats_reset(RRD_STATS *st)
 174 {
 175         st->last_collected_time.tv_sec = 0;
 176         st->last_collected_time.tv_usec = 0;
 177         st->last_updated.tv_sec = 0;
 178         st->last_updated.tv_usec = 0;
 179         st->current_entry = 0;
 180         st->counter_done = 0;
 181
 182         RRD_DIMENSION *rd;
 183         for(rd = st->dimensions; rd ; rd = rd->next) {
 184                 rd->last_collected_time.tv_sec = 0;
 185                 rd->last_collected_time.tv_usec = 0;
 186                 rd->current_entry = 0;
 187         }
 188 }
 189
 190 RRD_STATS *rrd_stats_create(const char *type, const char *id, const char *name, const char *family, const char *title, const char *units, long priority, int update_every, int chart_type)
 191 {
 192         if(!id || !id[0]) {
 193                 fatal("Cannot create rrd stats without an id.");
 194                 return NULL;
 195         }
 196
 197         char fullid[RRD_STATS_NAME_MAX + 1];
 198         char fullfilename[FILENAME_MAX + 1];
 199         RRD_STATS *st = NULL;
 200
 201         snprintf(fullid, RRD_STATS_NAME_MAX, "%s.%s", type, id);
 202
 203         long entries = config_get_number(fullid, "history", save_history);
 204         if(entries < 5) entries = config_set_number(fullid, "history", 5);
 205         if(entries > HISTORY_MAX) entries = config_set_number(fullid, "history", HISTORY_MAX);
 206
 207         int enabled = config_get_boolean(fullid, "enabled", 1);
 208         if(!enabled) entries = 5;
 209
 210         unsigned long size = sizeof(RRD_STATS);
 211         char *cache_dir = rrd_stats_cache_dir(fullid);
 212
 213         debug(D_RRD_STATS, "Creating RRD_STATS for '%s.%s'.", type, id);
 214
 215         snprintf(fullfilename, FILENAME_MAX, "%s/main.db", cache_dir);
 216         if(memory_mode != NETDATA_MEMORY_MODE_RAM) st = (RRD_STATS *)mymmap(fullfilename, size, ((memory_mode == NETDATA_MEMORY_MODE_MAP)?MAP_SHARED:MAP_PRIVATE));
 217         if(st) {
 218                 if(strcmp(st->magic, RRD_STATS_MAGIC) != 0) {
 219                         errno = 0;
 220                         error("File %s does not have our version. Clearing it.", fullfilename);
 221                         bzero(st, size);
 222                 }
 223                 else if(strcmp(st->id, fullid) != 0) {
 224                         errno = 0;
 225                         error("File %s does not have our id. Unmapping it.", fullfilename);
 226                         munmap(st, size);
 227                         st = NULL;
 228                 }
 229                 else if(st->memsize != size || st->entries != entries) {
 230                         errno = 0;
 231                         error("File %s does not have the desired size. Clearing it.", fullfilename);
 232                         bzero(st, size);
 233                 }
 234                 else if(st->update_every != update_every) {
 235                         errno = 0;
 236                         error("File %s does not have the desired update frequency. Clearing it.", fullfilename);
 237                         bzero(st, size);
 238                 }
 239         }
 240
 241         if(st) {
 242                 st->name = NULL;
 243                 st->type = NULL;
 244                 st->family = NULL;
 245                 st->title = NULL;
 246                 st->units = NULL;
 247                 st->dimensions = NULL;
 248                 st->next = NULL;
 249                 st->mapped = memory_mode;
 250         }
 251         else {
 252                 st = calloc(1, size);
 253                 if(!st) {
 254                         fatal("Cannot allocate memory for RRD_STATS %s.%s", type, id);
 255                         return NULL;
 256                 }
 257                 st->mapped = NETDATA_MEMORY_MODE_RAM;
 258         }
 259         st->memsize = size;
 260         st->entries = entries;
 261         st->update_every = update_every;
 262
 263         strcpy(st->cache_file, fullfilename);
 264         strcpy(st->magic, RRD_STATS_MAGIC);
 265
 266         strcpy(st->id, fullid);
 267         st->hash = simple_hash(st->id);
 268
 269         st->cache_dir = cache_dir;
 270
 271         st->family     = config_get(st->id, "family", family?family:st->id);
 272         st->units      = config_get(st->id, "units", units?units:"");
 273         st->type       = config_get(st->id, "type", type);
 274         st->chart_type = chart_type_id(config_get(st->id, "chart type", chart_type_name(chart_type)));
 275
 276         if(name && *name) rrd_stats_set_name(st, name);
 277         else rrd_stats_set_name(st, id);
 278
 279         {
 280                 char varvalue[CONFIG_MAX_VALUE + 1];
 281                 snprintf(varvalue, CONFIG_MAX_VALUE, "%s (%s)", title?title:"", st->name);
 282                 st->title = config_get(st->id, "title", varvalue);
 283         }
 284
 285         st->priority = config_get_number(st->id, "priority", priority);
 286         st->enabled = enabled;
 287
 288         st->isdetail = 0;
 289         st->debug = 0;
 290
 291         st->last_collected_time.tv_sec = 0;
 292         st->last_collected_time.tv_usec = 0;
 293         st->counter_done = 0;
 294
 295         pthread_rwlock_init(&st->rwlock, NULL);
 296         pthread_rwlock_wrlock(&root_rwlock);
 297
 298         st->next = root;
 299         root = st;
 300
 301         pthread_rwlock_unlock(&root_rwlock);
 302
 303         return(st);
 304 }
 305
 306 RRD_DIMENSION *rrd_stats_dimension_add(RRD_STATS *st, const char *id, const char *name, long multiplier, long divisor, int algorithm)
 307 {
 308         char filename[FILENAME_MAX + 1];
 309         char fullfilename[FILENAME_MAX + 1];
 310
 311         char varname[CONFIG_MAX_NAME + 1];
 312         RRD_DIMENSION *rd = NULL;
 313         unsigned long size = sizeof(RRD_DIMENSION) + (st->entries * sizeof(storage_number));
 314
 315         debug(D_RRD_STATS, "Adding dimension '%s/%s'.", st->id, id);
 316
 317         rrd_stats_strncpy_name(filename, id, FILENAME_MAX);
 318         snprintf(fullfilename, FILENAME_MAX, "%s/%s.db", st->cache_dir, filename);
 319         if(memory_mode != NETDATA_MEMORY_MODE_RAM) rd = (RRD_DIMENSION *)mymmap(fullfilename, size, ((memory_mode == NETDATA_MEMORY_MODE_MAP)?MAP_SHARED:MAP_PRIVATE));
 320         if(rd) {
 321                 struct timeval now;
 322                 gettimeofday(&now, NULL);
 323
 324                 if(strcmp(rd->magic, RRD_DIMENSION_MAGIC) != 0) {
 325                         errno = 0;
 326                         error("File %s does not have our version. Clearing it.", fullfilename);
 327                         bzero(rd, size);
 328                 }
 329                 else if(rd->memsize != size) {
 330                         errno = 0;
 331                         error("File %s does not have the desired size. Clearing it.", fullfilename);
 332                         bzero(rd, size);
 333                 }
 334                 else if(rd->multiplier != multiplier) {
 335                         errno = 0;
 336                         error("File %s does not have the same multiplier. Clearing it.", fullfilename);
 337                         bzero(rd, size);
 338                 }
 339                 else if(rd->divisor != divisor) {
 340                         errno = 0;
 341                         error("File %s does not have the same divisor. Clearing it.", fullfilename);
 342                         bzero(rd, size);
 343                 }
 344                 else if(rd->algorithm != algorithm) {
 345                         errno = 0;
 346                         error("File %s does not have the same algorithm. Clearing it.", fullfilename);
 347                         bzero(rd, size);
 348                 }
 349                 else if(rd->update_every != st->update_every) {
 350                         errno = 0;
 351                         error("File %s does not have the same refresh frequency. Clearing it.", fullfilename);
 352                         bzero(rd, size);
 353                 }
 354                 else if(usecdiff(&now, &rd->last_collected_time) > (rd->entries * rd->update_every * 1000000ULL)) {
 355                         errno = 0;
 356                         error("File %s is too old. Clearing it.", fullfilename);
 357                         bzero(rd, size);
 358                 }
 359                 else if(strcmp(rd->id, id) != 0) {
 360                         errno = 0;
 361                         error("File %s does not have our dimension id. Unmapping it.", fullfilename);
 362                         munmap(rd, size);
 363                         rd = NULL;
 364                 }
 365         }
 366
 367         if(rd) {
 368                 // we have a file mapped for rd
 369                 rd->mapped = memory_mode;
 370                 rd->hidden = 0;
 371                 rd->next = NULL;
 372                 rd->name = NULL;
 373         }
 374         else {
 375                 // if we didn't manage to get a mmap'd dimension, just create one
 376
 377                 rd = calloc(1, size);
 378                 if(!rd) {
 379                         fatal("Cannot allocate RRD_DIMENSION %s/%s.", st->id, id);
 380                         return NULL;
 381                 }
 382
 383                 rd->mapped = NETDATA_MEMORY_MODE_RAM;
 384         }
 385         rd->memsize = size;
 386
 387         strcpy(rd->magic, RRD_DIMENSION_MAGIC);
 388         strcpy(rd->cache_file, fullfilename);
 389         strncpy(rd->id, id, RRD_STATS_NAME_MAX);
 390         rd->hash = simple_hash(rd->id);
 391
 392         snprintf(varname, CONFIG_MAX_NAME, "dim %s name", rd->id);
 393         rd->name = config_get(st->id, varname, (name && *name)?name:rd->id);
 394
 395         snprintf(varname, CONFIG_MAX_NAME, "dim %s algorithm", rd->id);
 396         rd->algorithm = algorithm_id(config_get(st->id, varname, algorithm_name(algorithm)));
 397
 398         snprintf(varname, CONFIG_MAX_NAME, "dim %s multiplier", rd->id);
 399         rd->multiplier = config_get_number(st->id, varname, multiplier);
 400
 401         snprintf(varname, CONFIG_MAX_NAME, "dim %s divisor", rd->id);
 402         rd->divisor = config_get_number(st->id, varname, divisor);
 403         if(!rd->divisor) rd->divisor = 1;
 404
 405         rd->entries = st->entries;
 406         rd->update_every = st->update_every;
 407
 408         // append this dimension
 409         if(!st->dimensions)
 410                 st->dimensions = rd;
 411         else {
 412                 RRD_DIMENSION *td = st->dimensions;
 413                 for(; td->next; td = td->next) ;
 414                 td->next = rd;
 415         }
 416
 417         return(rd);
 418 }
 419
 420 void rrd_stats_dimension_set_name(RRD_STATS *st, RRD_DIMENSION *rd, const char *name)
 421 {
 422         char varname[CONFIG_MAX_NAME + 1];
 423         snprintf(varname, CONFIG_MAX_NAME, "dim %s name", rd->id);
 424         config_get(st->id, varname, name);
 425 }
 426
 427 void rrd_stats_dimension_free(RRD_DIMENSION *rd)
 428 {
 429         if(rd->next) rrd_stats_dimension_free(rd->next);
 430         // free(rd->annotations);
 431         if(rd->mapped == NETDATA_MEMORY_MODE_SAVE) {
 432                 debug(D_RRD_STATS, "Saving dimension '%s' to '%s'.", rd->name, rd->cache_file);
 433                 savememory(rd->cache_file, rd, rd->memsize);
 434
 435                 debug(D_RRD_STATS, "Unmapping dimension '%s'.", rd->name);
 436                 munmap(rd, rd->memsize);
 437         }
 438         else if(rd->mapped == NETDATA_MEMORY_MODE_MAP) {
 439                 debug(D_RRD_STATS, "Unmapping dimension '%s'.", rd->name);
 440                 munmap(rd, rd->memsize);
 441         }
 442         else {
 443                 debug(D_RRD_STATS, "Removing dimension '%s'.", rd->name);
 444                 free(rd);
 445         }
 446 }
 447
 448 void rrd_stats_free_all(void)
 449 {
 450         info("Freeing all memory...");
 451
 452         RRD_STATS *st;
 453         for(st = root; st ;) {
 454                 RRD_STATS *next = st->next;
 455
 456                 if(st->dimensions) rrd_stats_dimension_free(st->dimensions);
 457                 st->dimensions = NULL;
 458
 459                 if(st->mapped == NETDATA_MEMORY_MODE_SAVE) {
 460                         debug(D_RRD_STATS, "Saving stats '%s' to '%s'.", st->name, st->cache_file);
 461                         savememory(st->cache_file, st, st->memsize);
 462
 463                         debug(D_RRD_STATS, "Unmapping stats '%s'.", st->name);
 464                         munmap(st, st->memsize);
 465                 }
 466                 else if(st->mapped == NETDATA_MEMORY_MODE_MAP) {
 467                         debug(D_RRD_STATS, "Unmapping stats '%s'.", st->name);
 468                         munmap(st, st->memsize);
 469                 }
 470                 else
 471                         free(st);
 472
 473                 st = next;
 474         }
 475         root = NULL;
 476
 477         info("Memory cleanup completed...");
 478 }
 479
 480 void rrd_stats_save_all(void)
 481 {
 482         RRD_STATS *st;
 483         RRD_DIMENSION *rd;
 484
 485         pthread_rwlock_wrlock(&root_rwlock);
 486         for(st = root; st ; st = st->next) {
 487                 pthread_rwlock_wrlock(&st->rwlock);
 488
 489                 if(st->mapped == NETDATA_MEMORY_MODE_SAVE) {
 490                         debug(D_RRD_STATS, "Saving stats '%s' to '%s'.", st->name, st->cache_file);
 491                         savememory(st->cache_file, st, st->memsize);
 492                 }
 493
 494                 for(rd = st->dimensions; rd ; rd = rd->next) {
 495                         if(rd->mapped == NETDATA_MEMORY_MODE_SAVE) {
 496                                 debug(D_RRD_STATS, "Saving dimension '%s' to '%s'.", rd->name, rd->cache_file);
 497                                 savememory(rd->cache_file, rd, rd->memsize);
 498                         }
 499                 }
 500
 501                 pthread_rwlock_unlock(&st->rwlock);
 502         }
 503         pthread_rwlock_unlock(&root_rwlock);
 504 }
 505
 506
 507 RRD_STATS *rrd_stats_find(const char *id)
 508 {
 509         debug(D_RRD_STATS, "rrd_stats_find() for chart %s", id);
 510
 511         unsigned long hash = simple_hash(id);
 512
 513         pthread_rwlock_rdlock(&root_rwlock);
 514         RRD_STATS *st = root;
 515         for ( ; st ; st = st->next )
 516                 if(hash == st->hash)
 517                         if(strcmp(st->id, id) == 0)
 518                                 break;
 519         pthread_rwlock_unlock(&root_rwlock);
 520
 521         return(st);
 522 }
 523
 524 RRD_STATS *rrd_stats_find_bytype(const char *type, const char *id)
 525 {
 526         debug(D_RRD_STATS, "rrd_stats_find_bytype() for chart %s.%s", type, id);
 527
 528         char buf[RRD_STATS_NAME_MAX + 1];
 529
 530         strncpy(buf, type, RRD_STATS_NAME_MAX - 1);
 531         buf[RRD_STATS_NAME_MAX - 1] = '\0';
 532         strcat(buf, ".");
 533         int len = strlen(buf);
 534         strncpy(&buf[len], id, RRD_STATS_NAME_MAX - len);
 535         buf[RRD_STATS_NAME_MAX] = '\0';
 536
 537         return(rrd_stats_find(buf));
 538 }
 539
 540 RRD_STATS *rrd_stats_find_byname(const char *name)
 541 {
 542         debug(D_RRD_STATS, "rrd_stats_find_byname() for chart %s", name);
 543
 544         char b[CONFIG_MAX_VALUE + 1];
 545
 546         rrd_stats_strncpy_name(b, name, CONFIG_MAX_VALUE);
 547         unsigned long hash = simple_hash(b);
 548
 549         pthread_rwlock_rdlock(&root_rwlock);
 550         RRD_STATS *st = root;
 551         for ( ; st ; st = st->next ) {
 552                 if(hash == st->hash_name && strcmp(st->name, b) == 0) break;
 553         }
 554         pthread_rwlock_unlock(&root_rwlock);
 555
 556         return(st);
 557 }
 558
 559 RRD_DIMENSION *rrd_stats_dimension_find(RRD_STATS *st, const char *id)
 560 {
 561         debug(D_RRD_STATS, "rrd_stats_dimension_find() for chart %s, dimension %s", st->name, id);
 562
 563         unsigned long hash = simple_hash(id);
 564
 565         RRD_DIMENSION *rd = st->dimensions;
 566
 567         for ( ; rd ; rd = rd->next )
 568                 if(hash == rd->hash)
 569                         if(strcmp(rd->id, id) == 0)
 570                                 break;
 571
 572         return(rd);
 573 }
 574
 575 int rrd_stats_dimension_hide(RRD_STATS *st, const char *id)
 576 {
 577         debug(D_RRD_STATS, "rrd_stats_dimension_hide() for chart %s, dimension %s", st->name, id);
 578
 579         RRD_DIMENSION *rd = rrd_stats_dimension_find(st, id);
 580         if(!rd) {
 581                 error("Cannot find dimension with id '%s' on stats '%s' (%s).", id, st->name, st->id);
 582                 return 1;
 583         }
 584
 585         rd->hidden = 1;
 586         return 0;
 587 }
 588
 589 void rrd_stats_dimension_set_by_pointer(RRD_STATS *st, RRD_DIMENSION *rd, collected_number value)
 590 {
 591         debug(D_RRD_STATS, "rrd_stats_dimension_set() for chart %s, dimension %s, value " COLLECTED_NUMBER_FORMAT, st->name, rd->name, value);
 592
 593         gettimeofday(&rd->last_collected_time, NULL);
 594         rd->collected_value = value;
 595 }
 596
 597 int rrd_stats_dimension_set(RRD_STATS *st, const char *id, collected_number value)
 598 {
 599         RRD_DIMENSION *rd = rrd_stats_dimension_find(st, id);
 600         if(!rd) {
 601                 error("Cannot find dimension with id '%s' on stats '%s' (%s).", id, st->name, st->id);
 602                 return 1;
 603         }
 604
 605         rrd_stats_dimension_set_by_pointer(st, rd, value);
 606         return 0;
 607 }
 608
 609 void rrd_stats_next_usec(RRD_STATS *st, unsigned long long microseconds)
 610 {
 611         debug(D_RRD_STATS, "rrd_stats_next() for chart %s with microseconds %llu", st->name, microseconds);
 612
 613         if(st->debug) debug(D_RRD_STATS, "%s: NEXT: %llu microseconds", st->name, microseconds);
 614         st->usec_since_last_update = microseconds;
 615 }
 616
 617 void rrd_stats_next(RRD_STATS *st)
 618 {
 619         unsigned long long microseconds = 0;
 620
 621         if(st->last_collected_time.tv_sec) {
 622                 struct timeval now;
 623                 gettimeofday(&now, NULL);
 624                 microseconds = usecdiff(&now, &st->last_collected_time);
 625         }
 626
 627         rrd_stats_next_usec(st, microseconds);
 628 }
 629
 630 void rrd_stats_next_plugins(RRD_STATS *st)
 631 {
 632         rrd_stats_next(st);
 633 }
 634
 635 unsigned long long rrd_stats_done(RRD_STATS *st)
 636 {
 637         debug(D_RRD_STATS, "rrd_stats_done() for chart %s", st->name);
 638
 639         RRD_DIMENSION *rd, *last;
 640         int oldstate;
 641
 642         if(pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, &oldstate) != 0)
 643                 error("Cannot set pthread cancel state to DISABLE.");
 644
 645         // a read lock is OK here
 646         pthread_rwlock_rdlock(&st->rwlock);
 647
 648         if(st->usec_since_last_update > st->entries * st->update_every * 1000000ULL || usecdiff(&st->last_collected_time, &st->last_updated) > st->entries * st->update_every * 1000000ULL) {
 649                 info("History of chart %s too old. Reseting chart.", st->name);
 650                 rrd_stats_reset(st);
 651                 st->usec_since_last_update = st->update_every * 1000000ULL;
 652         }
 653
 654         if(st->debug) debug(D_RRD_STATS, "%s: microseconds since last update: %llu", st->name, st->usec_since_last_update);
 655
 656         if(!st->last_collected_time.tv_sec) gettimeofday(&st->last_collected_time, NULL);
 657         else {
 658                 unsigned long long ut = st->last_collected_time.tv_sec * 1000000ULL + st->last_collected_time.tv_usec + st->usec_since_last_update;
 659                 st->last_collected_time.tv_sec = ut / 1000000ULL;
 660                 st->last_collected_time.tv_usec = ut % 1000000ULL;
 661         }
 662
 663         if(!st->last_updated.tv_sec) {
 664                 unsigned long long ut = st->last_collected_time.tv_sec * 1000000ULL + st->last_collected_time.tv_usec - st->usec_since_last_update;
 665                 st->last_updated.tv_sec = ut / 1000000ULL;
 666                 st->last_updated.tv_usec = ut % 1000000ULL;
 667
 668                 if(st->debug) debug(D_RRD_STATS, "%s: initializing last_updated to now - %llu microseconds (%0.3Lf)", st->name, st->usec_since_last_update, (long double)ut/1000000.0);
 669         }
 670
 671         unsigned long long last_ut = st->last_updated.tv_sec * 1000000ULL + st->last_updated.tv_usec;
 672         unsigned long long now_ut = st->last_collected_time.tv_sec * 1000000ULL + st->last_collected_time.tv_usec;
 673         unsigned long long next_ut = (st->last_updated.tv_sec + st->update_every) * 1000000ULL;
 674
 675         if(st->debug) debug(D_RRD_STATS, "%s: last ut = %0.3Lf (last updated time)", st->name, (long double)last_ut/1000000.0);
 676         if(st->debug) debug(D_RRD_STATS, "%s: now  ut = %0.3Lf (current update time)", st->name, (long double)now_ut/1000000.0);
 677         if(st->debug) debug(D_RRD_STATS, "%s: next ut = %0.3Lf (next interpolation point)", st->name, (long double)next_ut/1000000.0);
 678
 679         st->counter_done++;
 680
 681         // calculate totals and count the dimensions
 682         int dimensions;
 683         st->collected_total = 0;
 684         for( rd = st->dimensions, dimensions = 0 ; rd ; rd = rd->next, dimensions++ )
 685                 st->collected_total += rd->collected_value;
 686
 687         // process all dimensions to calculate their values
 688         // based on the collected figures only
 689         // at this stage we do not interpolate anything
 690         for( rd = st->dimensions ; rd ; rd = rd->next ) {
 691                 if(st->debug) debug(D_RRD_STATS, "%s/%s: "
 692                         " last_collected_value = " COLLECTED_NUMBER_FORMAT
 693                         " collected_value = " COLLECTED_NUMBER_FORMAT
 694                         " last_calculated_value = " CALCULATED_NUMBER_FORMAT
 695                         " calculated_value = " CALCULATED_NUMBER_FORMAT
 696                         , st->id, rd->name
 697                         , rd->last_collected_value
 698                         , rd->collected_value
 699                         , rd->last_calculated_value
 700                         , rd->calculated_value
 701                         );
 702
 703                 switch(rd->algorithm) {
 704                         case RRD_DIMENSION_PCENT_OVER_DIFF_TOTAL:
 705                                 // the percentage of the current increment
 706                                 // over the increment of all dimensions together
 707                                 if(st->collected_total == st->last_collected_total) rd->calculated_value = rd->last_calculated_value;
 708                                 else rd->calculated_value =
 709                                           (calculated_number)100
 710                                         * (calculated_number)(rd->collected_value - rd->last_collected_value)
 711                                         / (calculated_number)(st->collected_total  - st->last_collected_total);
 712
 713                                 if(st->debug)
 714                                         debug(D_RRD_STATS, "%s/%s: CALC PCENT-DIFF "
 715                                                 CALCULATED_NUMBER_FORMAT " = 100"
 716                                                 " * (" COLLECTED_NUMBER_FORMAT " - " COLLECTED_NUMBER_FORMAT ")"
 717                                                 " / (" COLLECTED_NUMBER_FORMAT " - " COLLECTED_NUMBER_FORMAT ")"
 718                                                 , st->id, rd->name
 719                                                 , rd->calculated_value
 720                                                 , rd->collected_value, rd->last_collected_value
 721                                                 , st->collected_total, st->last_collected_total
 722                                                 );
 723                                 break;
 724
 725                         case RRD_DIMENSION_PCENT_OVER_ROW_TOTAL:
 726                                 if(!st->collected_total) rd->calculated_value = 0;
 727                                 else
 728                                 // the percentage of the current value
 729                                 // over the total of all dimensions
 730                                 rd->calculated_value =
 731                                           (calculated_number)100
 732                                         * (calculated_number)rd->collected_value
 733                                         / (calculated_number)st->collected_total;
 734
 735                                 if(st->debug)
 736                                         debug(D_RRD_STATS, "%s/%s: CALC PCENT-ROW "
 737                                                 CALCULATED_NUMBER_FORMAT " = 100"
 738                                                 " * " COLLECTED_NUMBER_FORMAT
 739                                                 " / " COLLECTED_NUMBER_FORMAT
 740                                                 , st->id, rd->name
 741                                                 , rd->calculated_value
 742                                                 , rd->collected_value
 743                                                 , st->collected_total
 744                                                 );
 745                                 break;
 746
 747                         case RRD_DIMENSION_INCREMENTAL:
 748                                 // if the new is smaller than the old (an overflow, or reset), set the old equal to the new
 749                                 // to reset the calculation (it will give zero as the calculation for this second)
 750                                 if(rd->last_collected_value > rd->collected_value) rd->last_collected_value = rd->collected_value;
 751
 752                                 rd->calculated_value += (calculated_number)(rd->collected_value - rd->last_collected_value);
 753
 754                                 if(st->debug)
 755                                         debug(D_RRD_STATS, "%s/%s: CALC INC "
 756                                                 CALCULATED_NUMBER_FORMAT " += "
 757                                                 COLLECTED_NUMBER_FORMAT " - " COLLECTED_NUMBER_FORMAT
 758                                                 , st->id, rd->name
 759                                                 , rd->calculated_value
 760                                                 , rd->collected_value, rd->last_collected_value
 761                                                 );
 762                                 break;
 763
 764                         case RRD_DIMENSION_ABSOLUTE:
 765                                 rd->calculated_value = (calculated_number)rd->collected_value;
 766
 767                                 if(st->debug)
 768                                         debug(D_RRD_STATS, "%s/%s: CALC ABS/ABS-NO-IN "
 769                                                 CALCULATED_NUMBER_FORMAT " = "
 770                                                 COLLECTED_NUMBER_FORMAT
 771                                                 , st->id, rd->name
 772                                                 , rd->calculated_value
 773                                                 , rd->collected_value
 774                                                 );
 775                                 break;
 776
 777                         default:
 778                                 // make the default zero, to make sure
 779                                 // it gets noticed when we add new types
 780                                 rd->calculated_value = 0;
 781
 782                                 if(st->debug)
 783                                         debug(D_RRD_STATS, "%s/%s: CALC "
 784                                                 CALCULATED_NUMBER_FORMAT " = 0"
 785                                                 , st->id, rd->name
 786                                                 , rd->calculated_value
 787                                                 );
 788                                 break;
 789                 }
 790         }
 791         // at this point we have all the calculated values ready
 792
 793         if(st->counter_done == 1 || next_ut > now_ut) {
 794                 // we don't have any usable data yet
 795                 if(st->debug) debug(D_RRD_STATS, "%s: Skipping collected values (usec since last update = %llu, counter_done = %lu)", st->name, st->usec_since_last_update, st->counter_done);
 796
 797                 for( rd = st->dimensions; rd ; rd = rd->next ) {
 798                         rd->last_calculated_value = rd->calculated_value;
 799                         rd->last_collected_value = rd->collected_value;
 800
 801                         switch(rd->algorithm) {
 802                                 case RRD_DIMENSION_PCENT_OVER_DIFF_TOTAL:
 803                                 case RRD_DIMENSION_INCREMENTAL:
 804                                         if(!st->usec_since_last_update) rd->calculated_value = 0;
 805                                         // keep the previous values
 806                                         // the next time, a new incremental total will be calculated
 807                                         break;
 808                         }
 809
 810                         rd->collected_value = 0;
 811                 }
 812                 st->last_collected_total  = st->collected_total;
 813
 814                 pthread_rwlock_unlock(&st->rwlock);
 815                 if(pthread_setcancelstate(oldstate, NULL) != 0)
 816                         error("Cannot set pthread cancel state to RESTORE (%d).", oldstate);
 817
 818                 return(st->usec_since_last_update);
 819         }
 820
 821         // it is now time to interpolate values on a second boundary
 822         unsigned long long first_ut = last_ut;
 823         for( ; next_ut <= now_ut ; next_ut += st->update_every * 1000000ULL ) {
 824                 if(st->debug) debug(D_RRD_STATS, "%s: last ut = %0.3Lf (last updated time)", st->name, (long double)last_ut/1000000.0);
 825                 if(st->debug) debug(D_RRD_STATS, "%s: next ut = %0.3Lf (next interpolation point)", st->name, (long double)next_ut/1000000.0);
 826
 827                 st->last_updated.tv_sec = next_ut / 1000000ULL;
 828                 st->last_updated.tv_usec = 0;
 829
 830                 for( rd = st->dimensions ; rd ; rd = rd->next ) {
 831                         calculated_number new_value;
 832
 833                         switch(rd->algorithm) {
 834                                 case RRD_DIMENSION_INCREMENTAL:
 835                                         new_value = (calculated_number)
 836                                                 (          rd->calculated_value
 837                                                         * (calculated_number)(next_ut - last_ut)
 838                                                         / (calculated_number)(now_ut - last_ut)
 839                                                 );
 840
 841                                         if(st->debug)
 842                                                 debug(D_RRD_STATS, "%s/%s: CALC2 INC "
 843                                                         CALCULATED_NUMBER_FORMAT " = "
 844                                                         CALCULATED_NUMBER_FORMAT
 845                                                         " * %llu"
 846                                                         " / %llu"
 847                                                         , st->id, rd->name
 848                                                         , new_value
 849                                                         , rd->calculated_value
 850                                                         , (unsigned long long)(next_ut - last_ut)
 851                                                         , (unsigned long long)(now_ut - last_ut)
 852                                                         );
 853
 854                                         rd->calculated_value -= new_value;
 855                                         break;
 856
 857                                 case RRD_DIMENSION_ABSOLUTE:
 858                                 case RRD_DIMENSION_PCENT_OVER_ROW_TOTAL:
 859                                 case RRD_DIMENSION_PCENT_OVER_DIFF_TOTAL:
 860                                 default:
 861                                         new_value = (calculated_number)
 862                                                 (       (         (rd->calculated_value - rd->last_calculated_value)
 863                                                                 * (calculated_number)(next_ut - first_ut)
 864                                                                 / (calculated_number)(now_ut - first_ut)
 865                                                         )
 866                                                         +  rd->last_calculated_value
 867                                                 );
 868
 869                                         if(st->debug)
 870                                                 debug(D_RRD_STATS, "%s/%s: CALC2 DEF "
 871                                                         CALCULATED_NUMBER_FORMAT " = ((("
 872                                                         "(" CALCULATED_NUMBER_FORMAT " - " CALCULATED_NUMBER_FORMAT ")"
 873                                                         " * %llu"
 874                                                         " / %llu) + " CALCULATED_NUMBER_FORMAT
 875                                                         , st->id, rd->name
 876                                                         , new_value
 877                                                         , rd->calculated_value, rd->last_calculated_value
 878                                                         , (next_ut - first_ut)
 879                                                         , (now_ut - first_ut), rd->last_calculated_value
 880                                                         );
 881
 882                                         if(next_ut + st->update_every * 1000000ULL > now_ut) rd->calculated_value = new_value;
 883                                         break;
 884
 885                         }
 886
 887
 888                         rd->values[st->current_entry] = pack_storage_number(
 889                                           new_value
 890                                         * (calculated_number)rd->multiplier
 891                                         / (calculated_number)rd->divisor
 892                                 );
 893
 894                         if(st->debug)
 895                                 debug(D_RRD_STATS, "%s/%s: STORE[%ld] "
 896                                         CALCULATED_NUMBER_FORMAT " = " CALCULATED_NUMBER_FORMAT
 897                                         " * %ld"
 898                                         " / %ld"
 899                                         , st->id, rd->name
 900                                         , st->current_entry
 901                                         , unpack_storage_number(rd->values[st->current_entry]), new_value
 902                                         , rd->multiplier
 903                                         , rd->divisor
 904                                         );
 905                 }
 906
 907                 if(st->first_entry_t && st->counter >= (unsigned long long)st->entries) {
 908                         // the db is overwriting values
 909                         // add the value we will overwrite
 910                         st->first_entry_t += st->update_every * 1000000ULL;
 911                 }
 912
 913                 st->counter++;
 914                 st->current_entry = ((st->current_entry + 1) >= st->entries) ? 0 : st->current_entry + 1;
 915                 if(!st->first_entry_t) st->first_entry_t = next_ut;
 916                 last_ut = next_ut;
 917         }
 918
 919         for( rd = st->dimensions; rd ; rd = rd->next ) {
 920                 rd->last_collected_value = rd->collected_value;
 921                 rd->last_calculated_value = rd->calculated_value;
 922                 rd->collected_value = 0;
 923         }
 924         st->last_collected_total  = st->collected_total;
 925
 926         // ALL DONE ABOUT THE DATA UPDATE
 927         // --------------------------------------------------------------------
 928
 929
 930         // find if there are any obsolete dimensions (not updated recently)
 931         for( rd = st->dimensions; rd ; rd = rd->next )
 932                 if((rd->last_collected_time.tv_sec + (10 * st->update_every)) < st->last_collected_time.tv_sec)
 933                         break;
 934
 935         if(rd) {
 936                 // there is dimension to free
 937                 // upgrade our read lock to a write lock
 938                 pthread_rwlock_unlock(&st->rwlock);
 939                 pthread_rwlock_wrlock(&st->rwlock);
 940
 941                 for( rd = st->dimensions, last = NULL ; rd ; ) {
 942                         if((rd->last_collected_time.tv_sec + (10 * st->update_every)) < st->last_collected_time.tv_sec) { // remove it only it is not updated in 10 seconds
 943                                 debug(D_RRD_STATS, "Removing obsolete dimension '%s' (%s) of '%s' (%s).", rd->name, rd->id, st->name, st->id);
 944
 945                                 if(!last) {
 946                                         st->dimensions = rd->next;
 947                                         rd->next = NULL;
 948                                         rrd_stats_dimension_free(rd);
 949                                         rd = st->dimensions;
 950                                         continue;
 951                                 }
 952                                 else {
 953                                         last->next = rd->next;
 954                                         rd->next = NULL;
 955                                         rrd_stats_dimension_free(rd);
 956                                         rd = last->next;
 957                                         continue;
 958                                 }
 959                         }
 960
 961                         last = rd;
 962                         rd = rd->next;
 963                 }
 964
 965                 if(!st->dimensions) st->enabled = 0;
 966         }
 967
 968         pthread_rwlock_unlock(&st->rwlock);
 969
 970         if(pthread_setcancelstate(oldstate, NULL) != 0)
 971                 error("Cannot set pthread cancel state to RESTORE (%d).", oldstate);
 972
 973         return(st->usec_since_last_update);
 974 }
 975
 976
 977 // find the oldest entry in the data, skipping all empty slots
 978 time_t rrd_stats_first_entry_t(RRD_STATS *st)
 979 {
 980         if(!st->first_entry_t) return st->last_updated.tv_sec;
 981
 982         return st->first_entry_t / 1000000;
 983 }