src/rrd.c

   1 #include <string.h>
   2 #include <unistd.h>
   3 #include <time.h>
   4 #include <sys/time.h>
   5 #include <sys/mman.h>
   6 #include <pthread.h>
   7 #include <errno.h>
   8 #include <ctype.h>
   9 #include <sys/stat.h>
  10 #include <sys/types.h>
  11 #include <stdlib.h>
  12
  13 #include "log.h"
  14 #include "config.h"
  15 #include "common.h"
  16
  17 #include "rrd.h"
  18
  19 int update_every = UPDATE_EVERY;
  20 int save_history = HISTORY;
  21
  22 // ----------------------------------------------------------------------------
  23 // chart types
  24
  25 int chart_type_id(const char *name)
  26 {
  27         if(strcmp(name, CHART_TYPE_AREA_NAME) == 0) return CHART_TYPE_AREA;
  28         if(strcmp(name, CHART_TYPE_STACKED_NAME) == 0) return CHART_TYPE_STACKED;
  29         if(strcmp(name, CHART_TYPE_LINE_NAME) == 0) return CHART_TYPE_LINE;
  30         return CHART_TYPE_LINE;
  31 }
  32
  33 const char *chart_type_name(int chart_type)
  34 {
  35         static char line[] = CHART_TYPE_LINE_NAME;
  36         static char area[] = CHART_TYPE_AREA_NAME;
  37         static char stacked[] = CHART_TYPE_STACKED_NAME;
  38
  39         switch(chart_type) {
  40                 case CHART_TYPE_LINE:
  41                         return line;
  42
  43                 case CHART_TYPE_AREA:
  44                         return area;
  45
  46                 case CHART_TYPE_STACKED:
  47                         return stacked;
  48         }
  49         return line;
  50 }
  51
  52 // ----------------------------------------------------------------------------
  53 // mmap() wrapper
  54
  55 int memory_mode = NETDATA_MEMORY_MODE_SAVE;
  56
  57 const char *memory_mode_name(int id)
  58 {
  59         static const char ram[] = NETDATA_MEMORY_MODE_RAM_NAME;
  60         static const char map[] = NETDATA_MEMORY_MODE_MAP_NAME;
  61         static const char save[] = NETDATA_MEMORY_MODE_SAVE_NAME;
  62
  63         switch(id) {
  64                 case NETDATA_MEMORY_MODE_RAM:
  65                         return ram;
  66
  67                 case NETDATA_MEMORY_MODE_MAP:
  68                         return map;
  69
  70                 case NETDATA_MEMORY_MODE_SAVE:
  71                 default:
  72                         return save;
  73         }
  74
  75         return save;
  76 }
  77
  78 int memory_mode_id(const char *name)
  79 {
  80         if(!strcmp(name, NETDATA_MEMORY_MODE_RAM_NAME))
  81                 return NETDATA_MEMORY_MODE_RAM;
  82         else if(!strcmp(name, NETDATA_MEMORY_MODE_MAP_NAME))
  83                 return NETDATA_MEMORY_MODE_MAP;
  84
  85         return NETDATA_MEMORY_MODE_SAVE;
  86 }
  87
  88 // ----------------------------------------------------------------------------
  89 // algorithms types
  90
  91 int algorithm_id(const char *name)
  92 {
  93         if(strcmp(name, RRD_DIMENSION_ABSOLUTE_NAME) == 0)                      return RRD_DIMENSION_ABSOLUTE;
  94         if(strcmp(name, RRD_DIMENSION_INCREMENTAL_NAME) == 0)                   return RRD_DIMENSION_INCREMENTAL;
  95         if(strcmp(name, RRD_DIMENSION_PCENT_OVER_ROW_TOTAL_NAME) == 0)          return RRD_DIMENSION_PCENT_OVER_ROW_TOTAL;
  96         if(strcmp(name, RRD_DIMENSION_PCENT_OVER_DIFF_TOTAL_NAME) == 0)         return RRD_DIMENSION_PCENT_OVER_DIFF_TOTAL;
  97         return RRD_DIMENSION_ABSOLUTE;
  98 }
  99
 100 const char *algorithm_name(int chart_type)
 101 {
 102         static char absolute[] = RRD_DIMENSION_ABSOLUTE_NAME;
 103         static char incremental[] = RRD_DIMENSION_INCREMENTAL_NAME;
 104         static char percentage_of_absolute_row[] = RRD_DIMENSION_PCENT_OVER_ROW_TOTAL_NAME;
 105         static char percentage_of_incremental_row[] = RRD_DIMENSION_PCENT_OVER_DIFF_TOTAL_NAME;
 106
 107         switch(chart_type) {
 108                 case RRD_DIMENSION_ABSOLUTE:
 109                         return absolute;
 110
 111                 case RRD_DIMENSION_INCREMENTAL:
 112                         return incremental;
 113
 114                 case RRD_DIMENSION_PCENT_OVER_ROW_TOTAL:
 115                         return percentage_of_absolute_row;
 116
 117                 case RRD_DIMENSION_PCENT_OVER_DIFF_TOTAL:
 118                         return percentage_of_incremental_row;
 119         }
 120         return absolute;
 121 }
 122
 123 RRD_STATS *root = NULL;
 124 pthread_rwlock_t root_rwlock = PTHREAD_RWLOCK_INITIALIZER;
 125
 126 char *rrd_stats_strncpy_name(char *to, const char *from, int length)
 127 {
 128         int i;
 129         for(i = 0; i < length && from[i] ;i++) {
 130                 if(from[i] == '.' || isalpha(from[i]) || isdigit(from[i])) to[i] = from[i];
 131                 else to[i] = '_';
 132         }
 133         if(i < length) to[i] = '\0';
 134         to[length - 1] = '\0';
 135
 136         return to;
 137 }
 138
 139 void rrd_stats_set_name(RRD_STATS *st, const char *name)
 140 {
 141         char b[CONFIG_MAX_VALUE + 1];
 142         char n[RRD_STATS_NAME_MAX + 1];
 143
 144         snprintf(n, RRD_STATS_NAME_MAX, "%s.%s", st->type, name);
 145         rrd_stats_strncpy_name(b, n, CONFIG_MAX_VALUE);
 146         st->name = config_get(st->id, "name", b);
 147         st->hash_name = simple_hash(st->name);
 148 }
 149
 150 char *rrd_stats_cache_dir(const char *id)
 151 {
 152         char *ret = NULL;
 153
 154         static char *cache_dir = NULL;
 155         if(!cache_dir) cache_dir = config_get("global", "database directory", "cache");
 156
 157         char b[FILENAME_MAX + 1];
 158         char n[FILENAME_MAX + 1];
 159         rrd_stats_strncpy_name(b, id, FILENAME_MAX);
 160
 161         snprintf(n, FILENAME_MAX, "%s/%s", cache_dir, b);
 162         ret = config_get(id, "database directory", n);
 163
 164         if(memory_mode == NETDATA_MEMORY_MODE_MAP || memory_mode == NETDATA_MEMORY_MODE_SAVE) {
 165                 int r = mkdir(ret, 0775);
 166                 if(r != 0 && errno != EEXIST)
 167                         error("Cannot create directory '%s'", ret);
 168         }
 169
 170         return ret;
 171 }
 172
 173 void rrd_stats_reset(RRD_STATS *st)
 174 {
 175         st->last_collected_time.tv_sec = 0;
 176         st->last_collected_time.tv_usec = 0;
 177         st->last_updated.tv_sec = 0;
 178         st->last_updated.tv_usec = 0;
 179         st->current_entry = 0;
 180         st->counter = 0;
 181         st->counter_done = 0;
 182
 183         RRD_DIMENSION *rd;
 184         for(rd = st->dimensions; rd ; rd = rd->next) {
 185                 rd->last_collected_time.tv_sec = 0;
 186                 rd->last_collected_time.tv_usec = 0;
 187                 bzero(rd->values, rd->entries * sizeof(storage_number));
 188         }
 189 }
 190
 191 RRD_STATS *rrd_stats_create(const char *type, const char *id, const char *name, const char *family, const char *title, const char *units, long priority, int update_every, int chart_type)
 192 {
 193         if(!id || !id[0]) {
 194                 fatal("Cannot create rrd stats without an id.");
 195                 return NULL;
 196         }
 197
 198         char fullid[RRD_STATS_NAME_MAX + 1];
 199         char fullfilename[FILENAME_MAX + 1];
 200         RRD_STATS *st = NULL;
 201
 202         snprintf(fullid, RRD_STATS_NAME_MAX, "%s.%s", type, id);
 203
 204         long entries = config_get_number(fullid, "history", save_history);
 205         if(entries < 5) entries = config_set_number(fullid, "history", 5);
 206         if(entries > HISTORY_MAX) entries = config_set_number(fullid, "history", HISTORY_MAX);
 207
 208         int enabled = config_get_boolean(fullid, "enabled", 1);
 209         if(!enabled) entries = 5;
 210
 211         unsigned long size = sizeof(RRD_STATS);
 212         char *cache_dir = rrd_stats_cache_dir(fullid);
 213
 214         debug(D_RRD_CALLS, "Creating RRD_STATS for '%s.%s'.", type, id);
 215
 216         snprintf(fullfilename, FILENAME_MAX, "%s/main.db", cache_dir);
 217         if(memory_mode != NETDATA_MEMORY_MODE_RAM) st = (RRD_STATS *)mymmap(fullfilename, size, ((memory_mode == NETDATA_MEMORY_MODE_MAP)?MAP_SHARED:MAP_PRIVATE));
 218         if(st) {
 219                 if(strcmp(st->magic, RRD_STATS_MAGIC) != 0) {
 220                         errno = 0;
 221                         info("Initializing file %s.", fullfilename);
 222                         bzero(st, size);
 223                 }
 224                 else if(strcmp(st->id, fullid) != 0) {
 225                         errno = 0;
 226                         error("File %s contents are not for chart %s. Clearing it.", fullfilename, fullid);
 227                         // munmap(st, size);
 228                         // st = NULL;
 229                         bzero(st, size);
 230                 }
 231                 else if(st->memsize != size || st->entries != entries) {
 232                         errno = 0;
 233                         error("File %s does not have the desired size. Clearing it.", fullfilename);
 234                         bzero(st, size);
 235                 }
 236                 else if(st->update_every != update_every) {
 237                         errno = 0;
 238                         error("File %s does not have the desired update frequency. Clearing it.", fullfilename);
 239                         bzero(st, size);
 240                 }
 241                 else if((time(NULL) - st->last_updated.tv_sec) > update_every * entries) {
 242                         errno = 0;
 243                         error("File %s is too old. Clearing it.", fullfilename);
 244                         bzero(st, size);
 245                 }
 246         }
 247
 248         if(st) {
 249                 st->name = NULL;
 250                 st->type = NULL;
 251                 st->family = NULL;
 252                 st->title = NULL;
 253                 st->units = NULL;
 254                 st->dimensions = NULL;
 255                 st->next = NULL;
 256                 st->mapped = memory_mode;
 257         }
 258         else {
 259                 st = calloc(1, size);
 260                 if(!st) {
 261                         fatal("Cannot allocate memory for RRD_STATS %s.%s", type, id);
 262                         return NULL;
 263                 }
 264                 st->mapped = NETDATA_MEMORY_MODE_RAM;
 265         }
 266         st->memsize = size;
 267         st->entries = entries;
 268         st->update_every = update_every;
 269
 270         strcpy(st->cache_file, fullfilename);
 271         strcpy(st->magic, RRD_STATS_MAGIC);
 272
 273         strcpy(st->id, fullid);
 274         st->hash = simple_hash(st->id);
 275
 276         st->cache_dir = cache_dir;
 277
 278         st->family     = config_get(st->id, "family", family?family:st->id);
 279         st->units      = config_get(st->id, "units", units?units:"");
 280         st->type       = config_get(st->id, "type", type);
 281         st->chart_type = chart_type_id(config_get(st->id, "chart type", chart_type_name(chart_type)));
 282
 283         if(name && *name) rrd_stats_set_name(st, name);
 284         else rrd_stats_set_name(st, id);
 285
 286         {
 287                 char varvalue[CONFIG_MAX_VALUE + 1];
 288                 snprintf(varvalue, CONFIG_MAX_VALUE, "%s (%s)", title?title:"", st->name);
 289                 st->title = config_get(st->id, "title", varvalue);
 290         }
 291
 292         st->priority = config_get_number(st->id, "priority", priority);
 293         st->enabled = enabled;
 294
 295         st->isdetail = 0;
 296         st->debug = 0;
 297
 298         st->last_collected_time.tv_sec = 0;
 299         st->last_collected_time.tv_usec = 0;
 300         st->counter_done = 0;
 301
 302         st->gap_when_lost_iterations = config_get_number(st->id, "gap when lost iterations above", DEFAULT_GAP_INTERPOLATIONS);
 303
 304         pthread_rwlock_init(&st->rwlock, NULL);
 305         pthread_rwlock_wrlock(&root_rwlock);
 306
 307         st->next = root;
 308         root = st;
 309
 310         pthread_rwlock_unlock(&root_rwlock);
 311
 312         return(st);
 313 }
 314
 315 RRD_DIMENSION *rrd_stats_dimension_add(RRD_STATS *st, const char *id, const char *name, long multiplier, long divisor, int algorithm)
 316 {
 317         char filename[FILENAME_MAX + 1];
 318         char fullfilename[FILENAME_MAX + 1];
 319
 320         char varname[CONFIG_MAX_NAME + 1];
 321         RRD_DIMENSION *rd = NULL;
 322         unsigned long size = sizeof(RRD_DIMENSION) + (st->entries * sizeof(storage_number));
 323
 324         debug(D_RRD_CALLS, "Adding dimension '%s/%s'.", st->id, id);
 325
 326         rrd_stats_strncpy_name(filename, id, FILENAME_MAX);
 327         snprintf(fullfilename, FILENAME_MAX, "%s/%s.db", st->cache_dir, filename);
 328         if(memory_mode != NETDATA_MEMORY_MODE_RAM) rd = (RRD_DIMENSION *)mymmap(fullfilename, size, ((memory_mode == NETDATA_MEMORY_MODE_MAP)?MAP_SHARED:MAP_PRIVATE));
 329         if(rd) {
 330                 struct timeval now;
 331                 gettimeofday(&now, NULL);
 332
 333                 if(strcmp(rd->magic, RRD_DIMENSION_MAGIC) != 0) {
 334                         errno = 0;
 335                         info("Initializing file %s.", fullfilename);
 336                         bzero(rd, size);
 337                 }
 338                 else if(rd->memsize != size) {
 339                         errno = 0;
 340                         error("File %s does not have the desired size. Clearing it.", fullfilename);
 341                         bzero(rd, size);
 342                 }
 343                 else if(rd->multiplier != multiplier) {
 344                         errno = 0;
 345                         error("File %s does not have the same multiplier. Clearing it.", fullfilename);
 346                         bzero(rd, size);
 347                 }
 348                 else if(rd->divisor != divisor) {
 349                         errno = 0;
 350                         error("File %s does not have the same divisor. Clearing it.", fullfilename);
 351                         bzero(rd, size);
 352                 }
 353                 else if(rd->algorithm != algorithm) {
 354                         errno = 0;
 355                         error("File %s does not have the same algorithm. Clearing it.", fullfilename);
 356                         bzero(rd, size);
 357                 }
 358                 else if(rd->update_every != st->update_every) {
 359                         errno = 0;
 360                         error("File %s does not have the same refresh frequency. Clearing it.", fullfilename);
 361                         bzero(rd, size);
 362                 }
 363                 else if(usecdiff(&now, &rd->last_collected_time) > (rd->entries * rd->update_every * 1000000ULL)) {
 364                         errno = 0;
 365                         error("File %s is too old. Clearing it.", fullfilename);
 366                         bzero(rd, size);
 367                 }
 368                 else if(strcmp(rd->id, id) != 0) {
 369                         errno = 0;
 370                         error("File %s contents are not for dimension %s. Clearing it.", fullfilename, id);
 371                         // munmap(rd, size);
 372                         // rd = NULL;
 373                         bzero(rd, size);
 374                 }
 375         }
 376
 377         if(rd) {
 378                 // we have a file mapped for rd
 379                 rd->mapped = memory_mode;
 380                 rd->hidden = 0;
 381                 rd->next = NULL;
 382                 rd->name = NULL;
 383         }
 384         else {
 385                 // if we didn't manage to get a mmap'd dimension, just create one
 386
 387                 rd = calloc(1, size);
 388                 if(!rd) {
 389                         fatal("Cannot allocate RRD_DIMENSION %s/%s.", st->id, id);
 390                         return NULL;
 391                 }
 392
 393                 rd->mapped = NETDATA_MEMORY_MODE_RAM;
 394         }
 395         rd->memsize = size;
 396
 397         strcpy(rd->magic, RRD_DIMENSION_MAGIC);
 398         strcpy(rd->cache_file, fullfilename);
 399         strncpy(rd->id, id, RRD_STATS_NAME_MAX);
 400         rd->hash = simple_hash(rd->id);
 401
 402         snprintf(varname, CONFIG_MAX_NAME, "dim %s name", rd->id);
 403         rd->name = config_get(st->id, varname, (name && *name)?name:rd->id);
 404
 405         snprintf(varname, CONFIG_MAX_NAME, "dim %s algorithm", rd->id);
 406         rd->algorithm = algorithm_id(config_get(st->id, varname, algorithm_name(algorithm)));
 407
 408         snprintf(varname, CONFIG_MAX_NAME, "dim %s multiplier", rd->id);
 409         rd->multiplier = config_get_number(st->id, varname, multiplier);
 410
 411         snprintf(varname, CONFIG_MAX_NAME, "dim %s divisor", rd->id);
 412         rd->divisor = config_get_number(st->id, varname, divisor);
 413         if(!rd->divisor) rd->divisor = 1;
 414
 415         rd->entries = st->entries;
 416         rd->update_every = st->update_every;
 417
 418         // append this dimension
 419         if(!st->dimensions)
 420                 st->dimensions = rd;
 421         else {
 422                 RRD_DIMENSION *td = st->dimensions;
 423                 for(; td->next; td = td->next) ;
 424                 td->next = rd;
 425         }
 426
 427         return(rd);
 428 }
 429
 430 void rrd_stats_dimension_set_name(RRD_STATS *st, RRD_DIMENSION *rd, const char *name)
 431 {
 432         char varname[CONFIG_MAX_NAME + 1];
 433         snprintf(varname, CONFIG_MAX_NAME, "dim %s name", rd->id);
 434         config_get(st->id, varname, name);
 435 }
 436
 437 void rrd_stats_dimension_free(RRD_DIMENSION *rd)
 438 {
 439         if(rd->next) rrd_stats_dimension_free(rd->next);
 440         // free(rd->annotations);
 441         if(rd->mapped == NETDATA_MEMORY_MODE_SAVE) {
 442                 debug(D_RRD_CALLS, "Saving dimension '%s' to '%s'.", rd->name, rd->cache_file);
 443                 savememory(rd->cache_file, rd, rd->memsize);
 444
 445                 debug(D_RRD_CALLS, "Unmapping dimension '%s'.", rd->name);
 446                 munmap(rd, rd->memsize);
 447         }
 448         else if(rd->mapped == NETDATA_MEMORY_MODE_MAP) {
 449                 debug(D_RRD_CALLS, "Unmapping dimension '%s'.", rd->name);
 450                 munmap(rd, rd->memsize);
 451         }
 452         else {
 453                 debug(D_RRD_CALLS, "Removing dimension '%s'.", rd->name);
 454                 free(rd);
 455         }
 456 }
 457
 458 void rrd_stats_free_all(void)
 459 {
 460         info("Freeing all memory...");
 461
 462         RRD_STATS *st;
 463         for(st = root; st ;) {
 464                 RRD_STATS *next = st->next;
 465
 466                 if(st->dimensions) rrd_stats_dimension_free(st->dimensions);
 467                 st->dimensions = NULL;
 468
 469                 if(st->mapped == NETDATA_MEMORY_MODE_SAVE) {
 470                         debug(D_RRD_CALLS, "Saving stats '%s' to '%s'.", st->name, st->cache_file);
 471                         savememory(st->cache_file, st, st->memsize);
 472
 473                         debug(D_RRD_CALLS, "Unmapping stats '%s'.", st->name);
 474                         munmap(st, st->memsize);
 475                 }
 476                 else if(st->mapped == NETDATA_MEMORY_MODE_MAP) {
 477                         debug(D_RRD_CALLS, "Unmapping stats '%s'.", st->name);
 478                         munmap(st, st->memsize);
 479                 }
 480                 else
 481                         free(st);
 482
 483                 st = next;
 484         }
 485         root = NULL;
 486
 487         info("Memory cleanup completed...");
 488 }
 489
 490 void rrd_stats_save_all(void)
 491 {
 492         RRD_STATS *st;
 493         RRD_DIMENSION *rd;
 494
 495         pthread_rwlock_wrlock(&root_rwlock);
 496         for(st = root; st ; st = st->next) {
 497                 pthread_rwlock_wrlock(&st->rwlock);
 498
 499                 if(st->mapped == NETDATA_MEMORY_MODE_SAVE) {
 500                         debug(D_RRD_CALLS, "Saving stats '%s' to '%s'.", st->name, st->cache_file);
 501                         savememory(st->cache_file, st, st->memsize);
 502                 }
 503
 504                 for(rd = st->dimensions; rd ; rd = rd->next) {
 505                         if(rd->mapped == NETDATA_MEMORY_MODE_SAVE) {
 506                                 debug(D_RRD_CALLS, "Saving dimension '%s' to '%s'.", rd->name, rd->cache_file);
 507                                 savememory(rd->cache_file, rd, rd->memsize);
 508                         }
 509                 }
 510
 511                 pthread_rwlock_unlock(&st->rwlock);
 512         }
 513         pthread_rwlock_unlock(&root_rwlock);
 514 }
 515
 516
 517 RRD_STATS *rrd_stats_find(const char *id)
 518 {
 519         debug(D_RRD_CALLS, "rrd_stats_find() for chart %s", id);
 520
 521         unsigned long hash = simple_hash(id);
 522
 523         pthread_rwlock_rdlock(&root_rwlock);
 524         RRD_STATS *st = root;
 525         for ( ; st ; st = st->next )
 526                 if(hash == st->hash)
 527                         if(strcmp(st->id, id) == 0)
 528                                 break;
 529         pthread_rwlock_unlock(&root_rwlock);
 530
 531         return(st);
 532 }
 533
 534 RRD_STATS *rrd_stats_find_bytype(const char *type, const char *id)
 535 {
 536         debug(D_RRD_CALLS, "rrd_stats_find_bytype() for chart %s.%s", type, id);
 537
 538         char buf[RRD_STATS_NAME_MAX + 1];
 539
 540         strncpy(buf, type, RRD_STATS_NAME_MAX - 1);
 541         buf[RRD_STATS_NAME_MAX - 1] = '\0';
 542         strcat(buf, ".");
 543         int len = strlen(buf);
 544         strncpy(&buf[len], id, RRD_STATS_NAME_MAX - len);
 545         buf[RRD_STATS_NAME_MAX] = '\0';
 546
 547         return(rrd_stats_find(buf));
 548 }
 549
 550 RRD_STATS *rrd_stats_find_byname(const char *name)
 551 {
 552         debug(D_RRD_CALLS, "rrd_stats_find_byname() for chart %s", name);
 553
 554         char b[CONFIG_MAX_VALUE + 1];
 555
 556         rrd_stats_strncpy_name(b, name, CONFIG_MAX_VALUE);
 557         unsigned long hash = simple_hash(b);
 558
 559         pthread_rwlock_rdlock(&root_rwlock);
 560         RRD_STATS *st = root;
 561         for ( ; st ; st = st->next ) {
 562                 if(hash == st->hash_name && strcmp(st->name, b) == 0) break;
 563         }
 564         pthread_rwlock_unlock(&root_rwlock);
 565
 566         return(st);
 567 }
 568
 569 RRD_DIMENSION *rrd_stats_dimension_find(RRD_STATS *st, const char *id)
 570 {
 571         debug(D_RRD_CALLS, "rrd_stats_dimension_find() for chart %s, dimension %s", st->name, id);
 572
 573         unsigned long hash = simple_hash(id);
 574
 575         RRD_DIMENSION *rd = st->dimensions;
 576
 577         for ( ; rd ; rd = rd->next )
 578                 if(hash == rd->hash)
 579                         if(strcmp(rd->id, id) == 0)
 580                                 break;
 581
 582         return(rd);
 583 }
 584
 585 int rrd_stats_dimension_hide(RRD_STATS *st, const char *id)
 586 {
 587         debug(D_RRD_CALLS, "rrd_stats_dimension_hide() for chart %s, dimension %s", st->name, id);
 588
 589         RRD_DIMENSION *rd = rrd_stats_dimension_find(st, id);
 590         if(!rd) {
 591                 error("Cannot find dimension with id '%s' on stats '%s' (%s).", id, st->name, st->id);
 592                 return 1;
 593         }
 594
 595         rd->hidden = 1;
 596         return 0;
 597 }
 598
 599 void rrd_stats_dimension_set_by_pointer(RRD_STATS *st, RRD_DIMENSION *rd, collected_number value)
 600 {
 601         debug(D_RRD_CALLS, "rrd_stats_dimension_set() for chart %s, dimension %s, value " COLLECTED_NUMBER_FORMAT, st->name, rd->name, value);
 602
 603         gettimeofday(&rd->last_collected_time, NULL);
 604         rd->collected_value = value;
 605         rd->updated = 1;
 606 }
 607
 608 int rrd_stats_dimension_set(RRD_STATS *st, const char *id, collected_number value)
 609 {
 610         RRD_DIMENSION *rd = rrd_stats_dimension_find(st, id);
 611         if(!rd) {
 612                 error("Cannot find dimension with id '%s' on stats '%s' (%s).", id, st->name, st->id);
 613                 return 1;
 614         }
 615
 616         rrd_stats_dimension_set_by_pointer(st, rd, value);
 617         return 0;
 618 }
 619
 620 void rrd_stats_next_usec(RRD_STATS *st, unsigned long long microseconds)
 621 {
 622         debug(D_RRD_CALLS, "rrd_stats_next() for chart %s with microseconds %llu", st->name, microseconds);
 623
 624         if(st->debug) debug(D_RRD_STATS, "%s: NEXT: %llu microseconds", st->name, microseconds);
 625         st->usec_since_last_update = microseconds;
 626 }
 627
 628 void rrd_stats_next(RRD_STATS *st)
 629 {
 630         unsigned long long microseconds = 0;
 631
 632         if(st->last_collected_time.tv_sec) {
 633                 struct timeval now;
 634                 gettimeofday(&now, NULL);
 635                 microseconds = usecdiff(&now, &st->last_collected_time);
 636         }
 637
 638         rrd_stats_next_usec(st, microseconds);
 639 }
 640
 641 void rrd_stats_next_plugins(RRD_STATS *st)
 642 {
 643         rrd_stats_next(st);
 644 }
 645
 646 unsigned long long rrd_stats_done(RRD_STATS *st)
 647 {
 648         debug(D_RRD_CALLS, "rrd_stats_done() for chart %s", st->name);
 649
 650         RRD_DIMENSION *rd, *last;
 651         int oldstate, store_this_entry = 1;
 652
 653         if(pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, &oldstate) != 0)
 654                 error("Cannot set pthread cancel state to DISABLE.");
 655
 656         // a read lock is OK here
 657         pthread_rwlock_rdlock(&st->rwlock);
 658
 659         // check if the chart has a long time to be refreshed
 660         if(st->usec_since_last_update > st->entries * st->update_every * 1000000ULL) {
 661                 info("%s: took too long to be updated (%0.3Lf secs). Reseting it.", st->name, (long double)(st->usec_since_last_update / 1000000.0));
 662                 rrd_stats_reset(st);
 663                 st->usec_since_last_update = st->update_every * 1000000ULL;
 664         }
 665         if(st->debug) debug(D_RRD_STATS, "%s: microseconds since last update: %llu", st->name, st->usec_since_last_update);
 666
 667         // set last_collected_time
 668         if(!st->last_collected_time.tv_sec) {
 669                 // it is the first entry
 670                 // set the last_collected_time to now
 671                 gettimeofday(&st->last_collected_time, NULL);
 672
 673                 // the first entry should not be stored
 674                 store_this_entry = 0;
 675
 676                 if(st->debug) debug(D_RRD_STATS, "%s: initializing last_collected to now. Will not store the next entry.", st->name);
 677         }
 678         else {
 679                 // it is not the first entry
 680                 // calculate the proper last_collected_time, using usec_since_last_update
 681                 unsigned long long ut = st->last_collected_time.tv_sec * 1000000ULL + st->last_collected_time.tv_usec + st->usec_since_last_update;
 682                 st->last_collected_time.tv_sec = ut / 1000000ULL;
 683                 st->last_collected_time.tv_usec = ut % 1000000ULL;
 684         }
 685
 686         // if this set has not been updated in the past
 687         // we fake the last_update time to be = now - usec_since_last_update
 688         if(!st->last_updated.tv_sec) {
 689                 // it has never been updated before
 690                 // set a fake last_updated, in the past using usec_since_last_update
 691                 unsigned long long ut = st->last_collected_time.tv_sec * 1000000ULL + st->last_collected_time.tv_usec - st->usec_since_last_update;
 692                 st->last_updated.tv_sec = ut / 1000000ULL;
 693                 st->last_updated.tv_usec = ut % 1000000ULL;
 694
 695                 // the first entry should not be stored
 696                 store_this_entry = 0;
 697
 698                 if(st->debug) debug(D_RRD_STATS, "%s: initializing last_updated to now - %llu microseconds (%0.3Lf). Will not store the next entry.", st->name, st->usec_since_last_update, (long double)ut/1000000.0);
 699         }
 700
 701         // check if we will re-write the entire data set
 702         if(usecdiff(&st->last_collected_time, &st->last_updated) > st->update_every * st->entries * 1000000ULL) {
 703                 info("%s: too old data (last updated at %u.%u, last collected at %u.%u). Reseting it. Will not store the next entry.", st->name, st->last_updated.tv_sec, st->last_updated.tv_usec, st->last_collected_time.tv_sec, st->last_collected_time.tv_usec);
 704                 rrd_stats_reset(st);
 705
 706                 st->usec_since_last_update = st->update_every * 1000000ULL;
 707
 708                 gettimeofday(&st->last_collected_time, NULL);
 709
 710                 unsigned long long ut = st->last_collected_time.tv_sec * 1000000ULL + st->last_collected_time.tv_usec - st->usec_since_last_update;
 711                 st->last_updated.tv_sec = ut / 1000000ULL;
 712                 st->last_updated.tv_usec = ut % 1000000ULL;
 713
 714                 // the first entry should not be stored
 715                 store_this_entry = 0;
 716         }
 717
 718         // these are the 3 variables that will help us in interpolation
 719         // last_ut = the last time we added a value to the storage
 720         //  now_ut = the time the current value is taken at
 721         // next_ut = the time of the next interpolation point
 722         unsigned long long last_ut = st->last_updated.tv_sec * 1000000ULL + st->last_updated.tv_usec;
 723         unsigned long long now_ut  = st->last_collected_time.tv_sec * 1000000ULL + st->last_collected_time.tv_usec;
 724         unsigned long long next_ut = (st->last_updated.tv_sec + st->update_every) * 1000000ULL;
 725
 726         if(st->debug) debug(D_RRD_STATS, "%s: last ut = %0.3Lf (last updated time)", st->name, (long double)last_ut/1000000.0);
 727         if(st->debug) debug(D_RRD_STATS, "%s: now  ut = %0.3Lf (current update time)", st->name, (long double)now_ut/1000000.0);
 728         if(st->debug) debug(D_RRD_STATS, "%s: next ut = %0.3Lf (next interpolation point)", st->name, (long double)next_ut/1000000.0);
 729
 730         if(!st->counter_done) {
 731                 store_this_entry = 0;
 732                 if(st->debug) debug(D_RRD_STATS, "%s: Will not store the next entry.", st->name);
 733         }
 734         st->counter_done++;
 735
 736         // calculate totals and count the dimensions
 737         int dimensions;
 738         st->collected_total = 0;
 739         for( rd = st->dimensions, dimensions = 0 ; rd ; rd = rd->next, dimensions++ )
 740                 st->collected_total += rd->collected_value;
 741
 742         uint32_t storage_flags = SN_EXISTS;
 743
 744         // process all dimensions to calculate their values
 745         // based on the collected figures only
 746         // at this stage we do not interpolate anything
 747         for( rd = st->dimensions ; rd ; rd = rd->next ) {
 748
 749                 if(st->debug) debug(D_RRD_STATS, "%s/%s: "
 750                         " last_collected_value = " COLLECTED_NUMBER_FORMAT
 751                         " collected_value = " COLLECTED_NUMBER_FORMAT
 752                         " last_calculated_value = " CALCULATED_NUMBER_FORMAT
 753                         " calculated_value = " CALCULATED_NUMBER_FORMAT
 754                         , st->id, rd->name
 755                         , rd->last_collected_value
 756                         , rd->collected_value
 757                         , rd->last_calculated_value
 758                         , rd->calculated_value
 759                         );
 760
 761                 switch(rd->algorithm) {
 762                         case RRD_DIMENSION_PCENT_OVER_DIFF_TOTAL:
 763                                 // the percentage of the current increment
 764                                 // over the increment of all dimensions together
 765                                 if(st->collected_total == st->last_collected_total) rd->calculated_value = rd->last_calculated_value;
 766                                 else rd->calculated_value =
 767                                           (calculated_number)100
 768                                         * (calculated_number)(rd->collected_value - rd->last_collected_value)
 769                                         / (calculated_number)(st->collected_total  - st->last_collected_total);
 770
 771                                 if(st->debug)
 772                                         debug(D_RRD_STATS, "%s/%s: CALC PCENT-DIFF "
 773                                                 CALCULATED_NUMBER_FORMAT " = 100"
 774                                                 " * (" COLLECTED_NUMBER_FORMAT " - " COLLECTED_NUMBER_FORMAT ")"
 775                                                 " / (" COLLECTED_NUMBER_FORMAT " - " COLLECTED_NUMBER_FORMAT ")"
 776                                                 , st->id, rd->name
 777                                                 , rd->calculated_value
 778                                                 , rd->collected_value, rd->last_collected_value
 779                                                 , st->collected_total, st->last_collected_total
 780                                                 );
 781                                 break;
 782
 783                         case RRD_DIMENSION_PCENT_OVER_ROW_TOTAL:
 784                                 if(!st->collected_total) rd->calculated_value = 0;
 785                                 else
 786                                 // the percentage of the current value
 787                                 // over the total of all dimensions
 788                                 rd->calculated_value =
 789                                           (calculated_number)100
 790                                         * (calculated_number)rd->collected_value
 791                                         / (calculated_number)st->collected_total;
 792
 793                                 if(st->debug)
 794                                         debug(D_RRD_STATS, "%s/%s: CALC PCENT-ROW "
 795                                                 CALCULATED_NUMBER_FORMAT " = 100"
 796                                                 " * " COLLECTED_NUMBER_FORMAT
 797                                                 " / " COLLECTED_NUMBER_FORMAT
 798                                                 , st->id, rd->name
 799                                                 , rd->calculated_value
 800                                                 , rd->collected_value
 801                                                 , st->collected_total
 802                                                 );
 803                                 break;
 804
 805                         case RRD_DIMENSION_INCREMENTAL:
 806                                 // if the new is smaller than the old (an overflow, or reset), set the old equal to the new
 807                                 // to reset the calculation (it will give zero as the calculation for this second)
 808                                 if(rd->last_collected_value > rd->collected_value) {
 809                                         storage_flags = SN_EXISTS_RESET;
 810                                         rd->last_collected_value = rd->collected_value;
 811                                 }
 812
 813                                 rd->calculated_value += (calculated_number)(rd->collected_value - rd->last_collected_value);
 814
 815                                 if(st->debug)
 816                                         debug(D_RRD_STATS, "%s/%s: CALC INC "
 817                                                 CALCULATED_NUMBER_FORMAT " += "
 818                                                 COLLECTED_NUMBER_FORMAT " - " COLLECTED_NUMBER_FORMAT
 819                                                 , st->id, rd->name
 820                                                 , rd->calculated_value
 821                                                 , rd->collected_value, rd->last_collected_value
 822                                                 );
 823                                 break;
 824
 825                         case RRD_DIMENSION_ABSOLUTE:
 826                                 rd->calculated_value = (calculated_number)rd->collected_value;
 827
 828                                 if(st->debug)
 829                                         debug(D_RRD_STATS, "%s/%s: CALC ABS/ABS-NO-IN "
 830                                                 CALCULATED_NUMBER_FORMAT " = "
 831                                                 COLLECTED_NUMBER_FORMAT
 832                                                 , st->id, rd->name
 833                                                 , rd->calculated_value
 834                                                 , rd->collected_value
 835                                                 );
 836                                 break;
 837
 838                         default:
 839                                 // make the default zero, to make sure
 840                                 // it gets noticed when we add new types
 841                                 rd->calculated_value = 0;
 842
 843                                 if(st->debug)
 844                                         debug(D_RRD_STATS, "%s/%s: CALC "
 845                                                 CALCULATED_NUMBER_FORMAT " = 0"
 846                                                 , st->id, rd->name
 847                                                 , rd->calculated_value
 848                                                 );
 849                                 break;
 850                 }
 851         }
 852
 853         // at this point we have all the calculated values ready
 854         // it is now time to interpolate values on a second boundary
 855
 856         unsigned long long first_ut = last_ut;
 857         int iterations = (now_ut - last_ut) / (st->update_every * 1000000ULL);
 858
 859         for( ; next_ut <= now_ut ; next_ut += st->update_every * 1000000ULL, iterations-- ) {
 860                 if(iterations < 0) error("iterations calculation wrapped!");
 861
 862                 if(st->debug) debug(D_RRD_STATS, "%s: last ut = %0.3Lf (last updated time)", st->name, (long double)last_ut/1000000.0);
 863                 if(st->debug) debug(D_RRD_STATS, "%s: next ut = %0.3Lf (next interpolation point)", st->name, (long double)next_ut/1000000.0);
 864
 865                 st->last_updated.tv_sec = next_ut / 1000000ULL;
 866                 st->last_updated.tv_usec = 0;
 867
 868                 for( rd = st->dimensions ; rd ; rd = rd->next ) {
 869                         calculated_number new_value;
 870
 871                         switch(rd->algorithm) {
 872                                 case RRD_DIMENSION_INCREMENTAL:
 873                                         new_value = (calculated_number)
 874                                                 (          rd->calculated_value
 875                                                         * (calculated_number)(next_ut - last_ut)
 876                                                         / (calculated_number)(now_ut - last_ut)
 877                                                 );
 878
 879                                         if(st->debug)
 880                                                 debug(D_RRD_STATS, "%s/%s: CALC2 INC "
 881                                                         CALCULATED_NUMBER_FORMAT " = "
 882                                                         CALCULATED_NUMBER_FORMAT
 883                                                         " * %llu"
 884                                                         " / %llu"
 885                                                         , st->id, rd->name
 886                                                         , new_value
 887                                                         , rd->calculated_value
 888                                                         , (unsigned long long)(next_ut - last_ut)
 889                                                         , (unsigned long long)(now_ut - last_ut)
 890                                                         );
 891
 892                                         rd->calculated_value -= new_value;
 893                                         break;
 894
 895                                 case RRD_DIMENSION_ABSOLUTE:
 896                                 case RRD_DIMENSION_PCENT_OVER_ROW_TOTAL:
 897                                 case RRD_DIMENSION_PCENT_OVER_DIFF_TOTAL:
 898                                 default:
 899                                         new_value = (calculated_number)
 900                                                 (       (         (rd->calculated_value - rd->last_calculated_value)
 901                                                                 * (calculated_number)(next_ut - first_ut)
 902                                                                 / (calculated_number)(now_ut - first_ut)
 903                                                         )
 904                                                         +  rd->last_calculated_value
 905                                                 );
 906
 907                                         if(st->debug)
 908                                                 debug(D_RRD_STATS, "%s/%s: CALC2 DEF "
 909                                                         CALCULATED_NUMBER_FORMAT " = ((("
 910                                                         "(" CALCULATED_NUMBER_FORMAT " - " CALCULATED_NUMBER_FORMAT ")"
 911                                                         " * %llu"
 912                                                         " / %llu) + " CALCULATED_NUMBER_FORMAT
 913                                                         , st->id, rd->name
 914                                                         , new_value
 915                                                         , rd->calculated_value, rd->last_calculated_value
 916                                                         , (next_ut - first_ut)
 917                                                         , (now_ut - first_ut), rd->last_calculated_value
 918                                                         );
 919
 920                                         if(next_ut + st->update_every * 1000000ULL > now_ut) rd->calculated_value = new_value;
 921                                         break;
 922                         }
 923
 924                         if(!store_this_entry) {
 925                                 store_this_entry = 1;
 926                                 continue;
 927                         }
 928
 929                         if(rd->updated && iterations < st->gap_when_lost_iterations) {
 930                                 rd->values[st->current_entry] = pack_storage_number(
 931                                                   new_value
 932                                                 * (calculated_number)rd->multiplier
 933                                                 / (calculated_number)rd->divisor
 934                                         , storage_flags );
 935
 936                                 if(st->debug)
 937                                         debug(D_RRD_STATS, "%s/%s: STORE[%ld] "
 938                                                 CALCULATED_NUMBER_FORMAT " = " CALCULATED_NUMBER_FORMAT
 939                                                 " * %ld"
 940                                                 " / %ld"
 941                                                 , st->id, rd->name
 942                                                 , st->current_entry
 943                                                 , unpack_storage_number(rd->values[st->current_entry]), new_value
 944                                                 , rd->multiplier
 945                                                 , rd->divisor
 946                                                 );
 947                         }
 948                         else {
 949                                 if(st->debug) debug(D_RRD_STATS, "%s/%s: STORE[%ld] = NON EXISTING "
 950                                                 , st->id, rd->name
 951                                                 , st->current_entry
 952                                                 );
 953                                 rd->values[st->current_entry] = pack_storage_number(0, SN_NOT_EXISTS);
 954                         }
 955
 956                         if(st->debug) {
 957                                 calculated_number t1 = new_value * (calculated_number)rd->multiplier / (calculated_number)rd->divisor;
 958                                 calculated_number t2 = unpack_storage_number(rd->values[st->current_entry]);
 959                                 calculated_number accuracy = accuracy_loss(t1, t2);
 960                                 debug(D_RRD_STATS, "%s/%s: UNPACK[%ld] = " CALCULATED_NUMBER_FORMAT " FLAGS=0x%08x (original = " CALCULATED_NUMBER_FORMAT ", accuracy loss = " CALCULATED_NUMBER_FORMAT "%%%s)"
 961                                                 , st->id, rd->name
 962                                                 , st->current_entry
 963                                                 , t2
 964                                                 , get_storage_number_flags(rd->values[st->current_entry])
 965                                                 , t1
 966                                                 , accuracy
 967                                                 , (accuracy > ACCURACY_LOSS) ? " **TOO BIG** " : ""
 968                                                 );
 969
 970                                 rd->collected_volume += t1;
 971                                 rd->stored_volume += t2;
 972                                 accuracy = accuracy_loss(rd->collected_volume, rd->stored_volume);
 973                                 debug(D_RRD_STATS, "%s/%s: VOLUME[%ld] = " CALCULATED_NUMBER_FORMAT ", calculated  = " CALCULATED_NUMBER_FORMAT ", accuracy loss = " CALCULATED_NUMBER_FORMAT "%%%s"
 974                                                 , st->id, rd->name
 975                                                 , st->current_entry
 976                                                 , rd->stored_volume
 977                                                 , rd->collected_volume
 978                                                 , accuracy
 979                                                 , (accuracy > ACCURACY_LOSS) ? " **TOO BIG** " : ""
 980                                                 );
 981
 982                         }
 983                 }
 984                 // reset the storage flags for the next point, if any;
 985                 storage_flags = SN_EXISTS;
 986
 987                 if(st->first_entry_t && st->counter >= (unsigned long long)st->entries) {
 988                         // the db is overwriting values
 989                         // add the value we will overwrite
 990                         st->first_entry_t += st->update_every * 1000000ULL;
 991                 }
 992
 993                 st->counter++;
 994                 st->current_entry = ((st->current_entry + 1) >= st->entries) ? 0 : st->current_entry + 1;
 995                 if(!st->first_entry_t) st->first_entry_t = next_ut;
 996                 last_ut = next_ut;
 997         }
 998
 999         for( rd = st->dimensions; rd ; rd = rd->next ) {
1000                 if(!rd->updated) continue;
1001                 rd->last_collected_value = rd->collected_value;
1002                 rd->last_calculated_value = rd->calculated_value;
1003                 rd->collected_value = 0;
1004                 rd->updated = 0;
1005
1006                 // if this is the first entry of incremental dimensions
1007                 // we have to set the first calculated_value to zero
1008                 // to eliminate the first spike
1009                 if(st->counter_done == 1) switch(rd->algorithm) {
1010                         case RRD_DIMENSION_PCENT_OVER_DIFF_TOTAL:
1011                         case RRD_DIMENSION_INCREMENTAL:
1012                                 rd->calculated_value = 0;
1013                                 // the next time, a new incremental total will be calculated
1014                                 break;
1015                 }
1016         }
1017         st->last_collected_total  = st->collected_total;
1018
1019         // ALL DONE ABOUT THE DATA UPDATE
1020         // --------------------------------------------------------------------
1021
1022         // find if there are any obsolete dimensions (not updated recently)
1023         for( rd = st->dimensions; rd ; rd = rd->next )
1024                 if((rd->last_collected_time.tv_sec + (10 * st->update_every)) < st->last_collected_time.tv_sec)
1025                         break;
1026
1027         if(rd) {
1028                 // there is dimension to free
1029                 // upgrade our read lock to a write lock
1030                 pthread_rwlock_unlock(&st->rwlock);
1031                 pthread_rwlock_wrlock(&st->rwlock);
1032
1033                 for( rd = st->dimensions, last = NULL ; rd ; ) {
1034                         if((rd->last_collected_time.tv_sec + (10 * st->update_every)) < st->last_collected_time.tv_sec) { // remove it only it is not updated in 10 seconds
1035                                 debug(D_RRD_STATS, "Removing obsolete dimension '%s' (%s) of '%s' (%s).", rd->name, rd->id, st->name, st->id);
1036
1037                                 if(!last) {
1038                                         st->dimensions = rd->next;
1039                                         rd->next = NULL;
1040                                         rrd_stats_dimension_free(rd);
1041                                         rd = st->dimensions;
1042                                         continue;
1043                                 }
1044                                 else {
1045                                         last->next = rd->next;
1046                                         rd->next = NULL;
1047                                         rrd_stats_dimension_free(rd);
1048                                         rd = last->next;
1049                                         continue;
1050                                 }
1051                         }
1052
1053                         last = rd;
1054                         rd = rd->next;
1055                 }
1056
1057                 if(!st->dimensions) st->enabled = 0;
1058         }
1059
1060         pthread_rwlock_unlock(&st->rwlock);
1061
1062         if(pthread_setcancelstate(oldstate, NULL) != 0)
1063                 error("Cannot set pthread cancel state to RESTORE (%d).", oldstate);
1064
1065         return(st->usec_since_last_update);
1066 }
1067
1068
1069 // find the oldest entry in the data, skipping all empty slots
1070 time_t rrd_stats_first_entry_t(RRD_STATS *st)
1071 {
1072         if(!st->first_entry_t) return st->last_updated.tv_sec;
1073
1074         return st->first_entry_t / 1000000;
1075 }