src/rrd.c

   1 #include <string.h>
   2 #include <unistd.h>
   3 #include <time.h>
   4 #include <sys/time.h>
   5 #include <sys/mman.h>
   6 #include <pthread.h>
   7 #include <errno.h>
   8 #include <ctype.h>
   9 #include <sys/stat.h>
  10 #include <sys/types.h>
  11 #include <stdlib.h>
  12
  13 #include "log.h"
  14 #include "config.h"
  15 #include "common.h"
  16
  17 #include "rrd.h"
  18
  19 int update_every = UPDATE_EVERY;
  20 int save_history = HISTORY;
  21
  22 // ----------------------------------------------------------------------------
  23 // chart types
  24
  25 int chart_type_id(const char *name)
  26 {
  27         if(strcmp(name, CHART_TYPE_AREA_NAME) == 0) return CHART_TYPE_AREA;
  28         if(strcmp(name, CHART_TYPE_STACKED_NAME) == 0) return CHART_TYPE_STACKED;
  29         if(strcmp(name, CHART_TYPE_LINE_NAME) == 0) return CHART_TYPE_LINE;
  30         return CHART_TYPE_LINE;
  31 }
  32
  33 const char *chart_type_name(int chart_type)
  34 {
  35         static char line[] = CHART_TYPE_LINE_NAME;
  36         static char area[] = CHART_TYPE_AREA_NAME;
  37         static char stacked[] = CHART_TYPE_STACKED_NAME;
  38
  39         switch(chart_type) {
  40                 case CHART_TYPE_LINE:
  41                         return line;
  42
  43                 case CHART_TYPE_AREA:
  44                         return area;
  45
  46                 case CHART_TYPE_STACKED:
  47                         return stacked;
  48         }
  49         return line;
  50 }
  51
  52 // ----------------------------------------------------------------------------
  53 // mmap() wrapper
  54
  55 int memory_mode = NETDATA_MEMORY_MODE_SAVE;
  56
  57 const char *memory_mode_name(int id)
  58 {
  59         static const char ram[] = NETDATA_MEMORY_MODE_RAM_NAME;
  60         static const char map[] = NETDATA_MEMORY_MODE_MAP_NAME;
  61         static const char save[] = NETDATA_MEMORY_MODE_SAVE_NAME;
  62
  63         switch(id) {
  64                 case NETDATA_MEMORY_MODE_RAM:
  65                         return ram;
  66
  67                 case NETDATA_MEMORY_MODE_MAP:
  68                         return map;
  69
  70                 case NETDATA_MEMORY_MODE_SAVE:
  71                 default:
  72                         return save;
  73         }
  74
  75         return save;
  76 }
  77
  78 int memory_mode_id(const char *name)
  79 {
  80         if(!strcmp(name, NETDATA_MEMORY_MODE_RAM_NAME))
  81                 return NETDATA_MEMORY_MODE_RAM;
  82         else if(!strcmp(name, NETDATA_MEMORY_MODE_MAP_NAME))
  83                 return NETDATA_MEMORY_MODE_MAP;
  84
  85         return NETDATA_MEMORY_MODE_SAVE;
  86 }
  87
  88 // ----------------------------------------------------------------------------
  89 // algorithms types
  90
  91 int algorithm_id(const char *name)
  92 {
  93         if(strcmp(name, RRD_DIMENSION_ABSOLUTE_NAME) == 0)                      return RRD_DIMENSION_ABSOLUTE;
  94         if(strcmp(name, RRD_DIMENSION_INCREMENTAL_NAME) == 0)                   return RRD_DIMENSION_INCREMENTAL;
  95         if(strcmp(name, RRD_DIMENSION_PCENT_OVER_ROW_TOTAL_NAME) == 0)          return RRD_DIMENSION_PCENT_OVER_ROW_TOTAL;
  96         if(strcmp(name, RRD_DIMENSION_PCENT_OVER_DIFF_TOTAL_NAME) == 0)         return RRD_DIMENSION_PCENT_OVER_DIFF_TOTAL;
  97         return RRD_DIMENSION_ABSOLUTE;
  98 }
  99
 100 const char *algorithm_name(int chart_type)
 101 {
 102         static char absolute[] = RRD_DIMENSION_ABSOLUTE_NAME;
 103         static char incremental[] = RRD_DIMENSION_INCREMENTAL_NAME;
 104         static char percentage_of_absolute_row[] = RRD_DIMENSION_PCENT_OVER_ROW_TOTAL_NAME;
 105         static char percentage_of_incremental_row[] = RRD_DIMENSION_PCENT_OVER_DIFF_TOTAL_NAME;
 106
 107         switch(chart_type) {
 108                 case RRD_DIMENSION_ABSOLUTE:
 109                         return absolute;
 110
 111                 case RRD_DIMENSION_INCREMENTAL:
 112                         return incremental;
 113
 114                 case RRD_DIMENSION_PCENT_OVER_ROW_TOTAL:
 115                         return percentage_of_absolute_row;
 116
 117                 case RRD_DIMENSION_PCENT_OVER_DIFF_TOTAL:
 118                         return percentage_of_incremental_row;
 119         }
 120         return absolute;
 121 }
 122
 123 RRD_STATS *root = NULL;
 124 pthread_rwlock_t root_rwlock = PTHREAD_RWLOCK_INITIALIZER;
 125
 126 char *rrd_stats_strncpy_name(char *to, const char *from, int length)
 127 {
 128         int i;
 129         for(i = 0; i < length && from[i] ;i++) {
 130                 if(from[i] == '.' || isalpha(from[i]) || isdigit(from[i])) to[i] = from[i];
 131                 else to[i] = '_';
 132         }
 133         if(i < length) to[i] = '\0';
 134         to[length - 1] = '\0';
 135
 136         return to;
 137 }
 138
 139 void rrd_stats_set_name(RRD_STATS *st, const char *name)
 140 {
 141         char b[CONFIG_MAX_VALUE + 1];
 142         char n[RRD_STATS_NAME_MAX + 1];
 143
 144         snprintf(n, RRD_STATS_NAME_MAX, "%s.%s", st->type, name);
 145         rrd_stats_strncpy_name(b, n, CONFIG_MAX_VALUE);
 146         st->name = config_get(st->id, "name", b);
 147         st->hash_name = simple_hash(st->name);
 148 }
 149
 150 char *rrd_stats_cache_dir(const char *id)
 151 {
 152         char *ret = NULL;
 153
 154         static char *cache_dir = NULL;
 155         if(!cache_dir) cache_dir = config_get("global", "database directory", "cache");
 156
 157         char b[FILENAME_MAX + 1];
 158         char n[FILENAME_MAX + 1];
 159         rrd_stats_strncpy_name(b, id, FILENAME_MAX);
 160
 161         snprintf(n, FILENAME_MAX, "%s/%s", cache_dir, b);
 162         ret = config_get(id, "database directory", n);
 163
 164         if(memory_mode == NETDATA_MEMORY_MODE_MAP || memory_mode == NETDATA_MEMORY_MODE_SAVE) {
 165                 int r = mkdir(ret, 0775);
 166                 if(r != 0 && errno != EEXIST)
 167                         error("Cannot create directory '%s'", ret);
 168         }
 169
 170         return ret;
 171 }
 172
 173 void rrd_stats_reset(RRD_STATS *st)
 174 {
 175         st->last_collected_time.tv_sec = 0;
 176         st->last_collected_time.tv_usec = 0;
 177         st->current_entry = 0;
 178         st->counter_done = 0;
 179
 180         RRD_DIMENSION *rd;
 181         for(rd = st->dimensions; rd ; rd = rd->next) {
 182                 rd->last_collected_time.tv_sec = 0;
 183                 rd->last_collected_time.tv_usec = 0;
 184                 rd->current_entry = 0;
 185         }
 186 }
 187
 188 RRD_STATS *rrd_stats_create(const char *type, const char *id, const char *name, const char *family, const char *title, const char *units, long priority, int update_every, int chart_type)
 189 {
 190         if(!id || !id[0]) {
 191                 fatal("Cannot create rrd stats without an id.");
 192                 return NULL;
 193         }
 194
 195         char fullid[RRD_STATS_NAME_MAX + 1];
 196         char fullfilename[FILENAME_MAX + 1];
 197         RRD_STATS *st = NULL;
 198
 199         snprintf(fullid, RRD_STATS_NAME_MAX, "%s.%s", type, id);
 200
 201         long entries = config_get_number(fullid, "history", save_history);
 202         if(entries < 5) entries = config_set_number(fullid, "history", 5);
 203         if(entries > HISTORY_MAX) entries = config_set_number(fullid, "history", HISTORY_MAX);
 204
 205         int enabled = config_get_boolean(fullid, "enabled", 1);
 206         if(!enabled) entries = 5;
 207
 208         unsigned long size = sizeof(RRD_STATS);
 209         char *cache_dir = rrd_stats_cache_dir(fullid);
 210
 211         debug(D_RRD_STATS, "Creating RRD_STATS for '%s.%s'.", type, id);
 212
 213         snprintf(fullfilename, FILENAME_MAX, "%s/main.db", cache_dir);
 214         if(memory_mode != NETDATA_MEMORY_MODE_RAM) st = (RRD_STATS *)mymmap(fullfilename, size, ((memory_mode == NETDATA_MEMORY_MODE_MAP)?MAP_SHARED:MAP_PRIVATE));
 215         if(st) {
 216                 if(strcmp(st->magic, RRD_STATS_MAGIC) != 0) {
 217                         errno = 0;
 218                         error("File %s does not have our version. Clearing it.", fullfilename);
 219                         bzero(st, size);
 220                 }
 221                 else if(strcmp(st->id, fullid) != 0) {
 222                         errno = 0;
 223                         error("File %s does not have our id. Unmapping it.", fullfilename);
 224                         munmap(st, size);
 225                         st = NULL;
 226                 }
 227                 else if(st->memsize != size || st->entries != entries) {
 228                         errno = 0;
 229                         error("File %s does not have the desired size. Clearing it.", fullfilename);
 230                         bzero(st, size);
 231                 }
 232                 else if(st->update_every != update_every) {
 233                         errno = 0;
 234                         error("File %s does not have the desired update frequency. Clearing it.", fullfilename);
 235                         bzero(st, size);
 236                 }
 237         }
 238
 239         if(st) {
 240                 st->name = NULL;
 241                 st->type = NULL;
 242                 st->family = NULL;
 243                 st->title = NULL;
 244                 st->units = NULL;
 245                 st->dimensions = NULL;
 246                 st->next = NULL;
 247                 st->mapped = memory_mode;
 248         }
 249         else {
 250                 st = calloc(1, size);
 251                 if(!st) {
 252                         fatal("Cannot allocate memory for RRD_STATS %s.%s", type, id);
 253                         return NULL;
 254                 }
 255                 st->mapped = NETDATA_MEMORY_MODE_RAM;
 256         }
 257         st->memsize = size;
 258         st->entries = entries;
 259         st->update_every = update_every;
 260
 261         strcpy(st->cache_file, fullfilename);
 262         strcpy(st->magic, RRD_STATS_MAGIC);
 263
 264         strcpy(st->id, fullid);
 265         st->hash = simple_hash(st->id);
 266
 267         st->cache_dir = cache_dir;
 268
 269         st->family     = config_get(st->id, "family", family?family:st->id);
 270         st->units      = config_get(st->id, "units", units?units:"");
 271         st->type       = config_get(st->id, "type", type);
 272         st->chart_type = chart_type_id(config_get(st->id, "chart type", chart_type_name(chart_type)));
 273
 274         if(name && *name) rrd_stats_set_name(st, name);
 275         else rrd_stats_set_name(st, id);
 276
 277         {
 278                 char varvalue[CONFIG_MAX_VALUE + 1];
 279                 snprintf(varvalue, CONFIG_MAX_VALUE, "%s (%s)", title?title:"", st->name);
 280                 st->title = config_get(st->id, "title", varvalue);
 281         }
 282
 283         st->priority = config_get_number(st->id, "priority", priority);
 284         st->enabled = enabled;
 285
 286         st->isdetail = 0;
 287         st->debug = 0;
 288
 289         st->last_collected_time.tv_sec = 0;
 290         st->last_collected_time.tv_usec = 0;
 291         st->counter_done = 0;
 292
 293         pthread_rwlock_init(&st->rwlock, NULL);
 294         pthread_rwlock_wrlock(&root_rwlock);
 295
 296         st->next = root;
 297         root = st;
 298
 299         pthread_rwlock_unlock(&root_rwlock);
 300
 301         return(st);
 302 }
 303
 304 RRD_DIMENSION *rrd_stats_dimension_add(RRD_STATS *st, const char *id, const char *name, long multiplier, long divisor, int algorithm)
 305 {
 306         char filename[FILENAME_MAX + 1];
 307         char fullfilename[FILENAME_MAX + 1];
 308
 309         char varname[CONFIG_MAX_NAME + 1];
 310         RRD_DIMENSION *rd = NULL;
 311         unsigned long size = sizeof(RRD_DIMENSION) + (st->entries * sizeof(storage_number));
 312
 313         debug(D_RRD_STATS, "Adding dimension '%s/%s'.", st->id, id);
 314
 315         rrd_stats_strncpy_name(filename, id, FILENAME_MAX);
 316         snprintf(fullfilename, FILENAME_MAX, "%s/%s.db", st->cache_dir, filename);
 317         if(memory_mode != NETDATA_MEMORY_MODE_RAM) rd = (RRD_DIMENSION *)mymmap(fullfilename, size, ((memory_mode == NETDATA_MEMORY_MODE_MAP)?MAP_SHARED:MAP_PRIVATE));
 318         if(rd) {
 319                 struct timeval now;
 320                 gettimeofday(&now, NULL);
 321
 322                 if(strcmp(rd->magic, RRD_DIMENSION_MAGIC) != 0) {
 323                         errno = 0;
 324                         error("File %s does not have our version. Clearing it.", fullfilename);
 325                         bzero(rd, size);
 326                 }
 327                 else if(rd->memsize != size) {
 328                         errno = 0;
 329                         error("File %s does not have the desired size. Clearing it.", fullfilename);
 330                         bzero(rd, size);
 331                 }
 332                 else if(rd->multiplier != multiplier) {
 333                         errno = 0;
 334                         error("File %s does not have the same multiplier. Clearing it.", fullfilename);
 335                         bzero(rd, size);
 336                 }
 337                 else if(rd->divisor != divisor) {
 338                         errno = 0;
 339                         error("File %s does not have the same divisor. Clearing it.", fullfilename);
 340                         bzero(rd, size);
 341                 }
 342                 else if(rd->algorithm != algorithm) {
 343                         errno = 0;
 344                         error("File %s does not have the same algorithm. Clearing it.", fullfilename);
 345                         bzero(rd, size);
 346                 }
 347                 else if(rd->update_every != st->update_every) {
 348                         errno = 0;
 349                         error("File %s does not have the same refresh frequency. Clearing it.", fullfilename);
 350                         bzero(rd, size);
 351                 }
 352                 else if(usecdiff(&now, &rd->last_collected_time) > (rd->entries * rd->update_every * 1000000ULL)) {
 353                         errno = 0;
 354                         error("File %s is too old. Clearing it.", fullfilename);
 355                         bzero(rd, size);
 356                 }
 357                 else if(strcmp(rd->id, id) != 0) {
 358                         errno = 0;
 359                         error("File %s does not have our dimension id. Unmapping it.", fullfilename);
 360                         munmap(rd, size);
 361                         rd = NULL;
 362                 }
 363         }
 364
 365         if(rd) {
 366                 // we have a file mapped for rd
 367                 rd->mapped = memory_mode;
 368                 rd->hidden = 0;
 369                 rd->next = NULL;
 370                 rd->name = NULL;
 371         }
 372         else {
 373                 // if we didn't manage to get a mmap'd dimension, just create one
 374
 375                 rd = calloc(1, size);
 376                 if(!rd) {
 377                         fatal("Cannot allocate RRD_DIMENSION %s/%s.", st->id, id);
 378                         return NULL;
 379                 }
 380
 381                 rd->mapped = NETDATA_MEMORY_MODE_RAM;
 382         }
 383         rd->memsize = size;
 384
 385         strcpy(rd->magic, RRD_DIMENSION_MAGIC);
 386         strcpy(rd->cache_file, fullfilename);
 387         strncpy(rd->id, id, RRD_STATS_NAME_MAX);
 388         rd->hash = simple_hash(rd->id);
 389
 390         snprintf(varname, CONFIG_MAX_NAME, "dim %s name", rd->id);
 391         rd->name = config_get(st->id, varname, (name && *name)?name:rd->id);
 392
 393         snprintf(varname, CONFIG_MAX_NAME, "dim %s algorithm", rd->id);
 394         rd->algorithm = algorithm_id(config_get(st->id, varname, algorithm_name(algorithm)));
 395
 396         snprintf(varname, CONFIG_MAX_NAME, "dim %s multiplier", rd->id);
 397         rd->multiplier = config_get_number(st->id, varname, multiplier);
 398
 399         snprintf(varname, CONFIG_MAX_NAME, "dim %s divisor", rd->id);
 400         rd->divisor = config_get_number(st->id, varname, divisor);
 401         if(!rd->divisor) rd->divisor = 1;
 402
 403         rd->entries = st->entries;
 404         rd->update_every = st->update_every;
 405
 406         // append this dimension
 407         if(!st->dimensions)
 408                 st->dimensions = rd;
 409         else {
 410                 RRD_DIMENSION *td = st->dimensions;
 411                 for(; td->next; td = td->next) ;
 412                 td->next = rd;
 413         }
 414
 415         return(rd);
 416 }
 417
 418 void rrd_stats_dimension_set_name(RRD_STATS *st, RRD_DIMENSION *rd, const char *name)
 419 {
 420         char varname[CONFIG_MAX_NAME + 1];
 421         snprintf(varname, CONFIG_MAX_NAME, "dim %s name", rd->id);
 422         config_get(st->id, varname, name);
 423 }
 424
 425 void rrd_stats_dimension_free(RRD_DIMENSION *rd)
 426 {
 427         if(rd->next) rrd_stats_dimension_free(rd->next);
 428         // free(rd->annotations);
 429         if(rd->mapped == NETDATA_MEMORY_MODE_SAVE) {
 430                 debug(D_RRD_STATS, "Saving dimension '%s' to '%s'.", rd->name, rd->cache_file);
 431                 savememory(rd->cache_file, rd, rd->memsize);
 432
 433                 debug(D_RRD_STATS, "Unmapping dimension '%s'.", rd->name);
 434                 munmap(rd, rd->memsize);
 435         }
 436         else if(rd->mapped == NETDATA_MEMORY_MODE_MAP) {
 437                 debug(D_RRD_STATS, "Unmapping dimension '%s'.", rd->name);
 438                 munmap(rd, rd->memsize);
 439         }
 440         else {
 441                 debug(D_RRD_STATS, "Removing dimension '%s'.", rd->name);
 442                 free(rd);
 443         }
 444 }
 445
 446 void rrd_stats_free_all(void)
 447 {
 448         info("Freeing all memory...");
 449
 450         RRD_STATS *st;
 451         for(st = root; st ;) {
 452                 RRD_STATS *next = st->next;
 453
 454                 if(st->dimensions) rrd_stats_dimension_free(st->dimensions);
 455                 st->dimensions = NULL;
 456
 457                 if(st->mapped == NETDATA_MEMORY_MODE_SAVE) {
 458                         debug(D_RRD_STATS, "Saving stats '%s' to '%s'.", st->name, st->cache_file);
 459                         savememory(st->cache_file, st, st->memsize);
 460
 461                         debug(D_RRD_STATS, "Unmapping stats '%s'.", st->name);
 462                         munmap(st, st->memsize);
 463                 }
 464                 else if(st->mapped == NETDATA_MEMORY_MODE_MAP) {
 465                         debug(D_RRD_STATS, "Unmapping stats '%s'.", st->name);
 466                         munmap(st, st->memsize);
 467                 }
 468                 else
 469                         free(st);
 470
 471                 st = next;
 472         }
 473         root = NULL;
 474
 475         info("Memory cleanup completed...");
 476 }
 477
 478 void rrd_stats_save_all(void)
 479 {
 480         RRD_STATS *st;
 481         RRD_DIMENSION *rd;
 482
 483         pthread_rwlock_wrlock(&root_rwlock);
 484         for(st = root; st ; st = st->next) {
 485                 pthread_rwlock_wrlock(&st->rwlock);
 486
 487                 if(st->mapped == NETDATA_MEMORY_MODE_SAVE) {
 488                         debug(D_RRD_STATS, "Saving stats '%s' to '%s'.", st->name, st->cache_file);
 489                         savememory(st->cache_file, st, st->memsize);
 490                 }
 491
 492                 for(rd = st->dimensions; rd ; rd = rd->next) {
 493                         if(rd->mapped == NETDATA_MEMORY_MODE_SAVE) {
 494                                 debug(D_RRD_STATS, "Saving dimension '%s' to '%s'.", rd->name, rd->cache_file);
 495                                 savememory(rd->cache_file, rd, rd->memsize);
 496                         }
 497                 }
 498
 499                 pthread_rwlock_unlock(&st->rwlock);
 500         }
 501         pthread_rwlock_unlock(&root_rwlock);
 502 }
 503
 504
 505 RRD_STATS *rrd_stats_find(const char *id)
 506 {
 507         debug(D_RRD_STATS, "rrd_stats_find() for chart %s", id);
 508
 509         unsigned long hash = simple_hash(id);
 510
 511         pthread_rwlock_rdlock(&root_rwlock);
 512         RRD_STATS *st = root;
 513         for ( ; st ; st = st->next )
 514                 if(hash == st->hash)
 515                         if(strcmp(st->id, id) == 0)
 516                                 break;
 517         pthread_rwlock_unlock(&root_rwlock);
 518
 519         return(st);
 520 }
 521
 522 RRD_STATS *rrd_stats_find_bytype(const char *type, const char *id)
 523 {
 524         debug(D_RRD_STATS, "rrd_stats_find_bytype() for chart %s.%s", type, id);
 525
 526         char buf[RRD_STATS_NAME_MAX + 1];
 527
 528         strncpy(buf, type, RRD_STATS_NAME_MAX - 1);
 529         buf[RRD_STATS_NAME_MAX - 1] = '\0';
 530         strcat(buf, ".");
 531         int len = strlen(buf);
 532         strncpy(&buf[len], id, RRD_STATS_NAME_MAX - len);
 533         buf[RRD_STATS_NAME_MAX] = '\0';
 534
 535         return(rrd_stats_find(buf));
 536 }
 537
 538 RRD_STATS *rrd_stats_find_byname(const char *name)
 539 {
 540         debug(D_RRD_STATS, "rrd_stats_find_byname() for chart %s", name);
 541
 542         char b[CONFIG_MAX_VALUE + 1];
 543
 544         rrd_stats_strncpy_name(b, name, CONFIG_MAX_VALUE);
 545         unsigned long hash = simple_hash(b);
 546
 547         pthread_rwlock_rdlock(&root_rwlock);
 548         RRD_STATS *st = root;
 549         for ( ; st ; st = st->next ) {
 550                 if(hash == st->hash_name && strcmp(st->name, b) == 0) break;
 551         }
 552         pthread_rwlock_unlock(&root_rwlock);
 553
 554         return(st);
 555 }
 556
 557 RRD_DIMENSION *rrd_stats_dimension_find(RRD_STATS *st, const char *id)
 558 {
 559         debug(D_RRD_STATS, "rrd_stats_dimension_find() for chart %s, dimension %s", st->name, id);
 560
 561         unsigned long hash = simple_hash(id);
 562
 563         RRD_DIMENSION *rd = st->dimensions;
 564
 565         for ( ; rd ; rd = rd->next )
 566                 if(hash == rd->hash)
 567                         if(strcmp(rd->id, id) == 0)
 568                                 break;
 569
 570         return(rd);
 571 }
 572
 573 int rrd_stats_dimension_hide(RRD_STATS *st, const char *id)
 574 {
 575         debug(D_RRD_STATS, "rrd_stats_dimension_hide() for chart %s, dimension %s", st->name, id);
 576
 577         RRD_DIMENSION *rd = rrd_stats_dimension_find(st, id);
 578         if(!rd) {
 579                 error("Cannot find dimension with id '%s' on stats '%s' (%s).", id, st->name, st->id);
 580                 return 1;
 581         }
 582
 583         rd->hidden = 1;
 584         return 0;
 585 }
 586
 587 void rrd_stats_dimension_set_by_pointer(RRD_STATS *st, RRD_DIMENSION *rd, collected_number value)
 588 {
 589         debug(D_RRD_STATS, "rrd_stats_dimension_set() for chart %s, dimension %s, value " COLLECTED_NUMBER_FORMAT, st->name, rd->name, value);
 590
 591         gettimeofday(&rd->last_collected_time, NULL);
 592         rd->collected_value = value;
 593 }
 594
 595 int rrd_stats_dimension_set(RRD_STATS *st, const char *id, collected_number value)
 596 {
 597         RRD_DIMENSION *rd = rrd_stats_dimension_find(st, id);
 598         if(!rd) {
 599                 error("Cannot find dimension with id '%s' on stats '%s' (%s).", id, st->name, st->id);
 600                 return 1;
 601         }
 602
 603         rrd_stats_dimension_set_by_pointer(st, rd, value);
 604         return 0;
 605 }
 606
 607 void rrd_stats_next_usec(RRD_STATS *st, unsigned long long microseconds)
 608 {
 609         debug(D_RRD_STATS, "rrd_stats_next() for chart %s with microseconds %llu", st->name, microseconds);
 610
 611         if(st->debug) debug(D_RRD_STATS, "%s: NEXT: %llu microseconds", st->name, microseconds);
 612         if(microseconds > st->entries * st->update_every * 1000000ULL) {
 613                 info("History of chart %s too old. Reseting chart.", st->name);
 614                 rrd_stats_reset(st);
 615                 microseconds = st->update_every * 1000000ULL;
 616         }
 617         st->usec_since_last_update = microseconds;
 618 }
 619
 620 void rrd_stats_next(RRD_STATS *st)
 621 {
 622         unsigned long long microseconds = 0;
 623
 624         if(st->last_collected_time.tv_sec) {
 625                 struct timeval now;
 626                 gettimeofday(&now, NULL);
 627                 microseconds = usecdiff(&now, &st->last_collected_time);
 628         }
 629
 630         rrd_stats_next_usec(st, microseconds);
 631 }
 632
 633 void rrd_stats_next_plugins(RRD_STATS *st)
 634 {
 635         rrd_stats_next(st);
 636 }
 637
 638 unsigned long long rrd_stats_done(RRD_STATS *st)
 639 {
 640         debug(D_RRD_STATS, "rrd_stats_done() for chart %s", st->name);
 641
 642         RRD_DIMENSION *rd, *last;
 643         int oldstate;
 644
 645         if(pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, &oldstate) != 0)
 646                 error("Cannot set pthread cancel state to DISABLE.");
 647
 648         // a read lock is OK here
 649         pthread_rwlock_rdlock(&st->rwlock);
 650
 651         if(st->debug) debug(D_RRD_STATS, "%s: microseconds since last update: %llu", st->name, st->usec_since_last_update);
 652
 653         if(!st->last_collected_time.tv_sec) gettimeofday(&st->last_collected_time, NULL);
 654         else {
 655                 unsigned long long ut = st->last_collected_time.tv_sec * 1000000ULL + st->last_collected_time.tv_usec + st->usec_since_last_update;
 656                 st->last_collected_time.tv_sec = ut / 1000000ULL;
 657                 st->last_collected_time.tv_usec = ut % 1000000ULL;
 658         }
 659
 660         if(!st->last_updated.tv_sec) {
 661                 unsigned long long ut = st->last_collected_time.tv_sec * 1000000ULL + st->last_collected_time.tv_usec - st->usec_since_last_update;
 662                 st->last_updated.tv_sec = ut / 1000000ULL;
 663                 st->last_updated.tv_usec = ut % 1000000ULL;
 664
 665                 if(st->debug) debug(D_RRD_STATS, "%s: initializing last_updated to now - %llu microseconds (%0.3Lf)", st->name, st->usec_since_last_update, (long double)ut/1000000.0);
 666         }
 667
 668         unsigned long long last_ut = st->last_updated.tv_sec * 1000000ULL + st->last_updated.tv_usec;
 669         unsigned long long now_ut = st->last_collected_time.tv_sec * 1000000ULL + st->last_collected_time.tv_usec;
 670         unsigned long long next_ut = (st->last_updated.tv_sec + st->update_every) * 1000000ULL;
 671
 672         if(st->debug) debug(D_RRD_STATS, "%s: last ut = %0.3Lf (last updated time)", st->name, (long double)last_ut/1000000.0);
 673         if(st->debug) debug(D_RRD_STATS, "%s: now  ut = %0.3Lf (current update time)", st->name, (long double)now_ut/1000000.0);
 674         if(st->debug) debug(D_RRD_STATS, "%s: next ut = %0.3Lf (next interpolation point)", st->name, (long double)next_ut/1000000.0);
 675
 676         st->counter_done++;
 677
 678         // calculate totals and count the dimensions
 679         int dimensions;
 680         st->collected_total = 0;
 681         for( rd = st->dimensions, dimensions = 0 ; rd ; rd = rd->next, dimensions++ )
 682                 st->collected_total += rd->collected_value;
 683
 684         // process all dimensions to calculate their values
 685         // based on the collected figures only
 686         // at this stage we do not interpolate anything
 687         for( rd = st->dimensions ; rd ; rd = rd->next ) {
 688                 if(st->debug) debug(D_RRD_STATS, "%s/%s: "
 689                         " last_collected_value = " COLLECTED_NUMBER_FORMAT
 690                         " collected_value = " COLLECTED_NUMBER_FORMAT
 691                         " last_calculated_value = " CALCULATED_NUMBER_FORMAT
 692                         " calculated_value = " CALCULATED_NUMBER_FORMAT
 693                         , st->id, rd->name
 694                         , rd->last_collected_value
 695                         , rd->collected_value
 696                         , rd->last_calculated_value
 697                         , rd->calculated_value
 698                         );
 699
 700                 switch(rd->algorithm) {
 701                         case RRD_DIMENSION_PCENT_OVER_DIFF_TOTAL:
 702                                 // the percentage of the current increment
 703                                 // over the increment of all dimensions together
 704                                 if(st->collected_total == st->last_collected_total) rd->calculated_value = rd->last_calculated_value;
 705                                 else rd->calculated_value =
 706                                           (calculated_number)100
 707                                         * (calculated_number)(rd->collected_value - rd->last_collected_value)
 708                                         / (calculated_number)(st->collected_total  - st->last_collected_total);
 709
 710                                 if(st->debug)
 711                                         debug(D_RRD_STATS, "%s/%s: CALC PCENT-DIFF "
 712                                                 CALCULATED_NUMBER_FORMAT " = 100"
 713                                                 " * (" COLLECTED_NUMBER_FORMAT " - " COLLECTED_NUMBER_FORMAT ")"
 714                                                 " / (" COLLECTED_NUMBER_FORMAT " - " COLLECTED_NUMBER_FORMAT ")"
 715                                                 , st->id, rd->name
 716                                                 , rd->calculated_value
 717                                                 , rd->collected_value, rd->last_collected_value
 718                                                 , st->collected_total, st->last_collected_total
 719                                                 );
 720                                 break;
 721
 722                         case RRD_DIMENSION_PCENT_OVER_ROW_TOTAL:
 723                                 if(!st->collected_total) rd->calculated_value = 0;
 724                                 else
 725                                 // the percentage of the current value
 726                                 // over the total of all dimensions
 727                                 rd->calculated_value =
 728                                           (calculated_number)100
 729                                         * (calculated_number)rd->collected_value
 730                                         / (calculated_number)st->collected_total;
 731
 732                                 if(st->debug)
 733                                         debug(D_RRD_STATS, "%s/%s: CALC PCENT-ROW "
 734                                                 CALCULATED_NUMBER_FORMAT " = 100"
 735                                                 " * " COLLECTED_NUMBER_FORMAT
 736                                                 " / " COLLECTED_NUMBER_FORMAT
 737                                                 , st->id, rd->name
 738                                                 , rd->calculated_value
 739                                                 , rd->collected_value
 740                                                 , st->collected_total
 741                                                 );
 742                                 break;
 743
 744                         case RRD_DIMENSION_INCREMENTAL:
 745                                 // if the new is smaller than the old (an overflow, or reset), set the old equal to the new
 746                                 // to reset the calculation (it will give zero as the calculation for this second)
 747                                 if(rd->last_collected_value > rd->collected_value) rd->last_collected_value = rd->collected_value;
 748
 749                                 rd->calculated_value += (calculated_number)(rd->collected_value - rd->last_collected_value);
 750
 751                                 if(st->debug)
 752                                         debug(D_RRD_STATS, "%s/%s: CALC INC "
 753                                                 CALCULATED_NUMBER_FORMAT " += "
 754                                                 COLLECTED_NUMBER_FORMAT " - " COLLECTED_NUMBER_FORMAT
 755                                                 , st->id, rd->name
 756                                                 , rd->calculated_value
 757                                                 , rd->collected_value, rd->last_collected_value
 758                                                 );
 759                                 break;
 760
 761                         case RRD_DIMENSION_ABSOLUTE:
 762                                 rd->calculated_value = (calculated_number)rd->collected_value;
 763
 764                                 if(st->debug)
 765                                         debug(D_RRD_STATS, "%s/%s: CALC ABS/ABS-NO-IN "
 766                                                 CALCULATED_NUMBER_FORMAT " = "
 767                                                 COLLECTED_NUMBER_FORMAT
 768                                                 , st->id, rd->name
 769                                                 , rd->calculated_value
 770                                                 , rd->collected_value
 771                                                 );
 772                                 break;
 773
 774                         default:
 775                                 // make the default zero, to make sure
 776                                 // it gets noticed when we add new types
 777                                 rd->calculated_value = 0;
 778
 779                                 if(st->debug)
 780                                         debug(D_RRD_STATS, "%s/%s: CALC "
 781                                                 CALCULATED_NUMBER_FORMAT " = 0"
 782                                                 , st->id, rd->name
 783                                                 , rd->calculated_value
 784                                                 );
 785                                 break;
 786                 }
 787         }
 788         // at this point we have all the calculated values ready
 789
 790         if(st->counter_done == 1 || next_ut > now_ut) {
 791                 // we don't have any usable data yet
 792                 if(st->debug) debug(D_RRD_STATS, "%s: Skipping collected values (usec since last update = %llu, counter_done = %lu)", st->name, st->usec_since_last_update, st->counter_done);
 793
 794                 for( rd = st->dimensions; rd ; rd = rd->next ) {
 795                         rd->last_calculated_value = rd->calculated_value;
 796                         rd->last_collected_value = rd->collected_value;
 797
 798                         switch(rd->algorithm) {
 799                                 case RRD_DIMENSION_PCENT_OVER_DIFF_TOTAL:
 800                                 case RRD_DIMENSION_INCREMENTAL:
 801                                         if(!st->usec_since_last_update) rd->calculated_value = 0;
 802                                         // keep the previous values
 803                                         // the next time, a new incremental total will be calculated
 804                                         break;
 805                         }
 806
 807                         rd->collected_value = 0;
 808                 }
 809                 st->last_collected_total  = st->collected_total;
 810
 811                 pthread_rwlock_unlock(&st->rwlock);
 812                 if(pthread_setcancelstate(oldstate, NULL) != 0)
 813                         error("Cannot set pthread cancel state to RESTORE (%d).", oldstate);
 814
 815                 return(st->usec_since_last_update);
 816         }
 817
 818         // it is now time to interpolate values on a second boundary
 819         unsigned long long first_ut = last_ut;
 820         for( ; next_ut <= now_ut ; next_ut += st->update_every * 1000000ULL ) {
 821                 if(st->debug) debug(D_RRD_STATS, "%s: last ut = %0.3Lf (last updated time)", st->name, (long double)last_ut/1000000.0);
 822                 if(st->debug) debug(D_RRD_STATS, "%s: next ut = %0.3Lf (next interpolation point)", st->name, (long double)next_ut/1000000.0);
 823
 824                 st->last_updated.tv_sec = next_ut / 1000000ULL;
 825                 st->last_updated.tv_usec = 0;
 826
 827                 for( rd = st->dimensions ; rd ; rd = rd->next ) {
 828                         calculated_number new_value;
 829
 830                         switch(rd->algorithm) {
 831                                 case RRD_DIMENSION_INCREMENTAL:
 832                                         new_value = (calculated_number)
 833                                                 (          rd->calculated_value
 834                                                         * (calculated_number)(next_ut - last_ut)
 835                                                         / (calculated_number)(now_ut - last_ut)
 836                                                 );
 837
 838                                         if(st->debug)
 839                                                 debug(D_RRD_STATS, "%s/%s: CALC2 INC "
 840                                                         CALCULATED_NUMBER_FORMAT " = "
 841                                                         CALCULATED_NUMBER_FORMAT
 842                                                         " * %llu"
 843                                                         " / %llu"
 844                                                         , st->id, rd->name
 845                                                         , new_value
 846                                                         , rd->calculated_value
 847                                                         , (unsigned long long)(next_ut - last_ut)
 848                                                         , (unsigned long long)(now_ut - last_ut)
 849                                                         );
 850
 851                                         rd->calculated_value -= new_value;
 852                                         break;
 853
 854                                 case RRD_DIMENSION_ABSOLUTE:
 855                                 case RRD_DIMENSION_PCENT_OVER_ROW_TOTAL:
 856                                 case RRD_DIMENSION_PCENT_OVER_DIFF_TOTAL:
 857                                 default:
 858                                         new_value = (calculated_number)
 859                                                 (       (         (rd->calculated_value - rd->last_calculated_value)
 860                                                                 * (calculated_number)(next_ut - first_ut)
 861                                                                 / (calculated_number)(now_ut - first_ut)
 862                                                         )
 863                                                         +  rd->last_calculated_value
 864                                                 );
 865
 866                                         if(st->debug)
 867                                                 debug(D_RRD_STATS, "%s/%s: CALC2 DEF "
 868                                                         CALCULATED_NUMBER_FORMAT " = ((("
 869                                                         "(" CALCULATED_NUMBER_FORMAT " - " CALCULATED_NUMBER_FORMAT ")"
 870                                                         " * %llu"
 871                                                         " / %llu) + " CALCULATED_NUMBER_FORMAT
 872                                                         , st->id, rd->name
 873                                                         , new_value
 874                                                         , rd->calculated_value, rd->last_calculated_value
 875                                                         , (next_ut - first_ut)
 876                                                         , (now_ut - first_ut), rd->last_calculated_value
 877                                                         );
 878
 879                                         if(next_ut + st->update_every * 1000000ULL > now_ut) rd->calculated_value = new_value;
 880                                         break;
 881
 882                         }
 883
 884
 885                         rd->values[st->current_entry] = pack_storage_number(
 886                                           new_value
 887                                         * (calculated_number)rd->multiplier
 888                                         / (calculated_number)rd->divisor
 889                                 );
 890
 891                         if(st->debug)
 892                                 debug(D_RRD_STATS, "%s/%s: STORE[%ld] "
 893                                         CALCULATED_NUMBER_FORMAT " = " CALCULATED_NUMBER_FORMAT
 894                                         " * %ld"
 895                                         " / %ld"
 896                                         , st->id, rd->name
 897                                         , st->current_entry
 898                                         , unpack_storage_number(rd->values[st->current_entry]), new_value
 899                                         , rd->multiplier
 900                                         , rd->divisor
 901                                         );
 902                 }
 903
 904                 if(st->first_entry_t && st->counter >= (unsigned long long)st->entries) {
 905                         // the db is overwriting values
 906                         // add the value we will overwrite
 907                         st->first_entry_t += st->update_every * 1000000ULL;
 908                 }
 909
 910                 st->counter++;
 911                 st->current_entry = ((st->current_entry + 1) >= st->entries) ? 0 : st->current_entry + 1;
 912                 if(!st->first_entry_t) st->first_entry_t = next_ut;
 913                 last_ut = next_ut;
 914         }
 915
 916         for( rd = st->dimensions; rd ; rd = rd->next ) {
 917                 rd->last_collected_value = rd->collected_value;
 918                 rd->last_calculated_value = rd->calculated_value;
 919                 rd->collected_value = 0;
 920         }
 921         st->last_collected_total  = st->collected_total;
 922
 923         // ALL DONE ABOUT THE DATA UPDATE
 924         // --------------------------------------------------------------------
 925
 926
 927         // find if there are any obsolete dimensions (not updated recently)
 928         for( rd = st->dimensions; rd ; rd = rd->next )
 929                 if((rd->last_collected_time.tv_sec + (10 * st->update_every)) < st->last_collected_time.tv_sec)
 930                         break;
 931
 932         if(rd) {
 933                 // there is dimension to free
 934                 // upgrade our read lock to a write lock
 935                 pthread_rwlock_unlock(&st->rwlock);
 936                 pthread_rwlock_wrlock(&st->rwlock);
 937
 938                 for( rd = st->dimensions, last = NULL ; rd ; ) {
 939                         if((rd->last_collected_time.tv_sec + (10 * st->update_every)) < st->last_collected_time.tv_sec) { // remove it only it is not updated in 10 seconds
 940                                 debug(D_RRD_STATS, "Removing obsolete dimension '%s' (%s) of '%s' (%s).", rd->name, rd->id, st->name, st->id);
 941
 942                                 if(!last) {
 943                                         st->dimensions = rd->next;
 944                                         rd->next = NULL;
 945                                         rrd_stats_dimension_free(rd);
 946                                         rd = st->dimensions;
 947                                         continue;
 948                                 }
 949                                 else {
 950                                         last->next = rd->next;
 951                                         rd->next = NULL;
 952                                         rrd_stats_dimension_free(rd);
 953                                         rd = last->next;
 954                                         continue;
 955                                 }
 956                         }
 957
 958                         last = rd;
 959                         rd = rd->next;
 960                 }
 961
 962                 if(!st->dimensions) st->enabled = 0;
 963         }
 964
 965         pthread_rwlock_unlock(&st->rwlock);
 966
 967         if(pthread_setcancelstate(oldstate, NULL) != 0)
 968                 error("Cannot set pthread cancel state to RESTORE (%d).", oldstate);
 969
 970         return(st->usec_since_last_update);
 971 }
 972
 973
 974 // find the oldest entry in the data, skipping all empty slots
 975 time_t rrd_stats_first_entry_t(RRD_STATS *st)
 976 {
 977         if(!st->first_entry_t) return st->last_updated.tv_sec;
 978
 979         return st->first_entry_t / 1000000;
 980 }