src/rrd.c

   1 #include <stddef.h>
   2 #include <string.h>
   3 #include <unistd.h>
   4 #include <time.h>
   5 #include <sys/time.h>
   6 #include <sys/mman.h>
   7 #include <pthread.h>
   8 #include <errno.h>
   9 #include <ctype.h>
  10 #include <sys/stat.h>
  11 #include <sys/types.h>
  12 #include <stdlib.h>
  13
  14 #include "log.h"
  15 #include "config.h"
  16 #include "common.h"
  17
  18 #include "rrd.h"
  19
  20 #define RRD_DEFAULT_GAP_INTERPOLATIONS 10
  21
  22 // ----------------------------------------------------------------------------
  23 // globals
  24
  25 int rrd_update_every = UPDATE_EVERY;
  26 int rrd_default_history_entries = RRD_DEFAULT_HISTORY_ENTRIES;
  27
  28 RRDSET *rrdset_root = NULL;
  29 pthread_rwlock_t rrdset_root_rwlock = PTHREAD_RWLOCK_INITIALIZER;
  30
  31 int rrd_memory_mode = RRD_MEMORY_MODE_SAVE;
  32
  33
  34 // ----------------------------------------------------------------------------
  35 // RRDSET index
  36
  37 static int rrdset_iterator(avl *a) { if(a) {}; return 0; }
  38
  39 static int rrdset_compare(void* a, void* b) {
  40         if(((RRDSET *)a)->hash < ((RRDSET *)b)->hash) return -1;
  41         else if(((RRDSET *)a)->hash > ((RRDSET *)b)->hash) return 1;
  42         else return strcmp(((RRDSET *)a)->id, ((RRDSET *)b)->id);
  43 }
  44
  45 avl_tree rrdset_root_index = {
  46                 NULL,
  47                 rrdset_compare
  48 };
  49
  50 #define rrdset_index_add(st) avl_insert(&rrdset_root_index, (avl *)(st))
  51 #define rrdset_index_del(st) avl_remove(&rrdset_root_index, (avl *)(st))
  52
  53 static RRDSET *rrdset_index_find(const char *id, uint32_t hash) {
  54         RRDSET *result = NULL, tmp;
  55         strncpy(tmp.id, id, RRD_ID_LENGTH_MAX);
  56         tmp.id[RRD_ID_LENGTH_MAX] = '\0';
  57         tmp.hash = (hash)?hash:simple_hash(tmp.id);
  58
  59         avl_search(&(rrdset_root_index), (avl *)&tmp, rrdset_iterator, (avl **)&result);
  60         return result;
  61 }
  62
  63 // ----------------------------------------------------------------------------
  64 // RRDSET name index
  65
  66 #define rrdset_from_avlname(avlname_ptr) ((RRDSET *)((avlname_ptr) - offsetof(RRDSET, avlname)))
  67
  68 static int rrdset_iterator_name(avl *a) { if(a) {}; return 0; }
  69
  70 static int rrdset_compare_name(void* a, void* b) {
  71         RRDSET *A = rrdset_from_avlname(a);
  72         RRDSET *B = rrdset_from_avlname(b);
  73
  74         // fprintf(stderr, "COMPARING: %s with %s\n", A->name, B->name);
  75
  76         if(A->hash_name < B->hash_name) return -1;
  77         else if(A->hash_name > B->hash_name) return 1;
  78         else return strcmp(A->name, B->name);
  79 }
  80
  81 avl_tree rrdset_root_index_name = {
  82                 NULL,
  83                 rrdset_compare_name
  84 };
  85
  86 int rrdset_index_add_name(RRDSET *st) {
  87         // fprintf(stderr, "ADDING: %s (name: %s)\n", st->id, st->name);
  88         return avl_insert(&rrdset_root_index_name, (avl *)(&st->avlname));
  89 }
  90
  91 #define rrdset_index_del_name(st) avl_remove(&rrdset_root_index_name, (avl *)(&st->avlname))
  92
  93 static RRDSET *rrdset_index_find_name(const char *name, uint32_t hash) {
  94         void *result = NULL;
  95         RRDSET tmp;
  96         tmp.name = name;
  97         tmp.hash_name = (hash)?hash:simple_hash(tmp.name);
  98
  99         // fprintf(stderr, "SEARCHING: %s\n", name);
 100         avl_search(&(rrdset_root_index_name), (avl *)(&(tmp.avlname)), rrdset_iterator_name, (avl **)&result);
 101         if(result) {
 102                 RRDSET *st = rrdset_from_avlname(result);
 103                 if(strcmp(st->magic, RRDSET_MAGIC))
 104                         error("Search for RRDSET %s returned an invalid RRDSET %s (name %s)", name, st->id, st->name);
 105
 106                 // fprintf(stderr, "FOUND: %s\n", name);
 107                 return rrdset_from_avlname(result);
 108         }
 109         // fprintf(stderr, "NOT FOUND: %s\n", name);
 110         return NULL;
 111 }
 112
 113
 114 // ----------------------------------------------------------------------------
 115 // RRDDIM index
 116
 117 static int rrddim_iterator(avl *a) { if(a) {}; return 0; }
 118
 119 static int rrddim_compare(void* a, void* b) {
 120         if(((RRDDIM *)a)->hash < ((RRDDIM *)b)->hash) return -1;
 121         else if(((RRDDIM *)a)->hash > ((RRDDIM *)b)->hash) return 1;
 122         else return strcmp(((RRDDIM *)a)->id, ((RRDDIM *)b)->id);
 123 }
 124
 125 #define rrddim_index_add(st, rd) avl_insert(&((st)->dimensions_index), (avl *)(rd))
 126 #define rrddim_index_del(st,rd ) avl_remove(&((st)->dimensions_index), (avl *)(rd))
 127
 128 static RRDDIM *rrddim_index_find(RRDSET *st, const char *id, uint32_t hash) {
 129         RRDDIM *result = NULL, tmp;
 130         tmp.hash = (hash)?hash:simple_hash(id);
 131         strncpy(tmp.id, id, RRD_ID_LENGTH_MAX);
 132         tmp.id[RRD_ID_LENGTH_MAX] = '\0';
 133         tmp.hash = (hash)?hash:simple_hash(tmp.id);
 134
 135         avl_search(&(st->dimensions_index), (avl *)&tmp, rrddim_iterator, (avl **)&result);
 136         return result;
 137 }
 138
 139 // ----------------------------------------------------------------------------
 140 // chart types
 141
 142 int rrdset_type_id(const char *name)
 143 {
 144         if(strcmp(name, RRDSET_TYPE_AREA_NAME) == 0) return RRDSET_TYPE_AREA;
 145         if(strcmp(name, RRDSET_TYPE_STACKED_NAME) == 0) return RRDSET_TYPE_STACKED;
 146         if(strcmp(name, RRDSET_TYPE_LINE_NAME) == 0) return RRDSET_TYPE_LINE;
 147         return RRDSET_TYPE_LINE;
 148 }
 149
 150 const char *rrdset_type_name(int chart_type)
 151 {
 152         static char line[] = RRDSET_TYPE_LINE_NAME;
 153         static char area[] = RRDSET_TYPE_AREA_NAME;
 154         static char stacked[] = RRDSET_TYPE_STACKED_NAME;
 155
 156         switch(chart_type) {
 157                 case RRDSET_TYPE_LINE:
 158                         return line;
 159
 160                 case RRDSET_TYPE_AREA:
 161                         return area;
 162
 163                 case RRDSET_TYPE_STACKED:
 164                         return stacked;
 165         }
 166         return line;
 167 }
 168
 169 // ----------------------------------------------------------------------------
 170 // load / save
 171
 172 const char *rrd_memory_mode_name(int id)
 173 {
 174         static const char ram[] = RRD_MEMORY_MODE_RAM_NAME;
 175         static const char map[] = RRD_MEMORY_MODE_MAP_NAME;
 176         static const char save[] = RRD_MEMORY_MODE_SAVE_NAME;
 177
 178         switch(id) {
 179                 case RRD_MEMORY_MODE_RAM:
 180                         return ram;
 181
 182                 case RRD_MEMORY_MODE_MAP:
 183                         return map;
 184
 185                 case RRD_MEMORY_MODE_SAVE:
 186                 default:
 187                         return save;
 188         }
 189
 190         return save;
 191 }
 192
 193 int rrd_memory_mode_id(const char *name)
 194 {
 195         if(!strcmp(name, RRD_MEMORY_MODE_RAM_NAME))
 196                 return RRD_MEMORY_MODE_RAM;
 197         else if(!strcmp(name, RRD_MEMORY_MODE_MAP_NAME))
 198                 return RRD_MEMORY_MODE_MAP;
 199
 200         return RRD_MEMORY_MODE_SAVE;
 201 }
 202
 203 // ----------------------------------------------------------------------------
 204 // algorithms types
 205
 206 int rrddim_algorithm_id(const char *name)
 207 {
 208         if(strcmp(name, RRDDIM_ABSOLUTE_NAME) == 0)                     return RRDDIM_ABSOLUTE;
 209         if(strcmp(name, RRDDIM_INCREMENTAL_NAME) == 0)                  return RRDDIM_INCREMENTAL;
 210         if(strcmp(name, RRDDIM_PCENT_OVER_ROW_TOTAL_NAME) == 0)                 return RRDDIM_PCENT_OVER_ROW_TOTAL;
 211         if(strcmp(name, RRDDIM_PCENT_OVER_DIFF_TOTAL_NAME) == 0)        return RRDDIM_PCENT_OVER_DIFF_TOTAL;
 212         return RRDDIM_ABSOLUTE;
 213 }
 214
 215 const char *rrddim_algorithm_name(int chart_type)
 216 {
 217         static char absolute[] = RRDDIM_ABSOLUTE_NAME;
 218         static char incremental[] = RRDDIM_INCREMENTAL_NAME;
 219         static char percentage_of_absolute_row[] = RRDDIM_PCENT_OVER_ROW_TOTAL_NAME;
 220         static char percentage_of_incremental_row[] = RRDDIM_PCENT_OVER_DIFF_TOTAL_NAME;
 221
 222         switch(chart_type) {
 223                 case RRDDIM_ABSOLUTE:
 224                         return absolute;
 225
 226                 case RRDDIM_INCREMENTAL:
 227                         return incremental;
 228
 229                 case RRDDIM_PCENT_OVER_ROW_TOTAL:
 230                         return percentage_of_absolute_row;
 231
 232                 case RRDDIM_PCENT_OVER_DIFF_TOTAL:
 233                         return percentage_of_incremental_row;
 234         }
 235         return absolute;
 236 }
 237
 238 // ----------------------------------------------------------------------------
 239 // chart names
 240
 241 char *rrdset_strncpy_name(char *to, const char *from, int length)
 242 {
 243         int i;
 244         for(i = 0; i < length && from[i] ;i++) {
 245                 if(from[i] == '.' || isalpha(from[i]) || isdigit(from[i])) to[i] = from[i];
 246                 else to[i] = '_';
 247         }
 248         if(i < length) to[i] = '\0';
 249         to[length - 1] = '\0';
 250
 251         return to;
 252 }
 253
 254 void rrdset_set_name(RRDSET *st, const char *name)
 255 {
 256         debug(D_RRD_CALLS, "rrdset_set_name() old: %s, new: %s", st->name, name);
 257
 258         if(st->name) rrdset_index_del_name(st);
 259
 260         char b[CONFIG_MAX_VALUE + 1];
 261         char n[RRD_ID_LENGTH_MAX + 1];
 262
 263         snprintf(n, RRD_ID_LENGTH_MAX, "%s.%s", st->type, name);
 264         rrdset_strncpy_name(b, n, CONFIG_MAX_VALUE);
 265         st->name = config_get(st->id, "name", b);
 266         st->hash_name = simple_hash(st->name);
 267
 268         rrdset_index_add_name(st);
 269 }
 270
 271 // ----------------------------------------------------------------------------
 272 // cache directory
 273
 274 char *rrdset_cache_dir(const char *id)
 275 {
 276         char *ret = NULL;
 277
 278         static char *cache_dir = NULL;
 279         if(!cache_dir) cache_dir = config_get("global", "database directory", "cache");
 280
 281         char b[FILENAME_MAX + 1];
 282         char n[FILENAME_MAX + 1];
 283         rrdset_strncpy_name(b, id, FILENAME_MAX);
 284
 285         snprintf(n, FILENAME_MAX, "%s/%s", cache_dir, b);
 286         ret = config_get(id, "database directory", n);
 287
 288         if(rrd_memory_mode == RRD_MEMORY_MODE_MAP || rrd_memory_mode == RRD_MEMORY_MODE_SAVE) {
 289                 int r = mkdir(ret, 0775);
 290                 if(r != 0 && errno != EEXIST)
 291                         error("Cannot create directory '%s'", ret);
 292         }
 293
 294         return ret;
 295 }
 296
 297 // ----------------------------------------------------------------------------
 298 // core functions
 299
 300 void rrdset_reset(RRDSET *st)
 301 {
 302         debug(D_RRD_CALLS, "rrdset_reset() %s", st->name);
 303
 304         st->last_collected_time.tv_sec = 0;
 305         st->last_collected_time.tv_usec = 0;
 306         st->last_updated.tv_sec = 0;
 307         st->last_updated.tv_usec = 0;
 308         st->current_entry = 0;
 309         st->counter = 0;
 310         st->counter_done = 0;
 311
 312         RRDDIM *rd;
 313         for(rd = st->dimensions; rd ; rd = rd->next) {
 314                 rd->last_collected_time.tv_sec = 0;
 315                 rd->last_collected_time.tv_usec = 0;
 316                 bzero(rd->values, rd->entries * sizeof(storage_number));
 317         }
 318 }
 319
 320 RRDSET *rrdset_create(const char *type, const char *id, const char *name, const char *family, const char *title, const char *units, long priority, int update_every, int chart_type)
 321 {
 322         if(!id || !id[0]) {
 323                 fatal("Cannot create rrd stats without an id.");
 324                 return NULL;
 325         }
 326
 327         char fullid[RRD_ID_LENGTH_MAX + 1];
 328         char fullfilename[FILENAME_MAX + 1];
 329         RRDSET *st = NULL;
 330
 331         snprintf(fullid, RRD_ID_LENGTH_MAX, "%s.%s", type, id);
 332
 333         long entries = config_get_number(fullid, "history", rrd_default_history_entries);
 334         if(entries < 5) entries = config_set_number(fullid, "history", 5);
 335         if(entries > RRD_HISTORY_ENTRIES_MAX) entries = config_set_number(fullid, "history", RRD_HISTORY_ENTRIES_MAX);
 336
 337         int enabled = config_get_boolean(fullid, "enabled", 1);
 338         if(!enabled) entries = 5;
 339
 340         unsigned long size = sizeof(RRDSET);
 341         char *cache_dir = rrdset_cache_dir(fullid);
 342
 343         debug(D_RRD_CALLS, "Creating RRD_STATS for '%s.%s'.", type, id);
 344
 345         snprintf(fullfilename, FILENAME_MAX, "%s/main.db", cache_dir);
 346         if(rrd_memory_mode != RRD_MEMORY_MODE_RAM) st = (RRDSET *)mymmap(fullfilename, size, ((rrd_memory_mode == RRD_MEMORY_MODE_MAP)?MAP_SHARED:MAP_PRIVATE));
 347         if(st) {
 348                 if(strcmp(st->magic, RRDSET_MAGIC) != 0) {
 349                         errno = 0;
 350                         info("Initializing file %s.", fullfilename);
 351                         bzero(st, size);
 352                 }
 353                 else if(strcmp(st->id, fullid) != 0) {
 354                         errno = 0;
 355                         error("File %s contents are not for chart %s. Clearing it.", fullfilename, fullid);
 356                         // munmap(st, size);
 357                         // st = NULL;
 358                         bzero(st, size);
 359                 }
 360                 else if(st->memsize != size || st->entries != entries) {
 361                         errno = 0;
 362                         error("File %s does not have the desired size. Clearing it.", fullfilename);
 363                         bzero(st, size);
 364                 }
 365                 else if(st->update_every != update_every) {
 366                         errno = 0;
 367                         error("File %s does not have the desired update frequency. Clearing it.", fullfilename);
 368                         bzero(st, size);
 369                 }
 370                 else if((time(NULL) - st->last_updated.tv_sec) > update_every * entries) {
 371                         errno = 0;
 372                         error("File %s is too old. Clearing it.", fullfilename);
 373                         bzero(st, size);
 374                 }
 375         }
 376
 377         if(st) {
 378                 st->name = NULL;
 379                 st->type = NULL;
 380                 st->family = NULL;
 381                 st->title = NULL;
 382                 st->units = NULL;
 383                 st->dimensions = NULL;
 384                 st->next = NULL;
 385                 st->mapped = rrd_memory_mode;
 386         }
 387         else {
 388                 st = calloc(1, size);
 389                 if(!st) {
 390                         fatal("Cannot allocate memory for RRD_STATS %s.%s", type, id);
 391                         return NULL;
 392                 }
 393                 st->mapped = RRD_MEMORY_MODE_RAM;
 394         }
 395         st->memsize = size;
 396         st->entries = entries;
 397         st->update_every = update_every;
 398
 399         strcpy(st->cache_filename, fullfilename);
 400         strcpy(st->magic, RRDSET_MAGIC);
 401
 402         strcpy(st->id, fullid);
 403         st->hash = simple_hash(st->id);
 404
 405         st->cache_dir = cache_dir;
 406
 407         st->family     = config_get(st->id, "family", family?family:st->id);
 408         st->units      = config_get(st->id, "units", units?units:"");
 409         st->type       = config_get(st->id, "type", type);
 410         st->chart_type = rrdset_type_id(config_get(st->id, "chart type", rrdset_type_name(chart_type)));
 411
 412         st->priority = config_get_number(st->id, "priority", priority);
 413         st->enabled = enabled;
 414
 415         st->isdetail = 0;
 416         st->debug = 0;
 417
 418         st->last_collected_time.tv_sec = 0;
 419         st->last_collected_time.tv_usec = 0;
 420         st->counter_done = 0;
 421
 422         st->gap_when_lost_iterations_above = config_get_number(st->id, "gap when lost iterations above", RRD_DEFAULT_GAP_INTERPOLATIONS);
 423
 424         st->dimensions_index.root = NULL;
 425         st->dimensions_index.compar = rrddim_compare;
 426
 427         pthread_rwlock_init(&st->rwlock, NULL);
 428         pthread_rwlock_wrlock(&rrdset_root_rwlock);
 429
 430         if(name && *name) rrdset_set_name(st, name);
 431         else rrdset_set_name(st, id);
 432
 433         {
 434                 char varvalue[CONFIG_MAX_VALUE + 1];
 435                 snprintf(varvalue, CONFIG_MAX_VALUE, "%s (%s)", title?title:"", st->name);
 436                 st->title = config_get(st->id, "title", varvalue);
 437         }
 438
 439         st->next = rrdset_root;
 440         rrdset_root = st;
 441
 442         rrdset_index_add(st);
 443
 444         pthread_rwlock_unlock(&rrdset_root_rwlock);
 445
 446         return(st);
 447 }
 448
 449 RRDDIM *rrddim_add(RRDSET *st, const char *id, const char *name, long multiplier, long divisor, int algorithm)
 450 {
 451         char filename[FILENAME_MAX + 1];
 452         char fullfilename[FILENAME_MAX + 1];
 453
 454         char varname[CONFIG_MAX_NAME + 1];
 455         RRDDIM *rd = NULL;
 456         unsigned long size = sizeof(RRDDIM) + (st->entries * sizeof(storage_number));
 457
 458         debug(D_RRD_CALLS, "Adding dimension '%s/%s'.", st->id, id);
 459
 460         rrdset_strncpy_name(filename, id, FILENAME_MAX);
 461         snprintf(fullfilename, FILENAME_MAX, "%s/%s.db", st->cache_dir, filename);
 462         if(rrd_memory_mode != RRD_MEMORY_MODE_RAM) rd = (RRDDIM *)mymmap(fullfilename, size, ((rrd_memory_mode == RRD_MEMORY_MODE_MAP)?MAP_SHARED:MAP_PRIVATE));
 463         if(rd) {
 464                 struct timeval now;
 465                 gettimeofday(&now, NULL);
 466
 467                 if(strcmp(rd->magic, RRDDIMENSION_MAGIC) != 0) {
 468                         errno = 0;
 469                         info("Initializing file %s.", fullfilename);
 470                         bzero(rd, size);
 471                 }
 472                 else if(rd->memsize != size) {
 473                         errno = 0;
 474                         error("File %s does not have the desired size. Clearing it.", fullfilename);
 475                         bzero(rd, size);
 476                 }
 477                 else if(rd->multiplier != multiplier) {
 478                         errno = 0;
 479                         error("File %s does not have the same multiplier. Clearing it.", fullfilename);
 480                         bzero(rd, size);
 481                 }
 482                 else if(rd->divisor != divisor) {
 483                         errno = 0;
 484                         error("File %s does not have the same divisor. Clearing it.", fullfilename);
 485                         bzero(rd, size);
 486                 }
 487                 else if(rd->algorithm != algorithm) {
 488                         errno = 0;
 489                         error("File %s does not have the same algorithm. Clearing it.", fullfilename);
 490                         bzero(rd, size);
 491                 }
 492                 else if(rd->update_every != st->update_every) {
 493                         errno = 0;
 494                         error("File %s does not have the same refresh frequency. Clearing it.", fullfilename);
 495                         bzero(rd, size);
 496                 }
 497                 else if(usecdiff(&now, &rd->last_collected_time) > (rd->entries * rd->update_every * 1000000ULL)) {
 498                         errno = 0;
 499                         error("File %s is too old. Clearing it.", fullfilename);
 500                         bzero(rd, size);
 501                 }
 502                 else if(strcmp(rd->id, id) != 0) {
 503                         errno = 0;
 504                         error("File %s contents are not for dimension %s. Clearing it.", fullfilename, id);
 505                         // munmap(rd, size);
 506                         // rd = NULL;
 507                         bzero(rd, size);
 508                 }
 509         }
 510
 511         if(rd) {
 512                 // we have a file mapped for rd
 513                 rd->mapped = rrd_memory_mode;
 514                 rd->hidden = 0;
 515                 rd->next = NULL;
 516                 rd->name = NULL;
 517         }
 518         else {
 519                 // if we didn't manage to get a mmap'd dimension, just create one
 520
 521                 rd = calloc(1, size);
 522                 if(!rd) {
 523                         fatal("Cannot allocate RRD_DIMENSION %s/%s.", st->id, id);
 524                         return NULL;
 525                 }
 526
 527                 rd->mapped = RRD_MEMORY_MODE_RAM;
 528         }
 529         rd->memsize = size;
 530
 531         strcpy(rd->magic, RRDDIMENSION_MAGIC);
 532         strcpy(rd->cache_filename, fullfilename);
 533         strncpy(rd->id, id, RRD_ID_LENGTH_MAX);
 534         rd->hash = simple_hash(rd->id);
 535
 536         snprintf(varname, CONFIG_MAX_NAME, "dim %s name", rd->id);
 537         rd->name = config_get(st->id, varname, (name && *name)?name:rd->id);
 538
 539         snprintf(varname, CONFIG_MAX_NAME, "dim %s algorithm", rd->id);
 540         rd->algorithm = rrddim_algorithm_id(config_get(st->id, varname, rrddim_algorithm_name(algorithm)));
 541
 542         snprintf(varname, CONFIG_MAX_NAME, "dim %s multiplier", rd->id);
 543         rd->multiplier = config_get_number(st->id, varname, multiplier);
 544
 545         snprintf(varname, CONFIG_MAX_NAME, "dim %s divisor", rd->id);
 546         rd->divisor = config_get_number(st->id, varname, divisor);
 547         if(!rd->divisor) rd->divisor = 1;
 548
 549         rd->entries = st->entries;
 550         rd->update_every = st->update_every;
 551
 552         // append this dimension
 553         if(!st->dimensions)
 554                 st->dimensions = rd;
 555         else {
 556                 RRDDIM *td = st->dimensions;
 557                 for(; td->next; td = td->next) ;
 558                 td->next = rd;
 559         }
 560
 561         rrddim_index_add(st, rd);
 562
 563         return(rd);
 564 }
 565
 566 void rrddim_set_name(RRDSET *st, RRDDIM *rd, const char *name)
 567 {
 568         debug(D_RRD_CALLS, "rrddim_set_name() %s.%s", st->name, rd->name);
 569
 570         char varname[CONFIG_MAX_NAME + 1];
 571         snprintf(varname, CONFIG_MAX_NAME, "dim %s name", rd->id);
 572         config_get(st->id, varname, name);
 573 }
 574
 575 void rrddim_free(RRDSET *st, RRDDIM *rd)
 576 {
 577         debug(D_RRD_CALLS, "rrddim_free() %s.%s", st->name, rd->name);
 578
 579         RRDDIM *i = st->dimensions, *last = NULL;
 580         for(i = st->dimensions; i && i != rd ; i = i->next) last = i;
 581
 582         if(!i) {
 583                 error("Request to free dimension %s.%s but it is not linked.", st->id, rd->name);
 584                 return;
 585         }
 586
 587         if(last) last = i->next;
 588         else st->dimensions = i->next;
 589
 590         rrddim_index_del(st, rd);
 591
 592         // free(rd->annotations);
 593         if(rd->mapped == RRD_MEMORY_MODE_SAVE) {
 594                 debug(D_RRD_CALLS, "Saving dimension '%s' to '%s'.", rd->name, rd->cache_filename);
 595                 savememory(rd->cache_filename, rd, rd->memsize);
 596
 597                 debug(D_RRD_CALLS, "Unmapping dimension '%s'.", rd->name);
 598                 munmap(rd, rd->memsize);
 599         }
 600         else if(rd->mapped == RRD_MEMORY_MODE_MAP) {
 601                 debug(D_RRD_CALLS, "Unmapping dimension '%s'.", rd->name);
 602                 munmap(rd, rd->memsize);
 603         }
 604         else {
 605                 debug(D_RRD_CALLS, "Removing dimension '%s'.", rd->name);
 606                 free(rd);
 607         }
 608 }
 609
 610 void rrdset_free_all(void)
 611 {
 612         info("Freeing all memory...");
 613
 614         RRDSET *st;
 615         for(st = rrdset_root; st ;) {
 616                 RRDSET *next = st->next;
 617
 618                 while(st->dimensions)
 619                         rrddim_free(st, st->dimensions);
 620
 621                 rrdset_index_del(st);
 622
 623                 if(st->mapped == RRD_MEMORY_MODE_SAVE) {
 624                         debug(D_RRD_CALLS, "Saving stats '%s' to '%s'.", st->name, st->cache_filename);
 625                         savememory(st->cache_filename, st, st->memsize);
 626
 627                         debug(D_RRD_CALLS, "Unmapping stats '%s'.", st->name);
 628                         munmap(st, st->memsize);
 629                 }
 630                 else if(st->mapped == RRD_MEMORY_MODE_MAP) {
 631                         debug(D_RRD_CALLS, "Unmapping stats '%s'.", st->name);
 632                         munmap(st, st->memsize);
 633                 }
 634                 else
 635                         free(st);
 636
 637                 st = next;
 638         }
 639         rrdset_root = NULL;
 640
 641         info("Memory cleanup completed...");
 642 }
 643
 644 void rrdset_save_all(void)
 645 {
 646         debug(D_RRD_CALLS, "rrdset_save_all()");
 647
 648         RRDSET *st;
 649         RRDDIM *rd;
 650
 651         pthread_rwlock_wrlock(&rrdset_root_rwlock);
 652         for(st = rrdset_root; st ; st = st->next) {
 653                 pthread_rwlock_wrlock(&st->rwlock);
 654
 655                 if(st->mapped == RRD_MEMORY_MODE_SAVE) {
 656                         debug(D_RRD_CALLS, "Saving stats '%s' to '%s'.", st->name, st->cache_filename);
 657                         savememory(st->cache_filename, st, st->memsize);
 658                 }
 659
 660                 for(rd = st->dimensions; rd ; rd = rd->next) {
 661                         if(rd->mapped == RRD_MEMORY_MODE_SAVE) {
 662                                 debug(D_RRD_CALLS, "Saving dimension '%s' to '%s'.", rd->name, rd->cache_filename);
 663                                 savememory(rd->cache_filename, rd, rd->memsize);
 664                         }
 665                 }
 666
 667                 pthread_rwlock_unlock(&st->rwlock);
 668         }
 669         pthread_rwlock_unlock(&rrdset_root_rwlock);
 670 }
 671
 672
 673 RRDSET *rrdset_find(const char *id)
 674 {
 675         debug(D_RRD_CALLS, "rrdset_find() for chart %s", id);
 676
 677         pthread_rwlock_rdlock(&rrdset_root_rwlock);
 678         RRDSET *st = rrdset_index_find(id, 0);
 679         pthread_rwlock_unlock(&rrdset_root_rwlock);
 680
 681         return(st);
 682 }
 683
 684 RRDSET *rrdset_find_bytype(const char *type, const char *id)
 685 {
 686         debug(D_RRD_CALLS, "rrdset_find_bytype() for chart %s.%s", type, id);
 687
 688         char buf[RRD_ID_LENGTH_MAX + 1];
 689
 690         strncpy(buf, type, RRD_ID_LENGTH_MAX - 1);
 691         buf[RRD_ID_LENGTH_MAX - 1] = '\0';
 692         strcat(buf, ".");
 693         int len = strlen(buf);
 694         strncpy(&buf[len], id, RRD_ID_LENGTH_MAX - len);
 695         buf[RRD_ID_LENGTH_MAX] = '\0';
 696
 697         return(rrdset_find(buf));
 698 }
 699
 700 RRDSET *rrdset_find_byname(const char *name)
 701 {
 702         debug(D_RRD_CALLS, "rrdset_find_byname() for chart %s", name);
 703
 704         pthread_rwlock_rdlock(&rrdset_root_rwlock);
 705         RRDSET *st = rrdset_index_find_name(name, 0);
 706         pthread_rwlock_unlock(&rrdset_root_rwlock);
 707
 708         return(st);
 709 }
 710
 711 RRDDIM *rrddim_find(RRDSET *st, const char *id)
 712 {
 713         debug(D_RRD_CALLS, "rrddim_find() for chart %s, dimension %s", st->name, id);
 714
 715         return rrddim_index_find(st, id, 0);
 716 }
 717
 718 int rrddim_hide(RRDSET *st, const char *id)
 719 {
 720         debug(D_RRD_CALLS, "rrddim_hide() for chart %s, dimension %s", st->name, id);
 721
 722         RRDDIM *rd = rrddim_find(st, id);
 723         if(!rd) {
 724                 error("Cannot find dimension with id '%s' on stats '%s' (%s).", id, st->name, st->id);
 725                 return 1;
 726         }
 727
 728         rd->hidden = 1;
 729         return 0;
 730 }
 731
 732 void rrddim_set_by_pointer(RRDSET *st, RRDDIM *rd, collected_number value)
 733 {
 734         debug(D_RRD_CALLS, "rrddim_set_by_pointer() for chart %s, dimension %s, value " COLLECTED_NUMBER_FORMAT, st->name, rd->name, value);
 735
 736         gettimeofday(&rd->last_collected_time, NULL);
 737         rd->collected_value = value;
 738         rd->updated = 1;
 739 }
 740
 741 int rrddim_set(RRDSET *st, const char *id, collected_number value)
 742 {
 743         RRDDIM *rd = rrddim_find(st, id);
 744         if(!rd) {
 745                 error("Cannot find dimension with id '%s' on stats '%s' (%s).", id, st->name, st->id);
 746                 return 1;
 747         }
 748
 749         rrddim_set_by_pointer(st, rd, value);
 750         return 0;
 751 }
 752
 753 void rrdset_next_usec(RRDSET *st, unsigned long long microseconds)
 754 {
 755         debug(D_RRD_CALLS, "rrdset_next_usec() for chart %s with microseconds %llu", st->name, microseconds);
 756
 757         if(st->debug) debug(D_RRD_STATS, "%s: NEXT: %llu microseconds", st->name, microseconds);
 758         st->usec_since_last_update = microseconds;
 759 }
 760
 761 void rrdset_next(RRDSET *st)
 762 {
 763         unsigned long long microseconds = 0;
 764
 765         if(st->last_collected_time.tv_sec) {
 766                 struct timeval now;
 767                 gettimeofday(&now, NULL);
 768                 microseconds = usecdiff(&now, &st->last_collected_time);
 769         }
 770
 771         rrdset_next_usec(st, microseconds);
 772 }
 773
 774 void rrdset_next_plugins(RRDSET *st)
 775 {
 776         rrdset_next(st);
 777 }
 778
 779 unsigned long long rrdset_done(RRDSET *st)
 780 {
 781         debug(D_RRD_CALLS, "rrdset_done() for chart %s", st->name);
 782
 783         RRDDIM *rd, *last;
 784         int oldstate, store_this_entry = 1;
 785
 786         if(pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, &oldstate) != 0)
 787                 error("Cannot set pthread cancel state to DISABLE.");
 788
 789         // a read lock is OK here
 790         pthread_rwlock_rdlock(&st->rwlock);
 791
 792         // check if the chart has a long time to be refreshed
 793         if(st->usec_since_last_update > st->entries * st->update_every * 1000000ULL) {
 794                 info("%s: took too long to be updated (%0.3Lf secs). Reseting it.", st->name, (long double)(st->usec_since_last_update / 1000000.0));
 795                 rrdset_reset(st);
 796                 st->usec_since_last_update = st->update_every * 1000000ULL;
 797         }
 798         if(st->debug) debug(D_RRD_STATS, "%s: microseconds since last update: %llu", st->name, st->usec_since_last_update);
 799
 800         // set last_collected_time
 801         if(!st->last_collected_time.tv_sec) {
 802                 // it is the first entry
 803                 // set the last_collected_time to now
 804                 gettimeofday(&st->last_collected_time, NULL);
 805
 806                 // the first entry should not be stored
 807                 store_this_entry = 0;
 808
 809                 if(st->debug) debug(D_RRD_STATS, "%s: initializing last_collected to now. Will not store the next entry.", st->name);
 810         }
 811         else {
 812                 // it is not the first entry
 813                 // calculate the proper last_collected_time, using usec_since_last_update
 814                 unsigned long long ut = st->last_collected_time.tv_sec * 1000000ULL + st->last_collected_time.tv_usec + st->usec_since_last_update;
 815                 st->last_collected_time.tv_sec = ut / 1000000ULL;
 816                 st->last_collected_time.tv_usec = ut % 1000000ULL;
 817         }
 818
 819         // if this set has not been updated in the past
 820         // we fake the last_update time to be = now - usec_since_last_update
 821         if(!st->last_updated.tv_sec) {
 822                 // it has never been updated before
 823                 // set a fake last_updated, in the past using usec_since_last_update
 824                 unsigned long long ut = st->last_collected_time.tv_sec * 1000000ULL + st->last_collected_time.tv_usec - st->usec_since_last_update;
 825                 st->last_updated.tv_sec = ut / 1000000ULL;
 826                 st->last_updated.tv_usec = ut % 1000000ULL;
 827
 828                 // the first entry should not be stored
 829                 store_this_entry = 0;
 830
 831                 if(st->debug) debug(D_RRD_STATS, "%s: initializing last_updated to now - %llu microseconds (%0.3Lf). Will not store the next entry.", st->name, st->usec_since_last_update, (long double)ut/1000000.0);
 832         }
 833
 834         // check if we will re-write the entire data set
 835         if(usecdiff(&st->last_collected_time, &st->last_updated) > st->update_every * st->entries * 1000000ULL) {
 836                 info("%s: too old data (last updated at %u.%u, last collected at %u.%u). Reseting it. Will not store the next entry.", st->name, st->last_updated.tv_sec, st->last_updated.tv_usec, st->last_collected_time.tv_sec, st->last_collected_time.tv_usec);
 837                 rrdset_reset(st);
 838
 839                 st->usec_since_last_update = st->update_every * 1000000ULL;
 840
 841                 gettimeofday(&st->last_collected_time, NULL);
 842
 843                 unsigned long long ut = st->last_collected_time.tv_sec * 1000000ULL + st->last_collected_time.tv_usec - st->usec_since_last_update;
 844                 st->last_updated.tv_sec = ut / 1000000ULL;
 845                 st->last_updated.tv_usec = ut % 1000000ULL;
 846
 847                 // the first entry should not be stored
 848                 store_this_entry = 0;
 849         }
 850
 851         // these are the 3 variables that will help us in interpolation
 852         // last_ut = the last time we added a value to the storage
 853         //  now_ut = the time the current value is taken at
 854         // next_ut = the time of the next interpolation point
 855         unsigned long long last_ut = st->last_updated.tv_sec * 1000000ULL + st->last_updated.tv_usec;
 856         unsigned long long now_ut  = st->last_collected_time.tv_sec * 1000000ULL + st->last_collected_time.tv_usec;
 857         unsigned long long next_ut = (st->last_updated.tv_sec + st->update_every) * 1000000ULL;
 858
 859         if(st->debug) debug(D_RRD_STATS, "%s: last ut = %0.3Lf (last updated time)", st->name, (long double)last_ut/1000000.0);
 860         if(st->debug) debug(D_RRD_STATS, "%s: now  ut = %0.3Lf (current update time)", st->name, (long double)now_ut/1000000.0);
 861         if(st->debug) debug(D_RRD_STATS, "%s: next ut = %0.3Lf (next interpolation point)", st->name, (long double)next_ut/1000000.0);
 862
 863         if(!st->counter_done) {
 864                 store_this_entry = 0;
 865                 if(st->debug) debug(D_RRD_STATS, "%s: Will not store the next entry.", st->name);
 866         }
 867         st->counter_done++;
 868
 869         // calculate totals and count the dimensions
 870         int dimensions;
 871         st->collected_total = 0;
 872         for( rd = st->dimensions, dimensions = 0 ; rd ; rd = rd->next, dimensions++ )
 873                 st->collected_total += rd->collected_value;
 874
 875         uint32_t storage_flags = SN_EXISTS;
 876
 877         // process all dimensions to calculate their values
 878         // based on the collected figures only
 879         // at this stage we do not interpolate anything
 880         for( rd = st->dimensions ; rd ; rd = rd->next ) {
 881
 882                 if(st->debug) debug(D_RRD_STATS, "%s/%s: "
 883                         " last_collected_value = " COLLECTED_NUMBER_FORMAT
 884                         " collected_value = " COLLECTED_NUMBER_FORMAT
 885                         " last_calculated_value = " CALCULATED_NUMBER_FORMAT
 886                         " calculated_value = " CALCULATED_NUMBER_FORMAT
 887                         , st->id, rd->name
 888                         , rd->last_collected_value
 889                         , rd->collected_value
 890                         , rd->last_calculated_value
 891                         , rd->calculated_value
 892                         );
 893
 894                 switch(rd->algorithm) {
 895                         case RRDDIM_PCENT_OVER_DIFF_TOTAL:
 896                                 // the percentage of the current increment
 897                                 // over the increment of all dimensions together
 898                                 if(st->collected_total == st->last_collected_total) rd->calculated_value = rd->last_calculated_value;
 899                                 else rd->calculated_value =
 900                                           (calculated_number)100
 901                                         * (calculated_number)(rd->collected_value - rd->last_collected_value)
 902                                         / (calculated_number)(st->collected_total  - st->last_collected_total);
 903
 904                                 if(st->debug)
 905                                         debug(D_RRD_STATS, "%s/%s: CALC PCENT-DIFF "
 906                                                 CALCULATED_NUMBER_FORMAT " = 100"
 907                                                 " * (" COLLECTED_NUMBER_FORMAT " - " COLLECTED_NUMBER_FORMAT ")"
 908                                                 " / (" COLLECTED_NUMBER_FORMAT " - " COLLECTED_NUMBER_FORMAT ")"
 909                                                 , st->id, rd->name
 910                                                 , rd->calculated_value
 911                                                 , rd->collected_value, rd->last_collected_value
 912                                                 , st->collected_total, st->last_collected_total
 913                                                 );
 914                                 break;
 915
 916                         case RRDDIM_PCENT_OVER_ROW_TOTAL:
 917                                 if(!st->collected_total) rd->calculated_value = 0;
 918                                 else
 919                                 // the percentage of the current value
 920                                 // over the total of all dimensions
 921                                 rd->calculated_value =
 922                                           (calculated_number)100
 923                                         * (calculated_number)rd->collected_value
 924                                         / (calculated_number)st->collected_total;
 925
 926                                 if(st->debug)
 927                                         debug(D_RRD_STATS, "%s/%s: CALC PCENT-ROW "
 928                                                 CALCULATED_NUMBER_FORMAT " = 100"
 929                                                 " * " COLLECTED_NUMBER_FORMAT
 930                                                 " / " COLLECTED_NUMBER_FORMAT
 931                                                 , st->id, rd->name
 932                                                 , rd->calculated_value
 933                                                 , rd->collected_value
 934                                                 , st->collected_total
 935                                                 );
 936                                 break;
 937
 938                         case RRDDIM_INCREMENTAL:
 939                                 // if the new is smaller than the old (an overflow, or reset), set the old equal to the new
 940                                 // to reset the calculation (it will give zero as the calculation for this second)
 941                                 if(rd->last_collected_value > rd->collected_value) {
 942                                         info("%s.%s: Detect RESET or OVERFLOW. Last collected value = " COLLECTED_NUMBER_FORMAT ", current = " COLLECTED_NUMBER_FORMAT
 943                                                         , st->name, rd->name
 944                                                         , rd->last_collected_value
 945                                                         , rd->collected_value);
 946                                         storage_flags = SN_EXISTS_RESET;
 947                                         rd->last_collected_value = rd->collected_value;
 948                                 }
 949
 950                                 rd->calculated_value += (calculated_number)(rd->collected_value - rd->last_collected_value);
 951
 952                                 if(st->debug)
 953                                         debug(D_RRD_STATS, "%s/%s: CALC INC "
 954                                                 CALCULATED_NUMBER_FORMAT " += "
 955                                                 COLLECTED_NUMBER_FORMAT " - " COLLECTED_NUMBER_FORMAT
 956                                                 , st->id, rd->name
 957                                                 , rd->calculated_value
 958                                                 , rd->collected_value, rd->last_collected_value
 959                                                 );
 960                                 break;
 961
 962                         case RRDDIM_ABSOLUTE:
 963                                 rd->calculated_value = (calculated_number)rd->collected_value;
 964
 965                                 if(st->debug)
 966                                         debug(D_RRD_STATS, "%s/%s: CALC ABS/ABS-NO-IN "
 967                                                 CALCULATED_NUMBER_FORMAT " = "
 968                                                 COLLECTED_NUMBER_FORMAT
 969                                                 , st->id, rd->name
 970                                                 , rd->calculated_value
 971                                                 , rd->collected_value
 972                                                 );
 973                                 break;
 974
 975                         default:
 976                                 // make the default zero, to make sure
 977                                 // it gets noticed when we add new types
 978                                 rd->calculated_value = 0;
 979
 980                                 if(st->debug)
 981                                         debug(D_RRD_STATS, "%s/%s: CALC "
 982                                                 CALCULATED_NUMBER_FORMAT " = 0"
 983                                                 , st->id, rd->name
 984                                                 , rd->calculated_value
 985                                                 );
 986                                 break;
 987                 }
 988         }
 989
 990         // at this point we have all the calculated values ready
 991         // it is now time to interpolate values on a second boundary
 992
 993         unsigned long long first_ut = last_ut;
 994         int iterations = (now_ut - last_ut) / (st->update_every * 1000000ULL);
 995
 996         for( ; next_ut <= now_ut ; next_ut += st->update_every * 1000000ULL, iterations-- ) {
 997                 if(iterations < 0) error("iterations calculation wrapped!");
 998
 999                 if(st->debug) debug(D_RRD_STATS, "%s: last ut = %0.3Lf (last updated time)", st->name, (long double)last_ut/1000000.0);
1000                 if(st->debug) debug(D_RRD_STATS, "%s: next ut = %0.3Lf (next interpolation point)", st->name, (long double)next_ut/1000000.0);
1001
1002                 st->last_updated.tv_sec = next_ut / 1000000ULL;
1003                 st->last_updated.tv_usec = 0;
1004
1005                 for( rd = st->dimensions ; rd ; rd = rd->next ) {
1006                         calculated_number new_value;
1007
1008                         switch(rd->algorithm) {
1009                                 case RRDDIM_INCREMENTAL:
1010                                         new_value = (calculated_number)
1011                                                 (          rd->calculated_value
1012                                                         * (calculated_number)(next_ut - last_ut)
1013                                                         / (calculated_number)(now_ut - last_ut)
1014                                                 );
1015
1016                                         if(st->debug)
1017                                                 debug(D_RRD_STATS, "%s/%s: CALC2 INC "
1018                                                         CALCULATED_NUMBER_FORMAT " = "
1019                                                         CALCULATED_NUMBER_FORMAT
1020                                                         " * %llu"
1021                                                         " / %llu"
1022                                                         , st->id, rd->name
1023                                                         , new_value
1024                                                         , rd->calculated_value
1025                                                         , (unsigned long long)(next_ut - last_ut)
1026                                                         , (unsigned long long)(now_ut - last_ut)
1027                                                         );
1028
1029                                         rd->calculated_value -= new_value;
1030                                         break;
1031
1032                                 case RRDDIM_ABSOLUTE:
1033                                 case RRDDIM_PCENT_OVER_ROW_TOTAL:
1034                                 case RRDDIM_PCENT_OVER_DIFF_TOTAL:
1035                                 default:
1036                                         new_value = (calculated_number)
1037                                                 (       (         (rd->calculated_value - rd->last_calculated_value)
1038                                                                 * (calculated_number)(next_ut - first_ut)
1039                                                                 / (calculated_number)(now_ut - first_ut)
1040                                                         )
1041                                                         +  rd->last_calculated_value
1042                                                 );
1043
1044                                         if(st->debug)
1045                                                 debug(D_RRD_STATS, "%s/%s: CALC2 DEF "
1046                                                         CALCULATED_NUMBER_FORMAT " = ((("
1047                                                         "(" CALCULATED_NUMBER_FORMAT " - " CALCULATED_NUMBER_FORMAT ")"
1048                                                         " * %llu"
1049                                                         " / %llu) + " CALCULATED_NUMBER_FORMAT
1050                                                         , st->id, rd->name
1051                                                         , new_value
1052                                                         , rd->calculated_value, rd->last_calculated_value
1053                                                         , (next_ut - first_ut)
1054                                                         , (now_ut - first_ut), rd->last_calculated_value
1055                                                         );
1056
1057                                         if(next_ut + st->update_every * 1000000ULL > now_ut) rd->calculated_value = new_value;
1058                                         break;
1059                         }
1060
1061                         if(!store_this_entry) {
1062                                 store_this_entry = 1;
1063                                 continue;
1064                         }
1065
1066                         if(rd->updated && iterations < st->gap_when_lost_iterations_above) {
1067                                 rd->values[st->current_entry] = pack_storage_number(
1068                                                   new_value
1069                                                 * (calculated_number)rd->multiplier
1070                                                 / (calculated_number)rd->divisor
1071                                         , storage_flags );
1072
1073                                 if(st->debug)
1074                                         debug(D_RRD_STATS, "%s/%s: STORE[%ld] "
1075                                                 CALCULATED_NUMBER_FORMAT " = " CALCULATED_NUMBER_FORMAT
1076                                                 " * %ld"
1077                                                 " / %ld"
1078                                                 , st->id, rd->name
1079                                                 , st->current_entry
1080                                                 , unpack_storage_number(rd->values[st->current_entry]), new_value
1081                                                 , rd->multiplier
1082                                                 , rd->divisor
1083                                                 );
1084                         }
1085                         else {
1086                                 if(st->debug) debug(D_RRD_STATS, "%s/%s: STORE[%ld] = NON EXISTING "
1087                                                 , st->id, rd->name
1088                                                 , st->current_entry
1089                                                 );
1090                                 rd->values[st->current_entry] = pack_storage_number(0, SN_NOT_EXISTS);
1091                         }
1092
1093                         if(st->debug) {
1094                                 calculated_number t1 = new_value * (calculated_number)rd->multiplier / (calculated_number)rd->divisor;
1095                                 calculated_number t2 = unpack_storage_number(rd->values[st->current_entry]);
1096                                 calculated_number accuracy = accuracy_loss(t1, t2);
1097                                 debug(D_RRD_STATS, "%s/%s: UNPACK[%ld] = " CALCULATED_NUMBER_FORMAT " FLAGS=0x%08x (original = " CALCULATED_NUMBER_FORMAT ", accuracy loss = " CALCULATED_NUMBER_FORMAT "%%%s)"
1098                                                 , st->id, rd->name
1099                                                 , st->current_entry
1100                                                 , t2
1101                                                 , get_storage_number_flags(rd->values[st->current_entry])
1102                                                 , t1
1103                                                 , accuracy
1104                                                 , (accuracy > ACCURACY_LOSS) ? " **TOO BIG** " : ""
1105                                                 );
1106
1107                                 rd->collected_volume += t1;
1108                                 rd->stored_volume += t2;
1109                                 accuracy = accuracy_loss(rd->collected_volume, rd->stored_volume);
1110                                 debug(D_RRD_STATS, "%s/%s: VOLUME[%ld] = " CALCULATED_NUMBER_FORMAT ", calculated  = " CALCULATED_NUMBER_FORMAT ", accuracy loss = " CALCULATED_NUMBER_FORMAT "%%%s"
1111                                                 , st->id, rd->name
1112                                                 , st->current_entry
1113                                                 , rd->stored_volume
1114                                                 , rd->collected_volume
1115                                                 , accuracy
1116                                                 , (accuracy > ACCURACY_LOSS) ? " **TOO BIG** " : ""
1117                                                 );
1118
1119                         }
1120                 }
1121                 // reset the storage flags for the next point, if any;
1122                 storage_flags = SN_EXISTS;
1123
1124                 if(st->first_entry_t && st->counter >= (unsigned long long)st->entries) {
1125                         // the db is overwriting values
1126                         // add the value we will overwrite
1127                         st->first_entry_t += st->update_every * 1000000ULL;
1128                 }
1129
1130                 st->counter++;
1131                 st->current_entry = ((st->current_entry + 1) >= st->entries) ? 0 : st->current_entry + 1;
1132                 if(!st->first_entry_t) st->first_entry_t = next_ut;
1133                 last_ut = next_ut;
1134         }
1135
1136         for( rd = st->dimensions; rd ; rd = rd->next ) {
1137                 if(!rd->updated) continue;
1138                 rd->last_collected_value = rd->collected_value;
1139                 rd->last_calculated_value = rd->calculated_value;
1140                 rd->collected_value = 0;
1141                 rd->updated = 0;
1142
1143                 // if this is the first entry of incremental dimensions
1144                 // we have to set the first calculated_value to zero
1145                 // to eliminate the first spike
1146                 if(st->counter_done == 1) switch(rd->algorithm) {
1147                         case RRDDIM_PCENT_OVER_DIFF_TOTAL:
1148                         case RRDDIM_INCREMENTAL:
1149                                 rd->calculated_value = 0;
1150                                 // the next time, a new incremental total will be calculated
1151                                 break;
1152                 }
1153         }
1154         st->last_collected_total  = st->collected_total;
1155
1156         // ALL DONE ABOUT THE DATA UPDATE
1157         // --------------------------------------------------------------------
1158
1159         // find if there are any obsolete dimensions (not updated recently)
1160         for( rd = st->dimensions; rd ; rd = rd->next )
1161                 if((rd->last_collected_time.tv_sec + (10 * st->update_every)) < st->last_collected_time.tv_sec)
1162                         break;
1163
1164         if(rd) {
1165                 // there is dimension to free
1166                 // upgrade our read lock to a write lock
1167                 pthread_rwlock_unlock(&st->rwlock);
1168                 pthread_rwlock_wrlock(&st->rwlock);
1169
1170                 for( rd = st->dimensions, last = NULL ; rd ; ) {
1171                         if((rd->last_collected_time.tv_sec + (10 * st->update_every)) < st->last_collected_time.tv_sec) { // remove it only it is not updated in 10 seconds
1172                                 debug(D_RRD_STATS, "Removing obsolete dimension '%s' (%s) of '%s' (%s).", rd->name, rd->id, st->name, st->id);
1173
1174                                 if(!last) {
1175                                         st->dimensions = rd->next;
1176                                         rd->next = NULL;
1177                                         rrddim_free(st, rd);
1178                                         rd = st->dimensions;
1179                                         continue;
1180                                 }
1181                                 else {
1182                                         last->next = rd->next;
1183                                         rd->next = NULL;
1184                                         rrddim_free(st, rd);
1185                                         rd = last->next;
1186                                         continue;
1187                                 }
1188                         }
1189
1190                         last = rd;
1191                         rd = rd->next;
1192                 }
1193
1194                 if(!st->dimensions) st->enabled = 0;
1195         }
1196
1197         pthread_rwlock_unlock(&st->rwlock);
1198
1199         if(pthread_setcancelstate(oldstate, NULL) != 0)
1200                 error("Cannot set pthread cancel state to RESTORE (%d).", oldstate);
1201
1202         return(st->usec_since_last_update);
1203 }
1204
1205
1206 // find the oldest entry in the data, skipping all empty slots
1207 time_t rrdset_first_entry_t(RRDSET *st)
1208 {
1209         if(!st->first_entry_t) return st->last_updated.tv_sec;
1210
1211         return st->first_entry_t / 1000000;
1212 }