10 #include <sys/types.h>
19 int update_every = UPDATE_EVERY;
20 int save_history = HISTORY;
22 // ----------------------------------------------------------------------------
25 int chart_type_id(const char *name)
27 if(strcmp(name, CHART_TYPE_AREA_NAME) == 0) return CHART_TYPE_AREA;
28 if(strcmp(name, CHART_TYPE_STACKED_NAME) == 0) return CHART_TYPE_STACKED;
29 if(strcmp(name, CHART_TYPE_LINE_NAME) == 0) return CHART_TYPE_LINE;
30 return CHART_TYPE_LINE;
33 const char *chart_type_name(int chart_type)
35 static char line[] = CHART_TYPE_LINE_NAME;
36 static char area[] = CHART_TYPE_AREA_NAME;
37 static char stacked[] = CHART_TYPE_STACKED_NAME;
46 case CHART_TYPE_STACKED:
52 // ----------------------------------------------------------------------------
55 int memory_mode = NETDATA_MEMORY_MODE_SAVE;
57 const char *memory_mode_name(int id)
59 static const char ram[] = NETDATA_MEMORY_MODE_RAM_NAME;
60 static const char map[] = NETDATA_MEMORY_MODE_MAP_NAME;
61 static const char save[] = NETDATA_MEMORY_MODE_SAVE_NAME;
64 case NETDATA_MEMORY_MODE_RAM:
67 case NETDATA_MEMORY_MODE_MAP:
70 case NETDATA_MEMORY_MODE_SAVE:
78 int memory_mode_id(const char *name)
80 if(!strcmp(name, NETDATA_MEMORY_MODE_RAM_NAME))
81 return NETDATA_MEMORY_MODE_RAM;
82 else if(!strcmp(name, NETDATA_MEMORY_MODE_MAP_NAME))
83 return NETDATA_MEMORY_MODE_MAP;
85 return NETDATA_MEMORY_MODE_SAVE;
88 // ----------------------------------------------------------------------------
91 int algorithm_id(const char *name)
93 if(strcmp(name, RRD_DIMENSION_ABSOLUTE_NAME) == 0) return RRD_DIMENSION_ABSOLUTE;
94 if(strcmp(name, RRD_DIMENSION_INCREMENTAL_NAME) == 0) return RRD_DIMENSION_INCREMENTAL;
95 if(strcmp(name, RRD_DIMENSION_PCENT_OVER_ROW_TOTAL_NAME) == 0) return RRD_DIMENSION_PCENT_OVER_ROW_TOTAL;
96 if(strcmp(name, RRD_DIMENSION_PCENT_OVER_DIFF_TOTAL_NAME) == 0) return RRD_DIMENSION_PCENT_OVER_DIFF_TOTAL;
97 return RRD_DIMENSION_ABSOLUTE;
100 const char *algorithm_name(int chart_type)
102 static char absolute[] = RRD_DIMENSION_ABSOLUTE_NAME;
103 static char incremental[] = RRD_DIMENSION_INCREMENTAL_NAME;
104 static char percentage_of_absolute_row[] = RRD_DIMENSION_PCENT_OVER_ROW_TOTAL_NAME;
105 static char percentage_of_incremental_row[] = RRD_DIMENSION_PCENT_OVER_DIFF_TOTAL_NAME;
108 case RRD_DIMENSION_ABSOLUTE:
111 case RRD_DIMENSION_INCREMENTAL:
114 case RRD_DIMENSION_PCENT_OVER_ROW_TOTAL:
115 return percentage_of_absolute_row;
117 case RRD_DIMENSION_PCENT_OVER_DIFF_TOTAL:
118 return percentage_of_incremental_row;
123 RRD_STATS *root = NULL;
124 pthread_rwlock_t root_rwlock = PTHREAD_RWLOCK_INITIALIZER;
126 char *rrd_stats_strncpy_name(char *to, const char *from, int length)
129 for(i = 0; i < length && from[i] ;i++) {
130 if(from[i] == '.' || isalpha(from[i]) || isdigit(from[i])) to[i] = from[i];
133 if(i < length) to[i] = '\0';
134 to[length - 1] = '\0';
139 void rrd_stats_set_name(RRD_STATS *st, const char *name)
141 char b[CONFIG_MAX_VALUE + 1];
142 char n[RRD_STATS_NAME_MAX + 1];
144 snprintf(n, RRD_STATS_NAME_MAX, "%s.%s", st->type, name);
145 rrd_stats_strncpy_name(b, n, CONFIG_MAX_VALUE);
146 st->name = config_get(st->id, "name", b);
147 st->hash_name = simple_hash(st->name);
150 char *rrd_stats_cache_dir(const char *id)
154 static char *cache_dir = NULL;
155 if(!cache_dir) cache_dir = config_get("global", "database directory", "cache");
157 char b[FILENAME_MAX + 1];
158 char n[FILENAME_MAX + 1];
159 rrd_stats_strncpy_name(b, id, FILENAME_MAX);
161 snprintf(n, FILENAME_MAX, "%s/%s", cache_dir, b);
162 ret = config_get(id, "database directory", n);
164 if(memory_mode == NETDATA_MEMORY_MODE_MAP || memory_mode == NETDATA_MEMORY_MODE_SAVE) {
165 int r = mkdir(ret, 0775);
166 if(r != 0 && errno != EEXIST)
167 error("Cannot create directory '%s'", ret);
173 void rrd_stats_reset(RRD_STATS *st)
175 st->last_collected_time.tv_sec = 0;
176 st->last_collected_time.tv_usec = 0;
177 st->last_updated.tv_sec = 0;
178 st->last_updated.tv_usec = 0;
179 st->current_entry = 0;
181 st->counter_done = 0;
184 for(rd = st->dimensions; rd ; rd = rd->next) {
185 rd->last_collected_time.tv_sec = 0;
186 rd->last_collected_time.tv_usec = 0;
187 bzero(rd->values, rd->entries * sizeof(storage_number));
191 RRD_STATS *rrd_stats_create(const char *type, const char *id, const char *name, const char *family, const char *title, const char *units, long priority, int update_every, int chart_type)
194 fatal("Cannot create rrd stats without an id.");
198 char fullid[RRD_STATS_NAME_MAX + 1];
199 char fullfilename[FILENAME_MAX + 1];
200 RRD_STATS *st = NULL;
202 snprintf(fullid, RRD_STATS_NAME_MAX, "%s.%s", type, id);
204 long entries = config_get_number(fullid, "history", save_history);
205 if(entries < 5) entries = config_set_number(fullid, "history", 5);
206 if(entries > HISTORY_MAX) entries = config_set_number(fullid, "history", HISTORY_MAX);
208 int enabled = config_get_boolean(fullid, "enabled", 1);
209 if(!enabled) entries = 5;
211 unsigned long size = sizeof(RRD_STATS);
212 char *cache_dir = rrd_stats_cache_dir(fullid);
214 debug(D_RRD_CALLS, "Creating RRD_STATS for '%s.%s'.", type, id);
216 snprintf(fullfilename, FILENAME_MAX, "%s/main.db", cache_dir);
217 if(memory_mode != NETDATA_MEMORY_MODE_RAM) st = (RRD_STATS *)mymmap(fullfilename, size, ((memory_mode == NETDATA_MEMORY_MODE_MAP)?MAP_SHARED:MAP_PRIVATE));
219 if(strcmp(st->magic, RRD_STATS_MAGIC) != 0) {
221 info("Initializing file %s.", fullfilename);
224 else if(strcmp(st->id, fullid) != 0) {
226 error("File %s contents are not for chart %s. Clearing it.", fullfilename, fullid);
231 else if(st->memsize != size || st->entries != entries) {
233 error("File %s does not have the desired size. Clearing it.", fullfilename);
236 else if(st->update_every != update_every) {
238 error("File %s does not have the desired update frequency. Clearing it.", fullfilename);
241 else if((time(NULL) - st->last_updated.tv_sec) > update_every * entries) {
243 error("File %s is too old. Clearing it.", fullfilename);
254 st->dimensions = NULL;
256 st->mapped = memory_mode;
259 st = calloc(1, size);
261 fatal("Cannot allocate memory for RRD_STATS %s.%s", type, id);
264 st->mapped = NETDATA_MEMORY_MODE_RAM;
267 st->entries = entries;
268 st->update_every = update_every;
270 strcpy(st->cache_file, fullfilename);
271 strcpy(st->magic, RRD_STATS_MAGIC);
273 strcpy(st->id, fullid);
274 st->hash = simple_hash(st->id);
276 st->cache_dir = cache_dir;
278 st->family = config_get(st->id, "family", family?family:st->id);
279 st->units = config_get(st->id, "units", units?units:"");
280 st->type = config_get(st->id, "type", type);
281 st->chart_type = chart_type_id(config_get(st->id, "chart type", chart_type_name(chart_type)));
283 if(name && *name) rrd_stats_set_name(st, name);
284 else rrd_stats_set_name(st, id);
287 char varvalue[CONFIG_MAX_VALUE + 1];
288 snprintf(varvalue, CONFIG_MAX_VALUE, "%s (%s)", title?title:"", st->name);
289 st->title = config_get(st->id, "title", varvalue);
292 st->priority = config_get_number(st->id, "priority", priority);
293 st->enabled = enabled;
298 st->last_collected_time.tv_sec = 0;
299 st->last_collected_time.tv_usec = 0;
300 st->counter_done = 0;
302 st->gap_when_lost_iterations = config_get_number(st->id, "gap when lost iterations above", DEFAULT_GAP_INTERPOLATIONS);
304 pthread_rwlock_init(&st->rwlock, NULL);
305 pthread_rwlock_wrlock(&root_rwlock);
310 pthread_rwlock_unlock(&root_rwlock);
315 RRD_DIMENSION *rrd_stats_dimension_add(RRD_STATS *st, const char *id, const char *name, long multiplier, long divisor, int algorithm)
317 char filename[FILENAME_MAX + 1];
318 char fullfilename[FILENAME_MAX + 1];
320 char varname[CONFIG_MAX_NAME + 1];
321 RRD_DIMENSION *rd = NULL;
322 unsigned long size = sizeof(RRD_DIMENSION) + (st->entries * sizeof(storage_number));
324 debug(D_RRD_CALLS, "Adding dimension '%s/%s'.", st->id, id);
326 rrd_stats_strncpy_name(filename, id, FILENAME_MAX);
327 snprintf(fullfilename, FILENAME_MAX, "%s/%s.db", st->cache_dir, filename);
328 if(memory_mode != NETDATA_MEMORY_MODE_RAM) rd = (RRD_DIMENSION *)mymmap(fullfilename, size, ((memory_mode == NETDATA_MEMORY_MODE_MAP)?MAP_SHARED:MAP_PRIVATE));
331 gettimeofday(&now, NULL);
333 if(strcmp(rd->magic, RRD_DIMENSION_MAGIC) != 0) {
335 info("Initializing file %s.", fullfilename);
338 else if(rd->memsize != size) {
340 error("File %s does not have the desired size. Clearing it.", fullfilename);
343 else if(rd->multiplier != multiplier) {
345 error("File %s does not have the same multiplier. Clearing it.", fullfilename);
348 else if(rd->divisor != divisor) {
350 error("File %s does not have the same divisor. Clearing it.", fullfilename);
353 else if(rd->algorithm != algorithm) {
355 error("File %s does not have the same algorithm. Clearing it.", fullfilename);
358 else if(rd->update_every != st->update_every) {
360 error("File %s does not have the same refresh frequency. Clearing it.", fullfilename);
363 else if(usecdiff(&now, &rd->last_collected_time) > (rd->entries * rd->update_every * 1000000ULL)) {
365 error("File %s is too old. Clearing it.", fullfilename);
368 else if(strcmp(rd->id, id) != 0) {
370 error("File %s contents are not for dimension %s. Clearing it.", fullfilename, id);
378 // we have a file mapped for rd
379 rd->mapped = memory_mode;
385 // if we didn't manage to get a mmap'd dimension, just create one
387 rd = calloc(1, size);
389 fatal("Cannot allocate RRD_DIMENSION %s/%s.", st->id, id);
393 rd->mapped = NETDATA_MEMORY_MODE_RAM;
397 strcpy(rd->magic, RRD_DIMENSION_MAGIC);
398 strcpy(rd->cache_file, fullfilename);
399 strncpy(rd->id, id, RRD_STATS_NAME_MAX);
400 rd->hash = simple_hash(rd->id);
402 snprintf(varname, CONFIG_MAX_NAME, "dim %s name", rd->id);
403 rd->name = config_get(st->id, varname, (name && *name)?name:rd->id);
405 snprintf(varname, CONFIG_MAX_NAME, "dim %s algorithm", rd->id);
406 rd->algorithm = algorithm_id(config_get(st->id, varname, algorithm_name(algorithm)));
408 snprintf(varname, CONFIG_MAX_NAME, "dim %s multiplier", rd->id);
409 rd->multiplier = config_get_number(st->id, varname, multiplier);
411 snprintf(varname, CONFIG_MAX_NAME, "dim %s divisor", rd->id);
412 rd->divisor = config_get_number(st->id, varname, divisor);
413 if(!rd->divisor) rd->divisor = 1;
415 rd->entries = st->entries;
416 rd->update_every = st->update_every;
418 // append this dimension
422 RRD_DIMENSION *td = st->dimensions;
423 for(; td->next; td = td->next) ;
430 void rrd_stats_dimension_set_name(RRD_STATS *st, RRD_DIMENSION *rd, const char *name)
432 char varname[CONFIG_MAX_NAME + 1];
433 snprintf(varname, CONFIG_MAX_NAME, "dim %s name", rd->id);
434 config_get(st->id, varname, name);
437 void rrd_stats_dimension_free(RRD_DIMENSION *rd)
439 if(rd->next) rrd_stats_dimension_free(rd->next);
440 // free(rd->annotations);
441 if(rd->mapped == NETDATA_MEMORY_MODE_SAVE) {
442 debug(D_RRD_CALLS, "Saving dimension '%s' to '%s'.", rd->name, rd->cache_file);
443 savememory(rd->cache_file, rd, rd->memsize);
445 debug(D_RRD_CALLS, "Unmapping dimension '%s'.", rd->name);
446 munmap(rd, rd->memsize);
448 else if(rd->mapped == NETDATA_MEMORY_MODE_MAP) {
449 debug(D_RRD_CALLS, "Unmapping dimension '%s'.", rd->name);
450 munmap(rd, rd->memsize);
453 debug(D_RRD_CALLS, "Removing dimension '%s'.", rd->name);
458 void rrd_stats_free_all(void)
460 info("Freeing all memory...");
463 for(st = root; st ;) {
464 RRD_STATS *next = st->next;
466 if(st->dimensions) rrd_stats_dimension_free(st->dimensions);
467 st->dimensions = NULL;
469 if(st->mapped == NETDATA_MEMORY_MODE_SAVE) {
470 debug(D_RRD_CALLS, "Saving stats '%s' to '%s'.", st->name, st->cache_file);
471 savememory(st->cache_file, st, st->memsize);
473 debug(D_RRD_CALLS, "Unmapping stats '%s'.", st->name);
474 munmap(st, st->memsize);
476 else if(st->mapped == NETDATA_MEMORY_MODE_MAP) {
477 debug(D_RRD_CALLS, "Unmapping stats '%s'.", st->name);
478 munmap(st, st->memsize);
487 info("Memory cleanup completed...");
490 void rrd_stats_save_all(void)
495 pthread_rwlock_wrlock(&root_rwlock);
496 for(st = root; st ; st = st->next) {
497 pthread_rwlock_wrlock(&st->rwlock);
499 if(st->mapped == NETDATA_MEMORY_MODE_SAVE) {
500 debug(D_RRD_CALLS, "Saving stats '%s' to '%s'.", st->name, st->cache_file);
501 savememory(st->cache_file, st, st->memsize);
504 for(rd = st->dimensions; rd ; rd = rd->next) {
505 if(rd->mapped == NETDATA_MEMORY_MODE_SAVE) {
506 debug(D_RRD_CALLS, "Saving dimension '%s' to '%s'.", rd->name, rd->cache_file);
507 savememory(rd->cache_file, rd, rd->memsize);
511 pthread_rwlock_unlock(&st->rwlock);
513 pthread_rwlock_unlock(&root_rwlock);
517 RRD_STATS *rrd_stats_find(const char *id)
519 debug(D_RRD_CALLS, "rrd_stats_find() for chart %s", id);
521 unsigned long hash = simple_hash(id);
523 pthread_rwlock_rdlock(&root_rwlock);
524 RRD_STATS *st = root;
525 for ( ; st ; st = st->next )
527 if(strcmp(st->id, id) == 0)
529 pthread_rwlock_unlock(&root_rwlock);
534 RRD_STATS *rrd_stats_find_bytype(const char *type, const char *id)
536 debug(D_RRD_CALLS, "rrd_stats_find_bytype() for chart %s.%s", type, id);
538 char buf[RRD_STATS_NAME_MAX + 1];
540 strncpy(buf, type, RRD_STATS_NAME_MAX - 1);
541 buf[RRD_STATS_NAME_MAX - 1] = '\0';
543 int len = strlen(buf);
544 strncpy(&buf[len], id, RRD_STATS_NAME_MAX - len);
545 buf[RRD_STATS_NAME_MAX] = '\0';
547 return(rrd_stats_find(buf));
550 RRD_STATS *rrd_stats_find_byname(const char *name)
552 debug(D_RRD_CALLS, "rrd_stats_find_byname() for chart %s", name);
554 char b[CONFIG_MAX_VALUE + 1];
556 rrd_stats_strncpy_name(b, name, CONFIG_MAX_VALUE);
557 unsigned long hash = simple_hash(b);
559 pthread_rwlock_rdlock(&root_rwlock);
560 RRD_STATS *st = root;
561 for ( ; st ; st = st->next ) {
562 if(hash == st->hash_name && strcmp(st->name, b) == 0) break;
564 pthread_rwlock_unlock(&root_rwlock);
569 RRD_DIMENSION *rrd_stats_dimension_find(RRD_STATS *st, const char *id)
571 debug(D_RRD_CALLS, "rrd_stats_dimension_find() for chart %s, dimension %s", st->name, id);
573 unsigned long hash = simple_hash(id);
575 RRD_DIMENSION *rd = st->dimensions;
577 for ( ; rd ; rd = rd->next )
579 if(strcmp(rd->id, id) == 0)
585 int rrd_stats_dimension_hide(RRD_STATS *st, const char *id)
587 debug(D_RRD_CALLS, "rrd_stats_dimension_hide() for chart %s, dimension %s", st->name, id);
589 RRD_DIMENSION *rd = rrd_stats_dimension_find(st, id);
591 error("Cannot find dimension with id '%s' on stats '%s' (%s).", id, st->name, st->id);
599 void rrd_stats_dimension_set_by_pointer(RRD_STATS *st, RRD_DIMENSION *rd, collected_number value)
601 debug(D_RRD_CALLS, "rrd_stats_dimension_set() for chart %s, dimension %s, value " COLLECTED_NUMBER_FORMAT, st->name, rd->name, value);
603 gettimeofday(&rd->last_collected_time, NULL);
604 rd->collected_value = value;
608 int rrd_stats_dimension_set(RRD_STATS *st, const char *id, collected_number value)
610 RRD_DIMENSION *rd = rrd_stats_dimension_find(st, id);
612 error("Cannot find dimension with id '%s' on stats '%s' (%s).", id, st->name, st->id);
616 rrd_stats_dimension_set_by_pointer(st, rd, value);
620 void rrd_stats_next_usec(RRD_STATS *st, unsigned long long microseconds)
622 debug(D_RRD_CALLS, "rrd_stats_next() for chart %s with microseconds %llu", st->name, microseconds);
624 if(st->debug) debug(D_RRD_STATS, "%s: NEXT: %llu microseconds", st->name, microseconds);
625 st->usec_since_last_update = microseconds;
628 void rrd_stats_next(RRD_STATS *st)
630 unsigned long long microseconds = 0;
632 if(st->last_collected_time.tv_sec) {
634 gettimeofday(&now, NULL);
635 microseconds = usecdiff(&now, &st->last_collected_time);
638 rrd_stats_next_usec(st, microseconds);
641 void rrd_stats_next_plugins(RRD_STATS *st)
646 unsigned long long rrd_stats_done(RRD_STATS *st)
648 debug(D_RRD_CALLS, "rrd_stats_done() for chart %s", st->name);
650 RRD_DIMENSION *rd, *last;
651 int oldstate, store_this_entry = 1;
653 if(pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, &oldstate) != 0)
654 error("Cannot set pthread cancel state to DISABLE.");
656 // a read lock is OK here
657 pthread_rwlock_rdlock(&st->rwlock);
659 // check if the chart has a long time to be refreshed
660 if(st->usec_since_last_update > st->entries * st->update_every * 1000000ULL) {
661 info("%s: took too long to be updated (%0.3Lf secs). Reseting it.", st->name, (long double)(st->usec_since_last_update / 1000000.0));
663 st->usec_since_last_update = st->update_every * 1000000ULL;
665 if(st->debug) debug(D_RRD_STATS, "%s: microseconds since last update: %llu", st->name, st->usec_since_last_update);
667 // set last_collected_time
668 if(!st->last_collected_time.tv_sec) {
669 // it is the first entry
670 // set the last_collected_time to now
671 gettimeofday(&st->last_collected_time, NULL);
673 // the first entry should not be stored
674 store_this_entry = 0;
676 if(st->debug) debug(D_RRD_STATS, "%s: initializing last_collected to now. Will not store the next entry.", st->name);
679 // it is not the first entry
680 // calculate the proper last_collected_time, using usec_since_last_update
681 unsigned long long ut = st->last_collected_time.tv_sec * 1000000ULL + st->last_collected_time.tv_usec + st->usec_since_last_update;
682 st->last_collected_time.tv_sec = ut / 1000000ULL;
683 st->last_collected_time.tv_usec = ut % 1000000ULL;
686 // if this set has not been updated in the past
687 // we fake the last_update time to be = now - usec_since_last_update
688 if(!st->last_updated.tv_sec) {
689 // it has never been updated before
690 // set a fake last_updated, in the past using usec_since_last_update
691 unsigned long long ut = st->last_collected_time.tv_sec * 1000000ULL + st->last_collected_time.tv_usec - st->usec_since_last_update;
692 st->last_updated.tv_sec = ut / 1000000ULL;
693 st->last_updated.tv_usec = ut % 1000000ULL;
695 // the first entry should not be stored
696 store_this_entry = 0;
698 if(st->debug) debug(D_RRD_STATS, "%s: initializing last_updated to now - %llu microseconds (%0.3Lf). Will not store the next entry.", st->name, st->usec_since_last_update, (long double)ut/1000000.0);
701 // check if we will re-write the entire data set
702 if(usecdiff(&st->last_collected_time, &st->last_updated) > st->update_every * st->entries * 1000000ULL) {
703 info("%s: too old data (last updated at %u.%u, last collected at %u.%u). Reseting it. Will not store the next entry.", st->name, st->last_updated.tv_sec, st->last_updated.tv_usec, st->last_collected_time.tv_sec, st->last_collected_time.tv_usec);
706 st->usec_since_last_update = st->update_every * 1000000ULL;
708 gettimeofday(&st->last_collected_time, NULL);
710 unsigned long long ut = st->last_collected_time.tv_sec * 1000000ULL + st->last_collected_time.tv_usec - st->usec_since_last_update;
711 st->last_updated.tv_sec = ut / 1000000ULL;
712 st->last_updated.tv_usec = ut % 1000000ULL;
714 // the first entry should not be stored
715 store_this_entry = 0;
718 // these are the 3 variables that will help us in interpolation
719 // last_ut = the last time we added a value to the storage
720 // now_ut = the time the current value is taken at
721 // next_ut = the time of the next interpolation point
722 unsigned long long last_ut = st->last_updated.tv_sec * 1000000ULL + st->last_updated.tv_usec;
723 unsigned long long now_ut = st->last_collected_time.tv_sec * 1000000ULL + st->last_collected_time.tv_usec;
724 unsigned long long next_ut = (st->last_updated.tv_sec + st->update_every) * 1000000ULL;
726 if(st->debug) debug(D_RRD_STATS, "%s: last ut = %0.3Lf (last updated time)", st->name, (long double)last_ut/1000000.0);
727 if(st->debug) debug(D_RRD_STATS, "%s: now ut = %0.3Lf (current update time)", st->name, (long double)now_ut/1000000.0);
728 if(st->debug) debug(D_RRD_STATS, "%s: next ut = %0.3Lf (next interpolation point)", st->name, (long double)next_ut/1000000.0);
730 if(!st->counter_done) {
731 store_this_entry = 0;
732 if(st->debug) debug(D_RRD_STATS, "%s: Will not store the next entry.", st->name);
736 // calculate totals and count the dimensions
738 st->collected_total = 0;
739 for( rd = st->dimensions, dimensions = 0 ; rd ; rd = rd->next, dimensions++ )
740 st->collected_total += rd->collected_value;
742 uint32_t storage_flags = SN_EXISTS;
744 // process all dimensions to calculate their values
745 // based on the collected figures only
746 // at this stage we do not interpolate anything
747 for( rd = st->dimensions ; rd ; rd = rd->next ) {
749 if(st->debug) debug(D_RRD_STATS, "%s/%s: "
750 " last_collected_value = " COLLECTED_NUMBER_FORMAT
751 " collected_value = " COLLECTED_NUMBER_FORMAT
752 " last_calculated_value = " CALCULATED_NUMBER_FORMAT
753 " calculated_value = " CALCULATED_NUMBER_FORMAT
755 , rd->last_collected_value
756 , rd->collected_value
757 , rd->last_calculated_value
758 , rd->calculated_value
761 switch(rd->algorithm) {
762 case RRD_DIMENSION_PCENT_OVER_DIFF_TOTAL:
763 // the percentage of the current increment
764 // over the increment of all dimensions together
765 if(st->collected_total == st->last_collected_total) rd->calculated_value = rd->last_calculated_value;
766 else rd->calculated_value =
767 (calculated_number)100
768 * (calculated_number)(rd->collected_value - rd->last_collected_value)
769 / (calculated_number)(st->collected_total - st->last_collected_total);
772 debug(D_RRD_STATS, "%s/%s: CALC PCENT-DIFF "
773 CALCULATED_NUMBER_FORMAT " = 100"
774 " * (" COLLECTED_NUMBER_FORMAT " - " COLLECTED_NUMBER_FORMAT ")"
775 " / (" COLLECTED_NUMBER_FORMAT " - " COLLECTED_NUMBER_FORMAT ")"
777 , rd->calculated_value
778 , rd->collected_value, rd->last_collected_value
779 , st->collected_total, st->last_collected_total
783 case RRD_DIMENSION_PCENT_OVER_ROW_TOTAL:
784 if(!st->collected_total) rd->calculated_value = 0;
786 // the percentage of the current value
787 // over the total of all dimensions
788 rd->calculated_value =
789 (calculated_number)100
790 * (calculated_number)rd->collected_value
791 / (calculated_number)st->collected_total;
794 debug(D_RRD_STATS, "%s/%s: CALC PCENT-ROW "
795 CALCULATED_NUMBER_FORMAT " = 100"
796 " * " COLLECTED_NUMBER_FORMAT
797 " / " COLLECTED_NUMBER_FORMAT
799 , rd->calculated_value
800 , rd->collected_value
801 , st->collected_total
805 case RRD_DIMENSION_INCREMENTAL:
806 // if the new is smaller than the old (an overflow, or reset), set the old equal to the new
807 // to reset the calculation (it will give zero as the calculation for this second)
808 if(rd->last_collected_value > rd->collected_value) {
809 storage_flags = SN_EXISTS_RESET;
810 rd->last_collected_value = rd->collected_value;
813 rd->calculated_value += (calculated_number)(rd->collected_value - rd->last_collected_value);
816 debug(D_RRD_STATS, "%s/%s: CALC INC "
817 CALCULATED_NUMBER_FORMAT " += "
818 COLLECTED_NUMBER_FORMAT " - " COLLECTED_NUMBER_FORMAT
820 , rd->calculated_value
821 , rd->collected_value, rd->last_collected_value
825 case RRD_DIMENSION_ABSOLUTE:
826 rd->calculated_value = (calculated_number)rd->collected_value;
829 debug(D_RRD_STATS, "%s/%s: CALC ABS/ABS-NO-IN "
830 CALCULATED_NUMBER_FORMAT " = "
831 COLLECTED_NUMBER_FORMAT
833 , rd->calculated_value
834 , rd->collected_value
839 // make the default zero, to make sure
840 // it gets noticed when we add new types
841 rd->calculated_value = 0;
844 debug(D_RRD_STATS, "%s/%s: CALC "
845 CALCULATED_NUMBER_FORMAT " = 0"
847 , rd->calculated_value
853 // at this point we have all the calculated values ready
854 // it is now time to interpolate values on a second boundary
856 unsigned long long first_ut = last_ut;
857 int iterations = (now_ut - last_ut) / (st->update_every * 1000000ULL);
859 for( ; next_ut <= now_ut ; next_ut += st->update_every * 1000000ULL, iterations-- ) {
860 if(iterations < 0) error("iterations calculation wrapped!");
862 if(st->debug) debug(D_RRD_STATS, "%s: last ut = %0.3Lf (last updated time)", st->name, (long double)last_ut/1000000.0);
863 if(st->debug) debug(D_RRD_STATS, "%s: next ut = %0.3Lf (next interpolation point)", st->name, (long double)next_ut/1000000.0);
865 st->last_updated.tv_sec = next_ut / 1000000ULL;
866 st->last_updated.tv_usec = 0;
868 for( rd = st->dimensions ; rd ; rd = rd->next ) {
869 calculated_number new_value;
871 switch(rd->algorithm) {
872 case RRD_DIMENSION_INCREMENTAL:
873 new_value = (calculated_number)
874 ( rd->calculated_value
875 * (calculated_number)(next_ut - last_ut)
876 / (calculated_number)(now_ut - last_ut)
880 debug(D_RRD_STATS, "%s/%s: CALC2 INC "
881 CALCULATED_NUMBER_FORMAT " = "
882 CALCULATED_NUMBER_FORMAT
887 , rd->calculated_value
888 , (unsigned long long)(next_ut - last_ut)
889 , (unsigned long long)(now_ut - last_ut)
892 rd->calculated_value -= new_value;
895 case RRD_DIMENSION_ABSOLUTE:
896 case RRD_DIMENSION_PCENT_OVER_ROW_TOTAL:
897 case RRD_DIMENSION_PCENT_OVER_DIFF_TOTAL:
899 new_value = (calculated_number)
900 ( ( (rd->calculated_value - rd->last_calculated_value)
901 * (calculated_number)(next_ut - first_ut)
902 / (calculated_number)(now_ut - first_ut)
904 + rd->last_calculated_value
908 debug(D_RRD_STATS, "%s/%s: CALC2 DEF "
909 CALCULATED_NUMBER_FORMAT " = ((("
910 "(" CALCULATED_NUMBER_FORMAT " - " CALCULATED_NUMBER_FORMAT ")"
912 " / %llu) + " CALCULATED_NUMBER_FORMAT
915 , rd->calculated_value, rd->last_calculated_value
916 , (next_ut - first_ut)
917 , (now_ut - first_ut), rd->last_calculated_value
920 if(next_ut + st->update_every * 1000000ULL > now_ut) rd->calculated_value = new_value;
924 if(!store_this_entry) {
925 store_this_entry = 1;
929 if(rd->updated && iterations < st->gap_when_lost_iterations) {
930 rd->values[st->current_entry] = pack_storage_number(
932 * (calculated_number)rd->multiplier
933 / (calculated_number)rd->divisor
937 debug(D_RRD_STATS, "%s/%s: STORE[%ld] "
938 CALCULATED_NUMBER_FORMAT " = " CALCULATED_NUMBER_FORMAT
943 , unpack_storage_number(rd->values[st->current_entry]), new_value
949 if(st->debug) debug(D_RRD_STATS, "%s/%s: STORE[%ld] = NON EXISTING "
953 rd->values[st->current_entry] = pack_storage_number(0, SN_NOT_EXISTS);
957 calculated_number t1 = new_value * (calculated_number)rd->multiplier / (calculated_number)rd->divisor;
958 calculated_number t2 = unpack_storage_number(rd->values[st->current_entry]);
959 calculated_number accuracy = accuracy_loss(t1, t2);
960 debug(D_RRD_STATS, "%s/%s: UNPACK[%ld] = " CALCULATED_NUMBER_FORMAT " FLAGS=0x%08x (original = " CALCULATED_NUMBER_FORMAT ", accuracy loss = " CALCULATED_NUMBER_FORMAT "%%%s)"
964 , get_storage_number_flags(rd->values[st->current_entry])
967 , (accuracy > ACCURACY_LOSS) ? " **TOO BIG** " : ""
970 rd->collected_volume += t1;
971 rd->stored_volume += t2;
972 accuracy = accuracy_loss(rd->collected_volume, rd->stored_volume);
973 debug(D_RRD_STATS, "%s/%s: VOLUME[%ld] = " CALCULATED_NUMBER_FORMAT ", calculated = " CALCULATED_NUMBER_FORMAT ", accuracy loss = " CALCULATED_NUMBER_FORMAT "%%%s"
977 , rd->collected_volume
979 , (accuracy > ACCURACY_LOSS) ? " **TOO BIG** " : ""
984 // reset the storage flags for the next point, if any;
985 storage_flags = SN_EXISTS;
987 if(st->first_entry_t && st->counter >= (unsigned long long)st->entries) {
988 // the db is overwriting values
989 // add the value we will overwrite
990 st->first_entry_t += st->update_every * 1000000ULL;
994 st->current_entry = ((st->current_entry + 1) >= st->entries) ? 0 : st->current_entry + 1;
995 if(!st->first_entry_t) st->first_entry_t = next_ut;
999 for( rd = st->dimensions; rd ; rd = rd->next ) {
1000 if(!rd->updated) continue;
1001 rd->last_collected_value = rd->collected_value;
1002 rd->last_calculated_value = rd->calculated_value;
1003 rd->collected_value = 0;
1006 // if this is the first entry of incremental dimensions
1007 // we have to set the first calculated_value to zero
1008 // to eliminate the first spike
1009 if(st->counter_done == 1) switch(rd->algorithm) {
1010 case RRD_DIMENSION_PCENT_OVER_DIFF_TOTAL:
1011 case RRD_DIMENSION_INCREMENTAL:
1012 rd->calculated_value = 0;
1013 // the next time, a new incremental total will be calculated
1017 st->last_collected_total = st->collected_total;
1019 // ALL DONE ABOUT THE DATA UPDATE
1020 // --------------------------------------------------------------------
1022 // find if there are any obsolete dimensions (not updated recently)
1023 for( rd = st->dimensions; rd ; rd = rd->next )
1024 if((rd->last_collected_time.tv_sec + (10 * st->update_every)) < st->last_collected_time.tv_sec)
1028 // there is dimension to free
1029 // upgrade our read lock to a write lock
1030 pthread_rwlock_unlock(&st->rwlock);
1031 pthread_rwlock_wrlock(&st->rwlock);
1033 for( rd = st->dimensions, last = NULL ; rd ; ) {
1034 if((rd->last_collected_time.tv_sec + (10 * st->update_every)) < st->last_collected_time.tv_sec) { // remove it only it is not updated in 10 seconds
1035 debug(D_RRD_STATS, "Removing obsolete dimension '%s' (%s) of '%s' (%s).", rd->name, rd->id, st->name, st->id);
1038 st->dimensions = rd->next;
1040 rrd_stats_dimension_free(rd);
1041 rd = st->dimensions;
1045 last->next = rd->next;
1047 rrd_stats_dimension_free(rd);
1057 if(!st->dimensions) st->enabled = 0;
1060 pthread_rwlock_unlock(&st->rwlock);
1062 if(pthread_setcancelstate(oldstate, NULL) != 0)
1063 error("Cannot set pthread cancel state to RESTORE (%d).", oldstate);
1065 return(st->usec_since_last_update);
1069 // find the oldest entry in the data, skipping all empty slots
1070 time_t rrd_stats_first_entry_t(RRD_STATS *st)
1072 if(!st->first_entry_t) return st->last_updated.tv_sec;
1074 return st->first_entry_t / 1000000;