]> arthur.barton.de Git - netdata.git/blob - src/rrd.c
detect too big updates correctly
[netdata.git] / src / rrd.c
1 #include <string.h>
2 #include <unistd.h>
3 #include <time.h>
4 #include <sys/time.h>
5 #include <sys/mman.h>
6 #include <pthread.h>
7 #include <errno.h>
8 #include <ctype.h>
9 #include <sys/stat.h>
10 #include <sys/types.h>
11 #include <stdlib.h>
12
13 #include "log.h"
14 #include "config.h"
15 #include "common.h"
16
17 #include "rrd.h"
18
19 int update_every = UPDATE_EVERY;
20 int save_history = HISTORY;
21
22 // ----------------------------------------------------------------------------
23 // chart types
24
25 int chart_type_id(const char *name)
26 {
27         if(strcmp(name, CHART_TYPE_AREA_NAME) == 0) return CHART_TYPE_AREA;
28         if(strcmp(name, CHART_TYPE_STACKED_NAME) == 0) return CHART_TYPE_STACKED;
29         if(strcmp(name, CHART_TYPE_LINE_NAME) == 0) return CHART_TYPE_LINE;
30         return CHART_TYPE_LINE;
31 }
32
33 const char *chart_type_name(int chart_type)
34 {
35         static char line[] = CHART_TYPE_LINE_NAME;
36         static char area[] = CHART_TYPE_AREA_NAME;
37         static char stacked[] = CHART_TYPE_STACKED_NAME;
38
39         switch(chart_type) {
40                 case CHART_TYPE_LINE:
41                         return line;
42
43                 case CHART_TYPE_AREA:
44                         return area;
45
46                 case CHART_TYPE_STACKED:
47                         return stacked;
48         }
49         return line;
50 }
51
52 // ----------------------------------------------------------------------------
53 // mmap() wrapper
54
55 int memory_mode = NETDATA_MEMORY_MODE_SAVE;
56
57 const char *memory_mode_name(int id)
58 {
59         static const char ram[] = NETDATA_MEMORY_MODE_RAM_NAME;
60         static const char map[] = NETDATA_MEMORY_MODE_MAP_NAME;
61         static const char save[] = NETDATA_MEMORY_MODE_SAVE_NAME;
62
63         switch(id) {
64                 case NETDATA_MEMORY_MODE_RAM:
65                         return ram;
66
67                 case NETDATA_MEMORY_MODE_MAP:
68                         return map;
69
70                 case NETDATA_MEMORY_MODE_SAVE:
71                 default:
72                         return save;
73         }
74
75         return save;
76 }
77
78 int memory_mode_id(const char *name)
79 {
80         if(!strcmp(name, NETDATA_MEMORY_MODE_RAM_NAME))
81                 return NETDATA_MEMORY_MODE_RAM;
82         else if(!strcmp(name, NETDATA_MEMORY_MODE_MAP_NAME))
83                 return NETDATA_MEMORY_MODE_MAP;
84
85         return NETDATA_MEMORY_MODE_SAVE;
86 }
87
88 // ----------------------------------------------------------------------------
89 // algorithms types
90
91 int algorithm_id(const char *name)
92 {
93         if(strcmp(name, RRD_DIMENSION_ABSOLUTE_NAME) == 0)                      return RRD_DIMENSION_ABSOLUTE;
94         if(strcmp(name, RRD_DIMENSION_INCREMENTAL_NAME) == 0)                   return RRD_DIMENSION_INCREMENTAL;
95         if(strcmp(name, RRD_DIMENSION_PCENT_OVER_ROW_TOTAL_NAME) == 0)          return RRD_DIMENSION_PCENT_OVER_ROW_TOTAL;
96         if(strcmp(name, RRD_DIMENSION_PCENT_OVER_DIFF_TOTAL_NAME) == 0)         return RRD_DIMENSION_PCENT_OVER_DIFF_TOTAL;
97         return RRD_DIMENSION_ABSOLUTE;
98 }
99
100 const char *algorithm_name(int chart_type)
101 {
102         static char absolute[] = RRD_DIMENSION_ABSOLUTE_NAME;
103         static char incremental[] = RRD_DIMENSION_INCREMENTAL_NAME;
104         static char percentage_of_absolute_row[] = RRD_DIMENSION_PCENT_OVER_ROW_TOTAL_NAME;
105         static char percentage_of_incremental_row[] = RRD_DIMENSION_PCENT_OVER_DIFF_TOTAL_NAME;
106
107         switch(chart_type) {
108                 case RRD_DIMENSION_ABSOLUTE:
109                         return absolute;
110
111                 case RRD_DIMENSION_INCREMENTAL:
112                         return incremental;
113
114                 case RRD_DIMENSION_PCENT_OVER_ROW_TOTAL:
115                         return percentage_of_absolute_row;
116
117                 case RRD_DIMENSION_PCENT_OVER_DIFF_TOTAL:
118                         return percentage_of_incremental_row;
119         }
120         return absolute;
121 }
122
123 RRD_STATS *root = NULL;
124 pthread_rwlock_t root_rwlock = PTHREAD_RWLOCK_INITIALIZER;
125
126 char *rrd_stats_strncpy_name(char *to, const char *from, int length)
127 {
128         int i;
129         for(i = 0; i < length && from[i] ;i++) {
130                 if(from[i] == '.' || isalpha(from[i]) || isdigit(from[i])) to[i] = from[i];
131                 else to[i] = '_';
132         }
133         if(i < length) to[i] = '\0';
134         to[length - 1] = '\0';
135
136         return to;
137 }
138
139 void rrd_stats_set_name(RRD_STATS *st, const char *name)
140 {
141         char b[CONFIG_MAX_VALUE + 1];
142         char n[RRD_STATS_NAME_MAX + 1];
143
144         snprintf(n, RRD_STATS_NAME_MAX, "%s.%s", st->type, name);
145         rrd_stats_strncpy_name(b, n, CONFIG_MAX_VALUE);
146         st->name = config_get(st->id, "name", b);
147         st->hash_name = simple_hash(st->name);
148 }
149
150 char *rrd_stats_cache_dir(const char *id)
151 {
152         char *ret = NULL;
153
154         static char *cache_dir = NULL;
155         if(!cache_dir) cache_dir = config_get("global", "database directory", "cache");
156
157         char b[FILENAME_MAX + 1];
158         char n[FILENAME_MAX + 1];
159         rrd_stats_strncpy_name(b, id, FILENAME_MAX);
160
161         snprintf(n, FILENAME_MAX, "%s/%s", cache_dir, b);
162         ret = config_get(id, "database directory", n);
163
164         if(memory_mode == NETDATA_MEMORY_MODE_MAP || memory_mode == NETDATA_MEMORY_MODE_SAVE) {
165                 int r = mkdir(ret, 0775);
166                 if(r != 0 && errno != EEXIST)
167                         error("Cannot create directory '%s'", ret);
168         }
169
170         return ret;
171 }
172
173 void rrd_stats_reset(RRD_STATS *st)
174 {
175         st->last_collected_time.tv_sec = 0;
176         st->last_collected_time.tv_usec = 0;
177         st->last_updated.tv_sec = 0;
178         st->last_updated.tv_usec = 0;
179         st->current_entry = 0;
180         st->counter_done = 0;
181
182         RRD_DIMENSION *rd;
183         for(rd = st->dimensions; rd ; rd = rd->next) {
184                 rd->last_collected_time.tv_sec = 0;
185                 rd->last_collected_time.tv_usec = 0;
186                 rd->current_entry = 0;
187         }
188 }
189
190 RRD_STATS *rrd_stats_create(const char *type, const char *id, const char *name, const char *family, const char *title, const char *units, long priority, int update_every, int chart_type)
191 {
192         if(!id || !id[0]) {
193                 fatal("Cannot create rrd stats without an id.");
194                 return NULL;
195         }
196
197         char fullid[RRD_STATS_NAME_MAX + 1];
198         char fullfilename[FILENAME_MAX + 1];
199         RRD_STATS *st = NULL;
200
201         snprintf(fullid, RRD_STATS_NAME_MAX, "%s.%s", type, id);
202
203         long entries = config_get_number(fullid, "history", save_history);
204         if(entries < 5) entries = config_set_number(fullid, "history", 5);
205         if(entries > HISTORY_MAX) entries = config_set_number(fullid, "history", HISTORY_MAX);
206
207         int enabled = config_get_boolean(fullid, "enabled", 1);
208         if(!enabled) entries = 5;
209
210         unsigned long size = sizeof(RRD_STATS);
211         char *cache_dir = rrd_stats_cache_dir(fullid);
212
213         debug(D_RRD_CALLS, "Creating RRD_STATS for '%s.%s'.", type, id);
214
215         snprintf(fullfilename, FILENAME_MAX, "%s/main.db", cache_dir);
216         if(memory_mode != NETDATA_MEMORY_MODE_RAM) st = (RRD_STATS *)mymmap(fullfilename, size, ((memory_mode == NETDATA_MEMORY_MODE_MAP)?MAP_SHARED:MAP_PRIVATE));
217         if(st) {
218                 if(strcmp(st->magic, RRD_STATS_MAGIC) != 0) {
219                         errno = 0;
220                         error("File %s does not have our version. Clearing it.", fullfilename);
221                         bzero(st, size);
222                 }
223                 else if(strcmp(st->id, fullid) != 0) {
224                         errno = 0;
225                         error("File %s does not have our id. Unmapping it.", fullfilename);
226                         munmap(st, size);
227                         st = NULL;
228                 }
229                 else if(st->memsize != size || st->entries != entries) {
230                         errno = 0;
231                         error("File %s does not have the desired size. Clearing it.", fullfilename);
232                         bzero(st, size);
233                 }
234                 else if(st->update_every != update_every) {
235                         errno = 0;
236                         error("File %s does not have the desired update frequency. Clearing it.", fullfilename);
237                         bzero(st, size);
238                 }
239                 else if((time(NULL) - st->last_updated.tv_sec) > update_every * entries) {
240                         errno = 0;
241                         error("File %s is too old. Clearing it.", fullfilename);
242                         bzero(st, size);
243                 }
244         }
245
246         if(st) {
247                 st->name = NULL;
248                 st->type = NULL;
249                 st->family = NULL;
250                 st->title = NULL;
251                 st->units = NULL;
252                 st->dimensions = NULL;
253                 st->next = NULL;
254                 st->mapped = memory_mode;
255         }
256         else {
257                 st = calloc(1, size);
258                 if(!st) {
259                         fatal("Cannot allocate memory for RRD_STATS %s.%s", type, id);
260                         return NULL;
261                 }
262                 st->mapped = NETDATA_MEMORY_MODE_RAM;
263         }
264         st->memsize = size;
265         st->entries = entries;
266         st->update_every = update_every;
267
268         strcpy(st->cache_file, fullfilename);
269         strcpy(st->magic, RRD_STATS_MAGIC);
270
271         strcpy(st->id, fullid);
272         st->hash = simple_hash(st->id);
273
274         st->cache_dir = cache_dir;
275
276         st->family     = config_get(st->id, "family", family?family:st->id);
277         st->units      = config_get(st->id, "units", units?units:"");
278         st->type       = config_get(st->id, "type", type);
279         st->chart_type = chart_type_id(config_get(st->id, "chart type", chart_type_name(chart_type)));
280
281         if(name && *name) rrd_stats_set_name(st, name);
282         else rrd_stats_set_name(st, id);
283
284         {
285                 char varvalue[CONFIG_MAX_VALUE + 1];
286                 snprintf(varvalue, CONFIG_MAX_VALUE, "%s (%s)", title?title:"", st->name);
287                 st->title = config_get(st->id, "title", varvalue);
288         }
289
290         st->priority = config_get_number(st->id, "priority", priority);
291         st->enabled = enabled;
292         
293         st->isdetail = 0;
294         st->debug = 0;
295
296         st->last_collected_time.tv_sec = 0;
297         st->last_collected_time.tv_usec = 0;
298         st->counter_done = 0;
299
300         pthread_rwlock_init(&st->rwlock, NULL);
301         pthread_rwlock_wrlock(&root_rwlock);
302
303         st->next = root;
304         root = st;
305
306         pthread_rwlock_unlock(&root_rwlock);
307
308         return(st);
309 }
310
311 RRD_DIMENSION *rrd_stats_dimension_add(RRD_STATS *st, const char *id, const char *name, long multiplier, long divisor, int algorithm)
312 {
313         char filename[FILENAME_MAX + 1];
314         char fullfilename[FILENAME_MAX + 1];
315
316         char varname[CONFIG_MAX_NAME + 1];
317         RRD_DIMENSION *rd = NULL;
318         unsigned long size = sizeof(RRD_DIMENSION) + (st->entries * sizeof(storage_number));
319
320         debug(D_RRD_CALLS, "Adding dimension '%s/%s'.", st->id, id);
321
322         rrd_stats_strncpy_name(filename, id, FILENAME_MAX);
323         snprintf(fullfilename, FILENAME_MAX, "%s/%s.db", st->cache_dir, filename);
324         if(memory_mode != NETDATA_MEMORY_MODE_RAM) rd = (RRD_DIMENSION *)mymmap(fullfilename, size, ((memory_mode == NETDATA_MEMORY_MODE_MAP)?MAP_SHARED:MAP_PRIVATE));
325         if(rd) {
326                 struct timeval now;
327                 gettimeofday(&now, NULL);
328
329                 if(strcmp(rd->magic, RRD_DIMENSION_MAGIC) != 0) {
330                         errno = 0;
331                         error("File %s does not have our version. Clearing it.", fullfilename);
332                         bzero(rd, size);
333                 }
334                 else if(rd->memsize != size) {
335                         errno = 0;
336                         error("File %s does not have the desired size. Clearing it.", fullfilename);
337                         bzero(rd, size);
338                 }
339                 else if(rd->multiplier != multiplier) {
340                         errno = 0;
341                         error("File %s does not have the same multiplier. Clearing it.", fullfilename);
342                         bzero(rd, size);
343                 }
344                 else if(rd->divisor != divisor) {
345                         errno = 0;
346                         error("File %s does not have the same divisor. Clearing it.", fullfilename);
347                         bzero(rd, size);
348                 }
349                 else if(rd->algorithm != algorithm) {
350                         errno = 0;
351                         error("File %s does not have the same algorithm. Clearing it.", fullfilename);
352                         bzero(rd, size);
353                 }
354                 else if(rd->update_every != st->update_every) {
355                         errno = 0;
356                         error("File %s does not have the same refresh frequency. Clearing it.", fullfilename);
357                         bzero(rd, size);
358                 }
359                 else if(usecdiff(&now, &rd->last_collected_time) > (rd->entries * rd->update_every * 1000000ULL)) {
360                         errno = 0;
361                         error("File %s is too old. Clearing it.", fullfilename);
362                         bzero(rd, size);
363                 }
364                 else if(strcmp(rd->id, id) != 0) {
365                         errno = 0;
366                         error("File %s does not have our dimension id. Unmapping it.", fullfilename);
367                         munmap(rd, size);
368                         rd = NULL;
369                 }
370         }
371
372         if(rd) {
373                 // we have a file mapped for rd
374                 rd->mapped = memory_mode;
375                 rd->hidden = 0;
376                 rd->next = NULL;
377                 rd->name = NULL;
378         }
379         else {
380                 // if we didn't manage to get a mmap'd dimension, just create one
381
382                 rd = calloc(1, size);
383                 if(!rd) {
384                         fatal("Cannot allocate RRD_DIMENSION %s/%s.", st->id, id);
385                         return NULL;
386                 }
387
388                 rd->mapped = NETDATA_MEMORY_MODE_RAM;
389         }
390         rd->memsize = size;
391
392         strcpy(rd->magic, RRD_DIMENSION_MAGIC);
393         strcpy(rd->cache_file, fullfilename);
394         strncpy(rd->id, id, RRD_STATS_NAME_MAX);
395         rd->hash = simple_hash(rd->id);
396
397         snprintf(varname, CONFIG_MAX_NAME, "dim %s name", rd->id);
398         rd->name = config_get(st->id, varname, (name && *name)?name:rd->id);
399
400         snprintf(varname, CONFIG_MAX_NAME, "dim %s algorithm", rd->id);
401         rd->algorithm = algorithm_id(config_get(st->id, varname, algorithm_name(algorithm)));
402
403         snprintf(varname, CONFIG_MAX_NAME, "dim %s multiplier", rd->id);
404         rd->multiplier = config_get_number(st->id, varname, multiplier);
405
406         snprintf(varname, CONFIG_MAX_NAME, "dim %s divisor", rd->id);
407         rd->divisor = config_get_number(st->id, varname, divisor);
408         if(!rd->divisor) rd->divisor = 1;
409
410         rd->entries = st->entries;
411         rd->update_every = st->update_every;
412         
413         // append this dimension
414         if(!st->dimensions)
415                 st->dimensions = rd;
416         else {
417                 RRD_DIMENSION *td = st->dimensions;
418                 for(; td->next; td = td->next) ;
419                 td->next = rd;
420         }
421
422         return(rd);
423 }
424
425 void rrd_stats_dimension_set_name(RRD_STATS *st, RRD_DIMENSION *rd, const char *name)
426 {
427         char varname[CONFIG_MAX_NAME + 1];
428         snprintf(varname, CONFIG_MAX_NAME, "dim %s name", rd->id);
429         config_get(st->id, varname, name);
430 }
431
432 void rrd_stats_dimension_free(RRD_DIMENSION *rd)
433 {
434         if(rd->next) rrd_stats_dimension_free(rd->next);
435         // free(rd->annotations);
436         if(rd->mapped == NETDATA_MEMORY_MODE_SAVE) {
437                 debug(D_RRD_CALLS, "Saving dimension '%s' to '%s'.", rd->name, rd->cache_file);
438                 savememory(rd->cache_file, rd, rd->memsize);
439
440                 debug(D_RRD_CALLS, "Unmapping dimension '%s'.", rd->name);
441                 munmap(rd, rd->memsize);
442         }
443         else if(rd->mapped == NETDATA_MEMORY_MODE_MAP) {
444                 debug(D_RRD_CALLS, "Unmapping dimension '%s'.", rd->name);
445                 munmap(rd, rd->memsize);
446         }
447         else {
448                 debug(D_RRD_CALLS, "Removing dimension '%s'.", rd->name);
449                 free(rd);
450         }
451 }
452
453 void rrd_stats_free_all(void)
454 {
455         info("Freeing all memory...");
456
457         RRD_STATS *st;
458         for(st = root; st ;) {
459                 RRD_STATS *next = st->next;
460
461                 if(st->dimensions) rrd_stats_dimension_free(st->dimensions);
462                 st->dimensions = NULL;
463
464                 if(st->mapped == NETDATA_MEMORY_MODE_SAVE) {
465                         debug(D_RRD_CALLS, "Saving stats '%s' to '%s'.", st->name, st->cache_file);
466                         savememory(st->cache_file, st, st->memsize);
467
468                         debug(D_RRD_CALLS, "Unmapping stats '%s'.", st->name);
469                         munmap(st, st->memsize);
470                 }
471                 else if(st->mapped == NETDATA_MEMORY_MODE_MAP) {
472                         debug(D_RRD_CALLS, "Unmapping stats '%s'.", st->name);
473                         munmap(st, st->memsize);
474                 }
475                 else
476                         free(st);
477
478                 st = next;
479         }
480         root = NULL;
481
482         info("Memory cleanup completed...");
483 }
484
485 void rrd_stats_save_all(void)
486 {
487         RRD_STATS *st;
488         RRD_DIMENSION *rd;
489
490         pthread_rwlock_wrlock(&root_rwlock);
491         for(st = root; st ; st = st->next) {
492                 pthread_rwlock_wrlock(&st->rwlock);
493
494                 if(st->mapped == NETDATA_MEMORY_MODE_SAVE) {
495                         debug(D_RRD_CALLS, "Saving stats '%s' to '%s'.", st->name, st->cache_file);
496                         savememory(st->cache_file, st, st->memsize);
497                 }
498
499                 for(rd = st->dimensions; rd ; rd = rd->next) {
500                         if(rd->mapped == NETDATA_MEMORY_MODE_SAVE) {
501                                 debug(D_RRD_CALLS, "Saving dimension '%s' to '%s'.", rd->name, rd->cache_file);
502                                 savememory(rd->cache_file, rd, rd->memsize);
503                         }
504                 }
505
506                 pthread_rwlock_unlock(&st->rwlock);
507         }
508         pthread_rwlock_unlock(&root_rwlock);
509 }
510
511
512 RRD_STATS *rrd_stats_find(const char *id)
513 {
514         debug(D_RRD_CALLS, "rrd_stats_find() for chart %s", id);
515
516         unsigned long hash = simple_hash(id);
517
518         pthread_rwlock_rdlock(&root_rwlock);
519         RRD_STATS *st = root;
520         for ( ; st ; st = st->next )
521                 if(hash == st->hash)
522                         if(strcmp(st->id, id) == 0)
523                                 break;
524         pthread_rwlock_unlock(&root_rwlock);
525
526         return(st);
527 }
528
529 RRD_STATS *rrd_stats_find_bytype(const char *type, const char *id)
530 {
531         debug(D_RRD_CALLS, "rrd_stats_find_bytype() for chart %s.%s", type, id);
532
533         char buf[RRD_STATS_NAME_MAX + 1];
534
535         strncpy(buf, type, RRD_STATS_NAME_MAX - 1);
536         buf[RRD_STATS_NAME_MAX - 1] = '\0';
537         strcat(buf, ".");
538         int len = strlen(buf);
539         strncpy(&buf[len], id, RRD_STATS_NAME_MAX - len);
540         buf[RRD_STATS_NAME_MAX] = '\0';
541
542         return(rrd_stats_find(buf));
543 }
544
545 RRD_STATS *rrd_stats_find_byname(const char *name)
546 {
547         debug(D_RRD_CALLS, "rrd_stats_find_byname() for chart %s", name);
548
549         char b[CONFIG_MAX_VALUE + 1];
550
551         rrd_stats_strncpy_name(b, name, CONFIG_MAX_VALUE);
552         unsigned long hash = simple_hash(b);
553
554         pthread_rwlock_rdlock(&root_rwlock);
555         RRD_STATS *st = root;
556         for ( ; st ; st = st->next ) {
557                 if(hash == st->hash_name && strcmp(st->name, b) == 0) break;
558         }
559         pthread_rwlock_unlock(&root_rwlock);
560
561         return(st);
562 }
563
564 RRD_DIMENSION *rrd_stats_dimension_find(RRD_STATS *st, const char *id)
565 {
566         debug(D_RRD_CALLS, "rrd_stats_dimension_find() for chart %s, dimension %s", st->name, id);
567
568         unsigned long hash = simple_hash(id);
569
570         RRD_DIMENSION *rd = st->dimensions;
571
572         for ( ; rd ; rd = rd->next )
573                 if(hash == rd->hash)
574                         if(strcmp(rd->id, id) == 0)
575                                 break;
576
577         return(rd);
578 }
579
580 int rrd_stats_dimension_hide(RRD_STATS *st, const char *id)
581 {
582         debug(D_RRD_CALLS, "rrd_stats_dimension_hide() for chart %s, dimension %s", st->name, id);
583
584         RRD_DIMENSION *rd = rrd_stats_dimension_find(st, id);
585         if(!rd) {
586                 error("Cannot find dimension with id '%s' on stats '%s' (%s).", id, st->name, st->id);
587                 return 1;
588         }
589
590         rd->hidden = 1;
591         return 0;
592 }
593
594 void rrd_stats_dimension_set_by_pointer(RRD_STATS *st, RRD_DIMENSION *rd, collected_number value)
595 {
596         debug(D_RRD_CALLS, "rrd_stats_dimension_set() for chart %s, dimension %s, value " COLLECTED_NUMBER_FORMAT, st->name, rd->name, value);
597         
598         gettimeofday(&rd->last_collected_time, NULL);
599         rd->collected_value = value;
600 }
601
602 int rrd_stats_dimension_set(RRD_STATS *st, const char *id, collected_number value)
603 {
604         RRD_DIMENSION *rd = rrd_stats_dimension_find(st, id);
605         if(!rd) {
606                 error("Cannot find dimension with id '%s' on stats '%s' (%s).", id, st->name, st->id);
607                 return 1;
608         }
609
610         rrd_stats_dimension_set_by_pointer(st, rd, value);
611         return 0;
612 }
613
614 void rrd_stats_next_usec(RRD_STATS *st, unsigned long long microseconds)
615 {
616         debug(D_RRD_CALLS, "rrd_stats_next() for chart %s with microseconds %llu", st->name, microseconds);
617
618         if(st->debug) debug(D_RRD_STATS, "%s: NEXT: %llu microseconds", st->name, microseconds);
619         st->usec_since_last_update = microseconds;
620 }
621
622 void rrd_stats_next(RRD_STATS *st)
623 {
624         unsigned long long microseconds = 0;
625
626         if(st->last_collected_time.tv_sec) {
627                 struct timeval now;
628                 gettimeofday(&now, NULL);
629                 microseconds = usecdiff(&now, &st->last_collected_time);
630         }
631
632         rrd_stats_next_usec(st, microseconds);
633 }
634
635 void rrd_stats_next_plugins(RRD_STATS *st)
636 {
637         rrd_stats_next(st);
638 }
639
640 unsigned long long rrd_stats_done(RRD_STATS *st)
641 {
642         debug(D_RRD_CALLS, "rrd_stats_done() for chart %s", st->name);
643
644         RRD_DIMENSION *rd, *last;
645         int oldstate;
646
647         if(pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, &oldstate) != 0)
648                 error("Cannot set pthread cancel state to DISABLE.");
649
650         // a read lock is OK here
651         pthread_rwlock_rdlock(&st->rwlock);
652
653         if(st->usec_since_last_update > st->entries * st->update_every * 1000000ULL) {
654                 info("Chart chart %s took too long to be updated (%0.3Lf secs). Reseting chart history.", st->name, (long double)(st->usec_since_last_update / 1000000.0));
655                 rrd_stats_reset(st);
656                 st->usec_since_last_update = st->update_every * 1000000ULL;
657         }
658
659         if(st->debug) debug(D_RRD_STATS, "%s: microseconds since last update: %llu", st->name, st->usec_since_last_update);
660
661         if(!st->last_collected_time.tv_sec) gettimeofday(&st->last_collected_time, NULL);
662         else {
663                 unsigned long long ut = st->last_collected_time.tv_sec * 1000000ULL + st->last_collected_time.tv_usec + st->usec_since_last_update;
664                 st->last_collected_time.tv_sec = ut / 1000000ULL;
665                 st->last_collected_time.tv_usec = ut % 1000000ULL;
666         }
667
668         if(!st->last_updated.tv_sec) {
669                 unsigned long long ut = st->last_collected_time.tv_sec * 1000000ULL + st->last_collected_time.tv_usec - st->usec_since_last_update;
670                 st->last_updated.tv_sec = ut / 1000000ULL;
671                 st->last_updated.tv_usec = ut % 1000000ULL;
672
673                 if(st->debug) debug(D_RRD_STATS, "%s: initializing last_updated to now - %llu microseconds (%0.3Lf)", st->name, st->usec_since_last_update, (long double)ut/1000000.0);
674         }
675
676         if(usecdiff(&st->last_collected_time, &st->last_updated) > st->entries * st->update_every * 1000000ULL) {
677                 info("History of chart %s too old (last updated at %u, last collected at %u). Reseting chart.", st->name, st->last_updated.tv_sec, st->last_collected_time.tv_sec);
678                 rrd_stats_reset(st);
679                 st->usec_since_last_update = st->update_every * 1000000ULL;
680                 return(st->usec_since_last_update);
681         }
682
683         unsigned long long last_ut = st->last_updated.tv_sec * 1000000ULL + st->last_updated.tv_usec;
684         unsigned long long now_ut = st->last_collected_time.tv_sec * 1000000ULL + st->last_collected_time.tv_usec;
685         unsigned long long next_ut = (st->last_updated.tv_sec + st->update_every) * 1000000ULL;
686
687         if(st->debug) debug(D_RRD_STATS, "%s: last ut = %0.3Lf (last updated time)", st->name, (long double)last_ut/1000000.0);
688         if(st->debug) debug(D_RRD_STATS, "%s: now  ut = %0.3Lf (current update time)", st->name, (long double)now_ut/1000000.0);
689         if(st->debug) debug(D_RRD_STATS, "%s: next ut = %0.3Lf (next interpolation point)", st->name, (long double)next_ut/1000000.0);
690
691         st->counter_done++;
692
693         // calculate totals and count the dimensions
694         int dimensions;
695         st->collected_total = 0;
696         for( rd = st->dimensions, dimensions = 0 ; rd ; rd = rd->next, dimensions++ )
697                 st->collected_total += rd->collected_value;
698
699         // process all dimensions to calculate their values
700         // based on the collected figures only
701         // at this stage we do not interpolate anything
702         for( rd = st->dimensions ; rd ; rd = rd->next ) {
703                 if(st->debug) debug(D_RRD_STATS, "%s/%s: "
704                         " last_collected_value = " COLLECTED_NUMBER_FORMAT
705                         " collected_value = " COLLECTED_NUMBER_FORMAT
706                         " last_calculated_value = " CALCULATED_NUMBER_FORMAT
707                         " calculated_value = " CALCULATED_NUMBER_FORMAT
708                         , st->id, rd->name
709                         , rd->last_collected_value
710                         , rd->collected_value
711                         , rd->last_calculated_value
712                         , rd->calculated_value
713                         );
714
715                 switch(rd->algorithm) {
716                         case RRD_DIMENSION_PCENT_OVER_DIFF_TOTAL:
717                                 // the percentage of the current increment
718                                 // over the increment of all dimensions together
719                                 if(st->collected_total == st->last_collected_total) rd->calculated_value = rd->last_calculated_value;
720                                 else rd->calculated_value =
721                                           (calculated_number)100
722                                         * (calculated_number)(rd->collected_value - rd->last_collected_value)
723                                         / (calculated_number)(st->collected_total  - st->last_collected_total);
724
725                                 if(st->debug)
726                                         debug(D_RRD_STATS, "%s/%s: CALC PCENT-DIFF "
727                                                 CALCULATED_NUMBER_FORMAT " = 100"
728                                                 " * (" COLLECTED_NUMBER_FORMAT " - " COLLECTED_NUMBER_FORMAT ")"
729                                                 " / (" COLLECTED_NUMBER_FORMAT " - " COLLECTED_NUMBER_FORMAT ")"
730                                                 , st->id, rd->name
731                                                 , rd->calculated_value
732                                                 , rd->collected_value, rd->last_collected_value
733                                                 , st->collected_total, st->last_collected_total
734                                                 );
735                                 break;
736
737                         case RRD_DIMENSION_PCENT_OVER_ROW_TOTAL:
738                                 if(!st->collected_total) rd->calculated_value = 0;
739                                 else
740                                 // the percentage of the current value
741                                 // over the total of all dimensions
742                                 rd->calculated_value =
743                                           (calculated_number)100
744                                         * (calculated_number)rd->collected_value
745                                         / (calculated_number)st->collected_total;
746
747                                 if(st->debug)
748                                         debug(D_RRD_STATS, "%s/%s: CALC PCENT-ROW "
749                                                 CALCULATED_NUMBER_FORMAT " = 100"
750                                                 " * " COLLECTED_NUMBER_FORMAT
751                                                 " / " COLLECTED_NUMBER_FORMAT
752                                                 , st->id, rd->name
753                                                 , rd->calculated_value
754                                                 , rd->collected_value
755                                                 , st->collected_total
756                                                 );
757                                 break;
758
759                         case RRD_DIMENSION_INCREMENTAL:
760                                 // if the new is smaller than the old (an overflow, or reset), set the old equal to the new
761                                 // to reset the calculation (it will give zero as the calculation for this second)
762                                 if(rd->last_collected_value > rd->collected_value) rd->last_collected_value = rd->collected_value;
763
764                                 rd->calculated_value += (calculated_number)(rd->collected_value - rd->last_collected_value);
765
766                                 if(st->debug)
767                                         debug(D_RRD_STATS, "%s/%s: CALC INC "
768                                                 CALCULATED_NUMBER_FORMAT " += "
769                                                 COLLECTED_NUMBER_FORMAT " - " COLLECTED_NUMBER_FORMAT
770                                                 , st->id, rd->name
771                                                 , rd->calculated_value
772                                                 , rd->collected_value, rd->last_collected_value
773                                                 );
774                                 break;
775
776                         case RRD_DIMENSION_ABSOLUTE:
777                                 rd->calculated_value = (calculated_number)rd->collected_value;
778
779                                 if(st->debug)
780                                         debug(D_RRD_STATS, "%s/%s: CALC ABS/ABS-NO-IN "
781                                                 CALCULATED_NUMBER_FORMAT " = "
782                                                 COLLECTED_NUMBER_FORMAT
783                                                 , st->id, rd->name
784                                                 , rd->calculated_value
785                                                 , rd->collected_value
786                                                 );
787                                 break;
788
789                         default:
790                                 // make the default zero, to make sure
791                                 // it gets noticed when we add new types
792                                 rd->calculated_value = 0;
793
794                                 if(st->debug)
795                                         debug(D_RRD_STATS, "%s/%s: CALC "
796                                                 CALCULATED_NUMBER_FORMAT " = 0"
797                                                 , st->id, rd->name
798                                                 , rd->calculated_value
799                                                 );
800                                 break;
801                 }
802         }
803         // at this point we have all the calculated values ready
804
805         if(st->counter_done == 1 || next_ut > now_ut) {
806                 // we don't have any usable data yet
807                 if(st->debug) debug(D_RRD_STATS, "%s: Skipping collected values (usec since last update = %llu, counter_done = %lu)", st->name, st->usec_since_last_update, st->counter_done);
808
809                 for( rd = st->dimensions; rd ; rd = rd->next ) {
810                         rd->last_calculated_value = rd->calculated_value;
811                         rd->last_collected_value = rd->collected_value;
812
813                         switch(rd->algorithm) {
814                                 case RRD_DIMENSION_PCENT_OVER_DIFF_TOTAL:
815                                 case RRD_DIMENSION_INCREMENTAL:
816                                         if(!st->usec_since_last_update) rd->calculated_value = 0;
817                                         // keep the previous values
818                                         // the next time, a new incremental total will be calculated
819                                         break;
820                         }
821
822                         rd->collected_value = 0;
823                 }
824                 st->last_collected_total  = st->collected_total;
825
826                 pthread_rwlock_unlock(&st->rwlock);
827                 if(pthread_setcancelstate(oldstate, NULL) != 0)
828                         error("Cannot set pthread cancel state to RESTORE (%d).", oldstate);
829
830                 return(st->usec_since_last_update);
831         }
832
833         // it is now time to interpolate values on a second boundary
834         unsigned long long first_ut = last_ut;
835         for( ; next_ut <= now_ut ; next_ut += st->update_every * 1000000ULL ) {
836                 if(st->debug) debug(D_RRD_STATS, "%s: last ut = %0.3Lf (last updated time)", st->name, (long double)last_ut/1000000.0);
837                 if(st->debug) debug(D_RRD_STATS, "%s: next ut = %0.3Lf (next interpolation point)", st->name, (long double)next_ut/1000000.0);
838
839                 st->last_updated.tv_sec = next_ut / 1000000ULL;
840                 st->last_updated.tv_usec = 0;
841
842                 for( rd = st->dimensions ; rd ; rd = rd->next ) {
843                         calculated_number new_value;
844
845                         switch(rd->algorithm) {
846                                 case RRD_DIMENSION_INCREMENTAL:
847                                         new_value = (calculated_number)
848                                                 (          rd->calculated_value
849                                                         * (calculated_number)(next_ut - last_ut)
850                                                         / (calculated_number)(now_ut - last_ut)
851                                                 );
852
853                                         if(st->debug)
854                                                 debug(D_RRD_STATS, "%s/%s: CALC2 INC "
855                                                         CALCULATED_NUMBER_FORMAT " = "
856                                                         CALCULATED_NUMBER_FORMAT
857                                                         " * %llu"
858                                                         " / %llu"
859                                                         , st->id, rd->name
860                                                         , new_value
861                                                         , rd->calculated_value
862                                                         , (unsigned long long)(next_ut - last_ut)
863                                                         , (unsigned long long)(now_ut - last_ut)
864                                                         );
865
866                                         rd->calculated_value -= new_value;
867                                         break;
868
869                                 case RRD_DIMENSION_ABSOLUTE:
870                                 case RRD_DIMENSION_PCENT_OVER_ROW_TOTAL:
871                                 case RRD_DIMENSION_PCENT_OVER_DIFF_TOTAL:
872                                 default:
873                                         new_value = (calculated_number)
874                                                 (       (         (rd->calculated_value - rd->last_calculated_value)
875                                                                 * (calculated_number)(next_ut - first_ut)
876                                                                 / (calculated_number)(now_ut - first_ut)
877                                                         )
878                                                         +  rd->last_calculated_value
879                                                 );
880
881                                         if(st->debug)
882                                                 debug(D_RRD_STATS, "%s/%s: CALC2 DEF "
883                                                         CALCULATED_NUMBER_FORMAT " = ((("
884                                                         "(" CALCULATED_NUMBER_FORMAT " - " CALCULATED_NUMBER_FORMAT ")"
885                                                         " * %llu"
886                                                         " / %llu) + " CALCULATED_NUMBER_FORMAT
887                                                         , st->id, rd->name
888                                                         , new_value
889                                                         , rd->calculated_value, rd->last_calculated_value
890                                                         , (next_ut - first_ut)
891                                                         , (now_ut - first_ut), rd->last_calculated_value
892                                                         );
893
894                                         if(next_ut + st->update_every * 1000000ULL > now_ut) rd->calculated_value = new_value;
895                                         break;
896
897                         }
898
899
900                         rd->values[st->current_entry] = pack_storage_number(
901                                           new_value
902                                         * (calculated_number)rd->multiplier
903                                         / (calculated_number)rd->divisor
904                                 );
905
906                         if(st->debug)
907                                 debug(D_RRD_STATS, "%s/%s: STORE[%ld] "
908                                         CALCULATED_NUMBER_FORMAT " = " CALCULATED_NUMBER_FORMAT
909                                         " * %ld"
910                                         " / %ld"
911                                         , st->id, rd->name
912                                         , st->current_entry
913                                         , unpack_storage_number(rd->values[st->current_entry]), new_value
914                                         , rd->multiplier
915                                         , rd->divisor
916                                         );
917                 }
918
919                 if(st->first_entry_t && st->counter >= (unsigned long long)st->entries) {
920                         // the db is overwriting values
921                         // add the value we will overwrite
922                         st->first_entry_t += st->update_every * 1000000ULL;
923                 }
924                 
925                 st->counter++;
926                 st->current_entry = ((st->current_entry + 1) >= st->entries) ? 0 : st->current_entry + 1;
927                 if(!st->first_entry_t) st->first_entry_t = next_ut;
928                 last_ut = next_ut;
929         }
930
931         for( rd = st->dimensions; rd ; rd = rd->next ) {
932                 rd->last_collected_value = rd->collected_value;
933                 rd->last_calculated_value = rd->calculated_value;
934                 rd->collected_value = 0;
935         }
936         st->last_collected_total  = st->collected_total;
937
938         // ALL DONE ABOUT THE DATA UPDATE
939         // --------------------------------------------------------------------
940
941
942         // find if there are any obsolete dimensions (not updated recently)
943         for( rd = st->dimensions; rd ; rd = rd->next )
944                 if((rd->last_collected_time.tv_sec + (10 * st->update_every)) < st->last_collected_time.tv_sec)
945                         break;
946
947         if(rd) {
948                 // there is dimension to free
949                 // upgrade our read lock to a write lock
950                 pthread_rwlock_unlock(&st->rwlock);
951                 pthread_rwlock_wrlock(&st->rwlock);
952
953                 for( rd = st->dimensions, last = NULL ; rd ; ) {
954                         if((rd->last_collected_time.tv_sec + (10 * st->update_every)) < st->last_collected_time.tv_sec) { // remove it only it is not updated in 10 seconds
955                                 debug(D_RRD_STATS, "Removing obsolete dimension '%s' (%s) of '%s' (%s).", rd->name, rd->id, st->name, st->id);
956
957                                 if(!last) {
958                                         st->dimensions = rd->next;
959                                         rd->next = NULL;
960                                         rrd_stats_dimension_free(rd);
961                                         rd = st->dimensions;
962                                         continue;
963                                 }
964                                 else {
965                                         last->next = rd->next;
966                                         rd->next = NULL;
967                                         rrd_stats_dimension_free(rd);
968                                         rd = last->next;
969                                         continue;
970                                 }
971                         }
972
973                         last = rd;
974                         rd = rd->next;
975                 }
976
977                 if(!st->dimensions) st->enabled = 0;
978         }
979
980         pthread_rwlock_unlock(&st->rwlock);
981
982         if(pthread_setcancelstate(oldstate, NULL) != 0)
983                 error("Cannot set pthread cancel state to RESTORE (%d).", oldstate);
984
985         return(st->usec_since_last_update);
986 }
987
988
989 // find the oldest entry in the data, skipping all empty slots
990 time_t rrd_stats_first_entry_t(RRD_STATS *st)
991 {
992         if(!st->first_entry_t) return st->last_updated.tv_sec;
993         
994         return st->first_entry_t / 1000000;
995 }