]> arthur.barton.de Git - netdata.git/blob - src/rrd2json.c
prevented buffer overflow when there are too many dimensions on a chart
[netdata.git] / src / rrd2json.c
1 #include <pthread.h>
2 #include <sys/time.h>
3 #include <stdlib.h>
4
5 #include "log.h"
6 #include "common.h"
7 #include "rrd2json.h"
8
9 #define HOSTNAME_MAX 1024
10 char *hostname = "unknown";
11
12 unsigned long rrd_stats_one_json(RRDSET *st, char *options, struct web_buffer *wb)
13 {
14         time_t now = time(NULL);
15         web_buffer_increase(wb, 65536);
16
17         pthread_rwlock_rdlock(&st->rwlock);
18
19         web_buffer_printf(wb,
20                 "\t\t{\n"
21                 "\t\t\t\"id\": \"%s\",\n"
22                 "\t\t\t\"name\": \"%s\",\n"
23                 "\t\t\t\"type\": \"%s\",\n"
24                 "\t\t\t\"family\": \"%s\",\n"
25                 "\t\t\t\"title\": \"%s\",\n"
26                 "\t\t\t\"priority\": %ld,\n"
27                 "\t\t\t\"enabled\": %d,\n"
28                 "\t\t\t\"units\": \"%s\",\n"
29                 "\t\t\t\"url\": \"/data/%s/%s\",\n"
30                 "\t\t\t\"chart_type\": \"%s\",\n"
31                 "\t\t\t\"counter\": %ld,\n"
32                 "\t\t\t\"entries\": %ld,\n"
33                 "\t\t\t\"first_entry_t\": %lu,\n"
34                 "\t\t\t\"last_entry\": %ld,\n"
35                 "\t\t\t\"last_entry_t\": %lu,\n"
36                 "\t\t\t\"last_entry_secs_ago\": %lu,\n"
37                 "\t\t\t\"update_every\": %d,\n"
38                 "\t\t\t\"isdetail\": %d,\n"
39                 "\t\t\t\"usec_since_last_update\": %llu,\n"
40                 "\t\t\t\"collected_total\": " TOTAL_NUMBER_FORMAT ",\n"
41                 "\t\t\t\"last_collected_total\": " TOTAL_NUMBER_FORMAT ",\n"
42                 "\t\t\t\"dimensions\": [\n"
43                 , st->id
44                 , st->name
45                 , st->type
46                 , st->family
47                 , st->title
48                 , st->priority
49                 , st->enabled
50                 , st->units
51                 , st->name, options?options:""
52                 , rrdset_type_name(st->chart_type)
53                 , st->counter
54                 , st->entries
55                 , rrdset_first_entry_t(st)
56                 , st->current_entry
57                 , st->last_updated.tv_sec
58                 , now - (st->last_updated.tv_sec > now) ? now : st->last_updated.tv_sec
59                 , st->update_every
60                 , st->isdetail
61                 , st->usec_since_last_update
62                 , st->collected_total
63                 , st->last_collected_total
64                 );
65
66         unsigned long memory = st->memsize;
67
68         RRDDIM *rd;
69         for(rd = st->dimensions; rd ; rd = rd->next) {
70                 web_buffer_increase(wb, 4096);
71
72                 memory += rd->memsize;
73
74                 web_buffer_printf(wb,
75                         "\t\t\t\t{\n"
76                         "\t\t\t\t\t\"id\": \"%s\",\n"
77                         "\t\t\t\t\t\"name\": \"%s\",\n"
78                         "\t\t\t\t\t\"entries\": %ld,\n"
79                         "\t\t\t\t\t\"isHidden\": %d,\n"
80                         "\t\t\t\t\t\"algorithm\": \"%s\",\n"
81                         "\t\t\t\t\t\"multiplier\": %ld,\n"
82                         "\t\t\t\t\t\"divisor\": %ld,\n"
83                         "\t\t\t\t\t\"last_entry_t\": %lu,\n"
84                         "\t\t\t\t\t\"collected_value\": " COLLECTED_NUMBER_FORMAT ",\n"
85                         "\t\t\t\t\t\"calculated_value\": " CALCULATED_NUMBER_FORMAT ",\n"
86                         "\t\t\t\t\t\"last_collected_value\": " COLLECTED_NUMBER_FORMAT ",\n"
87                         "\t\t\t\t\t\"last_calculated_value\": " CALCULATED_NUMBER_FORMAT ",\n"
88                         "\t\t\t\t\t\"memory\": %lu\n"
89                         "\t\t\t\t}%s\n"
90                         , rd->id
91                         , rd->name
92                         , rd->entries
93                         , rd->hidden
94                         , rrddim_algorithm_name(rd->algorithm)
95                         , rd->multiplier
96                         , rd->divisor
97                         , rd->last_collected_time.tv_sec
98                         , rd->collected_value
99                         , rd->calculated_value
100                         , rd->last_collected_value
101                         , rd->last_calculated_value
102                         , rd->memsize
103                         , rd->next?",":""
104                         );
105         }
106
107         web_buffer_printf(wb,
108                 "\t\t\t],\n"
109                 "\t\t\t\"memory\" : %lu\n"
110                 "\t\t}"
111                 , memory
112                 );
113
114         pthread_rwlock_unlock(&st->rwlock);
115         return memory;
116 }
117
118 #define RRD_GRAPH_JSON_HEADER "{\n\t\"charts\": [\n"
119 #define RRD_GRAPH_JSON_FOOTER "\n\t]\n}\n"
120
121 void rrd_stats_graph_json(RRDSET *st, char *options, struct web_buffer *wb)
122 {
123         web_buffer_increase(wb, 2048);
124
125         web_buffer_printf(wb, RRD_GRAPH_JSON_HEADER);
126         rrd_stats_one_json(st, options, wb);
127         web_buffer_printf(wb, RRD_GRAPH_JSON_FOOTER);
128 }
129
130 void rrd_stats_all_json(struct web_buffer *wb)
131 {
132         web_buffer_increase(wb, 2048);
133
134         unsigned long memory = 0;
135         long c;
136         RRDSET *st;
137
138         web_buffer_printf(wb, RRD_GRAPH_JSON_HEADER);
139
140         pthread_rwlock_rdlock(&rrdset_root_rwlock);
141         for(st = rrdset_root, c = 0; st ; st = st->next) {
142                 if(st->enabled) {
143                         if(c) web_buffer_printf(wb, "%s", ",\n");
144                         memory += rrd_stats_one_json(st, NULL, wb);
145                         c++;
146                 }
147         }
148         pthread_rwlock_unlock(&rrdset_root_rwlock);
149         
150         web_buffer_printf(wb, "\n\t],\n"
151                 "\t\"hostname\": \"%s\",\n"
152                 "\t\"update_every\": %d,\n"
153                 "\t\"history\": %d,\n"
154                 "\t\"memory\": %lu\n"
155                 "}\n"
156                 , hostname
157                 , rrd_update_every
158                 , rrd_default_history_entries
159                 , memory
160                 );
161 }
162
163 unsigned long rrd_stats_json(int type, RRDSET *st, struct web_buffer *wb, int entries_to_show, int group, int group_method, time_t after, time_t before, int only_non_zero)
164 {
165         int c;
166         pthread_rwlock_rdlock(&st->rwlock);
167
168
169         // -------------------------------------------------------------------------
170         // switch from JSON to google JSON
171         
172         char kq[2] = "\"";
173         char sq[2] = "\"";
174         switch(type) {
175                 case DATASOURCE_GOOGLE_JSON:
176                 case DATASOURCE_GOOGLE_JSONP:
177                         kq[0] = '\0';
178                         sq[0] = '\'';
179                         break;
180
181                 case DATASOURCE_JSON:
182                 default:
183                         break;
184         }
185
186
187         // -------------------------------------------------------------------------
188         // validate the parameters
189         
190         if(entries_to_show < 1) entries_to_show = 1;
191         if(group < 1) group = 1;
192         
193         // make sure current_entry is within limits
194         long current_entry = (long)st->current_entry - (long)1;
195         if(current_entry < 0) current_entry = 0;
196         else if(current_entry >= st->entries) current_entry = st->entries - 1;
197         
198         // find the oldest entry of the round-robin
199         long max_entries_init = (st->counter < (unsigned long)st->entries) ? st->counter : (unsigned long)st->entries;
200         
201         time_t time_init = st->last_updated.tv_sec;
202         
203         if(before == 0 || before > time_init) before = time_init;
204         if(after  == 0) after = rrdset_first_entry_t(st);
205
206
207         // ---
208
209         // our return value (the last timestamp printed)
210         // this is required to detect re-transmit in google JSONP
211         time_t last_timestamp = 0;                      
212
213
214         // -------------------------------------------------------------------------
215         // find how many dimensions we have
216         
217         int dimensions = 0;
218         RRDDIM *rd;
219         for( rd = st->dimensions ; rd ; rd = rd->next) dimensions++;
220         if(!dimensions) {
221                 pthread_rwlock_unlock(&st->rwlock);
222                 web_buffer_printf(wb, "No dimensions yet.");
223                 return 0;
224         }
225
226         
227         // -------------------------------------------------------------------------
228         // prepare various strings, to speed up the loop
229         
230         char overflow_annotation[201]; snprintf(overflow_annotation, 200, ",{%sv%s:%sRESET OR OVERFLOW%s},{%sv%s:%sThe counters have been wrapped.%s}", kq, kq, sq, sq, kq, kq, sq, sq);
231         char normal_annotation[201];   snprintf(normal_annotation,   200, ",{%sv%s:null},{%sv%s:null}", kq, kq, kq, kq);
232         char pre_date[51];             snprintf(pre_date,             50, "             {%sc%s:[{%sv%s:%s", kq, kq, kq, kq, sq);
233         char post_date[21];            snprintf(post_date,            20, "%s}", sq);
234         char pre_value[21];            snprintf(pre_value,            20, ",{%sv%s:", kq, kq);
235         char post_value[21];           snprintf(post_value,           20, "}");
236
237
238         // -------------------------------------------------------------------------
239         // checks for debuging
240         
241         if(st->debug) {
242                 debug(D_RRD_STATS, "%s first_entry_t = %lu, last_entry_t = %lu, duration = %lu, after = %lu, before = %lu, duration = %lu, entries_to_show = %lu, group = %lu, max_entries = %ld"
243                         , st->id
244                         , rrdset_first_entry_t(st)
245                         , st->last_updated.tv_sec
246                         , st->last_updated.tv_sec - rrdset_first_entry_t(st)
247                         , after
248                         , before
249                         , before - after
250                         , entries_to_show
251                         , group
252                         , max_entries_init
253                         );
254
255                 if(before < after)
256                         debug(D_RRD_STATS, "WARNING: %s The newest value in the database (%lu) is earlier than the oldest (%lu)", st->name, before, after);
257
258                 if((before - after) > st->entries * st->update_every)
259                         debug(D_RRD_STATS, "WARNING: %s The time difference between the oldest and the newest entries (%lu) is higher than the capacity of the database (%lu)", st->name, before - after, st->entries * st->update_every);
260         }
261
262
263         // -------------------------------------------------------------------------
264         // temp arrays for keeping values per dimension
265         
266         calculated_number group_values[dimensions]; // keep sums when grouping
267         calculated_number print_values[dimensions]; // keep the final value to be printed
268         int               print_hidden[dimensions]; // keep hidden flags
269         int               found_non_zero[dimensions];
270         int               found_non_existing[dimensions];
271
272         // initialize them
273         for( rd = st->dimensions, c = 0 ; rd && c < dimensions ; rd = rd->next, c++) {
274                 group_values[c] = print_values[c] = 0;
275                 print_hidden[c] = rd->hidden;
276                 found_non_zero[c] = 0;
277                 found_non_existing[c] = 0;
278         }
279
280         // -------------------------------------------------------------------------
281         // remove dimensions that contain only zeros
282
283         int max_loop = 1;
284         if(only_non_zero) max_loop = 2;
285
286         for(; max_loop ; max_loop--) {
287
288                 // -------------------------------------------------------------------------
289                 // print the JSON header
290                 
291                 web_buffer_printf(wb, "{\n      %scols%s:\n     [\n", kq, kq);
292                 web_buffer_printf(wb, "         {%sid%s:%s%s,%slabel%s:%stime%s,%spattern%s:%s%s,%stype%s:%sdatetime%s},\n", kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq);
293                 web_buffer_printf(wb, "         {%sid%s:%s%s,%slabel%s:%s%s,%spattern%s:%s%s,%stype%s:%sstring%s,%sp%s:{%srole%s:%sannotation%s}},\n", kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, kq, kq, sq, sq);
294                 web_buffer_printf(wb, "         {%sid%s:%s%s,%slabel%s:%s%s,%spattern%s:%s%s,%stype%s:%sstring%s,%sp%s:{%srole%s:%sannotationText%s}}", kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, kq, kq, sq, sq);
295
296                 // print the header for each dimension
297                 // and update the print_hidden array for the dimensions that should be hidden
298                 int pc = 0;
299                 for( rd = st->dimensions, c = 0 ; rd && c < dimensions ; rd = rd->next, c++) {
300                         if(!print_hidden[c]) {
301                                 pc++;
302                                 web_buffer_printf(wb, ",\n              {%sid%s:%s%s,%slabel%s:%s%s%s,%spattern%s:%s%s,%stype%s:%snumber%s}", kq, kq, sq, sq, kq, kq, sq, rd->name, sq, kq, kq, sq, sq, kq, kq, sq, sq);
303                         }
304                 }
305                 if(!pc) {
306                         web_buffer_printf(wb, ",\n              {%sid%s:%s%s,%slabel%s:%s%s%s,%spattern%s:%s%s,%stype%s:%snumber%s}", kq, kq, sq, sq, kq, kq, sq, "no data", sq, kq, kq, sq, sq, kq, kq, sq, sq);
307                 }
308
309                 // print the begin of row data
310                 web_buffer_printf(wb, "\n       ],\n    %srows%s:\n     [\n", kq, kq);
311
312
313                 // -------------------------------------------------------------------------
314                 // the main loop
315
316                 int annotate_reset = 0;
317                 int annotation_count = 0;
318                 
319                 // to allow grouping on the same values, we need a pad
320                 long pad = before % group;
321
322                 // the minimum line length we expect
323                 int line_size = 4096 + (dimensions * 200);
324
325                 time_t now = time_init;
326                 long max_entries = max_entries_init;
327
328                 long t;
329
330                 long count = 0, printed = 0, group_count = 0;
331                 last_timestamp = 0;
332                 for(t = current_entry; max_entries ; now -= st->update_every, t--, max_entries--) {
333                         if(t < 0) t = st->entries - 1;
334
335                         int print_this = 0;
336
337                         if(st->debug) {
338                                 debug(D_RRD_STATS, "%s t = %ld, count = %ld, group_count = %ld, printed = %ld, now = %lu, %s %s"
339                                         , st->id
340                                         , t
341                                         , count + 1
342                                         , group_count + 1
343                                         , printed
344                                         , now
345                                         , (((count + 1 - pad) % group) == 0)?"PRINT":"  -  "
346                                         , (now >= after && now <= before)?"RANGE":"  -  "
347                                         );
348                         }
349
350                         // make sure we return data in the proper time range
351                         if(now < after || now > before) continue;
352
353                         count++;
354                         group_count++;
355
356                         // check if we have to print this now
357                         if(((count - pad) % group) == 0) {
358                                 if(printed >= entries_to_show) {
359                                         // debug(D_RRD_STATS, "Already printed all rows. Stopping.");
360                                         break;
361                                 }
362                                 
363                                 if(group_count != group) {
364                                         // this is an incomplete group, skip it.
365                                         for( rd = st->dimensions, c = 0 ; rd && c < dimensions ; rd = rd->next, c++) {
366                                                 group_values[c] = 0;
367                                                 found_non_existing[c] = 0;
368                                         }
369                                                 
370                                         group_count = 0;
371                                         continue;
372                                 }
373
374                                 // check if we may exceed the buffer provided
375                                 web_buffer_increase(wb, line_size);
376
377                                 // generate the local date time
378                                 struct tm *tm = localtime(&now);
379                                 if(!tm) { error("localtime() failed."); continue; }
380                                 if(now > last_timestamp) last_timestamp = now;
381
382                                 if(printed) web_buffer_strcpy(wb, "]},\n");
383                                 web_buffer_strcpy(wb, pre_date);
384                                 web_buffer_jsdate(wb, tm->tm_year + 1900, tm->tm_mon, tm->tm_mday, tm->tm_hour, tm->tm_min, tm->tm_sec);
385                                 web_buffer_strcpy(wb, post_date);
386
387                                 print_this = 1;
388                         }
389
390                         // do the calculations
391                         for(rd = st->dimensions, c = 0 ; rd && c < dimensions ; rd = rd->next, c++) {
392                                 storage_number n = rd->values[t];
393                                 calculated_number value = unpack_storage_number(n);
394                                 
395                                 if(!does_storage_number_exist(n)) {
396                                         value = 0.0;
397                                         found_non_existing[c]++;
398                                 }
399                                 if(did_storage_number_reset(n)) annotate_reset = 1;
400
401                                 switch(group_method) {
402                                         case GROUP_MAX:
403                                                 if(abs(value) > abs(group_values[c])) group_values[c] = value;
404                                                 break;
405
406                                         case GROUP_SUM:
407                                                 group_values[c] += value;
408                                                 break;
409
410                                         default:
411                                         case GROUP_AVERAGE:
412                                                 group_values[c] += value;
413                                                 if(print_this) group_values[c] /= ( group_count - found_non_existing[c] );
414                                                 break;
415                                 }
416                         }
417
418                         if(print_this) {
419                                 if(annotate_reset) {
420                                         annotation_count++;
421                                         web_buffer_strcpy(wb, overflow_annotation);
422                                         annotate_reset = 0;
423                                 }
424                                 else
425                                         web_buffer_strcpy(wb, normal_annotation);
426
427                                 pc = 0;
428                                 for(c = 0 ; c < dimensions ; c++) {
429                                         if(found_non_existing[c] == group_count) {
430                                                 // all entries are non-existing
431                                                 pc++;
432                                                 web_buffer_strcpy(wb, pre_value);
433                                                 web_buffer_strcpy(wb, "null");
434                                                 web_buffer_strcpy(wb, post_value);
435                                         }
436                                         else if(!print_hidden[c]) {
437                                                 pc++;
438                                                 web_buffer_strcpy(wb, pre_value);
439                                                 web_buffer_rrd_value(wb, group_values[c]);
440                                                 web_buffer_strcpy(wb, post_value);
441
442                                                 if(group_values[c]) found_non_zero[c]++;
443                                         }
444
445                                         // reset them for the next loop
446                                         group_values[c] = 0;
447                                         found_non_existing[c] = 0;
448                                 }
449
450                                 // if all dimensions are hidden, print a null
451                                 if(!pc) {
452                                         web_buffer_strcpy(wb, pre_value);
453                                         web_buffer_strcpy(wb, "null");
454                                         web_buffer_strcpy(wb, post_value);
455                                 }
456
457                                 printed++;
458                                 group_count = 0;
459                         }
460                 }
461
462                 if(printed) web_buffer_printf(wb, "]}");
463                 web_buffer_printf(wb, "\n       ]\n}\n");
464
465                 if(only_non_zero && max_loop > 1) {
466                         int changed = 0;
467                         for(rd = st->dimensions, c = 0 ; rd && c < dimensions ; rd = rd->next, c++) {
468                                 group_values[c] = 0;
469                                 found_non_existing[c] = 0;
470
471                                 if(!print_hidden[c] && !found_non_zero[c]) {
472                                         changed = 1;
473                                         print_hidden[c] = 1;
474                                 }
475                         }
476
477                         if(changed) web_buffer_reset(wb);
478                         else break;
479                 }
480                 else break;
481                 
482         } // max_loop
483
484         debug(D_RRD_STATS, "RRD_STATS_JSON: %s total %ld bytes", st->name, wb->bytes);
485
486         pthread_rwlock_unlock(&st->rwlock);
487         return last_timestamp;
488 }
489
490
491 /*
492 unsigned long rrdset2json(int type, RRDSET *st, struct web_buffer *wb, int entries_to_show, int group, int group_method, time_t after, time_t before, int only_non_zero) {
493
494         if(!st->dimensions) {
495                 web_buffer_printf(wb, "No dimensions yet.");
496                 return 0;
497         }
498
499         int c;
500         pthread_rwlock_rdlock(&st->rwlock);
501
502         // -------------------------------------------------------------------------
503         // validate the parameters
504
505         if(entries_to_show < 1) entries_to_show = 1;
506         if(group < 1) group = 1;
507
508         // make sure current_entry is within limits
509         long current_entry = (long)st->current_entry - (long)1;
510         if(current_entry < 0) current_entry = 0;
511         else if(current_entry >= st->entries) current_entry = st->entries - 1;
512
513         // find the oldest entry of the round-robin
514         long max_entries = (st->counter < (unsigned long)st->entries) ? st->counter : (unsigned long)st->entries;
515         if(entries_to_show > max_entries) entries_to_show = max_entries;
516
517         if(before == 0 || before > st->last_updated.tv_sec) before = st->last_updated.tv_sec;
518         if(after  == 0 || after  < rrdset_first_entry_t(st)) after = rrdset_first_entry_t(st);
519
520         if(entries_to_show > (before - after) / st->update_every / group) entries_to_show = (before - after) / st->update_every / group;
521
522         // -------------------------------------------------------------------------
523         // checks for debuging
524
525         if(st->debug) {
526                 debug(D_RRD_STATS, "%s first_entry_t = %lu, last_entry_t = %lu, duration = %lu, after = %lu, before = %lu, duration = %lu, entries_to_show = %lu, group = %lu, max_entries = %ld"
527                         , st->id
528                         , rrdset_first_entry_t(st)
529                         , st->last_updated.tv_sec
530                         , st->last_updated.tv_sec - rrdset_first_entry_t(st)
531                         , after
532                         , before
533                         , before - after
534                         , entries_to_show
535                         , group
536                         , max_entries
537                         );
538
539                 if(before < after)
540                         debug(D_RRD_STATS, "WARNING: %s The newest value in the database (%lu) is earlier than the oldest (%lu)", st->name, before, after);
541
542                 if((before - after) > st->entries * st->update_every)
543                         debug(D_RRD_STATS, "WARNING: %s The time difference between the oldest and the newest entries (%lu) is higher than the capacity of the database (%lu)", st->name, before - after, st->entries * st->update_every);
544         }
545
546         // -------------------------------------------------------------------------
547
548         int dimensions = 0, i, j;
549         RRDDIM *rd;
550         for( rd = st->dimensions ; rd ; rd = rd->next) dimensions++;
551
552         uint32_t *annotations[dimensions];
553         calculated_number *values[dimensions];
554         int enabled[dimensions];
555
556         for( rd = st->dimensions, i = 0 ; rd && i < dimensions ; rd = rd->next, i++) {
557                 annotations[i] = malloc(sizeof(uint32_t) * entries_to_show);
558                 if(!annotations[i]) fatal("Cannot allocate %z bytes of memory.", sizeof(uint32_t) * entries_to_show);
559
560                 values[i] = malloc(sizeof(calculated_number) * entries_to_show);
561                 if(!values[i]) fatal("Cannot allocate %z bytes of memory.", sizeof(calculated_number) * entries_to_show);
562
563                 calculated_number sum = rrddim2values(st, rd, values, annotations, entries_to_show, group, group_method, after, before);
564                 if(only_non_zero && sum == 0) enabled[i] = 0;
565                 else enabled[i] = 1;
566         }
567
568         switch(type) {
569                 case DATASOURCE_GOOGLE_JSON:
570                 case DATASOURCE_GOOGLE_JSONP:
571                         //rrdvalues2googlejson(st, rd, values, annotations, enabled, entries_to_show, );
572                         break;
573
574                 case DATASOURCE_JSON:
575                 default:
576                         //rrdvalues2json();
577                         break;
578         }
579
580         //return ???;
581 }
582 */