src/rrd2json.c

   1 #include <pthread.h>
   2 #include <sys/time.h>
   3
   4 #include "log.h"
   5 #include "common.h"
   6 #include "rrd2json.h"
   7
   8 #define HOSTNAME_MAX 1024
   9 char *hostname = "unknown";
  10
  11 unsigned long rrd_stats_one_json(RRDSET *st, char *options, struct web_buffer *wb)
  12 {
  13         time_t now = time(NULL);
  14         web_buffer_increase(wb, 16384);
  15
  16         pthread_rwlock_rdlock(&st->rwlock);
  17
  18         web_buffer_printf(wb,
  19                 "\t\t{\n"
  20                 "\t\t\t\"id\": \"%s\",\n"
  21                 "\t\t\t\"name\": \"%s\",\n"
  22                 "\t\t\t\"type\": \"%s\",\n"
  23                 "\t\t\t\"family\": \"%s\",\n"
  24                 "\t\t\t\"title\": \"%s\",\n"
  25                 "\t\t\t\"priority\": %ld,\n"
  26                 "\t\t\t\"enabled\": %d,\n"
  27                 "\t\t\t\"units\": \"%s\",\n"
  28                 "\t\t\t\"url\": \"/data/%s/%s\",\n"
  29                 "\t\t\t\"chart_type\": \"%s\",\n"
  30                 "\t\t\t\"counter\": %ld,\n"
  31                 "\t\t\t\"entries\": %ld,\n"
  32                 "\t\t\t\"first_entry_t\": %lu,\n"
  33                 "\t\t\t\"last_entry\": %ld,\n"
  34                 "\t\t\t\"last_entry_t\": %lu,\n"
  35                 "\t\t\t\"last_entry_secs_ago\": %lu,\n"
  36                 "\t\t\t\"update_every\": %d,\n"
  37                 "\t\t\t\"isdetail\": %d,\n"
  38                 "\t\t\t\"usec_since_last_update\": %llu,\n"
  39                 "\t\t\t\"collected_total\": " TOTAL_NUMBER_FORMAT ",\n"
  40                 "\t\t\t\"last_collected_total\": " TOTAL_NUMBER_FORMAT ",\n"
  41                 "\t\t\t\"dimensions\": [\n"
  42                 , st->id
  43                 , st->name
  44                 , st->type
  45                 , st->family
  46                 , st->title
  47                 , st->priority
  48                 , st->enabled
  49                 , st->units
  50                 , st->name, options?options:""
  51                 , rrdset_type_name(st->chart_type)
  52                 , st->counter
  53                 , st->entries
  54                 , rrdset_first_entry_t(st)
  55                 , st->current_entry
  56                 , st->last_updated.tv_sec
  57                 , now - (st->last_updated.tv_sec > now) ? now : st->last_updated.tv_sec
  58                 , st->update_every
  59                 , st->isdetail
  60                 , st->usec_since_last_update
  61                 , st->collected_total
  62                 , st->last_collected_total
  63                 );
  64
  65         unsigned long memory = st->memsize;
  66
  67         RRDDIM *rd;
  68         for(rd = st->dimensions; rd ; rd = rd->next) {
  69                 memory += rd->memsize;
  70
  71                 web_buffer_printf(wb,
  72                         "\t\t\t\t{\n"
  73                         "\t\t\t\t\t\"id\": \"%s\",\n"
  74                         "\t\t\t\t\t\"name\": \"%s\",\n"
  75                         "\t\t\t\t\t\"entries\": %ld,\n"
  76                         "\t\t\t\t\t\"isHidden\": %d,\n"
  77                         "\t\t\t\t\t\"algorithm\": \"%s\",\n"
  78                         "\t\t\t\t\t\"multiplier\": %ld,\n"
  79                         "\t\t\t\t\t\"divisor\": %ld,\n"
  80                         "\t\t\t\t\t\"last_entry_t\": %lu,\n"
  81                         "\t\t\t\t\t\"collected_value\": " COLLECTED_NUMBER_FORMAT ",\n"
  82                         "\t\t\t\t\t\"calculated_value\": " CALCULATED_NUMBER_FORMAT ",\n"
  83                         "\t\t\t\t\t\"last_collected_value\": " COLLECTED_NUMBER_FORMAT ",\n"
  84                         "\t\t\t\t\t\"last_calculated_value\": " CALCULATED_NUMBER_FORMAT ",\n"
  85                         "\t\t\t\t\t\"memory\": %lu\n"
  86                         "\t\t\t\t}%s\n"
  87                         , rd->id
  88                         , rd->name
  89                         , rd->entries
  90                         , rd->hidden
  91                         , rrddim_algorithm_name(rd->algorithm)
  92                         , rd->multiplier
  93                         , rd->divisor
  94                         , rd->last_collected_time.tv_sec
  95                         , rd->collected_value
  96                         , rd->calculated_value
  97                         , rd->last_collected_value
  98                         , rd->last_calculated_value
  99                         , rd->memsize
 100                         , rd->next?",":""
 101                         );
 102         }
 103
 104         web_buffer_printf(wb,
 105                 "\t\t\t],\n"
 106                 "\t\t\t\"memory\" : %lu\n"
 107                 "\t\t}"
 108                 , memory
 109                 );
 110
 111         pthread_rwlock_unlock(&st->rwlock);
 112         return memory;
 113 }
 114
 115 #define RRD_GRAPH_JSON_HEADER "{\n\t\"charts\": [\n"
 116 #define RRD_GRAPH_JSON_FOOTER "\n\t]\n}\n"
 117
 118 void rrd_stats_graph_json(RRDSET *st, char *options, struct web_buffer *wb)
 119 {
 120         web_buffer_increase(wb, 16384);
 121
 122         web_buffer_printf(wb, RRD_GRAPH_JSON_HEADER);
 123         rrd_stats_one_json(st, options, wb);
 124         web_buffer_printf(wb, RRD_GRAPH_JSON_FOOTER);
 125 }
 126
 127 void rrd_stats_all_json(struct web_buffer *wb)
 128 {
 129         web_buffer_increase(wb, 1024);
 130
 131         unsigned long memory = 0;
 132         long c;
 133         RRDSET *st;
 134
 135         web_buffer_printf(wb, RRD_GRAPH_JSON_HEADER);
 136
 137         pthread_rwlock_rdlock(&rrdset_root_rwlock);
 138         for(st = rrdset_root, c = 0; st ; st = st->next) {
 139                 if(st->enabled) {
 140                         if(c) web_buffer_printf(wb, "%s", ",\n");
 141                         memory += rrd_stats_one_json(st, NULL, wb);
 142                         c++;
 143                 }
 144         }
 145         pthread_rwlock_unlock(&rrdset_root_rwlock);
 146
 147         web_buffer_printf(wb, "\n\t],\n"
 148                 "\t\"hostname\": \"%s\",\n"
 149                 "\t\"update_every\": %d,\n"
 150                 "\t\"history\": %d,\n"
 151                 "\t\"memory\": %lu\n"
 152                 "}\n"
 153                 , hostname
 154                 , rrd_update_every
 155                 , rrd_default_history_entries
 156                 , memory
 157                 );
 158 }
 159
 160 unsigned long rrd_stats_json(int type, RRDSET *st, struct web_buffer *wb, int entries_to_show, int group, int group_method, time_t after, time_t before, int only_non_zero)
 161 {
 162         int c;
 163         pthread_rwlock_rdlock(&st->rwlock);
 164
 165
 166         // -------------------------------------------------------------------------
 167         // switch from JSON to google JSON
 168
 169         char kq[2] = "\"";
 170         char sq[2] = "\"";
 171         switch(type) {
 172                 case DATASOURCE_GOOGLE_JSON:
 173                 case DATASOURCE_GOOGLE_JSONP:
 174                         kq[0] = '\0';
 175                         sq[0] = '\'';
 176                         break;
 177
 178                 case DATASOURCE_JSON:
 179                 default:
 180                         break;
 181         }
 182
 183
 184         // -------------------------------------------------------------------------
 185         // validate the parameters
 186
 187         if(entries_to_show < 1) entries_to_show = 1;
 188         if(group < 1) group = 1;
 189
 190         // make sure current_entry is within limits
 191         long current_entry = (long)st->current_entry - (long)1;
 192         if(current_entry < 0) current_entry = 0;
 193         else if(current_entry >= st->entries) current_entry = st->entries - 1;
 194
 195         // find the oldest entry of the round-robin
 196         long max_entries_init = (st->counter < (unsigned long)st->entries) ? st->counter : (unsigned long)st->entries;
 197
 198         time_t time_init = st->last_updated.tv_sec;
 199
 200         if(before == 0 || before > time_init) before = time_init;
 201         if(after  == 0) after = rrdset_first_entry_t(st);
 202
 203
 204         // ---
 205
 206         // our return value (the last timestamp printed)
 207         // this is required to detect re-transmit in google JSONP
 208         time_t last_timestamp = 0;
 209
 210
 211         // -------------------------------------------------------------------------
 212         // find how many dimensions we have
 213
 214         int dimensions = 0;
 215         RRDDIM *rd;
 216         for( rd = st->dimensions ; rd ; rd = rd->next) dimensions++;
 217         if(!dimensions) {
 218                 pthread_rwlock_unlock(&st->rwlock);
 219                 web_buffer_printf(wb, "No dimensions yet.");
 220                 return 0;
 221         }
 222
 223
 224         // -------------------------------------------------------------------------
 225         // prepare various strings, to speed up the loop
 226
 227         char overflow_annotation[201]; snprintf(overflow_annotation, 200, ",{%sv%s:%sRESET OR OVERFLOW%s},{%sv%s:%sThe counters have been wrapped.%s}", kq, kq, sq, sq, kq, kq, sq, sq);
 228         char normal_annotation[201];   snprintf(normal_annotation,   200, ",{%sv%s:null},{%sv%s:null}", kq, kq, kq, kq);
 229         char pre_date[51];             snprintf(pre_date,             50, "             {%sc%s:[{%sv%s:%s", kq, kq, kq, kq, sq);
 230         char post_date[21];            snprintf(post_date,            20, "%s}", sq);
 231         char pre_value[21];            snprintf(pre_value,            20, ",{%sv%s:", kq, kq);
 232         char post_value[21];           snprintf(post_value,           20, "}");
 233
 234
 235         // -------------------------------------------------------------------------
 236         // checks for debuging
 237
 238         if(st->debug) {
 239                 debug(D_RRD_STATS, "%s first_entry_t = %lu, last_entry_t = %lu, duration = %lu, after = %lu, before = %lu, duration = %lu, entries_to_show = %lu, group = %lu, max_entries = %ld"
 240                         , st->id
 241                         , rrdset_first_entry_t(st)
 242                         , st->last_updated.tv_sec
 243                         , st->last_updated.tv_sec - rrdset_first_entry_t(st)
 244                         , after
 245                         , before
 246                         , before - after
 247                         , entries_to_show
 248                         , group
 249                         , max_entries_init
 250                         );
 251
 252                 if(before < after)
 253                         debug(D_RRD_STATS, "WARNING: %s The newest value in the database (%lu) is earlier than the oldest (%lu)", st->name, before, after);
 254
 255                 if((before - after) > st->entries * st->update_every)
 256                         debug(D_RRD_STATS, "WARNING: %s The time difference between the oldest and the newest entries (%lu) is higher than the capacity of the database (%lu)", st->name, before - after, st->entries * st->update_every);
 257         }
 258
 259
 260         // -------------------------------------------------------------------------
 261         // temp arrays for keeping values per dimension
 262
 263         calculated_number group_values[dimensions]; // keep sums when grouping
 264         calculated_number print_values[dimensions]; // keep the final value to be printed
 265         int               print_hidden[dimensions]; // keep hidden flags
 266         int               found_non_zero[dimensions];
 267         int               found_non_existing[dimensions];
 268
 269         // initialize them
 270         for( rd = st->dimensions, c = 0 ; rd && c < dimensions ; rd = rd->next, c++) {
 271                 group_values[c] = print_values[c] = 0;
 272                 print_hidden[c] = rd->hidden;
 273                 found_non_zero[c] = 0;
 274                 found_non_existing[c] = 0;
 275         }
 276
 277         // -------------------------------------------------------------------------
 278         // remove dimensions that contain only zeros
 279
 280         int max_loop = 1;
 281         if(only_non_zero) max_loop = 2;
 282
 283         for(; max_loop ; max_loop--) {
 284
 285                 // -------------------------------------------------------------------------
 286                 // print the JSON header
 287
 288                 web_buffer_printf(wb, "{\n      %scols%s:\n     [\n", kq, kq);
 289                 web_buffer_printf(wb, "         {%sid%s:%s%s,%slabel%s:%stime%s,%spattern%s:%s%s,%stype%s:%sdatetime%s},\n", kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq);
 290                 web_buffer_printf(wb, "         {%sid%s:%s%s,%slabel%s:%s%s,%spattern%s:%s%s,%stype%s:%sstring%s,%sp%s:{%srole%s:%sannotation%s}},\n", kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, kq, kq, sq, sq);
 291                 web_buffer_printf(wb, "         {%sid%s:%s%s,%slabel%s:%s%s,%spattern%s:%s%s,%stype%s:%sstring%s,%sp%s:{%srole%s:%sannotationText%s}}", kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, kq, kq, sq, sq);
 292
 293                 // print the header for each dimension
 294                 // and update the print_hidden array for the dimensions that should be hidden
 295                 int pc = 0;
 296                 for( rd = st->dimensions, c = 0 ; rd && c < dimensions ; rd = rd->next, c++) {
 297                         if(!print_hidden[c]) {
 298                                 pc++;
 299                                 web_buffer_printf(wb, ",\n              {%sid%s:%s%s,%slabel%s:%s%s%s,%spattern%s:%s%s,%stype%s:%snumber%s}", kq, kq, sq, sq, kq, kq, sq, rd->name, sq, kq, kq, sq, sq, kq, kq, sq, sq);
 300                         }
 301                 }
 302                 if(!pc) {
 303                         web_buffer_printf(wb, ",\n              {%sid%s:%s%s,%slabel%s:%s%s%s,%spattern%s:%s%s,%stype%s:%snumber%s}", kq, kq, sq, sq, kq, kq, sq, "no data", sq, kq, kq, sq, sq, kq, kq, sq, sq);
 304                 }
 305
 306                 // print the begin of row data
 307                 web_buffer_printf(wb, "\n       ],\n    %srows%s:\n     [\n", kq, kq);
 308
 309
 310                 // -------------------------------------------------------------------------
 311                 // the main loop
 312
 313                 int annotate_reset = 0;
 314                 int annotation_count = 0;
 315
 316                 // to allow grouping on the same values, we need a pad
 317                 long pad = before % group;
 318
 319                 // the minimum line length we expect
 320                 int line_size = 4096 + (dimensions * 200);
 321
 322                 time_t now = time_init;
 323                 long max_entries = max_entries_init;
 324
 325                 long t;
 326
 327                 long count = 0, printed = 0, group_count = 0;
 328                 last_timestamp = 0;
 329                 for(t = current_entry; max_entries ; now -= st->update_every, t--, max_entries--) {
 330                         if(t < 0) t = st->entries - 1;
 331
 332                         int print_this = 0;
 333
 334                         if(st->debug) {
 335                                 debug(D_RRD_STATS, "%s t = %ld, count = %ld, group_count = %ld, printed = %ld, now = %lu, %s %s"
 336                                         , st->id
 337                                         , t
 338                                         , count + 1
 339                                         , group_count + 1
 340                                         , printed
 341                                         , now
 342                                         , (((count + 1 - pad) % group) == 0)?"PRINT":"  -  "
 343                                         , (now >= after && now <= before)?"RANGE":"  -  "
 344                                         );
 345                         }
 346
 347                         // make sure we return data in the proper time range
 348                         if(now < after || now > before) continue;
 349
 350                         count++;
 351                         group_count++;
 352
 353                         // check if we have to print this now
 354                         if(((count - pad) % group) == 0) {
 355                                 if(printed >= entries_to_show) {
 356                                         // debug(D_RRD_STATS, "Already printed all rows. Stopping.");
 357                                         break;
 358                                 }
 359
 360                                 if(group_count != group) {
 361                                         // this is an incomplete group, skip it.
 362                                         for( rd = st->dimensions, c = 0 ; rd && c < dimensions ; rd = rd->next, c++) {
 363                                                 group_values[c] = 0;
 364                                                 found_non_existing[c] = 0;
 365                                         }
 366
 367                                         group_count = 0;
 368                                         continue;
 369                                 }
 370
 371                                 // check if we may exceed the buffer provided
 372                                 web_buffer_increase(wb, line_size);
 373
 374                                 // generate the local date time
 375                                 struct tm *tm = localtime(&now);
 376                                 if(!tm) { error("localtime() failed."); continue; }
 377                                 if(now > last_timestamp) last_timestamp = now;
 378
 379                                 if(printed) web_buffer_strcpy(wb, "]},\n");
 380                                 web_buffer_strcpy(wb, pre_date);
 381                                 web_buffer_jsdate(wb, tm->tm_year + 1900, tm->tm_mon, tm->tm_mday, tm->tm_hour, tm->tm_min, tm->tm_sec);
 382                                 web_buffer_strcpy(wb, post_date);
 383
 384                                 print_this = 1;
 385                         }
 386
 387                         // do the calculations
 388                         for(rd = st->dimensions, c = 0 ; rd && c < dimensions ; rd = rd->next, c++) {
 389                                 storage_number n = rd->values[t];
 390                                 calculated_number value = unpack_storage_number(n);
 391
 392                                 if(!does_storage_number_exist(n)) {
 393                                         value = 0.0;
 394                                         found_non_existing[c]++;
 395                                 }
 396                                 if(did_storage_number_reset(n)) annotate_reset = 1;
 397
 398                                 switch(group_method) {
 399                                         case GROUP_MAX:
 400                                                 if(abs(value) > abs(group_values[c])) group_values[c] = value;
 401                                                 break;
 402
 403                                         case GROUP_SUM:
 404                                                 group_values[c] += value;
 405                                                 break;
 406
 407                                         default:
 408                                         case GROUP_AVERAGE:
 409                                                 group_values[c] += value;
 410                                                 if(print_this) group_values[c] /= ( group_count - found_non_existing[c] );
 411                                                 break;
 412                                 }
 413                         }
 414
 415                         if(print_this) {
 416                                 if(annotate_reset) {
 417                                         annotation_count++;
 418                                         web_buffer_strcpy(wb, overflow_annotation);
 419                                         annotate_reset = 0;
 420                                 }
 421                                 else
 422                                         web_buffer_strcpy(wb, normal_annotation);
 423
 424                                 pc = 0;
 425                                 for(c = 0 ; c < dimensions ; c++) {
 426                                         if(found_non_existing[c] == group_count) {
 427                                                 // all entries are non-existing
 428                                                 pc++;
 429                                                 web_buffer_strcpy(wb, pre_value);
 430                                                 web_buffer_strcpy(wb, "null");
 431                                                 web_buffer_strcpy(wb, post_value);
 432                                         }
 433                                         else if(!print_hidden[c]) {
 434                                                 pc++;
 435                                                 web_buffer_strcpy(wb, pre_value);
 436                                                 web_buffer_rrd_value(wb, group_values[c]);
 437                                                 web_buffer_strcpy(wb, post_value);
 438
 439                                                 if(group_values[c]) found_non_zero[c]++;
 440                                         }
 441
 442                                         // reset them for the next loop
 443                                         group_values[c] = 0;
 444                                         found_non_existing[c] = 0;
 445                                 }
 446
 447                                 // if all dimensions are hidden, print a null
 448                                 if(!pc) {
 449                                         web_buffer_strcpy(wb, pre_value);
 450                                         web_buffer_strcpy(wb, "null");
 451                                         web_buffer_strcpy(wb, post_value);
 452                                 }
 453
 454                                 printed++;
 455                                 group_count = 0;
 456                         }
 457                 }
 458
 459                 if(printed) web_buffer_printf(wb, "]}");
 460                 web_buffer_printf(wb, "\n       ]\n}\n");
 461
 462                 if(only_non_zero && max_loop > 1) {
 463                         int changed = 0;
 464                         for(rd = st->dimensions, c = 0 ; rd && c < dimensions ; rd = rd->next, c++) {
 465                                 group_values[c] = 0;
 466                                 found_non_existing[c] = 0;
 467
 468                                 if(!print_hidden[c] && !found_non_zero[c]) {
 469                                         changed = 1;
 470                                         print_hidden[c] = 1;
 471                                 }
 472                         }
 473
 474                         if(changed) web_buffer_reset(wb);
 475                         else break;
 476                 }
 477                 else break;
 478
 479         } // max_loop
 480
 481         debug(D_RRD_STATS, "RRD_STATS_JSON: %s total %ld bytes", st->name, wb->bytes);
 482
 483         pthread_rwlock_unlock(&st->rwlock);
 484         return last_timestamp;
 485 }