src/rrd2json.c

   1 #ifdef HAVE_CONFIG_H
   2 #include <config.h>
   3 #endif
   4 #include <pthread.h>
   5 #include <sys/time.h>
   6 #include <stdlib.h>
   7
   8 #include "log.h"
   9 #include "common.h"
  10 #include "rrd2json.h"
  11
  12 #define HOSTNAME_MAX 1024
  13 char *hostname = "unknown";
  14
  15 void rrd_stats_api_v1_chart(RRDSET *st, BUFFER *wb)
  16 {
  17         time_t now = time(NULL);
  18
  19         pthread_rwlock_rdlock(&st->rwlock);
  20
  21         buffer_sprintf(wb,
  22                 "\t\t{\n"
  23                 "\t\t\t\"id\": \"%s\",\n"
  24                 "\t\t\t\"name\": \"%s\",\n"
  25                 "\t\t\t\"type\": \"%s\",\n"
  26                 "\t\t\t\"family\": \"%s\",\n"
  27                 "\t\t\t\"title\": \"%s\",\n"
  28                 "\t\t\t\"priority\": %ld,\n"
  29                 "\t\t\t\"enabled\": %s,\n"
  30                 "\t\t\t\"units\": \"%s\",\n"
  31                 "\t\t\t\"data_url\": \"/api/v1/data?chart=%s\",\n"
  32                 "\t\t\t\"chart_type\": \"%s\",\n"
  33                 "\t\t\t\"duration\": %ld,\n"
  34                 "\t\t\t\"first_entry\": %lu,\n"
  35                 "\t\t\t\"last_entry\": %lu,\n"
  36                 "\t\t\t\"update_every\": %d,\n"
  37                 "\t\t\t\"dimensions\": {\n"
  38                 , st->id
  39                 , st->name
  40                 , st->type
  41                 , st->family
  42                 , st->title
  43                 , st->priority
  44                 , st->enabled?"true":"false"
  45                 , st->units
  46                 , st->name
  47                 , rrdset_type_name(st->chart_type)
  48                 , st->entries * st->update_every
  49                 , rrdset_first_entry_t(st)
  50                 , rrdset_last_entry_t(st)
  51                 , st->update_every
  52                 );
  53
  54         unsigned long memory = st->memsize;
  55
  56         int c = 0;
  57         RRDDIM *rd;
  58         for(rd = st->dimensions; rd ; rd = rd->next) {
  59                 if(rd->flags & RRDDIM_FLAG_HIDDEN) continue;
  60
  61                 memory += rd->memsize;
  62
  63                 buffer_sprintf(wb,
  64                         "%s"
  65                         "\t\t\t\t\"%s\": { \"name\": \"%s\" }"
  66                         , c?",\n":""
  67                         , rd->id
  68                         , rd->name
  69                         );
  70
  71                 c++;
  72         }
  73
  74         buffer_sprintf(wb,
  75                 "\n\t\t\t}\n"
  76                 "\t\t}"
  77                 );
  78
  79         pthread_rwlock_unlock(&st->rwlock);
  80 }
  81
  82 void rrd_stats_api_v1_charts(BUFFER *wb)
  83 {
  84         long c;
  85         RRDSET *st;
  86
  87         buffer_sprintf(wb, "{\n"
  88                    "\t\"hostname\": \"%s\""
  89                 ",\n\t\"update_every\": %d"
  90                 ",\n\t\"history\": %d"
  91                 ",\n\t\"charts\": {"
  92                 , hostname
  93                 , rrd_update_every
  94                 , rrd_default_history_entries
  95                 );
  96
  97         pthread_rwlock_rdlock(&rrdset_root_rwlock);
  98         for(st = rrdset_root, c = 0; st ; st = st->next) {
  99                 if(st->enabled) {
 100                         if(c) buffer_strcat(wb, ",");
 101                         buffer_strcat(wb, "\n\t\t\"");
 102                         buffer_strcat(wb, st->id);
 103                         buffer_strcat(wb, "\": ");
 104                         rrd_stats_api_v1_chart(st, wb);
 105                         c++;
 106                 }
 107         }
 108         pthread_rwlock_unlock(&rrdset_root_rwlock);
 109
 110         buffer_strcat(wb, "\n\t}\n}\n");
 111 }
 112
 113
 114 unsigned long rrd_stats_one_json(RRDSET *st, char *options, BUFFER *wb)
 115 {
 116         time_t now = time(NULL);
 117
 118         pthread_rwlock_rdlock(&st->rwlock);
 119
 120         buffer_sprintf(wb,
 121                 "\t\t{\n"
 122                 "\t\t\t\"id\": \"%s\",\n"
 123                 "\t\t\t\"name\": \"%s\",\n"
 124                 "\t\t\t\"type\": \"%s\",\n"
 125                 "\t\t\t\"family\": \"%s\",\n"
 126                 "\t\t\t\"title\": \"%s\",\n"
 127                 "\t\t\t\"priority\": %ld,\n"
 128                 "\t\t\t\"enabled\": %d,\n"
 129                 "\t\t\t\"units\": \"%s\",\n"
 130                 "\t\t\t\"url\": \"/data/%s/%s\",\n"
 131                 "\t\t\t\"chart_type\": \"%s\",\n"
 132                 "\t\t\t\"counter\": %ld,\n"
 133                 "\t\t\t\"entries\": %ld,\n"
 134                 "\t\t\t\"first_entry_t\": %lu,\n"
 135                 "\t\t\t\"last_entry\": %ld,\n"
 136                 "\t\t\t\"last_entry_t\": %lu,\n"
 137                 "\t\t\t\"last_entry_secs_ago\": %lu,\n"
 138                 "\t\t\t\"update_every\": %d,\n"
 139                 "\t\t\t\"isdetail\": %d,\n"
 140                 "\t\t\t\"usec_since_last_update\": %llu,\n"
 141                 "\t\t\t\"collected_total\": " TOTAL_NUMBER_FORMAT ",\n"
 142                 "\t\t\t\"last_collected_total\": " TOTAL_NUMBER_FORMAT ",\n"
 143                 "\t\t\t\"dimensions\": [\n"
 144                 , st->id
 145                 , st->name
 146                 , st->type
 147                 , st->family
 148                 , st->title
 149                 , st->priority
 150                 , st->enabled
 151                 , st->units
 152                 , st->name, options?options:""
 153                 , rrdset_type_name(st->chart_type)
 154                 , st->counter
 155                 , st->entries
 156                 , rrdset_first_entry_t(st)
 157                 , rrdset_last_slot(st)
 158                 , rrdset_last_entry_t(st)
 159                 , (now < rrdset_last_entry_t(st)) ? (time_t)0 : now - rrdset_last_entry_t(st)
 160                 , st->update_every
 161                 , st->isdetail
 162                 , st->usec_since_last_update
 163                 , st->collected_total
 164                 , st->last_collected_total
 165                 );
 166
 167         unsigned long memory = st->memsize;
 168
 169         RRDDIM *rd;
 170         for(rd = st->dimensions; rd ; rd = rd->next) {
 171
 172                 memory += rd->memsize;
 173
 174                 buffer_sprintf(wb,
 175                         "\t\t\t\t{\n"
 176                         "\t\t\t\t\t\"id\": \"%s\",\n"
 177                         "\t\t\t\t\t\"name\": \"%s\",\n"
 178                         "\t\t\t\t\t\"entries\": %ld,\n"
 179                         "\t\t\t\t\t\"isHidden\": %d,\n"
 180                         "\t\t\t\t\t\"algorithm\": \"%s\",\n"
 181                         "\t\t\t\t\t\"multiplier\": %ld,\n"
 182                         "\t\t\t\t\t\"divisor\": %ld,\n"
 183                         "\t\t\t\t\t\"last_entry_t\": %lu,\n"
 184                         "\t\t\t\t\t\"collected_value\": " COLLECTED_NUMBER_FORMAT ",\n"
 185                         "\t\t\t\t\t\"calculated_value\": " CALCULATED_NUMBER_FORMAT ",\n"
 186                         "\t\t\t\t\t\"last_collected_value\": " COLLECTED_NUMBER_FORMAT ",\n"
 187                         "\t\t\t\t\t\"last_calculated_value\": " CALCULATED_NUMBER_FORMAT ",\n"
 188                         "\t\t\t\t\t\"memory\": %lu\n"
 189                         "\t\t\t\t}%s\n"
 190                         , rd->id
 191                         , rd->name
 192                         , rd->entries
 193                         , (rd->flags & RRDDIM_FLAG_HIDDEN)?1:0
 194                         , rrddim_algorithm_name(rd->algorithm)
 195                         , rd->multiplier
 196                         , rd->divisor
 197                         , rd->last_collected_time.tv_sec
 198                         , rd->collected_value
 199                         , rd->calculated_value
 200                         , rd->last_collected_value
 201                         , rd->last_calculated_value
 202                         , rd->memsize
 203                         , rd->next?",":""
 204                         );
 205         }
 206
 207         buffer_sprintf(wb,
 208                 "\t\t\t],\n"
 209                 "\t\t\t\"memory\" : %lu\n"
 210                 "\t\t}"
 211                 , memory
 212                 );
 213
 214         pthread_rwlock_unlock(&st->rwlock);
 215         return memory;
 216 }
 217
 218 #define RRD_GRAPH_JSON_HEADER "{\n\t\"charts\": [\n"
 219 #define RRD_GRAPH_JSON_FOOTER "\n\t]\n}\n"
 220
 221 void rrd_stats_graph_json(RRDSET *st, char *options, BUFFER *wb)
 222 {
 223         buffer_strcat(wb, RRD_GRAPH_JSON_HEADER);
 224         rrd_stats_one_json(st, options, wb);
 225         buffer_strcat(wb, RRD_GRAPH_JSON_FOOTER);
 226 }
 227
 228 void rrd_stats_all_json(BUFFER *wb)
 229 {
 230         unsigned long memory = 0;
 231         long c;
 232         RRDSET *st;
 233
 234         buffer_strcat(wb, RRD_GRAPH_JSON_HEADER);
 235
 236         pthread_rwlock_rdlock(&rrdset_root_rwlock);
 237         for(st = rrdset_root, c = 0; st ; st = st->next) {
 238                 if(st->enabled) {
 239                         if(c) buffer_strcat(wb, ",\n");
 240                         memory += rrd_stats_one_json(st, NULL, wb);
 241                         c++;
 242                 }
 243         }
 244         pthread_rwlock_unlock(&rrdset_root_rwlock);
 245
 246         buffer_sprintf(wb, "\n\t],\n"
 247                 "\t\"hostname\": \"%s\",\n"
 248                 "\t\"update_every\": %d,\n"
 249                 "\t\"history\": %d,\n"
 250                 "\t\"memory\": %lu\n"
 251                 "}\n"
 252                 , hostname
 253                 , rrd_update_every
 254                 , rrd_default_history_entries
 255                 , memory
 256                 );
 257 }
 258
 259
 260
 261 // ----------------------------------------------------------------------------
 262
 263 // RRDR options
 264 #define RRDR_EMPTY      0x01
 265 #define RRDR_RESET      0x02
 266 #define RRDR_HIDDEN     0x04
 267 #define RRDR_NONZERO    0x08
 268
 269
 270 typedef struct rrdresult {
 271         RRDSET *st;                     // the chart this result refers to
 272
 273         int d;                                  // the number of dimensions
 274         int n;                                  // the number of values in the arrays
 275
 276         uint8_t *od;                    // the options for the dimensions
 277
 278         time_t *t;                              // array of n timestamps
 279         calculated_number *v;   // array n x d values
 280         uint8_t *o;                             // array n x d options
 281
 282         int c;                                  // current line ( 0 ~ n )
 283
 284         int has_st_lock;                // if st is read locked by us
 285 } RRDR;
 286
 287 static void rrdr_dump(RRDR *r)
 288 {
 289         long c, i;
 290         RRDDIM *d;
 291
 292         fprintf(stderr, "\nCHART %s (%s)\n", r->st->id, r->st->name);
 293
 294         for(c = 0, d = r->st->dimensions; d ;c++, d = d->next) {
 295                 fprintf(stderr, "DIMENSION %s (%s), %s%s%s%s\n"
 296                                 , d->id
 297                                 , d->name
 298                                 , (r->od[c] & RRDR_EMPTY)?"EMPTY ":""
 299                                 , (r->od[c] & RRDR_RESET)?"RESET ":""
 300                                 , (r->od[c] & RRDR_HIDDEN)?"HIDDEN ":""
 301                                 , (r->od[c] & RRDR_NONZERO)?"NONZERO ":""
 302                                 );
 303         }
 304
 305         if(r->c < 0) {
 306                 fprintf(stderr, "RRDR does not have any values in it.\n");
 307                 return;
 308         }
 309
 310         fprintf(stderr, "RRDR includes %d values in it:\n", r->c + 1);
 311
 312         // for each line in the array
 313         for(i = 0; i <= r->c ;i++) {
 314                 calculated_number *cn = &r->v[ i * r->d ];
 315                 uint8_t *co = &r->o[ i * r->d ];
 316
 317                 // print the id and the timestamp of the line
 318                 fprintf(stderr, "%ld %ld ", i + 1, r->t[i]);
 319
 320                 // for each dimension
 321                 for(c = 0, d = r->st->dimensions; d ;c++, d = d->next) {
 322                         if(unlikely(r->od[c] & RRDR_HIDDEN)) continue;
 323                         if(unlikely(!(r->od[c] & RRDR_NONZERO))) continue;
 324
 325                         if(co[c] & RRDR_EMPTY)
 326                                 fprintf(stderr, "null ");
 327                         else
 328                                 fprintf(stderr, CALCULATED_NUMBER_FORMAT " %s%s%s%s "
 329                                         , cn[c]
 330                                         , (co[c] & RRDR_EMPTY)?"E":" "
 331                                         , (co[c] & RRDR_RESET)?"R":" "
 332                                         , (co[c] & RRDR_HIDDEN)?"H":" "
 333                                         , (co[c] & RRDR_NONZERO)?"N":" "
 334                                         );
 335                 }
 336
 337                 fprintf(stderr, "\n");
 338         }
 339 }
 340
 341 inline static calculated_number *rrdr_line_values(RRDR *r)
 342 {
 343         return &r->v[ r->c * r->d ];
 344 }
 345
 346 inline static uint8_t *rrdr_line_options(RRDR *r)
 347 {
 348         return &r->o[ r->c * r->d ];
 349 }
 350
 351 inline static int rrdr_line_init(RRDR *r, time_t t)
 352 {
 353         r->c++;
 354         if(unlikely(r->c >= r->n)) {
 355                 r->c = r->n - 1;
 356                 return 0;
 357         }
 358
 359         // save the time
 360         r->t[r->c] = t;
 361
 362         return 1;
 363 }
 364
 365 inline static void rrdr_lock_rrdset(RRDR *r) {
 366         if(unlikely(!r)) {
 367                 error("NULL value given!");
 368                 return;
 369         }
 370
 371         pthread_rwlock_rdlock(&r->st->rwlock);
 372         r->has_st_lock = 1;
 373 }
 374
 375 inline static void rrdr_unlock_rrdset(RRDR *r) {
 376         if(unlikely(!r)) {
 377                 error("NULL value given!");
 378                 return;
 379         }
 380
 381         if(likely(r->has_st_lock)) {
 382                 pthread_rwlock_unlock(&r->st->rwlock);
 383                 r->has_st_lock = 0;
 384         }
 385 }
 386
 387 inline static void rrdr_free(RRDR *r)
 388 {
 389         if(unlikely(!r)) {
 390                 error("NULL value given!");
 391                 return;
 392         }
 393
 394         rrdr_unlock_rrdset(r);
 395         if(likely(r->t)) free(r->t);
 396         if(likely(r->v)) free(r->v);
 397         if(likely(r->o)) free(r->o);
 398         if(likely(r->od)) free(r->od);
 399         free(r);
 400 }
 401
 402 static RRDR *rrdr_create(RRDSET *st, int n)
 403 {
 404         if(unlikely(!st)) {
 405                 error("NULL value given!");
 406                 return NULL;
 407         }
 408
 409         RRDR *r = calloc(1, sizeof(RRDR));
 410         if(unlikely(!r)) goto cleanup;
 411
 412         r->st = st;
 413
 414         rrdr_lock_rrdset(r);
 415
 416         RRDDIM *rd;
 417         for(rd = st->dimensions ; rd ; rd = rd->next) r->d++;
 418
 419         r->n = n;
 420         r->t = malloc(n * sizeof(time_t));
 421         if(unlikely(!r->t)) goto cleanup;
 422
 423         r->t = malloc(n * sizeof(time_t));
 424         if(unlikely(!r->t)) goto cleanup;
 425
 426         r->v = malloc(n * r->d * sizeof(calculated_number));
 427         if(unlikely(!r->v)) goto cleanup;
 428
 429         r->o = malloc(n * r->d * sizeof(uint8_t));
 430         if(unlikely(!r->o)) goto cleanup;
 431
 432         r->od = calloc(r->d, sizeof(uint8_t));
 433         if(unlikely(!r->od)) goto cleanup;
 434
 435         r->c = -1;
 436
 437         return r;
 438
 439 cleanup:
 440         error("Cannot allocate memory");
 441         if(likely(r)) rrdr_free(r);
 442         return NULL;
 443 }
 444
 445 RRDR *rrd2rrdr(RRDSET *st, long points, time_t after, time_t before, int group_method)
 446 {
 447         int debug = st->debug;
 448
 449         time_t first_entry_t = rrdset_first_entry_t(st);
 450         time_t last_entry_t = rrdset_last_entry_t(st);
 451
 452         // allow relative for before and after
 453         if(before <= st->update_every * st->entries) before = last_entry_t + before;
 454         if(after <= st->update_every * st->entries) after = last_entry_t + after;
 455
 456         // make sure they are within our timeframe
 457         if(before > last_entry_t) before = last_entry_t;
 458         if(before < first_entry_t) before = first_entry_t;
 459
 460         if(after > last_entry_t) after = last_entry_t;
 461         if(after < first_entry_t) after = first_entry_t;
 462
 463         // check if they are upside down
 464         if(after > before) {
 465                 time_t t = before;
 466                 before = after;
 467                 after = t;
 468         }
 469
 470         // the duration of the chart
 471         time_t duration = before - after;
 472         if(duration <= 0) return NULL;
 473
 474         // check the required points
 475         if(points > duration / st->update_every) points = 0;
 476         if(points <= 0) points = duration / st->update_every;
 477
 478         // calculate proper grouping of source data
 479         long group = duration / points;
 480         if(group <= 0) group = 1;
 481         if(duration / group > points) group++;
 482
 483         // error("NEW: points=%d after=%d before=%d group=%d, duration=%d", points, after, before, group, duration);
 484
 485         // Now we have:
 486         // before = the end time of the calculation
 487         // after = the start time of the calculation
 488         // duration = the duration of the calculation
 489         // group = the number of source points to aggregate / group together
 490         // method = the method of grouping source points
 491         // points = the number of points to generate
 492
 493
 494         // -------------------------------------------------------------------------
 495         // initialize our result set
 496
 497         RRDR *r = rrdr_create(st, points);
 498         if(!r) return NULL;
 499         if(!r->d) {
 500                 rrdr_free(r);
 501                 return NULL;
 502         }
 503
 504         // find how many dimensions we have
 505         long dimensions = r->d;
 506
 507
 508         // -------------------------------------------------------------------------
 509         // checks for debugging
 510
 511         if(debug) debug(D_RRD_STATS, "INFO %s first_t: %lu, last_t: %lu, all_duration: %lu, after: %lu, before: %lu, duration: %lu, points: %ld, group: %ld"
 512                         , st->id
 513                         , first_entry_t
 514                         , last_entry_t
 515                         , last_entry_t - first_entry_t
 516                         , after
 517                         , before
 518                         , duration
 519                         , points
 520                         , group
 521                         );
 522
 523
 524         // -------------------------------------------------------------------------
 525         // temp arrays for keeping values per dimension
 526
 527         calculated_number       group_values[dimensions]; // keep sums when grouping
 528         long                            group_counts[dimensions]; // keep the number of values added to group_values
 529         uint8_t                         group_options[dimensions];
 530         uint8_t                         found_non_zero[dimensions];
 531
 532
 533         // initialize them
 534         RRDDIM *rd;
 535         long c;
 536         for( rd = st->dimensions, c = 0 ; rd && c < dimensions ; rd = rd->next, c++) {
 537                 group_values[c] = 0;
 538                 group_counts[c] = 0;
 539                 group_options[c] = 0;
 540                 found_non_zero[c] = 0;
 541         }
 542
 543
 544         // -------------------------------------------------------------------------
 545         // the main loop
 546
 547         long    t = rrdset_time2slot(st, before), // rrdset_last_slot(st),
 548                         stop_at_t = rrdset_time2slot(st, after),
 549                         added = 0,
 550                         group_count = 0,
 551                         add_this = 0,
 552                         stop_now = 0;
 553
 554         // align to group for proper panning of data
 555         t -= t % group;
 556
 557         time_t  now = rrdset_slot2time(st, t),
 558                         dt = st->update_every,
 559                         group_start_t = 0;
 560
 561         if(debug) debug(D_RRD_STATS, "BEGIN %s after_t: %lu (stop slot %ld), before_t: %lu (start slot %ld), start_t(now): %lu"
 562                         , st->id
 563                         , after
 564                         , stop_at_t
 565                         , before
 566                         , t
 567                         , now
 568                         );
 569
 570         for( ; !stop_now ; now -= dt, t--) {
 571                 if(unlikely(t < 0)) c = st->entries - 1;
 572                 if(t == stop_at_t) stop_now = 1;
 573
 574                 if(debug) debug(D_RRD_STATS, "ROW %s c: %ld, group_count: %ld, added: %ld, now: %lu, %s %s"
 575                                 , st->id
 576                                 , t
 577                                 , group_count + 1
 578                                 , added
 579                                 , now
 580                                 , (group_count + 1 == group)?"PRINT":"  -  "
 581                                 , (now >= after && now <= before)?"RANGE":"  -  "
 582                                 );
 583
 584                 // make sure we return data in the proper time range
 585                 if(unlikely(now > before)) continue;
 586                 if(unlikely(now < after)) break;
 587
 588                 if(group_count == 0) group_start_t = now;
 589                 group_count++;
 590
 591                 if(unlikely(group_count == group)) {
 592                         if(unlikely(added >= points)) break;
 593                         add_this = 1;
 594                 }
 595
 596                 // do the calculations
 597                 for(rd = st->dimensions, c = 0 ; likely(rd && c < dimensions) ; rd = rd->next, c++) {
 598                         storage_number n = rd->values[t];
 599                         if(unlikely(!does_storage_number_exist(n))) continue;
 600
 601                         group_counts[c]++;
 602
 603                         calculated_number value = unpack_storage_number(n);
 604                         if(value != 0.0) {
 605                                 group_options[c] |= RRDR_NONZERO;
 606                                 found_non_zero[c] = 1;
 607                         }
 608
 609                         if(unlikely(did_storage_number_reset(n)))
 610                                 group_options[c] |= RRDR_RESET;
 611
 612                         switch(group_method) {
 613                                 case GROUP_MAX:
 614                                         if(unlikely(abs(value) > abs(group_values[c])))
 615                                                 group_values[c] = value;
 616                                         break;
 617
 618                                 default:
 619                                 case GROUP_SUM:
 620                                 case GROUP_AVERAGE:
 621                                         group_values[c] += value;
 622                                         break;
 623                         }
 624                 }
 625
 626                 // added it
 627                 if(unlikely(add_this)) {
 628                         if(!rrdr_line_init(r, group_start_t)) break;
 629
 630                         calculated_number *cn = rrdr_line_values(r);
 631                         uint8_t *co = rrdr_line_options(r);
 632
 633                         for(rd = st->dimensions, c = 0 ; likely(rd && c < dimensions) ; rd = rd->next, c++) {
 634
 635                                 // update the dimension options
 636                                 if(found_non_zero[c]) r->od[c] |= RRDR_NONZERO;
 637                                 if(rd->flags & RRDDIM_FLAG_HIDDEN) r->od[c] |= RRDR_HIDDEN;
 638
 639                                 // store the specific point options
 640                                 co[c] = group_options[c];
 641
 642                                 // store the value
 643                                 if(group_counts[c] == 0) {
 644                                         cn[c] = 0.0;
 645                                         co[c] |= RRDR_EMPTY;
 646                                 }
 647                                 else if(unlikely(group_method == GROUP_AVERAGE)) {
 648                                         cn[c] = group_values[c] / group_counts[c];
 649                                 }
 650                                 else {
 651                                         cn[c] = group_values[c];
 652                                 }
 653
 654                                 // reset them for the next loop
 655                                 group_values[c] = 0;
 656                                 group_counts[c] = 0;
 657                                 group_options[c] = 0;
 658                         }
 659
 660                         added++;
 661                         group_count = 0;
 662                         add_this = 0;
 663                 }
 664         }
 665
 666         rrdr_dump(r);
 667         rrdr_free(r);
 668         return NULL;
 669 }
 670
 671 unsigned long rrd_stats_json(int type, RRDSET *st, BUFFER *wb, int points, int group, int group_method, time_t after, time_t before, int only_non_zero)
 672 {
 673         int c;
 674         pthread_rwlock_rdlock(&st->rwlock);
 675
 676
 677         // -------------------------------------------------------------------------
 678         // switch from JSON to google JSON
 679
 680         char kq[2] = "\"";
 681         char sq[2] = "\"";
 682         switch(type) {
 683                 case DATASOURCE_GOOGLE_JSON:
 684                 case DATASOURCE_GOOGLE_JSONP:
 685                         kq[0] = '\0';
 686                         sq[0] = '\'';
 687                         break;
 688
 689                 case DATASOURCE_JSON:
 690                 default:
 691                         break;
 692         }
 693
 694
 695         // -------------------------------------------------------------------------
 696         // validate the parameters
 697
 698         if(points < 1) points = 1;
 699         if(group < 1) group = 1;
 700
 701         if(before == 0 || before > rrdset_last_entry_t(st)) before = rrdset_last_entry_t(st);
 702         if(after  == 0 || after < rrdset_first_entry_t(st)) after = rrdset_first_entry_t(st);
 703
 704         // ---
 705
 706         // our return value (the last timestamp printed)
 707         // this is required to detect re-transmit in google JSONP
 708         time_t last_timestamp = 0;
 709
 710
 711         // -------------------------------------------------------------------------
 712         // find how many dimensions we have
 713
 714         int dimensions = 0;
 715         RRDDIM *rd;
 716         for( rd = st->dimensions ; rd ; rd = rd->next) dimensions++;
 717         if(!dimensions) {
 718                 pthread_rwlock_unlock(&st->rwlock);
 719                 buffer_strcat(wb, "No dimensions yet.");
 720                 return 0;
 721         }
 722
 723
 724         // -------------------------------------------------------------------------
 725         // prepare various strings, to speed up the loop
 726
 727         char overflow_annotation[201]; snprintf(overflow_annotation, 200, ",{%sv%s:%sRESET OR OVERFLOW%s},{%sv%s:%sThe counters have been wrapped.%s}", kq, kq, sq, sq, kq, kq, sq, sq);
 728         char normal_annotation[201];   snprintf(normal_annotation,   200, ",{%sv%s:null},{%sv%s:null}", kq, kq, kq, kq);
 729         char pre_date[51];             snprintf(pre_date,             50, "             {%sc%s:[{%sv%s:%s", kq, kq, kq, kq, sq);
 730         char post_date[21];            snprintf(post_date,            20, "%s}", sq);
 731         char pre_value[21];            snprintf(pre_value,            20, ",{%sv%s:", kq, kq);
 732         char post_value[21];           snprintf(post_value,           20, "}");
 733
 734
 735         // -------------------------------------------------------------------------
 736         // checks for debugging
 737
 738         if(st->debug) {
 739                 debug(D_RRD_STATS, "%s first_entry_t = %lu, last_entry_t = %lu, duration = %lu, after = %lu, before = %lu, duration = %lu, entries_to_show = %lu, group = %lu"
 740                         , st->id
 741                         , rrdset_first_entry_t(st)
 742                         , rrdset_last_entry_t(st)
 743                         , rrdset_last_entry_t(st) - rrdset_first_entry_t(st)
 744                         , after
 745                         , before
 746                         , before - after
 747                         , points
 748                         , group
 749                         );
 750
 751                 if(before < after)
 752                         debug(D_RRD_STATS, "WARNING: %s The newest value in the database (%lu) is earlier than the oldest (%lu)", st->name, before, after);
 753
 754                 if((before - after) > st->entries * st->update_every)
 755                         debug(D_RRD_STATS, "WARNING: %s The time difference between the oldest and the newest entries (%lu) is higher than the capacity of the database (%lu)", st->name, before - after, st->entries * st->update_every);
 756         }
 757
 758
 759         // -------------------------------------------------------------------------
 760         // temp arrays for keeping values per dimension
 761
 762         calculated_number group_values[dimensions]; // keep sums when grouping
 763         int               print_hidden[dimensions]; // keep hidden flags
 764         int               found_non_zero[dimensions];
 765         int               found_non_existing[dimensions];
 766
 767         // initialize them
 768         for( rd = st->dimensions, c = 0 ; rd && c < dimensions ; rd = rd->next, c++) {
 769                 group_values[c] = 0;
 770                 print_hidden[c] = (rd->flags & RRDDIM_FLAG_HIDDEN)?1:0;
 771                 found_non_zero[c] = 0;
 772                 found_non_existing[c] = 0;
 773         }
 774
 775
 776         // error("OLD: points=%d after=%d before=%d group=%d, duration=%d", entries_to_show, before - (st->update_every * group * entries_to_show), before, group, before - after + 1);
 777         // rrd2array(st, entries_to_show, before - (st->update_every * group * entries_to_show), before, group_method, only_non_zero);
 778         // rrd2rrdr(st, entries_to_show, before - (st->update_every * group * entries_to_show), before, group_method);
 779
 780         // -------------------------------------------------------------------------
 781         // remove dimensions that contain only zeros
 782
 783         int max_loop = 1;
 784         if(only_non_zero) max_loop = 2;
 785
 786         for(; max_loop ; max_loop--) {
 787
 788                 // -------------------------------------------------------------------------
 789                 // print the JSON header
 790
 791                 buffer_sprintf(wb, "{\n %scols%s:\n     [\n", kq, kq);
 792                 buffer_sprintf(wb, "            {%sid%s:%s%s,%slabel%s:%stime%s,%spattern%s:%s%s,%stype%s:%sdatetime%s},\n", kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq);
 793                 buffer_sprintf(wb, "            {%sid%s:%s%s,%slabel%s:%s%s,%spattern%s:%s%s,%stype%s:%sstring%s,%sp%s:{%srole%s:%sannotation%s}},\n", kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, kq, kq, sq, sq);
 794                 buffer_sprintf(wb, "            {%sid%s:%s%s,%slabel%s:%s%s,%spattern%s:%s%s,%stype%s:%sstring%s,%sp%s:{%srole%s:%sannotationText%s}}", kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, kq, kq, sq, sq);
 795
 796                 // print the header for each dimension
 797                 // and update the print_hidden array for the dimensions that should be hidden
 798                 int pc = 0;
 799                 for( rd = st->dimensions, c = 0 ; rd && c < dimensions ; rd = rd->next, c++) {
 800                         if(!print_hidden[c]) {
 801                                 pc++;
 802                                 buffer_sprintf(wb, ",\n         {%sid%s:%s%s,%slabel%s:%s%s%s,%spattern%s:%s%s,%stype%s:%snumber%s}", kq, kq, sq, sq, kq, kq, sq, rd->name, sq, kq, kq, sq, sq, kq, kq, sq, sq);
 803                         }
 804                 }
 805                 if(!pc) {
 806                         buffer_sprintf(wb, ",\n         {%sid%s:%s%s,%slabel%s:%s%s%s,%spattern%s:%s%s,%stype%s:%snumber%s}", kq, kq, sq, sq, kq, kq, sq, "no data", sq, kq, kq, sq, sq, kq, kq, sq, sq);
 807                 }
 808
 809                 // print the begin of row data
 810                 buffer_sprintf(wb, "\n  ],\n    %srows%s:\n     [\n", kq, kq);
 811
 812
 813                 // -------------------------------------------------------------------------
 814                 // the main loop
 815
 816                 int annotate_reset = 0;
 817                 int annotation_count = 0;
 818
 819                 // the minimum line length we expect
 820                 int line_size = 4096 + (dimensions * 200);
 821
 822                 long    t = rrdset_time2slot(st, before),
 823                                 stop_at_t = rrdset_time2slot(st, after),
 824                                 stop_now = 0;
 825
 826                 t -= t % group;
 827
 828                 time_t  now = rrdset_slot2time(st, t),
 829                                 dt = st->update_every;
 830
 831                 long count = 0, printed = 0, group_count = 0;
 832                 last_timestamp = 0;
 833
 834                 if(st->debug) debug(D_RRD_STATS, "%s: REQUEST after:%lu before:%lu, points:%d, group:%d, CHART cur:%ld first: %lu last:%lu, CALC start_t:%ld, stop_t:%ld"
 835                                         , st->id
 836                                         , after
 837                                         , before
 838                                         , points
 839                                         , group
 840                                         , st->current_entry
 841                                         , rrdset_first_entry_t(st)
 842                                         , rrdset_last_entry_t(st)
 843                                         , t
 844                                         , stop_at_t
 845                                         );
 846
 847                 for(; !stop_now ; now -= dt, t--) {
 848                         if(t < 0) t = st->entries - 1;
 849                         if(t == stop_at_t) stop_now = 1;
 850
 851                         int print_this = 0;
 852
 853                         if(st->debug) debug(D_RRD_STATS, "%s t = %ld, count = %ld, group_count = %ld, printed = %ld, now = %lu, %s %s"
 854                                         , st->id
 855                                         , t
 856                                         , count + 1
 857                                         , group_count + 1
 858                                         , printed
 859                                         , now
 860                                         , (group_count + 1 == group)?"PRINT":"  -  "
 861                                         , (now >= after && now <= before)?"RANGE":"  -  "
 862                                         );
 863
 864
 865                         // make sure we return data in the proper time range
 866                         if(now > before) continue;
 867                         if(now < after) break;
 868
 869                         //if(rrdset_slot2time(st, t) != now)
 870                         //      error("%s: slot=%ld, now=%ld, slot2time=%ld, diff=%ld, last_entry_t=%ld, rrdset_last_slot=%ld", st->id, t, now, rrdset_slot2time(st,t), now - rrdset_slot2time(st,t), rrdset_last_entry_t(st), rrdset_last_slot(st));
 871
 872                         count++;
 873                         group_count++;
 874
 875                         // check if we have to print this now
 876                         if(group_count == group) {
 877                                 if(printed >= points) {
 878                                         // debug(D_RRD_STATS, "Already printed all rows. Stopping.");
 879                                         break;
 880                                 }
 881
 882                                 // generate the local date time
 883                                 struct tm *tm = localtime(&now);
 884                                 if(!tm) { error("localtime() failed."); continue; }
 885                                 if(now > last_timestamp) last_timestamp = now;
 886
 887                                 if(printed) buffer_strcat(wb, "]},\n");
 888                                 buffer_strcat(wb, pre_date);
 889                                 buffer_jsdate(wb, tm->tm_year + 1900, tm->tm_mon, tm->tm_mday, tm->tm_hour, tm->tm_min, tm->tm_sec);
 890                                 buffer_strcat(wb, post_date);
 891
 892                                 print_this = 1;
 893                         }
 894
 895                         // do the calculations
 896                         for(rd = st->dimensions, c = 0 ; rd && c < dimensions ; rd = rd->next, c++) {
 897                                 storage_number n = rd->values[t];
 898                                 calculated_number value = unpack_storage_number(n);
 899
 900                                 if(!does_storage_number_exist(n)) {
 901                                         value = 0.0;
 902                                         found_non_existing[c]++;
 903                                 }
 904                                 if(did_storage_number_reset(n)) annotate_reset = 1;
 905
 906                                 switch(group_method) {
 907                                         case GROUP_MAX:
 908                                                 if(abs(value) > abs(group_values[c])) group_values[c] = value;
 909                                                 break;
 910
 911                                         case GROUP_SUM:
 912                                                 group_values[c] += value;
 913                                                 break;
 914
 915                                         default:
 916                                         case GROUP_AVERAGE:
 917                                                 group_values[c] += value;
 918                                                 if(print_this) group_values[c] /= ( group_count - found_non_existing[c] );
 919                                                 break;
 920                                 }
 921                         }
 922
 923                         if(print_this) {
 924                                 if(annotate_reset) {
 925                                         annotation_count++;
 926                                         buffer_strcat(wb, overflow_annotation);
 927                                         annotate_reset = 0;
 928                                 }
 929                                 else
 930                                         buffer_strcat(wb, normal_annotation);
 931
 932                                 pc = 0;
 933                                 for(c = 0 ; c < dimensions ; c++) {
 934                                         if(found_non_existing[c] == group_count) {
 935                                                 // all entries are non-existing
 936                                                 pc++;
 937                                                 buffer_strcat(wb, pre_value);
 938                                                 buffer_strcat(wb, "null");
 939                                                 buffer_strcat(wb, post_value);
 940                                         }
 941                                         else if(!print_hidden[c]) {
 942                                                 pc++;
 943                                                 buffer_strcat(wb, pre_value);
 944                                                 buffer_rrd_value(wb, group_values[c]);
 945                                                 buffer_strcat(wb, post_value);
 946
 947                                                 if(group_values[c]) found_non_zero[c]++;
 948                                         }
 949
 950                                         // reset them for the next loop
 951                                         group_values[c] = 0;
 952                                         found_non_existing[c] = 0;
 953                                 }
 954
 955                                 // if all dimensions are hidden, print a null
 956                                 if(!pc) {
 957                                         buffer_strcat(wb, pre_value);
 958                                         buffer_strcat(wb, "null");
 959                                         buffer_strcat(wb, post_value);
 960                                 }
 961
 962                                 printed++;
 963                                 group_count = 0;
 964                         }
 965                 }
 966
 967                 if(printed) buffer_strcat(wb, "]}");
 968                 buffer_strcat(wb, "\n   ]\n}\n");
 969
 970                 if(only_non_zero && max_loop > 1) {
 971                         int changed = 0;
 972                         for(rd = st->dimensions, c = 0 ; rd && c < dimensions ; rd = rd->next, c++) {
 973                                 group_values[c] = 0;
 974                                 found_non_existing[c] = 0;
 975
 976                                 if(!print_hidden[c] && !found_non_zero[c]) {
 977                                         changed = 1;
 978                                         print_hidden[c] = 1;
 979                                 }
 980                         }
 981
 982                         if(changed) buffer_flush(wb);
 983                         else break;
 984                 }
 985                 else break;
 986
 987         } // max_loop
 988
 989         debug(D_RRD_STATS, "RRD_STATS_JSON: %s total %ld bytes", st->name, wb->len);
 990
 991         pthread_rwlock_unlock(&st->rwlock);
 992         return last_timestamp;
 993 }