src/rrd2json.c

   1 #ifdef HAVE_CONFIG_H
   2 #include <config.h>
   3 #endif
   4 #include <pthread.h>
   5 #include <sys/time.h>
   6 #include <stdlib.h>
   7
   8 #include "log.h"
   9 #include "common.h"
  10 #include "rrd2json.h"
  11
  12 #define HOSTNAME_MAX 1024
  13 char *hostname = "unknown";
  14
  15 unsigned long rrd_stats_one_json(RRDSET *st, char *options, struct web_buffer *wb)
  16 {
  17         time_t now = time(NULL);
  18
  19         pthread_rwlock_rdlock(&st->rwlock);
  20
  21         web_buffer_snprintf(wb, 65536,
  22                 "\t\t{\n"
  23                 "\t\t\t\"id\": \"%s\",\n"
  24                 "\t\t\t\"name\": \"%s\",\n"
  25                 "\t\t\t\"type\": \"%s\",\n"
  26                 "\t\t\t\"family\": \"%s\",\n"
  27                 "\t\t\t\"title\": \"%s\",\n"
  28                 "\t\t\t\"priority\": %ld,\n"
  29                 "\t\t\t\"enabled\": %d,\n"
  30                 "\t\t\t\"units\": \"%s\",\n"
  31                 "\t\t\t\"url\": \"/data/%s/%s\",\n"
  32                 "\t\t\t\"chart_type\": \"%s\",\n"
  33                 "\t\t\t\"counter\": %ld,\n"
  34                 "\t\t\t\"entries\": %ld,\n"
  35                 "\t\t\t\"first_entry_t\": %lu,\n"
  36                 "\t\t\t\"last_entry\": %ld,\n"
  37                 "\t\t\t\"last_entry_t\": %lu,\n"
  38                 "\t\t\t\"last_entry_secs_ago\": %lu,\n"
  39                 "\t\t\t\"update_every\": %d,\n"
  40                 "\t\t\t\"isdetail\": %d,\n"
  41                 "\t\t\t\"usec_since_last_update\": %llu,\n"
  42                 "\t\t\t\"collected_total\": " TOTAL_NUMBER_FORMAT ",\n"
  43                 "\t\t\t\"last_collected_total\": " TOTAL_NUMBER_FORMAT ",\n"
  44                 "\t\t\t\"dimensions\": [\n"
  45                 , st->id
  46                 , st->name
  47                 , st->type
  48                 , st->family
  49                 , st->title
  50                 , st->priority
  51                 , st->enabled
  52                 , st->units
  53                 , st->name, options?options:""
  54                 , rrdset_type_name(st->chart_type)
  55                 , st->counter
  56                 , st->entries
  57                 , rrdset_first_entry_t(st)
  58                 , rrdset_last_slot(st)
  59                 , rrdset_last_entry_t(st)
  60                 , (now < rrdset_last_entry_t(st)) ? (time_t)0 : now - rrdset_last_entry_t(st)
  61                 , st->update_every
  62                 , st->isdetail
  63                 , st->usec_since_last_update
  64                 , st->collected_total
  65                 , st->last_collected_total
  66                 );
  67
  68         unsigned long memory = st->memsize;
  69
  70         RRDDIM *rd;
  71         for(rd = st->dimensions; rd ; rd = rd->next) {
  72
  73                 memory += rd->memsize;
  74
  75                 web_buffer_snprintf(wb, 4096,
  76                         "\t\t\t\t{\n"
  77                         "\t\t\t\t\t\"id\": \"%s\",\n"
  78                         "\t\t\t\t\t\"name\": \"%s\",\n"
  79                         "\t\t\t\t\t\"entries\": %ld,\n"
  80                         "\t\t\t\t\t\"isHidden\": %d,\n"
  81                         "\t\t\t\t\t\"algorithm\": \"%s\",\n"
  82                         "\t\t\t\t\t\"multiplier\": %ld,\n"
  83                         "\t\t\t\t\t\"divisor\": %ld,\n"
  84                         "\t\t\t\t\t\"last_entry_t\": %lu,\n"
  85                         "\t\t\t\t\t\"collected_value\": " COLLECTED_NUMBER_FORMAT ",\n"
  86                         "\t\t\t\t\t\"calculated_value\": " CALCULATED_NUMBER_FORMAT ",\n"
  87                         "\t\t\t\t\t\"last_collected_value\": " COLLECTED_NUMBER_FORMAT ",\n"
  88                         "\t\t\t\t\t\"last_calculated_value\": " CALCULATED_NUMBER_FORMAT ",\n"
  89                         "\t\t\t\t\t\"memory\": %lu\n"
  90                         "\t\t\t\t}%s\n"
  91                         , rd->id
  92                         , rd->name
  93                         , rd->entries
  94                         , (rd->flags & RRDDIM_FLAG_HIDDEN)?1:0
  95                         , rrddim_algorithm_name(rd->algorithm)
  96                         , rd->multiplier
  97                         , rd->divisor
  98                         , rd->last_collected_time.tv_sec
  99                         , rd->collected_value
 100                         , rd->calculated_value
 101                         , rd->last_collected_value
 102                         , rd->last_calculated_value
 103                         , rd->memsize
 104                         , rd->next?",":""
 105                         );
 106         }
 107
 108         web_buffer_snprintf(wb, 200,
 109                 "\t\t\t],\n"
 110                 "\t\t\t\"memory\" : %lu\n"
 111                 "\t\t}"
 112                 , memory
 113                 );
 114
 115         pthread_rwlock_unlock(&st->rwlock);
 116         return memory;
 117 }
 118
 119 #define RRD_GRAPH_JSON_HEADER "{\n\t\"charts\": [\n"
 120 #define RRD_GRAPH_JSON_FOOTER "\n\t]\n}\n"
 121
 122 void rrd_stats_graph_json(RRDSET *st, char *options, struct web_buffer *wb)
 123 {
 124         web_buffer_snprintf(wb, sizeof(RRD_GRAPH_JSON_HEADER), RRD_GRAPH_JSON_HEADER);
 125         rrd_stats_one_json(st, options, wb);
 126         web_buffer_snprintf(wb, sizeof(RRD_GRAPH_JSON_FOOTER), RRD_GRAPH_JSON_FOOTER);
 127 }
 128
 129 void rrd_stats_all_json(struct web_buffer *wb)
 130 {
 131         unsigned long memory = 0;
 132         long c;
 133         RRDSET *st;
 134
 135         web_buffer_snprintf(wb, sizeof(RRD_GRAPH_JSON_HEADER), RRD_GRAPH_JSON_HEADER);
 136
 137         pthread_rwlock_rdlock(&rrdset_root_rwlock);
 138         for(st = rrdset_root, c = 0; st ; st = st->next) {
 139                 if(st->enabled) {
 140                         if(c) web_buffer_strcat(wb, ",\n");
 141                         memory += rrd_stats_one_json(st, NULL, wb);
 142                         c++;
 143                 }
 144         }
 145         pthread_rwlock_unlock(&rrdset_root_rwlock);
 146
 147         web_buffer_snprintf(wb, 4096, "\n\t],\n"
 148                 "\t\"hostname\": \"%s\",\n"
 149                 "\t\"update_every\": %d,\n"
 150                 "\t\"history\": %d,\n"
 151                 "\t\"memory\": %lu\n"
 152                 "}\n"
 153                 , hostname
 154                 , rrd_update_every
 155                 , rrd_default_history_entries
 156                 , memory
 157                 );
 158 }
 159
 160
 161
 162 // ----------------------------------------------------------------------------
 163
 164 // RRDR options
 165 #define RRDR_EMPTY      0x01
 166 #define RRDR_RESET      0x02
 167 #define RRDR_HIDDEN     0x04
 168 #define RRDR_NONZERO    0x08
 169
 170
 171 typedef struct rrdresult {
 172         RRDSET *st;                     // the chart this result refers to
 173
 174         int d;                                  // the number of dimensions
 175         int n;                                  // the number of values in the arrays
 176
 177         uint8_t *od;                    // the options for the dimensions
 178
 179         time_t *t;                              // array of n timestamps
 180         calculated_number *v;   // array n x d values
 181         uint8_t *o;                             // array n x d options
 182
 183         int c;                                  // current line ( 0 ~ n )
 184
 185         int has_st_lock;                // if st is read locked by us
 186 } RRDR;
 187
 188 static void rrdr_dump(RRDR *r)
 189 {
 190         long c, i;
 191         RRDDIM *d;
 192
 193         fprintf(stderr, "\nCHART %s (%s)\n", r->st->id, r->st->name);
 194
 195         for(c = 0, d = r->st->dimensions; d ;c++, d = d->next) {
 196                 fprintf(stderr, "DIMENSION %s (%s), %s%s%s%s\n"
 197                                 , d->id
 198                                 , d->name
 199                                 , (r->od[c] & RRDR_EMPTY)?"EMPTY ":""
 200                                 , (r->od[c] & RRDR_RESET)?"RESET ":""
 201                                 , (r->od[c] & RRDR_HIDDEN)?"HIDDEN ":""
 202                                 , (r->od[c] & RRDR_NONZERO)?"NONZERO ":""
 203                                 );
 204         }
 205
 206         if(r->c < 0) {
 207                 fprintf(stderr, "RRDR does not have any values in it.\n");
 208                 return;
 209         }
 210
 211         fprintf(stderr, "RRDR includes %d values in it:\n", r->c + 1);
 212
 213         // for each line in the array
 214         for(i = 0; i <= r->c ;i++) {
 215                 calculated_number *cn = &r->v[ i * r->d ];
 216                 uint8_t *co = &r->o[ i * r->d ];
 217
 218                 // print the id and the timestamp of the line
 219                 fprintf(stderr, "%ld %ld ", i + 1, r->t[i]);
 220
 221                 // for each dimension
 222                 for(c = 0, d = r->st->dimensions; d ;c++, d = d->next) {
 223                         if(unlikely(r->od[c] & RRDR_HIDDEN)) continue;
 224                         if(unlikely(!(r->od[c] & RRDR_NONZERO))) continue;
 225
 226                         if(co[c] & RRDR_EMPTY)
 227                                 fprintf(stderr, "null ");
 228                         else
 229                                 fprintf(stderr, CALCULATED_NUMBER_FORMAT " %s%s%s%s "
 230                                         , cn[c]
 231                                         , (co[c] & RRDR_EMPTY)?"E":" "
 232                                         , (co[c] & RRDR_RESET)?"R":" "
 233                                         , (co[c] & RRDR_HIDDEN)?"H":" "
 234                                         , (co[c] & RRDR_NONZERO)?"N":" "
 235                                         );
 236                 }
 237
 238                 fprintf(stderr, "\n");
 239         }
 240 }
 241
 242 inline static calculated_number *rrdr_line_values(RRDR *r)
 243 {
 244         return &r->v[ r->c * r->d ];
 245 }
 246
 247 inline static uint8_t *rrdr_line_options(RRDR *r)
 248 {
 249         return &r->o[ r->c * r->d ];
 250 }
 251
 252 inline static int rrdr_line_init(RRDR *r, time_t t)
 253 {
 254         r->c++;
 255         if(unlikely(r->c >= r->n)) {
 256                 r->c = r->n - 1;
 257                 return 0;
 258         }
 259
 260         // save the time
 261         r->t[r->c] = t;
 262
 263         return 1;
 264 }
 265
 266 inline static void rrdr_lock_rrdset(RRDR *r) {
 267         if(unlikely(!r)) {
 268                 error("NULL value given!");
 269                 return;
 270         }
 271
 272         pthread_rwlock_rdlock(&r->st->rwlock);
 273         r->has_st_lock = 1;
 274 }
 275
 276 inline static void rrdr_unlock_rrdset(RRDR *r) {
 277         if(unlikely(!r)) {
 278                 error("NULL value given!");
 279                 return;
 280         }
 281
 282         if(likely(r->has_st_lock)) {
 283                 pthread_rwlock_unlock(&r->st->rwlock);
 284                 r->has_st_lock = 0;
 285         }
 286 }
 287
 288 inline static void rrdr_free(RRDR *r)
 289 {
 290         if(unlikely(!r)) {
 291                 error("NULL value given!");
 292                 return;
 293         }
 294
 295         rrdr_unlock_rrdset(r);
 296         if(likely(r->t)) free(r->t);
 297         if(likely(r->v)) free(r->v);
 298         if(likely(r->o)) free(r->o);
 299         if(likely(r->od)) free(r->od);
 300         free(r);
 301 }
 302
 303 static RRDR *rrdr_create(RRDSET *st, int n)
 304 {
 305         if(unlikely(!st)) {
 306                 error("NULL value given!");
 307                 return NULL;
 308         }
 309
 310         RRDR *r = calloc(1, sizeof(RRDR));
 311         if(unlikely(!r)) goto cleanup;
 312
 313         r->st = st;
 314
 315         rrdr_lock_rrdset(r);
 316
 317         RRDDIM *rd;
 318         for(rd = st->dimensions ; rd ; rd = rd->next) r->d++;
 319
 320         r->n = n;
 321         r->t = malloc(n * sizeof(time_t));
 322         if(unlikely(!r->t)) goto cleanup;
 323
 324         r->t = malloc(n * sizeof(time_t));
 325         if(unlikely(!r->t)) goto cleanup;
 326
 327         r->v = malloc(n * r->d * sizeof(calculated_number));
 328         if(unlikely(!r->v)) goto cleanup;
 329
 330         r->o = malloc(n * r->d * sizeof(uint8_t));
 331         if(unlikely(!r->o)) goto cleanup;
 332
 333         r->od = calloc(r->d, sizeof(uint8_t));
 334         if(unlikely(!r->od)) goto cleanup;
 335
 336         r->c = -1;
 337
 338         return r;
 339
 340 cleanup:
 341         error("Cannot allocate memory");
 342         if(likely(r)) rrdr_free(r);
 343         return NULL;
 344 }
 345
 346 RRDR *rrd2rrdr(RRDSET *st, long points, time_t after, time_t before, int group_method)
 347 {
 348         int debug = st->debug;
 349
 350         time_t first_entry_t = rrdset_first_entry_t(st);
 351         time_t last_entry_t = rrdset_last_entry_t(st);
 352
 353         // allow relative for before and after
 354         if(before <= st->update_every * st->entries) before = last_entry_t + before;
 355         if(after <= st->update_every * st->entries) after = last_entry_t + after;
 356
 357         // make sure they are within our timeframe
 358         if(before > last_entry_t) before = last_entry_t;
 359         if(before < first_entry_t) before = first_entry_t;
 360
 361         if(after > last_entry_t) after = last_entry_t;
 362         if(after < first_entry_t) after = first_entry_t;
 363
 364         // check if they are upside down
 365         if(after > before) {
 366                 time_t t = before;
 367                 before = after;
 368                 after = t;
 369         }
 370
 371         // the duration of the chart
 372         time_t duration = before - after;
 373         if(duration <= 0) return NULL;
 374
 375         // check the required points
 376         if(points > duration / st->update_every) points = 0;
 377         if(points <= 0) points = duration / st->update_every;
 378
 379         // calculate proper grouping of source data
 380         long group = duration / points;
 381         if(group <= 0) group = 1;
 382         if(duration / group > points) group++;
 383
 384         // error("NEW: points=%d after=%d before=%d group=%d, duration=%d", points, after, before, group, duration);
 385
 386         // Now we have:
 387         // before = the end time of the calculation
 388         // after = the start time of the calculation
 389         // duration = the duration of the calculation
 390         // group = the number of source points to aggregate / group together
 391         // method = the method of grouping source points
 392         // points = the number of points to generate
 393
 394
 395         // -------------------------------------------------------------------------
 396         // initialize our result set
 397
 398         RRDR *r = rrdr_create(st, points);
 399         if(!r) return NULL;
 400         if(!r->d) {
 401                 rrdr_free(r);
 402                 return NULL;
 403         }
 404
 405         // find how many dimensions we have
 406         long dimensions = r->d;
 407
 408
 409         // -------------------------------------------------------------------------
 410         // checks for debugging
 411
 412         if(debug) debug(D_RRD_STATS, "INFO %s first_t: %lu, last_t: %lu, all_duration: %lu, after: %lu, before: %lu, duration: %lu, points: %ld, group: %ld"
 413                         , st->id
 414                         , first_entry_t
 415                         , last_entry_t
 416                         , last_entry_t - first_entry_t
 417                         , after
 418                         , before
 419                         , duration
 420                         , points
 421                         , group
 422                         );
 423
 424
 425         // -------------------------------------------------------------------------
 426         // temp arrays for keeping values per dimension
 427
 428         calculated_number       group_values[dimensions]; // keep sums when grouping
 429         long                            group_counts[dimensions]; // keep the number of values added to group_values
 430         uint8_t                         group_options[dimensions];
 431         uint8_t                         found_non_zero[dimensions];
 432
 433
 434         // initialize them
 435         RRDDIM *rd;
 436         long c;
 437         for( rd = st->dimensions, c = 0 ; rd && c < dimensions ; rd = rd->next, c++) {
 438                 group_values[c] = 0;
 439                 group_counts[c] = 0;
 440                 group_options[c] = 0;
 441                 found_non_zero[c] = 0;
 442         }
 443
 444
 445         // -------------------------------------------------------------------------
 446         // the main loop
 447
 448         long    t = rrdset_time2slot(st, before), // rrdset_last_slot(st),
 449                         stop_at_t = rrdset_time2slot(st, after),
 450                         added = 0,
 451                         group_count = 0,
 452                         add_this = 0,
 453                         stop_now = 0;
 454
 455         // align to group for proper panning of data
 456         t -= t % group;
 457
 458         time_t  now = rrdset_slot2time(st, t),
 459                         dt = st->update_every,
 460                         group_start_t = 0;
 461
 462         if(debug) debug(D_RRD_STATS, "BEGIN %s after_t: %lu (stop slot %ld), before_t: %lu (start slot %ld), start_t(now): %lu"
 463                         , st->id
 464                         , after
 465                         , stop_at_t
 466                         , before
 467                         , t
 468                         , now
 469                         );
 470
 471         for( ; !stop_now ; now -= dt, t--) {
 472                 if(unlikely(t < 0)) c = st->entries - 1;
 473                 if(t == stop_at_t) stop_now = 1;
 474
 475                 if(debug) debug(D_RRD_STATS, "ROW %s c: %ld, group_count: %ld, added: %ld, now: %lu, %s %s"
 476                                 , st->id
 477                                 , t
 478                                 , group_count + 1
 479                                 , added
 480                                 , now
 481                                 , (group_count + 1 == group)?"PRINT":"  -  "
 482                                 , (now >= after && now <= before)?"RANGE":"  -  "
 483                                 );
 484
 485                 // make sure we return data in the proper time range
 486                 if(unlikely(now > before)) continue;
 487                 if(unlikely(now < after)) break;
 488
 489                 if(group_count == 0) group_start_t = now;
 490                 group_count++;
 491
 492                 if(unlikely(group_count == group)) {
 493                         if(unlikely(added >= points)) break;
 494                         add_this = 1;
 495                 }
 496
 497                 // do the calculations
 498                 for(rd = st->dimensions, c = 0 ; likely(rd && c < dimensions) ; rd = rd->next, c++) {
 499                         storage_number n = rd->values[t];
 500                         if(unlikely(!does_storage_number_exist(n))) continue;
 501
 502                         group_counts[c]++;
 503
 504                         calculated_number value = unpack_storage_number(n);
 505                         if(value != 0.0) {
 506                                 group_options[c] |= RRDR_NONZERO;
 507                                 found_non_zero[c] = 1;
 508                         }
 509
 510                         if(unlikely(did_storage_number_reset(n)))
 511                                 group_options[c] |= RRDR_RESET;
 512
 513                         switch(group_method) {
 514                                 case GROUP_MAX:
 515                                         if(unlikely(abs(value) > abs(group_values[c])))
 516                                                 group_values[c] = value;
 517                                         break;
 518
 519                                 default:
 520                                 case GROUP_SUM:
 521                                 case GROUP_AVERAGE:
 522                                         group_values[c] += value;
 523                                         break;
 524                         }
 525                 }
 526
 527                 // added it
 528                 if(unlikely(add_this)) {
 529                         if(!rrdr_line_init(r, group_start_t)) break;
 530
 531                         calculated_number *cn = rrdr_line_values(r);
 532                         uint8_t *co = rrdr_line_options(r);
 533
 534                         for(rd = st->dimensions, c = 0 ; likely(rd && c < dimensions) ; rd = rd->next, c++) {
 535
 536                                 // update the dimension options
 537                                 if(found_non_zero[c]) r->od[c] |= RRDR_NONZERO;
 538                                 if(rd->flags & RRDDIM_FLAG_HIDDEN) r->od[c] |= RRDR_HIDDEN;
 539
 540                                 // store the specific point options
 541                                 co[c] = group_options[c];
 542
 543                                 // store the value
 544                                 if(group_counts[c] == 0) {
 545                                         cn[c] = 0.0;
 546                                         co[c] |= RRDR_EMPTY;
 547                                 }
 548                                 else if(unlikely(group_method == GROUP_AVERAGE)) {
 549                                         cn[c] = group_values[c] / group_counts[c];
 550                                 }
 551                                 else {
 552                                         cn[c] = group_values[c];
 553                                 }
 554
 555                                 // reset them for the next loop
 556                                 group_values[c] = 0;
 557                                 group_counts[c] = 0;
 558                                 group_options[c] = 0;
 559                         }
 560
 561                         added++;
 562                         group_count = 0;
 563                         add_this = 0;
 564                 }
 565         }
 566
 567         rrdr_dump(r);
 568         rrdr_free(r);
 569         return NULL;
 570 }
 571
 572 unsigned long rrd_stats_json(int type, RRDSET *st, struct web_buffer *wb, int points, int group, int group_method, time_t after, time_t before, int only_non_zero)
 573 {
 574         int c;
 575         pthread_rwlock_rdlock(&st->rwlock);
 576
 577
 578         // -------------------------------------------------------------------------
 579         // switch from JSON to google JSON
 580
 581         char kq[2] = "\"";
 582         char sq[2] = "\"";
 583         switch(type) {
 584                 case DATASOURCE_GOOGLE_JSON:
 585                 case DATASOURCE_GOOGLE_JSONP:
 586                         kq[0] = '\0';
 587                         sq[0] = '\'';
 588                         break;
 589
 590                 case DATASOURCE_JSON:
 591                 default:
 592                         break;
 593         }
 594
 595
 596         // -------------------------------------------------------------------------
 597         // validate the parameters
 598
 599         if(points < 1) points = 1;
 600         if(group < 1) group = 1;
 601
 602         if(before == 0 || before > rrdset_last_entry_t(st)) before = rrdset_last_entry_t(st);
 603         if(after  == 0 || after < rrdset_first_entry_t(st)) after = rrdset_first_entry_t(st);
 604
 605         // ---
 606
 607         // our return value (the last timestamp printed)
 608         // this is required to detect re-transmit in google JSONP
 609         time_t last_timestamp = 0;
 610
 611
 612         // -------------------------------------------------------------------------
 613         // find how many dimensions we have
 614
 615         int dimensions = 0;
 616         RRDDIM *rd;
 617         for( rd = st->dimensions ; rd ; rd = rd->next) dimensions++;
 618         if(!dimensions) {
 619                 pthread_rwlock_unlock(&st->rwlock);
 620                 web_buffer_strcat(wb, "No dimensions yet.");
 621                 return 0;
 622         }
 623
 624
 625         // -------------------------------------------------------------------------
 626         // prepare various strings, to speed up the loop
 627
 628         char overflow_annotation[201]; snprintf(overflow_annotation, 200, ",{%sv%s:%sRESET OR OVERFLOW%s},{%sv%s:%sThe counters have been wrapped.%s}", kq, kq, sq, sq, kq, kq, sq, sq);
 629         char normal_annotation[201];   snprintf(normal_annotation,   200, ",{%sv%s:null},{%sv%s:null}", kq, kq, kq, kq);
 630         char pre_date[51];             snprintf(pre_date,             50, "             {%sc%s:[{%sv%s:%s", kq, kq, kq, kq, sq);
 631         char post_date[21];            snprintf(post_date,            20, "%s}", sq);
 632         char pre_value[21];            snprintf(pre_value,            20, ",{%sv%s:", kq, kq);
 633         char post_value[21];           snprintf(post_value,           20, "}");
 634
 635
 636         // -------------------------------------------------------------------------
 637         // checks for debugging
 638
 639         if(st->debug) {
 640                 debug(D_RRD_STATS, "%s first_entry_t = %lu, last_entry_t = %lu, duration = %lu, after = %lu, before = %lu, duration = %lu, entries_to_show = %lu, group = %lu"
 641                         , st->id
 642                         , rrdset_first_entry_t(st)
 643                         , rrdset_last_entry_t(st)
 644                         , rrdset_last_entry_t(st) - rrdset_first_entry_t(st)
 645                         , after
 646                         , before
 647                         , before - after
 648                         , points
 649                         , group
 650                         );
 651
 652                 if(before < after)
 653                         debug(D_RRD_STATS, "WARNING: %s The newest value in the database (%lu) is earlier than the oldest (%lu)", st->name, before, after);
 654
 655                 if((before - after) > st->entries * st->update_every)
 656                         debug(D_RRD_STATS, "WARNING: %s The time difference between the oldest and the newest entries (%lu) is higher than the capacity of the database (%lu)", st->name, before - after, st->entries * st->update_every);
 657         }
 658
 659
 660         // -------------------------------------------------------------------------
 661         // temp arrays for keeping values per dimension
 662
 663         calculated_number group_values[dimensions]; // keep sums when grouping
 664         int               print_hidden[dimensions]; // keep hidden flags
 665         int               found_non_zero[dimensions];
 666         int               found_non_existing[dimensions];
 667
 668         // initialize them
 669         for( rd = st->dimensions, c = 0 ; rd && c < dimensions ; rd = rd->next, c++) {
 670                 group_values[c] = 0;
 671                 print_hidden[c] = (rd->flags & RRDDIM_FLAG_HIDDEN)?1:0;
 672                 found_non_zero[c] = 0;
 673                 found_non_existing[c] = 0;
 674         }
 675
 676
 677         // error("OLD: points=%d after=%d before=%d group=%d, duration=%d", entries_to_show, before - (st->update_every * group * entries_to_show), before, group, before - after + 1);
 678         // rrd2array(st, entries_to_show, before - (st->update_every * group * entries_to_show), before, group_method, only_non_zero);
 679         // rrd2rrdr(st, entries_to_show, before - (st->update_every * group * entries_to_show), before, group_method);
 680
 681         // -------------------------------------------------------------------------
 682         // remove dimensions that contain only zeros
 683
 684         int max_loop = 1;
 685         if(only_non_zero) max_loop = 2;
 686
 687         for(; max_loop ; max_loop--) {
 688
 689                 // -------------------------------------------------------------------------
 690                 // print the JSON header
 691
 692                 web_buffer_snprintf(wb, 1024, "{\n      %scols%s:\n     [\n", kq, kq);
 693                 web_buffer_snprintf(wb, 1024, "         {%sid%s:%s%s,%slabel%s:%stime%s,%spattern%s:%s%s,%stype%s:%sdatetime%s},\n", kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq);
 694                 web_buffer_snprintf(wb, 1024, "         {%sid%s:%s%s,%slabel%s:%s%s,%spattern%s:%s%s,%stype%s:%sstring%s,%sp%s:{%srole%s:%sannotation%s}},\n", kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, kq, kq, sq, sq);
 695                 web_buffer_snprintf(wb, 1024, "         {%sid%s:%s%s,%slabel%s:%s%s,%spattern%s:%s%s,%stype%s:%sstring%s,%sp%s:{%srole%s:%sannotationText%s}}", kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, kq, kq, sq, sq);
 696
 697                 // print the header for each dimension
 698                 // and update the print_hidden array for the dimensions that should be hidden
 699                 int pc = 0;
 700                 for( rd = st->dimensions, c = 0 ; rd && c < dimensions ; rd = rd->next, c++) {
 701                         if(!print_hidden[c]) {
 702                                 pc++;
 703                                 web_buffer_snprintf(wb, 1024, ",\n              {%sid%s:%s%s,%slabel%s:%s%s%s,%spattern%s:%s%s,%stype%s:%snumber%s}", kq, kq, sq, sq, kq, kq, sq, rd->name, sq, kq, kq, sq, sq, kq, kq, sq, sq);
 704                         }
 705                 }
 706                 if(!pc) {
 707                         web_buffer_snprintf(wb, 1024, ",\n              {%sid%s:%s%s,%slabel%s:%s%s%s,%spattern%s:%s%s,%stype%s:%snumber%s}", kq, kq, sq, sq, kq, kq, sq, "no data", sq, kq, kq, sq, sq, kq, kq, sq, sq);
 708                 }
 709
 710                 // print the begin of row data
 711                 web_buffer_snprintf(wb, 1024, "\n       ],\n    %srows%s:\n     [\n", kq, kq);
 712
 713
 714                 // -------------------------------------------------------------------------
 715                 // the main loop
 716
 717                 int annotate_reset = 0;
 718                 int annotation_count = 0;
 719
 720                 // the minimum line length we expect
 721                 int line_size = 4096 + (dimensions * 200);
 722
 723                 long    t = rrdset_time2slot(st, before),
 724                                 stop_at_t = rrdset_time2slot(st, after),
 725                                 stop_now = 0;
 726
 727                 t -= t % group;
 728
 729                 time_t  now = rrdset_slot2time(st, t),
 730                                 dt = st->update_every;
 731
 732                 long count = 0, printed = 0, group_count = 0;
 733                 last_timestamp = 0;
 734
 735                 if(st->debug) debug(D_RRD_STATS, "%s: REQUEST after:%lu before:%lu, points:%d, group:%d, CHART cur:%ld first: %lu last:%lu, CALC start_t:%ld, stop_t:%ld"
 736                                         , st->id
 737                                         , after
 738                                         , before
 739                                         , points
 740                                         , group
 741                                         , st->current_entry
 742                                         , rrdset_first_entry_t(st)
 743                                         , rrdset_last_entry_t(st)
 744                                         , t
 745                                         , stop_at_t
 746                                         );
 747
 748                 for(; !stop_now ; now -= dt, t--) {
 749                         if(t < 0) t = st->entries - 1;
 750                         if(t == stop_at_t) stop_now = 1;
 751
 752                         int print_this = 0;
 753
 754                         if(st->debug) debug(D_RRD_STATS, "%s t = %ld, count = %ld, group_count = %ld, printed = %ld, now = %lu, %s %s"
 755                                         , st->id
 756                                         , t
 757                                         , count + 1
 758                                         , group_count + 1
 759                                         , printed
 760                                         , now
 761                                         , (group_count + 1 == group)?"PRINT":"  -  "
 762                                         , (now >= after && now <= before)?"RANGE":"  -  "
 763                                         );
 764
 765
 766                         // make sure we return data in the proper time range
 767                         if(now > before) continue;
 768                         if(now < after) break;
 769
 770                         //if(rrdset_slot2time(st, t) != now)
 771                         //      error("%s: slot=%ld, now=%ld, slot2time=%ld, diff=%ld, last_entry_t=%ld, rrdset_last_slot=%ld", st->id, t, now, rrdset_slot2time(st,t), now - rrdset_slot2time(st,t), rrdset_last_entry_t(st), rrdset_last_slot(st));
 772
 773                         count++;
 774                         group_count++;
 775
 776                         // check if we have to print this now
 777                         if(group_count == group) {
 778                                 if(printed >= points) {
 779                                         // debug(D_RRD_STATS, "Already printed all rows. Stopping.");
 780                                         break;
 781                                 }
 782
 783                                 // generate the local date time
 784                                 struct tm *tm = localtime(&now);
 785                                 if(!tm) { error("localtime() failed."); continue; }
 786                                 if(now > last_timestamp) last_timestamp = now;
 787
 788                                 if(printed) web_buffer_strcat(wb, "]},\n");
 789                                 web_buffer_strcat(wb, pre_date);
 790                                 web_buffer_jsdate(wb, tm->tm_year + 1900, tm->tm_mon, tm->tm_mday, tm->tm_hour, tm->tm_min, tm->tm_sec);
 791                                 web_buffer_strcat(wb, post_date);
 792
 793                                 print_this = 1;
 794                         }
 795
 796                         // do the calculations
 797                         for(rd = st->dimensions, c = 0 ; rd && c < dimensions ; rd = rd->next, c++) {
 798                                 storage_number n = rd->values[t];
 799                                 calculated_number value = unpack_storage_number(n);
 800
 801                                 if(!does_storage_number_exist(n)) {
 802                                         value = 0.0;
 803                                         found_non_existing[c]++;
 804                                 }
 805                                 if(did_storage_number_reset(n)) annotate_reset = 1;
 806
 807                                 switch(group_method) {
 808                                         case GROUP_MAX:
 809                                                 if(abs(value) > abs(group_values[c])) group_values[c] = value;
 810                                                 break;
 811
 812                                         case GROUP_SUM:
 813                                                 group_values[c] += value;
 814                                                 break;
 815
 816                                         default:
 817                                         case GROUP_AVERAGE:
 818                                                 group_values[c] += value;
 819                                                 if(print_this) group_values[c] /= ( group_count - found_non_existing[c] );
 820                                                 break;
 821                                 }
 822                         }
 823
 824                         if(print_this) {
 825                                 if(annotate_reset) {
 826                                         annotation_count++;
 827                                         web_buffer_strcat(wb, overflow_annotation);
 828                                         annotate_reset = 0;
 829                                 }
 830                                 else
 831                                         web_buffer_strcat(wb, normal_annotation);
 832
 833                                 pc = 0;
 834                                 for(c = 0 ; c < dimensions ; c++) {
 835                                         if(found_non_existing[c] == group_count) {
 836                                                 // all entries are non-existing
 837                                                 pc++;
 838                                                 web_buffer_strcat(wb, pre_value);
 839                                                 web_buffer_strcat(wb, "null");
 840                                                 web_buffer_strcat(wb, post_value);
 841                                         }
 842                                         else if(!print_hidden[c]) {
 843                                                 pc++;
 844                                                 web_buffer_strcat(wb, pre_value);
 845                                                 web_buffer_rrd_value(wb, group_values[c]);
 846                                                 web_buffer_strcat(wb, post_value);
 847
 848                                                 if(group_values[c]) found_non_zero[c]++;
 849                                         }
 850
 851                                         // reset them for the next loop
 852                                         group_values[c] = 0;
 853                                         found_non_existing[c] = 0;
 854                                 }
 855
 856                                 // if all dimensions are hidden, print a null
 857                                 if(!pc) {
 858                                         web_buffer_strcat(wb, pre_value);
 859                                         web_buffer_strcat(wb, "null");
 860                                         web_buffer_strcat(wb, post_value);
 861                                 }
 862
 863                                 printed++;
 864                                 group_count = 0;
 865                         }
 866                 }
 867
 868                 if(printed) web_buffer_strcat(wb, "]}");
 869                 web_buffer_strcat(wb, "\n       ]\n}\n");
 870
 871                 if(only_non_zero && max_loop > 1) {
 872                         int changed = 0;
 873                         for(rd = st->dimensions, c = 0 ; rd && c < dimensions ; rd = rd->next, c++) {
 874                                 group_values[c] = 0;
 875                                 found_non_existing[c] = 0;
 876
 877                                 if(!print_hidden[c] && !found_non_zero[c]) {
 878                                         changed = 1;
 879                                         print_hidden[c] = 1;
 880                                 }
 881                         }
 882
 883                         if(changed) web_buffer_flush(wb);
 884                         else break;
 885                 }
 886                 else break;
 887
 888         } // max_loop
 889
 890         debug(D_RRD_STATS, "RRD_STATS_JSON: %s total %ld bytes", st->name, wb->len);
 891
 892         pthread_rwlock_unlock(&st->rwlock);
 893         return last_timestamp;
 894 }