src/rrd2json.c

   1 #ifdef HAVE_CONFIG_H
   2 #include <config.h>
   3 #endif
   4 #include <pthread.h>
   5 #include <stdlib.h>
   6 #include <string.h>
   7 #include <stdint.h>
   8
   9 #include "log.h"
  10 #include "common.h"
  11 #include "rrd2json.h"
  12
  13 #define HOSTNAME_MAX 1024
  14 char *hostname = "unknown";
  15
  16 void rrd_stats_api_v1_chart(RRDSET *st, BUFFER *wb)
  17 {
  18         pthread_rwlock_rdlock(&st->rwlock);
  19
  20         buffer_sprintf(wb,
  21                 "\t\t{\n"
  22                 "\t\t\t\"id\": \"%s\",\n"
  23                 "\t\t\t\"name\": \"%s\",\n"
  24                 "\t\t\t\"type\": \"%s\",\n"
  25                 "\t\t\t\"family\": \"%s\",\n"
  26                 "\t\t\t\"title\": \"%s\",\n"
  27                 "\t\t\t\"priority\": %ld,\n"
  28                 "\t\t\t\"enabled\": %s,\n"
  29                 "\t\t\t\"units\": \"%s\",\n"
  30                 "\t\t\t\"data_url\": \"/api/v1/data?chart=%s\",\n"
  31                 "\t\t\t\"chart_type\": \"%s\",\n"
  32                 "\t\t\t\"duration\": %ld,\n"
  33                 "\t\t\t\"first_entry\": %lu,\n"
  34                 "\t\t\t\"last_entry\": %lu,\n"
  35                 "\t\t\t\"update_every\": %d,\n"
  36                 "\t\t\t\"dimensions\": {\n"
  37                 , st->id
  38                 , st->name
  39                 , st->type
  40                 , st->family
  41                 , st->title
  42                 , st->priority
  43                 , st->enabled?"true":"false"
  44                 , st->units
  45                 , st->name
  46                 , rrdset_type_name(st->chart_type)
  47                 , st->entries * st->update_every
  48                 , rrdset_first_entry_t(st)
  49                 , rrdset_last_entry_t(st)
  50                 , st->update_every
  51                 );
  52
  53         unsigned long memory = st->memsize;
  54
  55         int c = 0;
  56         RRDDIM *rd;
  57         for(rd = st->dimensions; rd ; rd = rd->next) {
  58                 if(rd->flags & RRDDIM_FLAG_HIDDEN) continue;
  59
  60                 memory += rd->memsize;
  61
  62                 buffer_sprintf(wb,
  63                         "%s"
  64                         "\t\t\t\t\"%s\": { \"name\": \"%s\" }"
  65                         , c?",\n":""
  66                         , rd->id
  67                         , rd->name
  68                         );
  69
  70                 c++;
  71         }
  72
  73         buffer_sprintf(wb,
  74                 "\n\t\t\t}\n"
  75                 "\t\t}"
  76                 );
  77
  78         pthread_rwlock_unlock(&st->rwlock);
  79 }
  80
  81 void rrd_stats_api_v1_charts(BUFFER *wb)
  82 {
  83         long c;
  84         RRDSET *st;
  85
  86         buffer_sprintf(wb, "{\n"
  87                    "\t\"hostname\": \"%s\""
  88                 ",\n\t\"update_every\": %d"
  89                 ",\n\t\"history\": %d"
  90                 ",\n\t\"charts\": {"
  91                 , hostname
  92                 , rrd_update_every
  93                 , rrd_default_history_entries
  94                 );
  95
  96         pthread_rwlock_rdlock(&rrdset_root_rwlock);
  97         for(st = rrdset_root, c = 0; st ; st = st->next) {
  98                 if(st->enabled) {
  99                         if(c) buffer_strcat(wb, ",");
 100                         buffer_strcat(wb, "\n\t\t\"");
 101                         buffer_strcat(wb, st->id);
 102                         buffer_strcat(wb, "\": ");
 103                         rrd_stats_api_v1_chart(st, wb);
 104                         c++;
 105                 }
 106         }
 107         pthread_rwlock_unlock(&rrdset_root_rwlock);
 108
 109         buffer_strcat(wb, "\n\t}\n}\n");
 110 }
 111
 112
 113 unsigned long rrd_stats_one_json(RRDSET *st, char *options, BUFFER *wb)
 114 {
 115         time_t now = time(NULL);
 116
 117         pthread_rwlock_rdlock(&st->rwlock);
 118
 119         buffer_sprintf(wb,
 120                 "\t\t{\n"
 121                 "\t\t\t\"id\": \"%s\",\n"
 122                 "\t\t\t\"name\": \"%s\",\n"
 123                 "\t\t\t\"type\": \"%s\",\n"
 124                 "\t\t\t\"family\": \"%s\",\n"
 125                 "\t\t\t\"title\": \"%s\",\n"
 126                 "\t\t\t\"priority\": %ld,\n"
 127                 "\t\t\t\"enabled\": %d,\n"
 128                 "\t\t\t\"units\": \"%s\",\n"
 129                 "\t\t\t\"url\": \"/data/%s/%s\",\n"
 130                 "\t\t\t\"chart_type\": \"%s\",\n"
 131                 "\t\t\t\"counter\": %ld,\n"
 132                 "\t\t\t\"entries\": %ld,\n"
 133                 "\t\t\t\"first_entry_t\": %lu,\n"
 134                 "\t\t\t\"last_entry\": %ld,\n"
 135                 "\t\t\t\"last_entry_t\": %lu,\n"
 136                 "\t\t\t\"last_entry_secs_ago\": %lu,\n"
 137                 "\t\t\t\"update_every\": %d,\n"
 138                 "\t\t\t\"isdetail\": %d,\n"
 139                 "\t\t\t\"usec_since_last_update\": %llu,\n"
 140                 "\t\t\t\"collected_total\": " TOTAL_NUMBER_FORMAT ",\n"
 141                 "\t\t\t\"last_collected_total\": " TOTAL_NUMBER_FORMAT ",\n"
 142                 "\t\t\t\"dimensions\": [\n"
 143                 , st->id
 144                 , st->name
 145                 , st->type
 146                 , st->family
 147                 , st->title
 148                 , st->priority
 149                 , st->enabled
 150                 , st->units
 151                 , st->name, options?options:""
 152                 , rrdset_type_name(st->chart_type)
 153                 , st->counter
 154                 , st->entries
 155                 , rrdset_first_entry_t(st)
 156                 , rrdset_last_slot(st)
 157                 , rrdset_last_entry_t(st)
 158                 , (now < rrdset_last_entry_t(st)) ? (time_t)0 : now - rrdset_last_entry_t(st)
 159                 , st->update_every
 160                 , st->isdetail
 161                 , st->usec_since_last_update
 162                 , st->collected_total
 163                 , st->last_collected_total
 164                 );
 165
 166         unsigned long memory = st->memsize;
 167
 168         RRDDIM *rd;
 169         for(rd = st->dimensions; rd ; rd = rd->next) {
 170
 171                 memory += rd->memsize;
 172
 173                 buffer_sprintf(wb,
 174                         "\t\t\t\t{\n"
 175                         "\t\t\t\t\t\"id\": \"%s\",\n"
 176                         "\t\t\t\t\t\"name\": \"%s\",\n"
 177                         "\t\t\t\t\t\"entries\": %ld,\n"
 178                         "\t\t\t\t\t\"isHidden\": %d,\n"
 179                         "\t\t\t\t\t\"algorithm\": \"%s\",\n"
 180                         "\t\t\t\t\t\"multiplier\": %ld,\n"
 181                         "\t\t\t\t\t\"divisor\": %ld,\n"
 182                         "\t\t\t\t\t\"last_entry_t\": %lu,\n"
 183                         "\t\t\t\t\t\"collected_value\": " COLLECTED_NUMBER_FORMAT ",\n"
 184                         "\t\t\t\t\t\"calculated_value\": " CALCULATED_NUMBER_FORMAT ",\n"
 185                         "\t\t\t\t\t\"last_collected_value\": " COLLECTED_NUMBER_FORMAT ",\n"
 186                         "\t\t\t\t\t\"last_calculated_value\": " CALCULATED_NUMBER_FORMAT ",\n"
 187                         "\t\t\t\t\t\"memory\": %lu\n"
 188                         "\t\t\t\t}%s\n"
 189                         , rd->id
 190                         , rd->name
 191                         , rd->entries
 192                         , (rd->flags & RRDDIM_FLAG_HIDDEN)?1:0
 193                         , rrddim_algorithm_name(rd->algorithm)
 194                         , rd->multiplier
 195                         , rd->divisor
 196                         , rd->last_collected_time.tv_sec
 197                         , rd->collected_value
 198                         , rd->calculated_value
 199                         , rd->last_collected_value
 200                         , rd->last_calculated_value
 201                         , rd->memsize
 202                         , rd->next?",":""
 203                         );
 204         }
 205
 206         buffer_sprintf(wb,
 207                 "\t\t\t],\n"
 208                 "\t\t\t\"memory\" : %lu\n"
 209                 "\t\t}"
 210                 , memory
 211                 );
 212
 213         pthread_rwlock_unlock(&st->rwlock);
 214         return memory;
 215 }
 216
 217 #define RRD_GRAPH_JSON_HEADER "{\n\t\"charts\": [\n"
 218 #define RRD_GRAPH_JSON_FOOTER "\n\t]\n}\n"
 219
 220 void rrd_stats_graph_json(RRDSET *st, char *options, BUFFER *wb)
 221 {
 222         buffer_strcat(wb, RRD_GRAPH_JSON_HEADER);
 223         rrd_stats_one_json(st, options, wb);
 224         buffer_strcat(wb, RRD_GRAPH_JSON_FOOTER);
 225 }
 226
 227 void rrd_stats_all_json(BUFFER *wb)
 228 {
 229         unsigned long memory = 0;
 230         long c;
 231         RRDSET *st;
 232
 233         buffer_strcat(wb, RRD_GRAPH_JSON_HEADER);
 234
 235         pthread_rwlock_rdlock(&rrdset_root_rwlock);
 236         for(st = rrdset_root, c = 0; st ; st = st->next) {
 237                 if(st->enabled) {
 238                         if(c) buffer_strcat(wb, ",\n");
 239                         memory += rrd_stats_one_json(st, NULL, wb);
 240                         c++;
 241                 }
 242         }
 243         pthread_rwlock_unlock(&rrdset_root_rwlock);
 244
 245         buffer_sprintf(wb, "\n\t],\n"
 246                 "\t\"hostname\": \"%s\",\n"
 247                 "\t\"update_every\": %d,\n"
 248                 "\t\"history\": %d,\n"
 249                 "\t\"memory\": %lu\n"
 250                 "}\n"
 251                 , hostname
 252                 , rrd_update_every
 253                 , rrd_default_history_entries
 254                 , memory
 255                 );
 256 }
 257
 258
 259
 260 // ----------------------------------------------------------------------------
 261
 262 // RRDR options
 263 #define RRDR_EMPTY      0x01 // the dimension contains / the value is empty (null)
 264 #define RRDR_RESET      0x02 // the dimension contains / the value is reset
 265 #define RRDR_HIDDEN     0x04 // the dimension contains / the value is hidden
 266 #define RRDR_NONZERO    0x08 // the dimension contains / the value is non-zero
 267
 268
 269 typedef struct rrdresult {
 270         RRDSET *st;                     // the chart this result refers to
 271
 272         int d;                                  // the number of dimensions
 273         long n;                                 // the number of values in the arrays
 274         long rows;                          // the number of rows used
 275
 276         uint8_t *od;                    // the options for the dimensions
 277
 278         time_t *t;                              // array of n timestamps
 279         calculated_number *v;   // array n x d values
 280         uint8_t *o;                             // array n x d options
 281
 282         long c;                                 // current line ( -1 ~ n ), ( -1 = none, use rrdr_rows() to get number of rows )
 283
 284         long group;                             // how many collected values were grouped for each row
 285         long update_every;              // what is the suggested update frequency in seconds
 286
 287         calculated_number min;
 288         calculated_number max;
 289
 290         time_t before;
 291         time_t after;
 292
 293         int has_st_lock;                // if st is read locked by us
 294 } RRDR;
 295
 296 #define rrdr_rows(r) ((r)->rows)
 297
 298 /*
 299 static void rrdr_dump(RRDR *r)
 300 {
 301         long c, i;
 302         RRDDIM *d;
 303
 304         fprintf(stderr, "\nCHART %s (%s)\n", r->st->id, r->st->name);
 305
 306         for(c = 0, d = r->st->dimensions; d ;c++, d = d->next) {
 307                 fprintf(stderr, "DIMENSION %s (%s), %s%s%s%s\n"
 308                                 , d->id
 309                                 , d->name
 310                                 , (r->od[c] & RRDR_EMPTY)?"EMPTY ":""
 311                                 , (r->od[c] & RRDR_RESET)?"RESET ":""
 312                                 , (r->od[c] & RRDR_HIDDEN)?"HIDDEN ":""
 313                                 , (r->od[c] & RRDR_NONZERO)?"NONZERO ":""
 314                                 );
 315         }
 316
 317         if(r->rows <= 0) {
 318                 fprintf(stderr, "RRDR does not have any values in it.\n");
 319                 return;
 320         }
 321
 322         fprintf(stderr, "RRDR includes %d values in it:\n", r->rows);
 323
 324         // for each line in the array
 325         for(i = 0; i < r->rows ;i++) {
 326                 calculated_number *cn = &r->v[ i * r->d ];
 327                 uint8_t *co = &r->o[ i * r->d ];
 328
 329                 // print the id and the timestamp of the line
 330                 fprintf(stderr, "%ld %ld ", i + 1, r->t[i]);
 331
 332                 // for each dimension
 333                 for(c = 0, d = r->st->dimensions; d ;c++, d = d->next) {
 334                         if(unlikely(r->od[c] & RRDR_HIDDEN)) continue;
 335                         if(unlikely(!(r->od[c] & RRDR_NONZERO))) continue;
 336
 337                         if(co[c] & RRDR_EMPTY)
 338                                 fprintf(stderr, "null ");
 339                         else
 340                                 fprintf(stderr, CALCULATED_NUMBER_FORMAT " %s%s%s%s "
 341                                         , cn[c]
 342                                         , (co[c] & RRDR_EMPTY)?"E":" "
 343                                         , (co[c] & RRDR_RESET)?"R":" "
 344                                         , (co[c] & RRDR_HIDDEN)?"H":" "
 345                                         , (co[c] & RRDR_NONZERO)?"N":" "
 346                                         );
 347                 }
 348
 349                 fprintf(stderr, "\n");
 350         }
 351 }
 352 */
 353
 354 void rrdr_disable_not_selected_dimensions(RRDR *r, const char *dims)
 355 {
 356         char b[strlen(dims) + 1];
 357         char *o = b, *tok;
 358         strcpy(o, dims);
 359
 360         long c;
 361         RRDDIM *d;
 362
 363         // disable all of them
 364         for(c = 0, d = r->st->dimensions; d ;c++, d = d->next)
 365                 r->od[c] |= RRDR_HIDDEN;
 366
 367         while(o && *o && (tok = mystrsep(&o, ", |"))) {
 368                 if(!*tok) continue;
 369
 370                 // find it and enable it
 371                 for(c = 0, d = r->st->dimensions; d ;c++, d = d->next) {
 372                         if(!strcmp(d->name, tok)) {
 373                                 r->od[c] &= ~RRDR_HIDDEN;
 374                         }
 375                 }
 376         }
 377 }
 378
 379 void rrdr_buffer_print_format(BUFFER *wb, uint32_t format)
 380 {
 381         switch(format) {
 382         case DATASOURCE_JSON:
 383                 buffer_strcat(wb, DATASOURCE_FORMAT_JSON);
 384                 break;
 385
 386         case DATASOURCE_DATATABLE_JSON:
 387                 buffer_strcat(wb, DATASOURCE_FORMAT_DATATABLE_JSON);
 388                 break;
 389
 390         case DATASOURCE_DATATABLE_JSONP:
 391                 buffer_strcat(wb, DATASOURCE_FORMAT_DATATABLE_JSONP);
 392                 break;
 393
 394         case DATASOURCE_JSONP:
 395                 buffer_strcat(wb, DATASOURCE_FORMAT_JSONP);
 396                 break;
 397
 398         case DATASOURCE_SSV:
 399                 buffer_strcat(wb, DATASOURCE_FORMAT_SSV);
 400                 break;
 401
 402         case DATASOURCE_CSV:
 403                 buffer_strcat(wb, DATASOURCE_FORMAT_CSV);
 404                 break;
 405
 406         case DATASOURCE_TSV:
 407                 buffer_strcat(wb, DATASOURCE_FORMAT_TSV);
 408                 break;
 409
 410         case DATASOURCE_HTML:
 411                 buffer_strcat(wb, DATASOURCE_FORMAT_HTML);
 412                 break;
 413
 414         case DATASOURCE_JS_ARRAY:
 415                 buffer_strcat(wb, DATASOURCE_FORMAT_JS_ARRAY);
 416                 break;
 417
 418         case DATASOURCE_SSV_COMMA:
 419                 buffer_strcat(wb, DATASOURCE_FORMAT_SSV_COMMA);
 420                 break;
 421
 422         default:
 423                 buffer_strcat(wb, "unknown");
 424                 break;
 425         }
 426 }
 427
 428 uint32_t rrdr_check_options(RRDR *r, uint32_t options)
 429 {
 430         if(options & RRDR_OPTION_NONZERO) {
 431                 long c, i;
 432                 RRDDIM *rd;
 433
 434                 // find how many dimensions are not zero
 435                 for(c = 0, i = 0, rd = r->st->dimensions; rd && c < r->d ; c++, rd = rd->next) {
 436                         if(unlikely(r->od[c] & RRDR_HIDDEN)) continue;
 437                         if(unlikely(!(r->od[c] & RRDR_NONZERO))) continue;
 438                         i++;
 439                 }
 440
 441                 // if with nonzero we get i = 0
 442                 // but c != 0, then disable nonzero
 443                 // to show all dimensions
 444                 if(!i) options &= ~RRDR_OPTION_NONZERO;
 445         }
 446
 447         return options;
 448 }
 449
 450 void rrdr_json_wrapper_begin(RRDR *r, BUFFER *wb, uint32_t format, uint32_t options, int string_value)
 451 {
 452         long rows = rrdr_rows(r);
 453         long c, i;
 454         RRDDIM *rd;
 455
 456         //info("JSONWRAPPER(): %s: BEGIN", r->st->id);
 457         char kq[2] = "",                                        // key quote
 458                 sq[2] = "";                                             // string quote
 459
 460         if( options & RRDR_OPTION_GOOGLE_JSON ) {
 461                 kq[0] = '\0';
 462                 sq[0] = '\'';
 463         }
 464         else {
 465                 kq[0] = '"';
 466                 sq[0] = '"';
 467         }
 468
 469         buffer_sprintf(wb, "{\n"
 470                         "       %sapi%s: 1,\n"
 471                         "       %sid%s: %s%s%s,\n"
 472                         "       %sname%s: %s%s%s,\n"
 473                         "       %sview_update_every%s: %d,\n"
 474                         "       %supdate_every%s: %d,\n"
 475                         "       %sfirst_entry%s: %u,\n"
 476                         "       %slast_entry%s: %u,\n"
 477                         "       %sbefore%s: %u,\n"
 478                         "       %safter%s: %u,\n"
 479                         "       %sdimension_names%s: ["
 480                         , kq, kq
 481                         , kq, kq, sq, r->st->id, sq
 482                         , kq, kq, sq, r->st->name, sq
 483                         , kq, kq, r->update_every
 484                         , kq, kq, r->st->update_every
 485                         , kq, kq, rrdset_first_entry_t(r->st)
 486                         , kq, kq, rrdset_last_entry_t(r->st)
 487                         , kq, kq, r->before
 488                         , kq, kq, r->after
 489                         , kq, kq);
 490
 491         for(c = 0, i = 0, rd = r->st->dimensions; rd && c < r->d ;c++, rd = rd->next) {
 492                 if(unlikely(r->od[c] & RRDR_HIDDEN)) continue;
 493                 if(unlikely((options & RRDR_OPTION_NONZERO) && !(r->od[c] & RRDR_NONZERO))) continue;
 494
 495                 if(i) buffer_strcat(wb, ", ");
 496                 buffer_strcat(wb, sq);
 497                 buffer_strcat(wb, rd->name);
 498                 buffer_strcat(wb, sq);
 499                 i++;
 500         }
 501         if(!i) {
 502 #ifdef NETDATA_INTERNAL_CHECKS
 503                 error("RRDR is empty for %s (RRDR has %d dimensions, options is 0x%08x)", r->st->id, r->d, options);
 504 #endif
 505                 rows = 0;
 506                 buffer_strcat(wb, sq);
 507                 buffer_strcat(wb, "no data");
 508                 buffer_strcat(wb, sq);
 509         }
 510
 511         buffer_sprintf(wb, "],\n"
 512                         "       %sdimension_ids%s: ["
 513                         , kq, kq);
 514
 515         for(c = 0, i = 0, rd = r->st->dimensions; rd && c < r->d ;c++, rd = rd->next) {
 516                 if(unlikely(r->od[c] & RRDR_HIDDEN)) continue;
 517                 if(unlikely((options & RRDR_OPTION_NONZERO) && !(r->od[c] & RRDR_NONZERO))) continue;
 518
 519                 if(i) buffer_strcat(wb, ", ");
 520                 buffer_strcat(wb, sq);
 521                 buffer_strcat(wb, rd->id);
 522                 buffer_strcat(wb, sq);
 523                 i++;
 524         }
 525         if(!i) {
 526                 rows = 0;
 527                 buffer_strcat(wb, sq);
 528                 buffer_strcat(wb, "no data");
 529                 buffer_strcat(wb, sq);
 530         }
 531
 532         buffer_sprintf(wb, "],\n"
 533                         "       %slatest_values%s: ["
 534                         , kq, kq);
 535
 536         for(c = 0, i = 0, rd = r->st->dimensions; rd && c < r->d ;c++, rd = rd->next) {
 537                 if(unlikely(r->od[c] & RRDR_HIDDEN)) continue;
 538                 if(unlikely((options & RRDR_OPTION_NONZERO) && !(r->od[c] & RRDR_NONZERO))) continue;
 539
 540                 if(i) buffer_strcat(wb, ", ");
 541                 i++;
 542
 543                 storage_number n = rd->values[rrdset_last_slot(r->st)];
 544
 545                 if(!does_storage_number_exist(n))
 546                         buffer_strcat(wb, "null");
 547                 else
 548                         buffer_rrd_value(wb, unpack_storage_number(n));
 549         }
 550         if(!i) {
 551                 rows = 0;
 552                 buffer_strcat(wb, "null");
 553         }
 554
 555         buffer_sprintf(wb, "],\n"
 556                         "       %sview_latest_values%s: ["
 557                         , kq, kq);
 558
 559         i = 0;
 560         if(rows) {
 561                 for(c = 0, i = 0, rd = r->st->dimensions; rd && c < r->d ;c++, rd = rd->next) {
 562                         if(unlikely(r->od[c] & RRDR_HIDDEN)) continue;
 563                         if(unlikely((options & RRDR_OPTION_NONZERO) && !(r->od[c] & RRDR_NONZERO))) continue;
 564
 565                         if(i) buffer_strcat(wb, ", ");
 566                         i++;
 567
 568                         calculated_number *cn = &r->v[ (0) * r->d ];
 569                         uint8_t *co = &r->o[ (0) * r->d ];
 570
 571                         if(co[c] & RRDR_EMPTY)
 572                                 buffer_strcat(wb, "null");
 573                         else
 574                                 buffer_rrd_value(wb, cn[c]);
 575                 }
 576         }
 577         if(!i) {
 578                 rows = 0;
 579                 buffer_strcat(wb, "null");
 580         }
 581
 582         buffer_sprintf(wb, "],\n"
 583                         "       %sdimensions%s: %d,\n"
 584                         "       %spoints%s: %d,\n"
 585                         "       %sformat%s: %s"
 586                         , kq, kq, i
 587                         , kq, kq, rows
 588                         , kq, kq, sq
 589                         );
 590
 591         rrdr_buffer_print_format(wb, format);
 592
 593         buffer_sprintf(wb, "%s,\n"
 594                         "       %sresult%s: "
 595                         , sq
 596                         , kq, kq
 597                         );
 598
 599         if(string_value) buffer_strcat(wb, sq);
 600         //info("JSONWRAPPER(): %s: END", r->st->id);
 601 }
 602
 603 void rrdr_json_wrapper_end(RRDR *r, BUFFER *wb, uint32_t format, uint32_t options, int string_value)
 604 {
 605         if(r) {;}
 606         if(format) {;}
 607
 608         char kq[2] = "",                                        // key quote
 609                 sq[2] = "";                                             // string quote
 610
 611         if( options & RRDR_OPTION_GOOGLE_JSON ) {
 612                 kq[0] = '\0';
 613                 sq[0] = '\'';
 614         }
 615         else {
 616                 kq[0] = '"';
 617                 sq[0] = '"';
 618         }
 619
 620         if(string_value) buffer_strcat(wb, sq);
 621
 622         buffer_sprintf(wb, ",\n %smin%s: ", kq, kq);
 623         buffer_rrd_value(wb, r->min);
 624         buffer_sprintf(wb, ",\n %smax%s: ", kq, kq);
 625         buffer_rrd_value(wb, r->max);
 626         buffer_strcat(wb, "\n}\n");
 627 }
 628
 629 #define JSON_DATES_JS 1
 630 #define JSON_DATES_TIMESTAMP 2
 631
 632 static void rrdr2json(RRDR *r, BUFFER *wb, uint32_t options, int datatable)
 633 {
 634         //info("RRD2JSON(): %s: BEGIN", r->st->id);
 635         int row_annotations = 0, dates, dates_with_new = 0;
 636         char kq[2] = "",                                        // key quote
 637                 sq[2] = "",                                             // string quote
 638                 pre_label[101] = "",                    // before each label
 639                 post_label[101] = "",                   // after each label
 640                 pre_date[101] = "",                             // the beginning of line, to the date
 641                 post_date[101] = "",                    // closing the date
 642                 pre_value[101] = "",                    // before each value
 643                 post_value[101] = "",                   // after each value
 644                 post_line[101] = "",                    // at the end of each row
 645                 normal_annotation[201] = "",    // default row annotation
 646                 overflow_annotation[201] = "",  // overflow row annotation
 647                 data_begin[101] = "",                   // between labels and values
 648                 finish[101] = "";                               // at the end of everything
 649
 650         if(datatable) {
 651                 dates = JSON_DATES_JS;
 652                 if( options & RRDR_OPTION_GOOGLE_JSON ) {
 653                         kq[0] = '\0';
 654                         sq[0] = '\'';
 655                 }
 656                 else {
 657                         kq[0] = '"';
 658                         sq[0] = '"';
 659                 }
 660                 row_annotations = 1;
 661                 snprintf(pre_date,   100, "             {%sc%s:[{%sv%s:%s", kq, kq, kq, kq, sq);
 662                 snprintf(post_date,  100, "%s}", sq);
 663                 snprintf(pre_label,  100, ",\n          {%sid%s:%s%s,%slabel%s:%s", kq, kq, sq, sq, kq, kq, sq);
 664                 snprintf(post_label, 100, "%s,%spattern%s:%s%s,%stype%s:%snumber%s}", sq, kq, kq, sq, sq, kq, kq, sq, sq);
 665                 snprintf(pre_value,  100, ",{%sv%s:", kq, kq);
 666                 snprintf(post_value, 100, "}");
 667                 snprintf(post_line,  100, "]}");
 668                 snprintf(data_begin, 100, "\n   ],\n    %srows%s:\n     [\n", kq, kq);
 669                 snprintf(finish,     100, "\n   ]\n}");
 670
 671                 snprintf(overflow_annotation, 200, ",{%sv%s:%sRESET OR OVERFLOW%s},{%sv%s:%sThe counters have been wrapped.%s}", kq, kq, sq, sq, kq, kq, sq, sq);
 672                 snprintf(normal_annotation,   200, ",{%sv%s:null},{%sv%s:null}", kq, kq, kq, kq);
 673
 674                 buffer_sprintf(wb, "{\n %scols%s:\n     [\n", kq, kq, kq, kq);
 675                 buffer_sprintf(wb, "            {%sid%s:%s%s,%slabel%s:%stime%s,%spattern%s:%s%s,%stype%s:%sdatetime%s},\n", kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq);
 676                 buffer_sprintf(wb, "            {%sid%s:%s%s,%slabel%s:%s%s,%spattern%s:%s%s,%stype%s:%sstring%s,%sp%s:{%srole%s:%sannotation%s}},\n", kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, kq, kq, sq, sq);
 677                 buffer_sprintf(wb, "            {%sid%s:%s%s,%slabel%s:%s%s,%spattern%s:%s%s,%stype%s:%sstring%s,%sp%s:{%srole%s:%sannotationText%s}}", kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, kq, kq, sq, sq);
 678
 679                 // remove the valueobjects flag
 680                 // google wants its own keys
 681                 if(options & RRDR_OPTION_OBJECTSROWS)
 682                         options &= ~RRDR_OPTION_OBJECTSROWS;
 683         }
 684         else {
 685                 kq[0] = '"';
 686                 sq[0] = '"';
 687                 if((options & RRDR_OPTION_SECONDS) || (options & RRDR_OPTION_MILLISECONDS)) {
 688                         dates = JSON_DATES_TIMESTAMP;
 689                         dates_with_new = 0;
 690                 }
 691                 else {
 692                         dates = JSON_DATES_JS;
 693                         dates_with_new = 1;
 694                 }
 695                 if( options & RRDR_OPTION_OBJECTSROWS )
 696                         snprintf(pre_date,   100, "             { ");
 697                 else
 698                         snprintf(pre_date,   100, "             [ ");
 699                 snprintf(pre_label,  100, ", \"");
 700                 snprintf(post_label, 100, "\"");
 701                 snprintf(pre_value,  100, ", ");
 702                 if( options & RRDR_OPTION_OBJECTSROWS )
 703                         snprintf(post_line,  100, "}");
 704                 else
 705                         snprintf(post_line,  100, "]");
 706                 snprintf(data_begin, 100, "],\n %sdata%s:\n     [\n", kq, kq);
 707                 snprintf(finish,     100, "\n   ]\n}");
 708
 709                 buffer_sprintf(wb, "{\n %slabels%s: [", kq, kq);
 710                 buffer_sprintf(wb, "%stime%s", sq, sq);
 711         }
 712
 713         // -------------------------------------------------------------------------
 714         // print the JSON header
 715
 716         long c, i;
 717         RRDDIM *rd;
 718
 719         // print the header lines
 720         for(c = 0, i = 0, rd = r->st->dimensions; rd && c < r->d ;c++, rd = rd->next) {
 721                 if(unlikely(r->od[c] & RRDR_HIDDEN)) continue;
 722                 if(unlikely((options & RRDR_OPTION_NONZERO) && !(r->od[c] & RRDR_NONZERO))) continue;
 723
 724                 buffer_strcat(wb, pre_label);
 725                 buffer_strcat(wb, rd->name);
 726                 buffer_strcat(wb, post_label);
 727                 i++;
 728         }
 729         if(!i) {
 730                 buffer_strcat(wb, pre_label);
 731                 buffer_strcat(wb, "no data");
 732                 buffer_strcat(wb, post_label);
 733         }
 734
 735         // print the begin of row data
 736         buffer_strcat(wb, data_begin);
 737
 738         // if all dimensions are hidden, print a null
 739         if(!i) {
 740                 buffer_strcat(wb, finish);
 741                 return;
 742         }
 743
 744         long start = 0, end = rrdr_rows(r), step = 1;
 745         if((options & RRDR_OPTION_REVERSED)) {
 746                 start = rrdr_rows(r) - 1;
 747                 end = -1;
 748                 step = -1;
 749         }
 750
 751         // for each line in the array
 752         for(i = start; i != end ;i += step) {
 753                 calculated_number *cn = &r->v[ i * r->d ];
 754                 uint8_t *co = &r->o[ i * r->d ];
 755
 756                 time_t now = r->t[i];
 757
 758                 if(dates == JSON_DATES_JS) {
 759                         // generate the local date time
 760                         struct tm tmbuf, *tm = localtime_r(&now, &tmbuf);
 761                         if(!tm) { error("localtime_r() failed."); continue; }
 762
 763                         if(likely(i != start)) buffer_strcat(wb, ",\n");
 764                         buffer_strcat(wb, pre_date);
 765
 766                         if( options & RRDR_OPTION_OBJECTSROWS )
 767                                 buffer_sprintf(wb, "%stime%s: ", kq, kq);
 768
 769                         if(dates_with_new)
 770                                 buffer_strcat(wb, "new ");
 771
 772                         buffer_jsdate(wb, tm->tm_year + 1900, tm->tm_mon, tm->tm_mday, tm->tm_hour, tm->tm_min, tm->tm_sec);
 773
 774                         buffer_strcat(wb, post_date);
 775
 776                         if(row_annotations) {
 777                                 // google supports one annotation per row
 778                                 int annotation_found = 0;
 779                                 for(c = 0, rd = r->st->dimensions; rd ;c++, rd = rd->next) {
 780                                         if(co[c] & RRDR_RESET) {
 781                                                 buffer_strcat(wb, overflow_annotation);
 782                                                 annotation_found = 1;
 783                                                 break;
 784                                         }
 785                                 }
 786                                 if(!annotation_found)
 787                                         buffer_strcat(wb, normal_annotation);
 788                         }
 789                 }
 790                 else {
 791                         // print the timestamp of the line
 792                         if(likely(i != start)) buffer_strcat(wb, ",\n");
 793                         buffer_strcat(wb, pre_date);
 794
 795                         if( options & RRDR_OPTION_OBJECTSROWS )
 796                                 buffer_sprintf(wb, "%stime%s: ", kq, kq);
 797
 798                         buffer_rrd_value(wb, (calculated_number)r->t[i]);
 799                         // in ms
 800                         if(options & RRDR_OPTION_MILLISECONDS) buffer_strcat(wb, "000");
 801
 802                         buffer_strcat(wb, post_date);
 803                 }
 804
 805                 // for each dimension
 806                 for(c = 0, rd = r->st->dimensions; rd && c < r->d ;c++, rd = rd->next) {
 807                         if(unlikely(r->od[c] & RRDR_HIDDEN)) continue;
 808                         if(unlikely((options & RRDR_OPTION_NONZERO) && !(r->od[c] & RRDR_NONZERO))) continue;
 809
 810                         calculated_number n = cn[c];
 811
 812                         buffer_strcat(wb, pre_value);
 813
 814                         if( options & RRDR_OPTION_OBJECTSROWS )
 815                                 buffer_sprintf(wb, "%s%s%s: ", kq, rd->name, kq);
 816
 817                         if(co[c] & RRDR_EMPTY) {
 818                                 if(options & RRDR_OPTION_NULL2ZERO)
 819                                         buffer_strcat(wb, "0");
 820                                 else
 821                                         buffer_strcat(wb, "null");
 822                         }
 823                         else if((options & RRDR_OPTION_ABSOLUTE))
 824                                 buffer_rrd_value(wb, (n<0)?-n:n);
 825                         else
 826                                 buffer_rrd_value(wb, n);
 827
 828                         buffer_strcat(wb, post_value);
 829                 }
 830
 831                 buffer_strcat(wb, post_line);
 832         }
 833
 834         buffer_strcat(wb, finish);
 835         //info("RRD2JSON(): %s: END", r->st->id);
 836 }
 837
 838 static void rrdr2csv(RRDR *r, BUFFER *wb, uint32_t options, const char *startline, const char *separator, const char *endline)
 839 {
 840         //info("RRD2CSV(): %s: BEGIN", r->st->id);
 841         long c, i;
 842         RRDDIM *d;
 843
 844         // print the csv header
 845         for(c = 0, i = 0, d = r->st->dimensions; d && c < r->d ;c++, d = d->next) {
 846                 if(unlikely(r->od[c] & RRDR_HIDDEN)) continue;
 847                 if(unlikely((options & RRDR_OPTION_NONZERO) && !(r->od[c] & RRDR_NONZERO))) continue;
 848
 849                 if(!i) {
 850                         buffer_strcat(wb, startline);
 851                         buffer_strcat(wb, "time");
 852                 }
 853                 buffer_strcat(wb, separator);
 854                 buffer_strcat(wb, d->name);
 855                 i++;
 856         }
 857         buffer_strcat(wb, endline);
 858
 859         if(!i) {
 860                 // no dimensions present
 861                 return;
 862         }
 863
 864         long start = 0, end = rrdr_rows(r), step = 1;
 865         if((options & RRDR_OPTION_REVERSED)) {
 866                 start = rrdr_rows(r) - 1;
 867                 end = -1;
 868                 step = -1;
 869         }
 870
 871         // for each line in the array
 872         for(i = start; i != end ;i += step) {
 873                 calculated_number *cn = &r->v[ i * r->d ];
 874                 uint8_t *co = &r->o[ i * r->d ];
 875
 876                 buffer_strcat(wb, startline);
 877
 878                 time_t now = r->t[i];
 879
 880                 if((options & RRDR_OPTION_SECONDS) || (options & RRDR_OPTION_MILLISECONDS)) {
 881                         // print the timestamp of the line
 882                         buffer_rrd_value(wb, (calculated_number)now);
 883                         // in ms
 884                         if(options & RRDR_OPTION_MILLISECONDS) buffer_strcat(wb, "000");
 885                 }
 886                 else {
 887                         // generate the local date time
 888                         struct tm tmbuf, *tm = localtime_r(&now, &tmbuf);
 889                         if(!tm) { error("localtime() failed."); continue; }
 890                         buffer_date(wb, tm->tm_year + 1900, tm->tm_mon, tm->tm_mday, tm->tm_hour, tm->tm_min, tm->tm_sec);
 891                 }
 892
 893                 // for each dimension
 894                 for(c = 0, d = r->st->dimensions; d && c < r->d ;c++, d = d->next) {
 895                         if(unlikely(r->od[c] & RRDR_HIDDEN)) continue;
 896                         if(unlikely((options & RRDR_OPTION_NONZERO) && !(r->od[c] & RRDR_NONZERO))) continue;
 897
 898                         buffer_strcat(wb, separator);
 899
 900                         calculated_number n = cn[c];
 901
 902                         if(co[c] & RRDR_EMPTY) {
 903                                 if(options & RRDR_OPTION_NULL2ZERO)
 904                                         buffer_strcat(wb, "0");
 905                                 else
 906                                         buffer_strcat(wb, "null");
 907                         }
 908                         else if((options & RRDR_OPTION_ABSOLUTE))
 909                                 buffer_rrd_value(wb, (n<0)?-n:n);
 910                         else
 911                                 buffer_rrd_value(wb, n);
 912                 }
 913
 914                 buffer_strcat(wb, endline);
 915         }
 916         //info("RRD2CSV(): %s: END", r->st->id);
 917 }
 918
 919 static void rrdr2ssv(RRDR *r, BUFFER *wb, uint32_t options, const char *prefix, const char *separator, const char *suffix)
 920 {
 921         //info("RRD2SSV(): %s: BEGIN", r->st->id);
 922         long c, i;
 923         RRDDIM *d;
 924
 925         buffer_strcat(wb, prefix);
 926         long start = 0, end = rrdr_rows(r), step = 1;
 927         if((options & RRDR_OPTION_REVERSED)) {
 928                 start = rrdr_rows(r) - 1;
 929                 end = -1;
 930                 step = -1;
 931         }
 932
 933         // for each line in the array
 934         for(i = start; i != end ;i += step) {
 935
 936                 calculated_number *cn = &r->v[ i * r->d ];
 937                 uint8_t *co = &r->o[ i * r->d ];
 938
 939                 calculated_number sum = 0, min = 0, max = 0, v;
 940                 int all_null = 1, init = 1;
 941
 942                 // for each dimension
 943                 for(c = 0, d = r->st->dimensions; d && c < r->d ;c++, d = d->next) {
 944                         if(unlikely(r->od[c] & RRDR_HIDDEN)) continue;
 945                         if(unlikely((options & RRDR_OPTION_NONZERO) && !(r->od[c] & RRDR_NONZERO))) continue;
 946
 947                         calculated_number n = cn[c];
 948
 949                         if(unlikely(init)) {
 950                                 if(n > 0) {
 951                                         min = 0;
 952                                         max = n;
 953                                 }
 954                                 else {
 955                                         min = n;
 956                                         max = 0;
 957                                 }
 958                                 init = 0;
 959                         }
 960
 961                         if(likely(!(co[c] & RRDR_EMPTY))) {
 962                                 all_null = 0;
 963                                 if((options & RRDR_OPTION_ABSOLUTE) && n < 0) n = -n;
 964                                 sum += n;
 965                         }
 966
 967                         if(n < min) min = n;
 968                         if(n > max) max = n;
 969                 }
 970
 971                 if(likely(i != start))
 972                         buffer_strcat(wb, separator);
 973
 974                 if(all_null) {
 975                         if(options & RRDR_OPTION_NULL2ZERO)
 976                                 buffer_strcat(wb, "0");
 977                         else
 978                                 buffer_strcat(wb, "null");
 979                 }
 980                 else {
 981                         if(options & RRDR_OPTION_MIN2MAX)
 982                                 v = max - min;
 983                         else
 984                                 v = sum;
 985
 986                         if(likely(i != start)) {
 987                                 if(r->min > v) r->min = v;
 988                                 if(r->max < v) r->max = v;
 989                         }
 990                         else {
 991                                 r->min = v;
 992                                 r->max = v;
 993                         }
 994
 995                         buffer_rrd_value(wb, v);
 996                 }
 997         }
 998         buffer_strcat(wb, suffix);
 999         //info("RRD2SSV(): %s: END", r->st->id);
1000 }
1001
1002 inline static calculated_number *rrdr_line_values(RRDR *r)
1003 {
1004         return &r->v[ r->c * r->d ];
1005 }
1006
1007 inline static uint8_t *rrdr_line_options(RRDR *r)
1008 {
1009         return &r->o[ r->c * r->d ];
1010 }
1011
1012 inline static int rrdr_line_init(RRDR *r, time_t t)
1013 {
1014         r->c++;
1015
1016         if(unlikely(r->c >= r->n)) {
1017                 error("requested to step above RRDR size for chart %s", r->st->name);
1018                 r->c = r->n - 1;
1019         }
1020
1021         // save the time
1022         r->t[r->c] = t;
1023
1024         return 1;
1025 }
1026
1027 inline static void rrdr_lock_rrdset(RRDR *r) {
1028         if(unlikely(!r)) {
1029                 error("NULL value given!");
1030                 return;
1031         }
1032
1033         pthread_rwlock_rdlock(&r->st->rwlock);
1034         r->has_st_lock = 1;
1035 }
1036
1037 inline static void rrdr_unlock_rrdset(RRDR *r) {
1038         if(unlikely(!r)) {
1039                 error("NULL value given!");
1040                 return;
1041         }
1042
1043         if(likely(r->has_st_lock)) {
1044                 pthread_rwlock_unlock(&r->st->rwlock);
1045                 r->has_st_lock = 0;
1046         }
1047 }
1048
1049 inline static void rrdr_free(RRDR *r)
1050 {
1051         if(unlikely(!r)) {
1052                 error("NULL value given!");
1053                 return;
1054         }
1055
1056         rrdr_unlock_rrdset(r);
1057         if(likely(r->t)) free(r->t);
1058         if(likely(r->v)) free(r->v);
1059         if(likely(r->o)) free(r->o);
1060         if(likely(r->od)) free(r->od);
1061         free(r);
1062 }
1063
1064 inline void rrdr_done(RRDR *r)
1065 {
1066         r->rows = r->c + 1;
1067         r->c = 0;
1068 }
1069
1070 static RRDR *rrdr_create(RRDSET *st, long n)
1071 {
1072         if(unlikely(!st)) {
1073                 error("NULL value given!");
1074                 return NULL;
1075         }
1076
1077         RRDR *r = calloc(1, sizeof(RRDR));
1078         if(unlikely(!r)) goto cleanup;
1079
1080         r->st = st;
1081
1082         rrdr_lock_rrdset(r);
1083
1084         RRDDIM *rd;
1085         for(rd = st->dimensions ; rd ; rd = rd->next) r->d++;
1086
1087         r->n = n;
1088
1089         r->t = malloc(n * sizeof(time_t));
1090         if(unlikely(!r->t)) goto cleanup;
1091
1092         r->v = malloc(n * r->d * sizeof(calculated_number));
1093         if(unlikely(!r->v)) goto cleanup;
1094
1095         r->o = malloc(n * r->d * sizeof(uint8_t));
1096         if(unlikely(!r->o)) goto cleanup;
1097
1098         r->od = malloc(r->d * sizeof(uint8_t));
1099         if(unlikely(!r->od)) goto cleanup;
1100
1101         // set the hidden flag on hidden dimensions
1102         int c;
1103         for(c = 0, rd = st->dimensions ; rd ; c++, rd = rd->next) {
1104                 if(unlikely(rd->flags & RRDDIM_FLAG_HIDDEN)) r->od[c] = RRDR_HIDDEN;
1105                 else r->od[c] = 0;
1106         }
1107
1108         r->c = -1;
1109
1110         r->group = 1;
1111         r->update_every = 1;
1112
1113         return r;
1114
1115 cleanup:
1116         error("Cannot allocate RRDR memory for %d entries", n);
1117         if(likely(r)) rrdr_free(r);
1118         return NULL;
1119 }
1120
1121 RRDR *rrd2rrdr(RRDSET *st, long points, long long after, long long before, int group_method)
1122 {
1123         int debug = st->debug;
1124
1125         time_t first_entry_t = rrdset_first_entry_t(st);
1126         time_t last_entry_t = rrdset_last_entry_t(st);
1127
1128         if(before == 0 && after == 0) {
1129                 before = last_entry_t;
1130                 after = first_entry_t;
1131         }
1132
1133         // allow relative for before and after
1134         if(before <= st->update_every * st->entries)
1135                 before = last_entry_t + before;
1136
1137         if(after <= st->update_every * st->entries)
1138                 after = last_entry_t + after;
1139
1140         // make sure they are within our timeframe
1141         if(before > last_entry_t) before = last_entry_t;
1142         if(before < first_entry_t) before = first_entry_t;
1143
1144         if(after > last_entry_t) after = last_entry_t;
1145         if(after < first_entry_t) after = first_entry_t;
1146
1147         // check if they are upside down
1148         if(after > before) {
1149                 time_t tmp = before;
1150                 before = after;
1151                 after = tmp;
1152         }
1153
1154         // the duration of the chart
1155         time_t duration = before - after;
1156         long available_points = duration / st->update_every;
1157
1158         if(duration <= 0 || available_points <= 0)
1159                 return rrdr_create(st, 1);
1160
1161         // check the wanted points
1162         if(points < 0) points = -points;
1163         if(points > available_points) points = available_points;
1164         if(points == 0) points = available_points;
1165
1166         // calculate proper grouping of source data
1167         long group = available_points / points;
1168         if(group <= 0) group = 1;
1169
1170         // round group to the closest integer
1171         if(available_points % points > points / 2) group++;
1172
1173         time_t after_new = after - (after % (group * st->update_every));
1174         time_t before_new = before - (before % (group * st->update_every));
1175         long points_new = (before_new - after_new) / st->update_every / group;
1176
1177         // find the starting and ending slots in our round robin db
1178         long    start_at_slot = rrdset_time2slot(st, before_new),
1179                         stop_at_slot = rrdset_time2slot(st, after_new);
1180
1181 #ifdef NETDATA_INTERNAL_CHECKS
1182         if(after_new < first_entry_t) {
1183                 error("after_new %u is too small, minimum %u", after_new, first_entry_t);
1184         }
1185         if(after_new > last_entry_t) {
1186                 error("after_new %u is too big, maximum %u", after_new, last_entry_t);
1187         }
1188         if(before_new < first_entry_t) {
1189                 error("before_new %u is too small, minimum %u", before_new, first_entry_t);
1190         }
1191         if(before_new > last_entry_t) {
1192                 error("before_new %u is too big, maximum %u", before_new, last_entry_t);
1193         }
1194         if(start_at_slot < 0 || start_at_slot >= st->entries) {
1195                 error("start_at_slot is invalid %ld, expected %ld to %ld", start_at_slot, 0, st->entries - 1);
1196         }
1197         if(stop_at_slot < 0 || stop_at_slot >= st->entries) {
1198                 error("stop_at_slot is invalid %ld, expected %ld to %ld", stop_at_slot, 0, st->entries - 1);
1199         }
1200         if(points_new > (before_new - after_new) / group / st->update_every + 1) {
1201                 error("points_new %ld is more than points %ld", points_new, (before_new - after_new) / group / st->update_every + 1);
1202         }
1203 #endif
1204
1205         //info("RRD2RRDR(): %s: wanted %ld points, got %ld - group=%ld, wanted duration=%u, got %u - wanted %ld - %ld, got %ld - %ld", st->id, points, points_new, group, before - after, before_new - after_new, after, before, after_new, before_new);
1206
1207         after = after_new;
1208         before = before_new;
1209         duration = before - after;
1210         points = points_new;
1211
1212         // Now we have:
1213         // before = the end time of the calculation
1214         // after = the start time of the calculation
1215         // duration = the duration of the calculation
1216         // group = the number of source points to aggregate / group together
1217         // method = the method of grouping source points
1218         // points = the number of points to generate
1219
1220
1221         // -------------------------------------------------------------------------
1222         // initialize our result set
1223
1224         RRDR *r = rrdr_create(st, points);
1225         if(!r) {
1226 #ifdef NETDATA_INTERNAL_CHECKS
1227                 error("Cannot create RRDR for %s, after=%u, before=%u, duration=%u, points=%d", st->id, after, before, duration, points);
1228 #endif
1229                 return NULL;
1230         }
1231         if(!r->d) {
1232 #ifdef NETDATA_INTERNAL_CHECKS
1233                 error("Returning empty RRDR (no dimensions in RRDSET) for %s, after=%u, before=%u, duration=%u, points=%d", st->id, after, before, duration, points);
1234 #endif
1235                 return r;
1236         }
1237
1238         // find how many dimensions we have
1239         long dimensions = r->d;
1240
1241
1242         // -------------------------------------------------------------------------
1243         // checks for debugging
1244
1245         if(debug) debug(D_RRD_STATS, "INFO %s first_t: %lu, last_t: %lu, all_duration: %lu, after: %lu, before: %lu, duration: %lu, points: %ld, group: %ld"
1246                         , st->id
1247                         , first_entry_t
1248                         , last_entry_t
1249                         , last_entry_t - first_entry_t
1250                         , after
1251                         , before
1252                         , duration
1253                         , points
1254                         , group
1255                         );
1256
1257
1258         // -------------------------------------------------------------------------
1259         // temp arrays for keeping values per dimension
1260
1261         calculated_number       group_values[dimensions]; // keep sums when grouping
1262         long                            group_counts[dimensions]; // keep the number of values added to group_values
1263         uint8_t                         group_options[dimensions];
1264         uint8_t                         found_non_zero[dimensions];
1265
1266
1267         // initialize them
1268         RRDDIM *rd;
1269         long c;
1270         for( rd = st->dimensions, c = 0 ; rd && c < dimensions ; rd = rd->next, c++) {
1271                 group_values[c] = 0;
1272                 group_counts[c] = 0;
1273                 group_options[c] = 0;
1274                 found_non_zero[c] = 0;
1275         }
1276
1277
1278         // -------------------------------------------------------------------------
1279         // the main loop
1280
1281         time_t  now = rrdset_slot2time(st, start_at_slot),
1282                         dt = st->update_every,
1283                         group_start_t = 0;
1284
1285         if(unlikely(debug)) debug(D_RRD_STATS, "BEGIN %s after_t: %lu (stop_at_t: %ld), before_t: %lu (start_at_t: %ld), start_t(now): %lu, current_entry: %ld, entries: %ld"
1286                         , st->id
1287                         , after
1288                         , stop_at_slot
1289                         , before
1290                         , start_at_slot
1291                         , now
1292                         , st->current_entry
1293                         , st->entries
1294                         );
1295
1296         r->group = group;
1297         r->update_every = group * st->update_every;
1298         r->before = now;
1299         r->after = now;
1300
1301         //info("RRD2RRDR(): %s: STARTING", st->id);
1302
1303         long slot = start_at_slot, counter = 0, stop_now = 0, added = 0, group_count = 0, add_this = 0;
1304         for(; !stop_now ; now -= dt, slot--, counter++) {
1305                 if(unlikely(slot < 0)) slot = st->entries - 1;
1306                 if(unlikely(slot == stop_at_slot)) stop_now = counter;
1307
1308                 if(unlikely(debug)) debug(D_RRD_STATS, "ROW %s slot: %ld, entries_counter: %ld, group_count: %ld, added: %ld, now: %lu, %s %s"
1309                                 , st->id
1310                                 , slot
1311                                 , counter
1312                                 , group_count + 1
1313                                 , added
1314                                 , now
1315                                 , (group_count + 1 == group)?"PRINT":"  -  "
1316                                 , (now >= after && now <= before)?"RANGE":"  -  "
1317                                 );
1318
1319                 // make sure we return data in the proper time range
1320                 if(unlikely(now > before)) continue;
1321                 if(unlikely(now < after)) break;
1322
1323                 if(unlikely(group_count == 0)) {
1324                         group_start_t = now;
1325                 }
1326                 group_count++;
1327
1328                 if(unlikely(group_count == group)) {
1329                         if(unlikely(added >= points)) break;
1330                         add_this = 1;
1331                 }
1332
1333                 // do the calculations
1334                 for(rd = st->dimensions, c = 0 ; rd && c < dimensions ; rd = rd->next, c++) {
1335                         storage_number n = rd->values[slot];
1336                         if(unlikely(!does_storage_number_exist(n))) continue;
1337
1338                         group_counts[c]++;
1339
1340                         calculated_number value = unpack_storage_number(n);
1341                         if(likely(value != 0.0)) {
1342                                 group_options[c] |= RRDR_NONZERO;
1343                                 found_non_zero[c] = 1;
1344                         }
1345
1346                         if(unlikely(did_storage_number_reset(n)))
1347                                 group_options[c] |= RRDR_RESET;
1348
1349                         switch(group_method) {
1350                                 case GROUP_MAX:
1351                                         if(unlikely(abs(value) > abs(group_values[c])))
1352                                                 group_values[c] = value;
1353                                         break;
1354
1355                                 default:
1356                                 case GROUP_SUM:
1357                                 case GROUP_AVERAGE:
1358                                         group_values[c] += value;
1359                                         break;
1360                         }
1361                 }
1362
1363                 // added it
1364                 if(unlikely(add_this)) {
1365                         if(unlikely(!rrdr_line_init(r, group_start_t))) break;
1366
1367                         r->after = now;
1368
1369                         calculated_number *cn = rrdr_line_values(r);
1370                         uint8_t *co = rrdr_line_options(r);
1371
1372                         for(rd = st->dimensions, c = 0 ; rd && c < dimensions ; rd = rd->next, c++) {
1373
1374                                 // update the dimension options
1375                                 if(likely(found_non_zero[c])) r->od[c] |= RRDR_NONZERO;
1376
1377                                 // store the specific point options
1378                                 co[c] = group_options[c];
1379
1380                                 // store the value
1381                                 if(unlikely(group_counts[c] == 0)) {
1382                                         cn[c] = 0.0;
1383                                         co[c] |= RRDR_EMPTY;
1384                                 }
1385                                 else if(unlikely(group_method == GROUP_AVERAGE)) {
1386                                         cn[c] = group_values[c] / group_counts[c];
1387                                 }
1388                                 else {
1389                                         cn[c] = group_values[c];
1390                                 }
1391
1392                                 if(cn[c] < r->min) r->min = cn[c];
1393                                 if(cn[c] > r->max) r->max = cn[c];
1394
1395                                 // reset them for the next loop
1396                                 group_values[c] = 0;
1397                                 group_counts[c] = 0;
1398                                 group_options[c] = 0;
1399                         }
1400
1401                         added++;
1402                         group_count = 0;
1403                         add_this = 0;
1404                 }
1405         }
1406
1407         rrdr_done(r);
1408         //info("RRD2RRDR(): %s: END %ld loops made, %ld points generated", st->id, counter, rrdr_rows(r));
1409         //error("SHIFT: %s: wanted %ld points, got %ld", st->id, points, rrdr_rows(r));
1410         return r;
1411 }
1412
1413 int rrd2format(RRDSET *st, BUFFER *wb, BUFFER *dimensions, uint32_t format, long points, long long after, long long before, int group_method, uint32_t options, time_t *latest_timestamp)
1414 {
1415         RRDR *r = rrd2rrdr(st, points, after, before, group_method);
1416         if(!r) {
1417                 buffer_strcat(wb, "Cannot generate output with these parameters on this chart.");
1418                 return 500;
1419         }
1420
1421         options = rrdr_check_options(r, options);
1422
1423         if(dimensions)
1424                 rrdr_disable_not_selected_dimensions(r, buffer_tostring(dimensions));
1425
1426         if(latest_timestamp && rrdr_rows(r) > 0)
1427                 *latest_timestamp = r->before;
1428
1429         switch(format) {
1430         case DATASOURCE_SSV:
1431                 if(options & RRDR_OPTION_JSON_WRAP) {
1432                         wb->contenttype = CT_APPLICATION_JSON;
1433                         rrdr_json_wrapper_begin(r, wb, format, options, 1);
1434                         rrdr2ssv(r, wb, options, "", " ", "");
1435                         rrdr_json_wrapper_end(r, wb, format, options, 1);
1436                 }
1437                 else {
1438                         wb->contenttype = CT_TEXT_PLAIN;
1439                         rrdr2ssv(r, wb, options, "", " ", "");
1440                 }
1441                 break;
1442
1443         case DATASOURCE_SSV_COMMA:
1444                 if(options & RRDR_OPTION_JSON_WRAP) {
1445                         wb->contenttype = CT_APPLICATION_JSON;
1446                         rrdr_json_wrapper_begin(r, wb, format, options, 1);
1447                         rrdr2ssv(r, wb, options, "", ",", "");
1448                         rrdr_json_wrapper_end(r, wb, format, options, 1);
1449                 }
1450                 else {
1451                         wb->contenttype = CT_TEXT_PLAIN;
1452                         rrdr2ssv(r, wb, options, "", ",", "");
1453                 }
1454                 break;
1455
1456         case DATASOURCE_JS_ARRAY:
1457                 if(options & RRDR_OPTION_JSON_WRAP) {
1458                         wb->contenttype = CT_APPLICATION_JSON;
1459                         rrdr_json_wrapper_begin(r, wb, format, options, 0);
1460                         rrdr2ssv(r, wb, options, "[", ",", "]");
1461                         rrdr_json_wrapper_end(r, wb, format, options, 0);
1462                 }
1463                 else {
1464                         wb->contenttype = CT_APPLICATION_JSON;
1465                         rrdr2ssv(r, wb, options, "[", ",", "]");
1466                 }
1467                 break;
1468
1469         case DATASOURCE_CSV:
1470                 if(options & RRDR_OPTION_JSON_WRAP) {
1471                         wb->contenttype = CT_APPLICATION_JSON;
1472                         rrdr_json_wrapper_begin(r, wb, format, options, 1);
1473                         rrdr2csv(r, wb, options, "", ",", "\\n");
1474                         rrdr_json_wrapper_end(r, wb, format, options, 1);
1475                 }
1476                 else {
1477                         wb->contenttype = CT_TEXT_PLAIN;
1478                         rrdr2csv(r, wb, options, "", ",", "\r\n");
1479                 }
1480                 break;
1481
1482         case DATASOURCE_TSV:
1483                 if(options & RRDR_OPTION_JSON_WRAP) {
1484                         wb->contenttype = CT_APPLICATION_JSON;
1485                         rrdr_json_wrapper_begin(r, wb, format, options, 1);
1486                         rrdr2csv(r, wb, options, "", "\t", "\\n");
1487                         rrdr_json_wrapper_end(r, wb, format, options, 1);
1488                 }
1489                 else {
1490                         wb->contenttype = CT_TEXT_PLAIN;
1491                         rrdr2csv(r, wb, options, "", "\t", "\r\n");
1492                 }
1493                 break;
1494
1495         case DATASOURCE_HTML:
1496                 if(options & RRDR_OPTION_JSON_WRAP) {
1497                         wb->contenttype = CT_APPLICATION_JSON;
1498                         rrdr_json_wrapper_begin(r, wb, format, options, 1);
1499                         buffer_strcat(wb, "<html>\\n<center>\\n<table border=\\\"0\\\" cellpadding=\\\"5\\\" cellspacing=\\\"5\\\">\\n");
1500                         rrdr2csv(r, wb, options, "<tr><td>", "</td><td>", "</td></tr>\\n");
1501                         buffer_strcat(wb, "</table>\\n</center>\\n</html>\\n");
1502                         rrdr_json_wrapper_end(r, wb, format, options, 1);
1503                 }
1504                 else {
1505                         wb->contenttype = CT_TEXT_HTML;
1506                         buffer_strcat(wb, "<html>\n<center>\n<table border=\"0\" cellpadding=\"5\" cellspacing=\"5\">\n");
1507                         rrdr2csv(r, wb, options, "<tr><td>", "</td><td>", "</td></tr>\n");
1508                         buffer_strcat(wb, "</table>\n</center>\n</html>\n");
1509                 }
1510                 break;
1511
1512         case DATASOURCE_DATATABLE_JSONP:
1513                 wb->contenttype = CT_APPLICATION_X_JAVASCRIPT;
1514
1515                 if(options & RRDR_OPTION_JSON_WRAP)
1516                         rrdr_json_wrapper_begin(r, wb, format, options, 0);
1517
1518                 rrdr2json(r, wb, options, 1);
1519
1520                 if(options & RRDR_OPTION_JSON_WRAP)
1521                         rrdr_json_wrapper_end(r, wb, format, options, 0);
1522                 break;
1523
1524         case DATASOURCE_DATATABLE_JSON:
1525                 wb->contenttype = CT_APPLICATION_JSON;
1526
1527                 if(options & RRDR_OPTION_JSON_WRAP)
1528                         rrdr_json_wrapper_begin(r, wb, format, options, 0);
1529
1530                 rrdr2json(r, wb, options, 1);
1531
1532                 if(options & RRDR_OPTION_JSON_WRAP)
1533                         rrdr_json_wrapper_end(r, wb, format, options, 0);
1534                 break;
1535
1536         case DATASOURCE_JSONP:
1537                 wb->contenttype = CT_APPLICATION_X_JAVASCRIPT;
1538                 if(options & RRDR_OPTION_JSON_WRAP)
1539                         rrdr_json_wrapper_begin(r, wb, format, options, 0);
1540
1541                 rrdr2json(r, wb, options, 0);
1542
1543                 if(options & RRDR_OPTION_JSON_WRAP)
1544                         rrdr_json_wrapper_end(r, wb, format, options, 0);
1545                 break;
1546
1547         case DATASOURCE_JSON:
1548         default:
1549                 wb->contenttype = CT_APPLICATION_JSON;
1550
1551                 if(options & RRDR_OPTION_JSON_WRAP)
1552                         rrdr_json_wrapper_begin(r, wb, format, options, 0);
1553
1554                 rrdr2json(r, wb, options, 0);
1555
1556                 if(options & RRDR_OPTION_JSON_WRAP)
1557                         rrdr_json_wrapper_end(r, wb, format, options, 0);
1558                 break;
1559         }
1560
1561         rrdr_free(r);
1562         return 200;
1563 }
1564
1565 time_t rrd_stats_json(int type, RRDSET *st, BUFFER *wb, long points, long group, int group_method, time_t after, time_t before, int only_non_zero)
1566 {
1567         int c;
1568         pthread_rwlock_rdlock(&st->rwlock);
1569
1570
1571         // -------------------------------------------------------------------------
1572         // switch from JSON to google JSON
1573
1574         char kq[2] = "\"";
1575         char sq[2] = "\"";
1576         switch(type) {
1577                 case DATASOURCE_DATATABLE_JSON:
1578                 case DATASOURCE_DATATABLE_JSONP:
1579                         kq[0] = '\0';
1580                         sq[0] = '\'';
1581                         break;
1582
1583                 case DATASOURCE_JSON:
1584                 default:
1585                         break;
1586         }
1587
1588
1589         // -------------------------------------------------------------------------
1590         // validate the parameters
1591
1592         if(points < 1) points = 1;
1593         if(group < 1) group = 1;
1594
1595         if(before == 0 || before > rrdset_last_entry_t(st)) before = rrdset_last_entry_t(st);
1596         if(after  == 0 || after < rrdset_first_entry_t(st)) after = rrdset_first_entry_t(st);
1597
1598         // ---
1599
1600         // our return value (the last timestamp printed)
1601         // this is required to detect re-transmit in google JSONP
1602         time_t last_timestamp = 0;
1603
1604
1605         // -------------------------------------------------------------------------
1606         // find how many dimensions we have
1607
1608         int dimensions = 0;
1609         RRDDIM *rd;
1610         for( rd = st->dimensions ; rd ; rd = rd->next) dimensions++;
1611         if(!dimensions) {
1612                 pthread_rwlock_unlock(&st->rwlock);
1613                 buffer_strcat(wb, "No dimensions yet.");
1614                 return 0;
1615         }
1616
1617
1618         // -------------------------------------------------------------------------
1619         // prepare various strings, to speed up the loop
1620
1621         char overflow_annotation[201]; snprintf(overflow_annotation, 200, ",{%sv%s:%sRESET OR OVERFLOW%s},{%sv%s:%sThe counters have been wrapped.%s}", kq, kq, sq, sq, kq, kq, sq, sq);
1622         char normal_annotation[201];   snprintf(normal_annotation,   200, ",{%sv%s:null},{%sv%s:null}", kq, kq, kq, kq);
1623         char pre_date[51];             snprintf(pre_date,             50, "             {%sc%s:[{%sv%s:%s", kq, kq, kq, kq, sq);
1624         char post_date[21];            snprintf(post_date,            20, "%s}", sq);
1625         char pre_value[21];            snprintf(pre_value,            20, ",{%sv%s:", kq, kq);
1626         char post_value[21];           snprintf(post_value,           20, "}");
1627
1628
1629         // -------------------------------------------------------------------------
1630         // checks for debugging
1631
1632         if(st->debug) {
1633                 debug(D_RRD_STATS, "%s first_entry_t = %lu, last_entry_t = %lu, duration = %lu, after = %lu, before = %lu, duration = %lu, entries_to_show = %lu, group = %lu"
1634                         , st->id
1635                         , rrdset_first_entry_t(st)
1636                         , rrdset_last_entry_t(st)
1637                         , rrdset_last_entry_t(st) - rrdset_first_entry_t(st)
1638                         , after
1639                         , before
1640                         , before - after
1641                         , points
1642                         , group
1643                         );
1644
1645                 if(before < after)
1646                         debug(D_RRD_STATS, "WARNING: %s The newest value in the database (%lu) is earlier than the oldest (%lu)", st->name, before, after);
1647
1648                 if((before - after) > st->entries * st->update_every)
1649                         debug(D_RRD_STATS, "WARNING: %s The time difference between the oldest and the newest entries (%lu) is higher than the capacity of the database (%lu)", st->name, before - after, st->entries * st->update_every);
1650         }
1651
1652
1653         // -------------------------------------------------------------------------
1654         // temp arrays for keeping values per dimension
1655
1656         calculated_number group_values[dimensions]; // keep sums when grouping
1657         int               print_hidden[dimensions]; // keep hidden flags
1658         int               found_non_zero[dimensions];
1659         int               found_non_existing[dimensions];
1660
1661         // initialize them
1662         for( rd = st->dimensions, c = 0 ; rd && c < dimensions ; rd = rd->next, c++) {
1663                 group_values[c] = 0;
1664                 print_hidden[c] = (rd->flags & RRDDIM_FLAG_HIDDEN)?1:0;
1665                 found_non_zero[c] = 0;
1666                 found_non_existing[c] = 0;
1667         }
1668
1669
1670         // error("OLD: points=%d after=%d before=%d group=%d, duration=%d", entries_to_show, before - (st->update_every * group * entries_to_show), before, group, before - after + 1);
1671         // rrd2array(st, entries_to_show, before - (st->update_every * group * entries_to_show), before, group_method, only_non_zero);
1672         // rrd2rrdr(st, entries_to_show, before - (st->update_every * group * entries_to_show), before, group_method);
1673
1674         // -------------------------------------------------------------------------
1675         // remove dimensions that contain only zeros
1676
1677         int max_loop = 1;
1678         if(only_non_zero) max_loop = 2;
1679
1680         for(; max_loop ; max_loop--) {
1681
1682                 // -------------------------------------------------------------------------
1683                 // print the JSON header
1684
1685                 buffer_sprintf(wb, "{\n %scols%s:\n     [\n", kq, kq);
1686                 buffer_sprintf(wb, "            {%sid%s:%s%s,%slabel%s:%stime%s,%spattern%s:%s%s,%stype%s:%sdatetime%s},\n", kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq);
1687                 buffer_sprintf(wb, "            {%sid%s:%s%s,%slabel%s:%s%s,%spattern%s:%s%s,%stype%s:%sstring%s,%sp%s:{%srole%s:%sannotation%s}},\n", kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, kq, kq, sq, sq);
1688                 buffer_sprintf(wb, "            {%sid%s:%s%s,%slabel%s:%s%s,%spattern%s:%s%s,%stype%s:%sstring%s,%sp%s:{%srole%s:%sannotationText%s}}", kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, kq, kq, sq, sq);
1689
1690                 // print the header for each dimension
1691                 // and update the print_hidden array for the dimensions that should be hidden
1692                 int pc = 0;
1693                 for( rd = st->dimensions, c = 0 ; rd && c < dimensions ; rd = rd->next, c++) {
1694                         if(!print_hidden[c]) {
1695                                 pc++;
1696                                 buffer_sprintf(wb, ",\n         {%sid%s:%s%s,%slabel%s:%s%s%s,%spattern%s:%s%s,%stype%s:%snumber%s}", kq, kq, sq, sq, kq, kq, sq, rd->name, sq, kq, kq, sq, sq, kq, kq, sq, sq);
1697                         }
1698                 }
1699                 if(!pc) {
1700                         buffer_sprintf(wb, ",\n         {%sid%s:%s%s,%slabel%s:%s%s%s,%spattern%s:%s%s,%stype%s:%snumber%s}", kq, kq, sq, sq, kq, kq, sq, "no data", sq, kq, kq, sq, sq, kq, kq, sq, sq);
1701                 }
1702
1703                 // print the begin of row data
1704                 buffer_sprintf(wb, "\n  ],\n    %srows%s:\n     [\n", kq, kq);
1705
1706
1707                 // -------------------------------------------------------------------------
1708                 // the main loop
1709
1710                 int annotate_reset = 0;
1711                 int annotation_count = 0;
1712
1713                 long    t = rrdset_time2slot(st, before),
1714                                 stop_at_t = rrdset_time2slot(st, after),
1715                                 stop_now = 0;
1716
1717                 t -= t % group;
1718
1719                 time_t  now = rrdset_slot2time(st, t),
1720                                 dt = st->update_every;
1721
1722                 long count = 0, printed = 0, group_count = 0;
1723                 last_timestamp = 0;
1724
1725                 if(st->debug) debug(D_RRD_STATS, "%s: REQUEST after:%lu before:%lu, points:%d, group:%d, CHART cur:%ld first: %lu last:%lu, CALC start_t:%ld, stop_t:%ld"
1726                                         , st->id
1727                                         , after
1728                                         , before
1729                                         , points
1730                                         , group
1731                                         , st->current_entry
1732                                         , rrdset_first_entry_t(st)
1733                                         , rrdset_last_entry_t(st)
1734                                         , t
1735                                         , stop_at_t
1736                                         );
1737
1738                 long counter = 0;
1739                 for(; !stop_now ; now -= dt, t--, counter++) {
1740                         if(t < 0) t = st->entries - 1;
1741                         if(t == stop_at_t) stop_now = counter;
1742
1743                         int print_this = 0;
1744
1745                         if(st->debug) debug(D_RRD_STATS, "%s t = %ld, count = %ld, group_count = %ld, printed = %ld, now = %lu, %s %s"
1746                                         , st->id
1747                                         , t
1748                                         , count + 1
1749                                         , group_count + 1
1750                                         , printed
1751                                         , now
1752                                         , (group_count + 1 == group)?"PRINT":"  -  "
1753                                         , (now >= after && now <= before)?"RANGE":"  -  "
1754                                         );
1755
1756
1757                         // make sure we return data in the proper time range
1758                         if(now > before) continue;
1759                         if(now < after) break;
1760
1761                         //if(rrdset_slot2time(st, t) != now)
1762                         //      error("%s: slot=%ld, now=%ld, slot2time=%ld, diff=%ld, last_entry_t=%ld, rrdset_last_slot=%ld", st->id, t, now, rrdset_slot2time(st,t), now - rrdset_slot2time(st,t), rrdset_last_entry_t(st), rrdset_last_slot(st));
1763
1764                         count++;
1765                         group_count++;
1766
1767                         // check if we have to print this now
1768                         if(group_count == group) {
1769                                 if(printed >= points) {
1770                                         // debug(D_RRD_STATS, "Already printed all rows. Stopping.");
1771                                         break;
1772                                 }
1773
1774                                 // generate the local date time
1775                                 struct tm tmbuf, *tm = localtime_r(&now, &tmbuf);
1776                                 if(!tm) { error("localtime() failed."); continue; }
1777                                 if(now > last_timestamp) last_timestamp = now;
1778
1779                                 if(printed) buffer_strcat(wb, "]},\n");
1780                                 buffer_strcat(wb, pre_date);
1781                                 buffer_jsdate(wb, tm->tm_year + 1900, tm->tm_mon, tm->tm_mday, tm->tm_hour, tm->tm_min, tm->tm_sec);
1782                                 buffer_strcat(wb, post_date);
1783
1784                                 print_this = 1;
1785                         }
1786
1787                         // do the calculations
1788                         for(rd = st->dimensions, c = 0 ; rd && c < dimensions ; rd = rd->next, c++) {
1789                                 storage_number n = rd->values[t];
1790                                 calculated_number value = unpack_storage_number(n);
1791
1792                                 if(!does_storage_number_exist(n)) {
1793                                         value = 0.0;
1794                                         found_non_existing[c]++;
1795                                 }
1796                                 if(did_storage_number_reset(n)) annotate_reset = 1;
1797
1798                                 switch(group_method) {
1799                                         case GROUP_MAX:
1800                                                 if(abs(value) > abs(group_values[c])) group_values[c] = value;
1801                                                 break;
1802
1803                                         case GROUP_SUM:
1804                                                 group_values[c] += value;
1805                                                 break;
1806
1807                                         default:
1808                                         case GROUP_AVERAGE:
1809                                                 group_values[c] += value;
1810                                                 if(print_this) group_values[c] /= ( group_count - found_non_existing[c] );
1811                                                 break;
1812                                 }
1813                         }
1814
1815                         if(print_this) {
1816                                 if(annotate_reset) {
1817                                         annotation_count++;
1818                                         buffer_strcat(wb, overflow_annotation);
1819                                         annotate_reset = 0;
1820                                 }
1821                                 else
1822                                         buffer_strcat(wb, normal_annotation);
1823
1824                                 pc = 0;
1825                                 for(c = 0 ; c < dimensions ; c++) {
1826                                         if(found_non_existing[c] == group_count) {
1827                                                 // all entries are non-existing
1828                                                 pc++;
1829                                                 buffer_strcat(wb, pre_value);
1830                                                 buffer_strcat(wb, "null");
1831                                                 buffer_strcat(wb, post_value);
1832                                         }
1833                                         else if(!print_hidden[c]) {
1834                                                 pc++;
1835                                                 buffer_strcat(wb, pre_value);
1836                                                 buffer_rrd_value(wb, group_values[c]);
1837                                                 buffer_strcat(wb, post_value);
1838
1839                                                 if(group_values[c]) found_non_zero[c]++;
1840                                         }
1841
1842                                         // reset them for the next loop
1843                                         group_values[c] = 0;
1844                                         found_non_existing[c] = 0;
1845                                 }
1846
1847                                 // if all dimensions are hidden, print a null
1848                                 if(!pc) {
1849                                         buffer_strcat(wb, pre_value);
1850                                         buffer_strcat(wb, "null");
1851                                         buffer_strcat(wb, post_value);
1852                                 }
1853
1854                                 printed++;
1855                                 group_count = 0;
1856                         }
1857                 }
1858
1859                 if(printed) buffer_strcat(wb, "]}");
1860                 buffer_strcat(wb, "\n   ]\n}\n");
1861
1862                 if(only_non_zero && max_loop > 1) {
1863                         int changed = 0;
1864                         for(rd = st->dimensions, c = 0 ; rd && c < dimensions ; rd = rd->next, c++) {
1865                                 group_values[c] = 0;
1866                                 found_non_existing[c] = 0;
1867
1868                                 if(!print_hidden[c] && !found_non_zero[c]) {
1869                                         changed = 1;
1870                                         print_hidden[c] = 1;
1871                                 }
1872                         }
1873
1874                         if(changed) buffer_flush(wb);
1875                         else break;
1876                 }
1877                 else break;
1878
1879         } // max_loop
1880
1881         debug(D_RRD_STATS, "RRD_STATS_JSON: %s total %ld bytes", st->name, wb->len);
1882
1883         pthread_rwlock_unlock(&st->rwlock);
1884         return last_timestamp;
1885 }