src/rrd2json.c

   1 #include "common.h"
   2
   3 inline void rrd_stats_api_v1_chart_with_data(RRDSET *st, BUFFER *wb, size_t *dimensions_count, size_t *memory_used)
   4 {
   5     pthread_rwlock_rdlock(&st->rwlock);
   6
   7     buffer_sprintf(wb,
   8         "\t\t{\n"
   9         "\t\t\t\"id\": \"%s\",\n"
  10         "\t\t\t\"name\": \"%s\",\n"
  11         "\t\t\t\"type\": \"%s\",\n"
  12         "\t\t\t\"family\": \"%s\",\n"
  13         "\t\t\t\"context\": \"%s\",\n"
  14         "\t\t\t\"title\": \"%s\",\n"
  15         "\t\t\t\"priority\": %ld,\n"
  16         "\t\t\t\"enabled\": %s,\n"
  17         "\t\t\t\"units\": \"%s\",\n"
  18         "\t\t\t\"data_url\": \"/api/v1/data?chart=%s\",\n"
  19         "\t\t\t\"chart_type\": \"%s\",\n"
  20         "\t\t\t\"duration\": %ld,\n"
  21         "\t\t\t\"first_entry\": %ld,\n"
  22         "\t\t\t\"last_entry\": %ld,\n"
  23         "\t\t\t\"update_every\": %d,\n"
  24         "\t\t\t\"dimensions\": {\n"
  25         , st->id
  26         , st->name
  27         , st->type
  28         , st->family
  29         , st->context
  30         , st->title
  31         , st->priority
  32         , st->enabled?"true":"false"
  33         , st->units
  34         , st->name
  35         , rrdset_type_name(st->chart_type)
  36         , st->entries * st->update_every
  37         , rrdset_first_entry_t(st)
  38         , rrdset_last_entry_t(st)
  39         , st->update_every
  40         );
  41
  42     unsigned long memory = st->memsize;
  43
  44     size_t c = 0;
  45     RRDDIM *rd;
  46     for(rd = st->dimensions; rd ; rd = rd->next) {
  47         if(rd->flags & RRDDIM_FLAG_HIDDEN) continue;
  48
  49         memory += rd->memsize;
  50
  51         buffer_sprintf(wb,
  52             "%s"
  53             "\t\t\t\t\"%s\": { \"name\": \"%s\" }"
  54             , c?",\n":""
  55             , rd->id
  56             , rd->name
  57             );
  58
  59         c++;
  60     }
  61     if(dimensions_count) *dimensions_count += c;
  62     if(memory_used) *memory_used += memory;
  63
  64     buffer_strcat(wb, "\n\t\t\t},\n\t\t\t\"green\": ");
  65     buffer_rrd_value(wb, st->green);
  66     buffer_strcat(wb, ",\n\t\t\t\"red\": ");
  67     buffer_rrd_value(wb, st->red);
  68
  69     buffer_sprintf(wb,
  70         "\n\t\t}"
  71         );
  72
  73     pthread_rwlock_unlock(&st->rwlock);
  74 }
  75
  76 void rrd_stats_api_v1_chart(RRDSET *st, BUFFER *wb) {
  77     rrd_stats_api_v1_chart_with_data(st, wb, NULL, NULL);
  78 }
  79
  80 void rrd_stats_api_v1_charts(BUFFER *wb)
  81 {
  82     size_t c, dimensions = 0, memory = 0;
  83     RRDSET *st;
  84
  85     buffer_sprintf(wb, "{\n"
  86            "\t\"hostname\": \"%s\""
  87         ",\n\t\"update_every\": %d"
  88         ",\n\t\"history\": %d"
  89         ",\n\t\"charts\": {"
  90         , localhost.hostname
  91         , rrd_update_every
  92         , rrd_default_history_entries
  93         );
  94
  95     pthread_rwlock_rdlock(&localhost.rrdset_root_rwlock);
  96     for(st = localhost.rrdset_root, c = 0; st ; st = st->next) {
  97         if(st->enabled && st->dimensions) {
  98             if(c) buffer_strcat(wb, ",");
  99             buffer_strcat(wb, "\n\t\t\"");
 100             buffer_strcat(wb, st->id);
 101             buffer_strcat(wb, "\": ");
 102             rrd_stats_api_v1_chart_with_data(st, wb, &dimensions, &memory);
 103             c++;
 104         }
 105     }
 106     pthread_rwlock_unlock(&localhost.rrdset_root_rwlock);
 107
 108     buffer_sprintf(wb, "\n\t}"
 109                     ",\n\t\"charts_count\": %zu"
 110                     ",\n\t\"dimensions_count\": %zu"
 111                     ",\n\t\"rrd_memory_bytes\": %zu"
 112                     "\n}\n"
 113                    , c
 114                    , dimensions
 115                    , memory
 116     );
 117 }
 118
 119
 120 unsigned long rrd_stats_one_json(RRDSET *st, char *options, BUFFER *wb)
 121 {
 122     time_t now = time(NULL);
 123
 124     pthread_rwlock_rdlock(&st->rwlock);
 125
 126     buffer_sprintf(wb,
 127         "\t\t{\n"
 128         "\t\t\t\"id\": \"%s\",\n"
 129         "\t\t\t\"name\": \"%s\",\n"
 130         "\t\t\t\"type\": \"%s\",\n"
 131         "\t\t\t\"family\": \"%s\",\n"
 132         "\t\t\t\"context\": \"%s\",\n"
 133         "\t\t\t\"title\": \"%s\",\n"
 134         "\t\t\t\"priority\": %ld,\n"
 135         "\t\t\t\"enabled\": %d,\n"
 136         "\t\t\t\"units\": \"%s\",\n"
 137         "\t\t\t\"url\": \"/data/%s/%s\",\n"
 138         "\t\t\t\"chart_type\": \"%s\",\n"
 139         "\t\t\t\"counter\": %lu,\n"
 140         "\t\t\t\"entries\": %ld,\n"
 141         "\t\t\t\"first_entry_t\": %ld,\n"
 142         "\t\t\t\"last_entry\": %lu,\n"
 143         "\t\t\t\"last_entry_t\": %ld,\n"
 144         "\t\t\t\"last_entry_secs_ago\": %ld,\n"
 145         "\t\t\t\"update_every\": %d,\n"
 146         "\t\t\t\"isdetail\": %d,\n"
 147         "\t\t\t\"usec_since_last_update\": %llu,\n"
 148         "\t\t\t\"collected_total\": " TOTAL_NUMBER_FORMAT ",\n"
 149         "\t\t\t\"last_collected_total\": " TOTAL_NUMBER_FORMAT ",\n"
 150         "\t\t\t\"dimensions\": [\n"
 151         , st->id
 152         , st->name
 153         , st->type
 154         , st->family
 155         , st->context
 156         , st->title
 157         , st->priority
 158         , st->enabled
 159         , st->units
 160         , st->name, options?options:""
 161         , rrdset_type_name(st->chart_type)
 162         , st->counter
 163         , st->entries
 164         , rrdset_first_entry_t(st)
 165         , rrdset_last_slot(st)
 166         , rrdset_last_entry_t(st)
 167         , (now < rrdset_last_entry_t(st)) ? (time_t)0 : now - rrdset_last_entry_t(st)
 168         , st->update_every
 169         , st->isdetail
 170         , st->usec_since_last_update
 171         , st->collected_total
 172         , st->last_collected_total
 173         );
 174
 175     unsigned long memory = st->memsize;
 176
 177     RRDDIM *rd;
 178     for(rd = st->dimensions; rd ; rd = rd->next) {
 179
 180         memory += rd->memsize;
 181
 182         buffer_sprintf(wb,
 183             "\t\t\t\t{\n"
 184             "\t\t\t\t\t\"id\": \"%s\",\n"
 185             "\t\t\t\t\t\"name\": \"%s\",\n"
 186             "\t\t\t\t\t\"entries\": %ld,\n"
 187             "\t\t\t\t\t\"isHidden\": %d,\n"
 188             "\t\t\t\t\t\"algorithm\": \"%s\",\n"
 189             "\t\t\t\t\t\"multiplier\": %ld,\n"
 190             "\t\t\t\t\t\"divisor\": %ld,\n"
 191             "\t\t\t\t\t\"last_entry_t\": %ld,\n"
 192             "\t\t\t\t\t\"collected_value\": " COLLECTED_NUMBER_FORMAT ",\n"
 193             "\t\t\t\t\t\"calculated_value\": " CALCULATED_NUMBER_FORMAT ",\n"
 194             "\t\t\t\t\t\"last_collected_value\": " COLLECTED_NUMBER_FORMAT ",\n"
 195             "\t\t\t\t\t\"last_calculated_value\": " CALCULATED_NUMBER_FORMAT ",\n"
 196             "\t\t\t\t\t\"memory\": %lu\n"
 197             "\t\t\t\t}%s\n"
 198             , rd->id
 199             , rd->name
 200             , rd->entries
 201             , (rd->flags & RRDDIM_FLAG_HIDDEN)?1:0
 202             , rrddim_algorithm_name(rd->algorithm)
 203             , rd->multiplier
 204             , rd->divisor
 205             , rd->last_collected_time.tv_sec
 206             , rd->collected_value
 207             , rd->calculated_value
 208             , rd->last_collected_value
 209             , rd->last_calculated_value
 210             , rd->memsize
 211             , rd->next?",":""
 212             );
 213     }
 214
 215     buffer_sprintf(wb,
 216         "\t\t\t],\n"
 217         "\t\t\t\"memory\" : %lu\n"
 218         "\t\t}"
 219         , memory
 220         );
 221
 222     pthread_rwlock_unlock(&st->rwlock);
 223     return memory;
 224 }
 225
 226 #define RRD_GRAPH_JSON_HEADER "{\n\t\"charts\": [\n"
 227 #define RRD_GRAPH_JSON_FOOTER "\n\t]\n}\n"
 228
 229 void rrd_stats_graph_json(RRDSET *st, char *options, BUFFER *wb)
 230 {
 231     buffer_strcat(wb, RRD_GRAPH_JSON_HEADER);
 232     rrd_stats_one_json(st, options, wb);
 233     buffer_strcat(wb, RRD_GRAPH_JSON_FOOTER);
 234 }
 235
 236 void rrd_stats_all_json(BUFFER *wb)
 237 {
 238     unsigned long memory = 0;
 239     long c;
 240     RRDSET *st;
 241
 242     buffer_strcat(wb, RRD_GRAPH_JSON_HEADER);
 243
 244     pthread_rwlock_rdlock(&localhost.rrdset_root_rwlock);
 245     for(st = localhost.rrdset_root, c = 0; st ; st = st->next) {
 246         if(st->enabled && st->dimensions) {
 247             if(c) buffer_strcat(wb, ",\n");
 248             memory += rrd_stats_one_json(st, NULL, wb);
 249             c++;
 250         }
 251     }
 252     pthread_rwlock_unlock(&localhost.rrdset_root_rwlock);
 253
 254     buffer_sprintf(wb, "\n\t],\n"
 255         "\t\"hostname\": \"%s\",\n"
 256         "\t\"update_every\": %d,\n"
 257         "\t\"history\": %d,\n"
 258         "\t\"memory\": %lu\n"
 259         "}\n"
 260         , localhost.hostname
 261         , rrd_update_every
 262         , rrd_default_history_entries
 263         , memory
 264         );
 265 }
 266
 267
 268
 269 // ----------------------------------------------------------------------------
 270
 271 // RRDR dimension options
 272 #define RRDR_EMPTY      0x01 // the dimension contains / the value is empty (null)
 273 #define RRDR_RESET      0x02 // the dimension contains / the value is reset
 274 #define RRDR_HIDDEN     0x04 // the dimension contains / the value is hidden
 275 #define RRDR_NONZERO    0x08 // the dimension contains / the value is non-zero
 276
 277 // RRDR result options
 278 #define RRDR_RESULT_OPTION_ABSOLUTE 0x00000001
 279 #define RRDR_RESULT_OPTION_RELATIVE 0x00000002
 280
 281 typedef struct rrdresult {
 282     RRDSET *st;         // the chart this result refers to
 283
 284     uint32_t result_options;    // RRDR_RESULT_OPTION_*
 285
 286     int d;                  // the number of dimensions
 287     long n;                 // the number of values in the arrays
 288     long rows;              // the number of rows used
 289
 290     uint8_t *od;            // the options for the dimensions
 291
 292     time_t *t;              // array of n timestamps
 293     calculated_number *v;   // array n x d values
 294     uint8_t *o;             // array n x d options
 295
 296     long c;                 // current line ( -1 ~ n ), ( -1 = none, use rrdr_rows() to get number of rows )
 297
 298     long group;             // how many collected values were grouped for each row
 299     int update_every;       // what is the suggested update frequency in seconds
 300
 301     calculated_number min;
 302     calculated_number max;
 303
 304     time_t before;
 305     time_t after;
 306
 307     int has_st_lock;        // if st is read locked by us
 308 } RRDR;
 309
 310 #define rrdr_rows(r) ((r)->rows)
 311
 312 /*
 313 static void rrdr_dump(RRDR *r)
 314 {
 315     long c, i;
 316     RRDDIM *d;
 317
 318     fprintf(stderr, "\nCHART %s (%s)\n", r->st->id, r->st->name);
 319
 320     for(c = 0, d = r->st->dimensions; d ;c++, d = d->next) {
 321         fprintf(stderr, "DIMENSION %s (%s), %s%s%s%s\n"
 322                 , d->id
 323                 , d->name
 324                 , (r->od[c] & RRDR_EMPTY)?"EMPTY ":""
 325                 , (r->od[c] & RRDR_RESET)?"RESET ":""
 326                 , (r->od[c] & RRDR_HIDDEN)?"HIDDEN ":""
 327                 , (r->od[c] & RRDR_NONZERO)?"NONZERO ":""
 328                 );
 329     }
 330
 331     if(r->rows <= 0) {
 332         fprintf(stderr, "RRDR does not have any values in it.\n");
 333         return;
 334     }
 335
 336     fprintf(stderr, "RRDR includes %d values in it:\n", r->rows);
 337
 338     // for each line in the array
 339     for(i = 0; i < r->rows ;i++) {
 340         calculated_number *cn = &r->v[ i * r->d ];
 341         uint8_t *co = &r->o[ i * r->d ];
 342
 343         // print the id and the timestamp of the line
 344         fprintf(stderr, "%ld %ld ", i + 1, r->t[i]);
 345
 346         // for each dimension
 347         for(c = 0, d = r->st->dimensions; d ;c++, d = d->next) {
 348             if(unlikely(r->od[c] & RRDR_HIDDEN)) continue;
 349             if(unlikely(!(r->od[c] & RRDR_NONZERO))) continue;
 350
 351             if(co[c] & RRDR_EMPTY)
 352                 fprintf(stderr, "null ");
 353             else
 354                 fprintf(stderr, CALCULATED_NUMBER_FORMAT " %s%s%s%s "
 355                     , cn[c]
 356                     , (co[c] & RRDR_EMPTY)?"E":" "
 357                     , (co[c] & RRDR_RESET)?"R":" "
 358                     , (co[c] & RRDR_HIDDEN)?"H":" "
 359                     , (co[c] & RRDR_NONZERO)?"N":" "
 360                     );
 361         }
 362
 363         fprintf(stderr, "\n");
 364     }
 365 }
 366 */
 367
 368 void rrdr_disable_not_selected_dimensions(RRDR *r, const char *dims)
 369 {
 370     char b[strlen(dims) + 1];
 371     char *o = b, *tok;
 372     strcpy(o, dims);
 373
 374     long c;
 375     RRDDIM *d;
 376
 377     // disable all of them
 378     for(c = 0, d = r->st->dimensions; d ;c++, d = d->next)
 379         r->od[c] |= RRDR_HIDDEN;
 380
 381     while(o && *o && (tok = mystrsep(&o, ",|"))) {
 382         if(!*tok) continue;
 383
 384         uint32_t hash = simple_hash(tok);
 385
 386         // find it and enable it
 387         for(c = 0, d = r->st->dimensions; d ;c++, d = d->next) {
 388             if(unlikely((hash == d->hash && !strcmp(d->id, tok)) || !strcmp(d->name, tok))) {
 389                 r->od[c] &= ~RRDR_HIDDEN;
 390
 391                 // since the user needs this dimension
 392                 // make it appear as NONZERO, to return it
 393                 // even if the dimension has only zeros
 394                 r->od[c] |= RRDR_NONZERO;
 395             }
 396         }
 397     }
 398 }
 399
 400 void rrdr_buffer_print_format(BUFFER *wb, uint32_t format)
 401 {
 402     switch(format) {
 403     case DATASOURCE_JSON:
 404         buffer_strcat(wb, DATASOURCE_FORMAT_JSON);
 405         break;
 406
 407     case DATASOURCE_DATATABLE_JSON:
 408         buffer_strcat(wb, DATASOURCE_FORMAT_DATATABLE_JSON);
 409         break;
 410
 411     case DATASOURCE_DATATABLE_JSONP:
 412         buffer_strcat(wb, DATASOURCE_FORMAT_DATATABLE_JSONP);
 413         break;
 414
 415     case DATASOURCE_JSONP:
 416         buffer_strcat(wb, DATASOURCE_FORMAT_JSONP);
 417         break;
 418
 419     case DATASOURCE_SSV:
 420         buffer_strcat(wb, DATASOURCE_FORMAT_SSV);
 421         break;
 422
 423     case DATASOURCE_CSV:
 424         buffer_strcat(wb, DATASOURCE_FORMAT_CSV);
 425         break;
 426
 427     case DATASOURCE_TSV:
 428         buffer_strcat(wb, DATASOURCE_FORMAT_TSV);
 429         break;
 430
 431     case DATASOURCE_HTML:
 432         buffer_strcat(wb, DATASOURCE_FORMAT_HTML);
 433         break;
 434
 435     case DATASOURCE_JS_ARRAY:
 436         buffer_strcat(wb, DATASOURCE_FORMAT_JS_ARRAY);
 437         break;
 438
 439     case DATASOURCE_SSV_COMMA:
 440         buffer_strcat(wb, DATASOURCE_FORMAT_SSV_COMMA);
 441         break;
 442
 443     default:
 444         buffer_strcat(wb, "unknown");
 445         break;
 446     }
 447 }
 448
 449 uint32_t rrdr_check_options(RRDR *r, uint32_t options, const char *dims)
 450 {
 451     if(options & RRDR_OPTION_NONZERO) {
 452         long i;
 453
 454         if(dims && *dims) {
 455             // the caller wants specific dimensions
 456             // disable NONZERO option
 457             // to make sure we don't accidentally prevent
 458             // the specific dimensions from being returned
 459             i = 0;
 460         }
 461         else {
 462             // find how many dimensions are not zero
 463             long c;
 464             RRDDIM *rd;
 465             for(c = 0, i = 0, rd = r->st->dimensions; rd && c < r->d ; c++, rd = rd->next) {
 466                 if(unlikely(r->od[c] & RRDR_HIDDEN)) continue;
 467                 if(unlikely(!(r->od[c] & RRDR_NONZERO))) continue;
 468                 i++;
 469             }
 470         }
 471
 472         // if with nonzero we get i = 0 (no dimensions will be returned)
 473         // disable nonzero to show all dimensions
 474         if(!i) options &= ~RRDR_OPTION_NONZERO;
 475     }
 476
 477     return options;
 478 }
 479
 480 void rrdr_json_wrapper_begin(RRDR *r, BUFFER *wb, uint32_t format, uint32_t options, int string_value)
 481 {
 482     long rows = rrdr_rows(r);
 483     long c, i;
 484     RRDDIM *rd;
 485
 486     //info("JSONWRAPPER(): %s: BEGIN", r->st->id);
 487     char kq[2] = "",                    // key quote
 488         sq[2] = "";                     // string quote
 489
 490     if( options & RRDR_OPTION_GOOGLE_JSON ) {
 491         kq[0] = '\0';
 492         sq[0] = '\'';
 493     }
 494     else {
 495         kq[0] = '"';
 496         sq[0] = '"';
 497     }
 498
 499     buffer_sprintf(wb, "{\n"
 500             "   %sapi%s: 1,\n"
 501             "   %sid%s: %s%s%s,\n"
 502             "   %sname%s: %s%s%s,\n"
 503             "   %sview_update_every%s: %d,\n"
 504             "   %supdate_every%s: %d,\n"
 505             "   %sfirst_entry%s: %u,\n"
 506             "   %slast_entry%s: %u,\n"
 507             "   %sbefore%s: %u,\n"
 508             "   %safter%s: %u,\n"
 509             "   %sdimension_names%s: ["
 510             , kq, kq
 511             , kq, kq, sq, r->st->id, sq
 512             , kq, kq, sq, r->st->name, sq
 513             , kq, kq, r->update_every
 514             , kq, kq, r->st->update_every
 515             , kq, kq, (uint32_t)rrdset_first_entry_t(r->st)
 516             , kq, kq, (uint32_t)rrdset_last_entry_t(r->st)
 517             , kq, kq, (uint32_t)r->before
 518             , kq, kq, (uint32_t)r->after
 519             , kq, kq);
 520
 521     for(c = 0, i = 0, rd = r->st->dimensions; rd && c < r->d ;c++, rd = rd->next) {
 522         if(unlikely(r->od[c] & RRDR_HIDDEN)) continue;
 523         if(unlikely((options & RRDR_OPTION_NONZERO) && !(r->od[c] & RRDR_NONZERO))) continue;
 524
 525         if(i) buffer_strcat(wb, ", ");
 526         buffer_strcat(wb, sq);
 527         buffer_strcat(wb, rd->name);
 528         buffer_strcat(wb, sq);
 529         i++;
 530     }
 531     if(!i) {
 532 #ifdef NETDATA_INTERNAL_CHECKS
 533         error("RRDR is empty for %s (RRDR has %d dimensions, options is 0x%08x)", r->st->id, r->d, options);
 534 #endif
 535         rows = 0;
 536         buffer_strcat(wb, sq);
 537         buffer_strcat(wb, "no data");
 538         buffer_strcat(wb, sq);
 539     }
 540
 541     buffer_sprintf(wb, "],\n"
 542             "   %sdimension_ids%s: ["
 543             , kq, kq);
 544
 545     for(c = 0, i = 0, rd = r->st->dimensions; rd && c < r->d ;c++, rd = rd->next) {
 546         if(unlikely(r->od[c] & RRDR_HIDDEN)) continue;
 547         if(unlikely((options & RRDR_OPTION_NONZERO) && !(r->od[c] & RRDR_NONZERO))) continue;
 548
 549         if(i) buffer_strcat(wb, ", ");
 550         buffer_strcat(wb, sq);
 551         buffer_strcat(wb, rd->id);
 552         buffer_strcat(wb, sq);
 553         i++;
 554     }
 555     if(!i) {
 556         rows = 0;
 557         buffer_strcat(wb, sq);
 558         buffer_strcat(wb, "no data");
 559         buffer_strcat(wb, sq);
 560     }
 561
 562     buffer_sprintf(wb, "],\n"
 563             "   %slatest_values%s: ["
 564             , kq, kq);
 565
 566     for(c = 0, i = 0, rd = r->st->dimensions; rd && c < r->d ;c++, rd = rd->next) {
 567         if(unlikely(r->od[c] & RRDR_HIDDEN)) continue;
 568         if(unlikely((options & RRDR_OPTION_NONZERO) && !(r->od[c] & RRDR_NONZERO))) continue;
 569
 570         if(i) buffer_strcat(wb, ", ");
 571         i++;
 572
 573         storage_number n = rd->values[rrdset_last_slot(r->st)];
 574
 575         if(!does_storage_number_exist(n))
 576             buffer_strcat(wb, "null");
 577         else
 578             buffer_rrd_value(wb, unpack_storage_number(n));
 579     }
 580     if(!i) {
 581         rows = 0;
 582         buffer_strcat(wb, "null");
 583     }
 584
 585     buffer_sprintf(wb, "],\n"
 586             "   %sview_latest_values%s: ["
 587             , kq, kq);
 588
 589     i = 0;
 590     if(rows) {
 591         for(c = 0, i = 0, rd = r->st->dimensions; rd && c < r->d ;c++, rd = rd->next) {
 592             if(unlikely(r->od[c] & RRDR_HIDDEN)) continue;
 593             if(unlikely((options & RRDR_OPTION_NONZERO) && !(r->od[c] & RRDR_NONZERO))) continue;
 594
 595             if(i) buffer_strcat(wb, ", ");
 596             i++;
 597
 598             calculated_number *cn = &r->v[ (0) * r->d ];
 599             uint8_t *co = &r->o[ (0) * r->d ];
 600
 601             if(co[c] & RRDR_EMPTY)
 602                 buffer_strcat(wb, "null");
 603             else
 604                 buffer_rrd_value(wb, cn[c]);
 605         }
 606     }
 607     if(!i) {
 608         rows = 0;
 609         buffer_strcat(wb, "null");
 610     }
 611
 612     buffer_sprintf(wb, "],\n"
 613             "   %sdimensions%s: %ld,\n"
 614             "   %spoints%s: %ld,\n"
 615             "   %sformat%s: %s"
 616             , kq, kq, i
 617             , kq, kq, rows
 618             , kq, kq, sq
 619             );
 620
 621     rrdr_buffer_print_format(wb, format);
 622
 623     buffer_sprintf(wb, "%s,\n"
 624             "   %sresult%s: "
 625             , sq
 626             , kq, kq
 627             );
 628
 629     if(string_value) buffer_strcat(wb, sq);
 630     //info("JSONWRAPPER(): %s: END", r->st->id);
 631 }
 632
 633 void rrdr_json_wrapper_end(RRDR *r, BUFFER *wb, uint32_t format, uint32_t options, int string_value)
 634 {
 635     (void)format;
 636
 637     char kq[2] = "",                    // key quote
 638         sq[2] = "";                     // string quote
 639
 640     if( options & RRDR_OPTION_GOOGLE_JSON ) {
 641         kq[0] = '\0';
 642         sq[0] = '\'';
 643     }
 644     else {
 645         kq[0] = '"';
 646         sq[0] = '"';
 647     }
 648
 649     if(string_value) buffer_strcat(wb, sq);
 650
 651     buffer_sprintf(wb, ",\n %smin%s: ", kq, kq);
 652     buffer_rrd_value(wb, r->min);
 653     buffer_sprintf(wb, ",\n %smax%s: ", kq, kq);
 654     buffer_rrd_value(wb, r->max);
 655     buffer_strcat(wb, "\n}\n");
 656 }
 657
 658 #define JSON_DATES_JS 1
 659 #define JSON_DATES_TIMESTAMP 2
 660
 661 static void rrdr2json(RRDR *r, BUFFER *wb, uint32_t options, int datatable)
 662 {
 663     //info("RRD2JSON(): %s: BEGIN", r->st->id);
 664     int row_annotations = 0, dates, dates_with_new = 0;
 665     char kq[2] = "",                    // key quote
 666         sq[2] = "",                     // string quote
 667         pre_label[101] = "",            // before each label
 668         post_label[101] = "",           // after each label
 669         pre_date[101] = "",             // the beginning of line, to the date
 670         post_date[101] = "",            // closing the date
 671         pre_value[101] = "",            // before each value
 672         post_value[101] = "",           // after each value
 673         post_line[101] = "",            // at the end of each row
 674         normal_annotation[201] = "",    // default row annotation
 675         overflow_annotation[201] = "",  // overflow row annotation
 676         data_begin[101] = "",           // between labels and values
 677         finish[101] = "";               // at the end of everything
 678
 679     if(datatable) {
 680         dates = JSON_DATES_JS;
 681         if( options & RRDR_OPTION_GOOGLE_JSON ) {
 682             kq[0] = '\0';
 683             sq[0] = '\'';
 684         }
 685         else {
 686             kq[0] = '"';
 687             sq[0] = '"';
 688         }
 689         row_annotations = 1;
 690         snprintfz(pre_date,   100, "        {%sc%s:[{%sv%s:%s", kq, kq, kq, kq, sq);
 691         snprintfz(post_date,  100, "%s}", sq);
 692         snprintfz(pre_label,  100, ",\n     {%sid%s:%s%s,%slabel%s:%s", kq, kq, sq, sq, kq, kq, sq);
 693         snprintfz(post_label, 100, "%s,%spattern%s:%s%s,%stype%s:%snumber%s}", sq, kq, kq, sq, sq, kq, kq, sq, sq);
 694         snprintfz(pre_value,  100, ",{%sv%s:", kq, kq);
 695         strcpy(post_value,         "}");
 696         strcpy(post_line,          "]}");
 697         snprintfz(data_begin, 100, "\n  ],\n    %srows%s:\n [\n", kq, kq);
 698         strcpy(finish,             "\n  ]\n}");
 699
 700         snprintfz(overflow_annotation, 200, ",{%sv%s:%sRESET OR OVERFLOW%s},{%sv%s:%sThe counters have been wrapped.%s}", kq, kq, sq, sq, kq, kq, sq, sq);
 701         snprintfz(normal_annotation,   200, ",{%sv%s:null},{%sv%s:null}", kq, kq, kq, kq);
 702
 703         buffer_sprintf(wb, "{\n %scols%s:\n [\n", kq, kq);
 704         buffer_sprintf(wb, "        {%sid%s:%s%s,%slabel%s:%stime%s,%spattern%s:%s%s,%stype%s:%sdatetime%s},\n", kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq);
 705         buffer_sprintf(wb, "        {%sid%s:%s%s,%slabel%s:%s%s,%spattern%s:%s%s,%stype%s:%sstring%s,%sp%s:{%srole%s:%sannotation%s}},\n", kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, kq, kq, sq, sq);
 706         buffer_sprintf(wb, "        {%sid%s:%s%s,%slabel%s:%s%s,%spattern%s:%s%s,%stype%s:%sstring%s,%sp%s:{%srole%s:%sannotationText%s}}", kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, kq, kq, sq, sq);
 707
 708         // remove the valueobjects flag
 709         // google wants its own keys
 710         if(options & RRDR_OPTION_OBJECTSROWS)
 711             options &= ~RRDR_OPTION_OBJECTSROWS;
 712     }
 713     else {
 714         kq[0] = '"';
 715         sq[0] = '"';
 716         if((options & RRDR_OPTION_SECONDS) || (options & RRDR_OPTION_MILLISECONDS)) {
 717             dates = JSON_DATES_TIMESTAMP;
 718             dates_with_new = 0;
 719         }
 720         else {
 721             dates = JSON_DATES_JS;
 722             dates_with_new = 1;
 723         }
 724         if( options & RRDR_OPTION_OBJECTSROWS )
 725             strcpy(pre_date, "      { ");
 726         else
 727             strcpy(pre_date, "      [ ");
 728         strcpy(pre_label,  ", \"");
 729         strcpy(post_label, "\"");
 730         strcpy(pre_value,  ", ");
 731         if( options & RRDR_OPTION_OBJECTSROWS )
 732             strcpy(post_line, "}");
 733         else
 734             strcpy(post_line, "]");
 735         snprintfz(data_begin, 100, "],\n    %sdata%s:\n [\n", kq, kq);
 736         strcpy(finish,             "\n  ]\n}");
 737
 738         buffer_sprintf(wb, "{\n %slabels%s: [", kq, kq);
 739         buffer_sprintf(wb, "%stime%s", sq, sq);
 740     }
 741
 742     // -------------------------------------------------------------------------
 743     // print the JSON header
 744
 745     long c, i;
 746     RRDDIM *rd;
 747
 748     // print the header lines
 749     for(c = 0, i = 0, rd = r->st->dimensions; rd && c < r->d ;c++, rd = rd->next) {
 750         if(unlikely(r->od[c] & RRDR_HIDDEN)) continue;
 751         if(unlikely((options & RRDR_OPTION_NONZERO) && !(r->od[c] & RRDR_NONZERO))) continue;
 752
 753         buffer_strcat(wb, pre_label);
 754         buffer_strcat(wb, rd->name);
 755         buffer_strcat(wb, post_label);
 756         i++;
 757     }
 758     if(!i) {
 759         buffer_strcat(wb, pre_label);
 760         buffer_strcat(wb, "no data");
 761         buffer_strcat(wb, post_label);
 762     }
 763
 764     // print the begin of row data
 765     buffer_strcat(wb, data_begin);
 766
 767     // if all dimensions are hidden, print a null
 768     if(!i) {
 769         buffer_strcat(wb, finish);
 770         return;
 771     }
 772
 773     long start = 0, end = rrdr_rows(r), step = 1;
 774     if((options & RRDR_OPTION_REVERSED)) {
 775         start = rrdr_rows(r) - 1;
 776         end = -1;
 777         step = -1;
 778     }
 779
 780     // for each line in the array
 781     calculated_number total = 1;
 782     for(i = start; i != end ;i += step) {
 783         calculated_number *cn = &r->v[ i * r->d ];
 784         uint8_t *co = &r->o[ i * r->d ];
 785
 786         time_t now = r->t[i];
 787
 788         if(dates == JSON_DATES_JS) {
 789             // generate the local date time
 790             struct tm tmbuf, *tm = localtime_r(&now, &tmbuf);
 791             if(!tm) { error("localtime_r() failed."); continue; }
 792
 793             if(likely(i != start)) buffer_strcat(wb, ",\n");
 794             buffer_strcat(wb, pre_date);
 795
 796             if( options & RRDR_OPTION_OBJECTSROWS )
 797                 buffer_sprintf(wb, "%stime%s: ", kq, kq);
 798
 799             if(dates_with_new)
 800                 buffer_strcat(wb, "new ");
 801
 802             buffer_jsdate(wb, tm->tm_year + 1900, tm->tm_mon, tm->tm_mday, tm->tm_hour, tm->tm_min, tm->tm_sec);
 803
 804             buffer_strcat(wb, post_date);
 805
 806             if(row_annotations) {
 807                 // google supports one annotation per row
 808                 int annotation_found = 0;
 809                 for(c = 0, rd = r->st->dimensions; rd ;c++, rd = rd->next) {
 810                     if(co[c] & RRDR_RESET) {
 811                         buffer_strcat(wb, overflow_annotation);
 812                         annotation_found = 1;
 813                         break;
 814                     }
 815                 }
 816                 if(!annotation_found)
 817                     buffer_strcat(wb, normal_annotation);
 818             }
 819         }
 820         else {
 821             // print the timestamp of the line
 822             if(likely(i != start)) buffer_strcat(wb, ",\n");
 823             buffer_strcat(wb, pre_date);
 824
 825             if( options & RRDR_OPTION_OBJECTSROWS )
 826                 buffer_sprintf(wb, "%stime%s: ", kq, kq);
 827
 828             buffer_rrd_value(wb, (calculated_number)r->t[i]);
 829             // in ms
 830             if(options & RRDR_OPTION_MILLISECONDS) buffer_strcat(wb, "000");
 831
 832             buffer_strcat(wb, post_date);
 833         }
 834
 835         if(unlikely(options & RRDR_OPTION_PERCENTAGE)) {
 836             total = 0;
 837             for(c = 0, rd = r->st->dimensions; rd && c < r->d ;c++, rd = rd->next) {
 838                 calculated_number n = cn[c];
 839
 840                 if(likely((options & RRDR_OPTION_ABSOLUTE) && n < 0))
 841                     n = -n;
 842
 843                 total += n;
 844             }
 845             // prevent a division by zero
 846             if(total == 0) total = 1;
 847         }
 848
 849         // for each dimension
 850         for(c = 0, rd = r->st->dimensions; rd && c < r->d ;c++, rd = rd->next) {
 851             if(unlikely(r->od[c] & RRDR_HIDDEN)) continue;
 852             if(unlikely((options & RRDR_OPTION_NONZERO) && !(r->od[c] & RRDR_NONZERO))) continue;
 853
 854             calculated_number n = cn[c];
 855
 856             buffer_strcat(wb, pre_value);
 857
 858             if( options & RRDR_OPTION_OBJECTSROWS )
 859                 buffer_sprintf(wb, "%s%s%s: ", kq, rd->name, kq);
 860
 861             if(co[c] & RRDR_EMPTY) {
 862                 if(options & RRDR_OPTION_NULL2ZERO)
 863                     buffer_strcat(wb, "0");
 864                 else
 865                     buffer_strcat(wb, "null");
 866             }
 867             else {
 868                 if(unlikely((options & RRDR_OPTION_ABSOLUTE) && n < 0))
 869                     n = -n;
 870
 871                 if(unlikely(options & RRDR_OPTION_PERCENTAGE))
 872                     n = n * 100 / total;
 873
 874                 buffer_rrd_value(wb, n);
 875             }
 876
 877             buffer_strcat(wb, post_value);
 878         }
 879
 880         buffer_strcat(wb, post_line);
 881     }
 882
 883     buffer_strcat(wb, finish);
 884     //info("RRD2JSON(): %s: END", r->st->id);
 885 }
 886
 887 static void rrdr2csv(RRDR *r, BUFFER *wb, uint32_t options, const char *startline, const char *separator, const char *endline, const char *betweenlines)
 888 {
 889     //info("RRD2CSV(): %s: BEGIN", r->st->id);
 890     long c, i;
 891     RRDDIM *d;
 892
 893     // print the csv header
 894     for(c = 0, i = 0, d = r->st->dimensions; d && c < r->d ;c++, d = d->next) {
 895         if(unlikely(r->od[c] & RRDR_HIDDEN)) continue;
 896         if(unlikely((options & RRDR_OPTION_NONZERO) && !(r->od[c] & RRDR_NONZERO))) continue;
 897
 898         if(!i) {
 899             buffer_strcat(wb, startline);
 900             if(options & RRDR_OPTION_LABEL_QUOTES) buffer_strcat(wb, "\"");
 901             buffer_strcat(wb, "time");
 902             if(options & RRDR_OPTION_LABEL_QUOTES) buffer_strcat(wb, "\"");
 903         }
 904         buffer_strcat(wb, separator);
 905         if(options & RRDR_OPTION_LABEL_QUOTES) buffer_strcat(wb, "\"");
 906         buffer_strcat(wb, d->name);
 907         if(options & RRDR_OPTION_LABEL_QUOTES) buffer_strcat(wb, "\"");
 908         i++;
 909     }
 910     buffer_strcat(wb, endline);
 911
 912     if(!i) {
 913         // no dimensions present
 914         return;
 915     }
 916
 917     long start = 0, end = rrdr_rows(r), step = 1;
 918     if((options & RRDR_OPTION_REVERSED)) {
 919         start = rrdr_rows(r) - 1;
 920         end = -1;
 921         step = -1;
 922     }
 923
 924     // for each line in the array
 925     calculated_number total = 1;
 926     for(i = start; i != end ;i += step) {
 927         calculated_number *cn = &r->v[ i * r->d ];
 928         uint8_t *co = &r->o[ i * r->d ];
 929
 930         buffer_strcat(wb, betweenlines);
 931         buffer_strcat(wb, startline);
 932
 933         time_t now = r->t[i];
 934
 935         if((options & RRDR_OPTION_SECONDS) || (options & RRDR_OPTION_MILLISECONDS)) {
 936             // print the timestamp of the line
 937             buffer_rrd_value(wb, (calculated_number)now);
 938             // in ms
 939             if(options & RRDR_OPTION_MILLISECONDS) buffer_strcat(wb, "000");
 940         }
 941         else {
 942             // generate the local date time
 943             struct tm tmbuf, *tm = localtime_r(&now, &tmbuf);
 944             if(!tm) { error("localtime() failed."); continue; }
 945             buffer_date(wb, tm->tm_year + 1900, tm->tm_mon + 1, tm->tm_mday, tm->tm_hour, tm->tm_min, tm->tm_sec);
 946         }
 947
 948         if(unlikely(options & RRDR_OPTION_PERCENTAGE)) {
 949             total = 0;
 950             for(c = 0, d = r->st->dimensions; d && c < r->d ;c++, d = d->next) {
 951                 calculated_number n = cn[c];
 952
 953                 if(likely((options & RRDR_OPTION_ABSOLUTE) && n < 0))
 954                     n = -n;
 955
 956                 total += n;
 957             }
 958             // prevent a division by zero
 959             if(total == 0) total = 1;
 960         }
 961
 962         // for each dimension
 963         for(c = 0, d = r->st->dimensions; d && c < r->d ;c++, d = d->next) {
 964             if(unlikely(r->od[c] & RRDR_HIDDEN)) continue;
 965             if(unlikely((options & RRDR_OPTION_NONZERO) && !(r->od[c] & RRDR_NONZERO))) continue;
 966
 967             buffer_strcat(wb, separator);
 968
 969             calculated_number n = cn[c];
 970
 971             if(co[c] & RRDR_EMPTY) {
 972                 if(options & RRDR_OPTION_NULL2ZERO)
 973                     buffer_strcat(wb, "0");
 974                 else
 975                     buffer_strcat(wb, "null");
 976             }
 977             else {
 978                 if(unlikely((options & RRDR_OPTION_ABSOLUTE) && n < 0))
 979                     n = -n;
 980
 981                 if(unlikely(options & RRDR_OPTION_PERCENTAGE))
 982                     n = n * 100 / total;
 983
 984                 buffer_rrd_value(wb, n);
 985             }
 986         }
 987
 988         buffer_strcat(wb, endline);
 989     }
 990     //info("RRD2CSV(): %s: END", r->st->id);
 991 }
 992
 993 inline static calculated_number rrdr2value(RRDR *r, long i, uint32_t options, int *all_values_are_null) {
 994     long c;
 995     RRDDIM *d;
 996
 997     calculated_number *cn = &r->v[ i * r->d ];
 998     uint8_t *co = &r->o[ i * r->d ];
 999
1000     calculated_number sum = 0, min = 0, max = 0, v;
1001     int all_null = 1, init = 1;
1002
1003     calculated_number total = 1;
1004     if(unlikely(options & RRDR_OPTION_PERCENTAGE)) {
1005         total = 0;
1006         for(c = 0, d = r->st->dimensions; d && c < r->d ;c++, d = d->next) {
1007             calculated_number n = cn[c];
1008
1009             if(likely((options & RRDR_OPTION_ABSOLUTE) && n < 0))
1010                 n = -n;
1011
1012             total += n;
1013         }
1014         // prevent a division by zero
1015         if(total == 0) total = 1;
1016     }
1017
1018     // for each dimension
1019     for(c = 0, d = r->st->dimensions; d && c < r->d ;c++, d = d->next) {
1020         if(unlikely(r->od[c] & RRDR_HIDDEN)) continue;
1021         if(unlikely((options & RRDR_OPTION_NONZERO) && !(r->od[c] & RRDR_NONZERO))) continue;
1022
1023         calculated_number n = cn[c];
1024
1025         if(likely((options & RRDR_OPTION_ABSOLUTE) && n < 0))
1026             n = -n;
1027
1028         if(unlikely(options & RRDR_OPTION_PERCENTAGE))
1029             n = n * 100 / total;
1030
1031         if(unlikely(init)) {
1032             if(n > 0) {
1033                 min = 0;
1034                 max = n;
1035             }
1036             else {
1037                 min = n;
1038                 max = 0;
1039             }
1040             init = 0;
1041         }
1042
1043         if(likely(!(co[c] & RRDR_EMPTY))) {
1044             all_null = 0;
1045             sum += n;
1046         }
1047
1048         if(n < min) min = n;
1049         if(n > max) max = n;
1050     }
1051
1052     if(unlikely(all_null)) {
1053         if(likely(all_values_are_null))
1054             *all_values_are_null = 1;
1055         return 0;
1056     }
1057     else {
1058         if(likely(all_values_are_null))
1059             *all_values_are_null = 0;
1060     }
1061
1062     if(options & RRDR_OPTION_MIN2MAX)
1063         v = max - min;
1064     else
1065         v = sum;
1066
1067     return v;
1068 }
1069
1070 static void rrdr2ssv(RRDR *r, BUFFER *wb, uint32_t options, const char *prefix, const char *separator, const char *suffix)
1071 {
1072     //info("RRD2SSV(): %s: BEGIN", r->st->id);
1073     long i;
1074
1075     buffer_strcat(wb, prefix);
1076     long start = 0, end = rrdr_rows(r), step = 1;
1077     if((options & RRDR_OPTION_REVERSED)) {
1078         start = rrdr_rows(r) - 1;
1079         end = -1;
1080         step = -1;
1081     }
1082
1083     // for each line in the array
1084     for(i = start; i != end ;i += step) {
1085         int all_values_are_null = 0;
1086         calculated_number v = rrdr2value(r, i, options, &all_values_are_null);
1087
1088         if(likely(i != start)) {
1089             if(r->min > v) r->min = v;
1090             if(r->max < v) r->max = v;
1091         }
1092         else {
1093             r->min = v;
1094             r->max = v;
1095         }
1096
1097         if(likely(i != start))
1098             buffer_strcat(wb, separator);
1099
1100         if(all_values_are_null) {
1101             if(options & RRDR_OPTION_NULL2ZERO)
1102                 buffer_strcat(wb, "0");
1103             else
1104                 buffer_strcat(wb, "null");
1105         }
1106         else
1107             buffer_rrd_value(wb, v);
1108     }
1109     buffer_strcat(wb, suffix);
1110     //info("RRD2SSV(): %s: END", r->st->id);
1111 }
1112
1113 inline static calculated_number *rrdr_line_values(RRDR *r)
1114 {
1115     return &r->v[ r->c * r->d ];
1116 }
1117
1118 inline static uint8_t *rrdr_line_options(RRDR *r)
1119 {
1120     return &r->o[ r->c * r->d ];
1121 }
1122
1123 inline static int rrdr_line_init(RRDR *r, time_t t)
1124 {
1125     r->c++;
1126
1127     if(unlikely(r->c >= r->n)) {
1128         error("requested to step above RRDR size for chart %s", r->st->name);
1129         r->c = r->n - 1;
1130     }
1131
1132     // save the time
1133     r->t[r->c] = t;
1134
1135     return 1;
1136 }
1137
1138 inline static void rrdr_lock_rrdset(RRDR *r) {
1139     if(unlikely(!r)) {
1140         error("NULL value given!");
1141         return;
1142     }
1143
1144     pthread_rwlock_rdlock(&r->st->rwlock);
1145     r->has_st_lock = 1;
1146 }
1147
1148 inline static void rrdr_unlock_rrdset(RRDR *r) {
1149     if(unlikely(!r)) {
1150         error("NULL value given!");
1151         return;
1152     }
1153
1154     if(likely(r->has_st_lock)) {
1155         pthread_rwlock_unlock(&r->st->rwlock);
1156         r->has_st_lock = 0;
1157     }
1158 }
1159
1160 inline static void rrdr_free(RRDR *r)
1161 {
1162     if(unlikely(!r)) {
1163         error("NULL value given!");
1164         return;
1165     }
1166
1167     rrdr_unlock_rrdset(r);
1168     freez(r->t);
1169     freez(r->v);
1170     freez(r->o);
1171     freez(r->od);
1172     freez(r);
1173 }
1174
1175 static inline void rrdr_done(RRDR *r)
1176 {
1177     r->rows = r->c + 1;
1178     r->c = 0;
1179 }
1180
1181 static RRDR *rrdr_create(RRDSET *st, long n)
1182 {
1183     if(unlikely(!st)) {
1184         error("NULL value given!");
1185         return NULL;
1186     }
1187
1188     RRDR *r = callocz(1, sizeof(RRDR));
1189     r->st = st;
1190
1191     rrdr_lock_rrdset(r);
1192
1193     RRDDIM *rd;
1194     for(rd = st->dimensions ; rd ; rd = rd->next) r->d++;
1195
1196     r->n = n;
1197
1198     r->t = mallocz(n * sizeof(time_t));
1199     r->v = mallocz(n * r->d * sizeof(calculated_number));
1200     r->o = mallocz(n * r->d * sizeof(uint8_t));
1201     r->od = mallocz(r->d * sizeof(uint8_t));
1202
1203     // set the hidden flag on hidden dimensions
1204     int c;
1205     for(c = 0, rd = st->dimensions ; rd ; c++, rd = rd->next) {
1206         if(unlikely(rd->flags & RRDDIM_FLAG_HIDDEN)) r->od[c] = RRDR_HIDDEN;
1207         else r->od[c] = 0;
1208     }
1209
1210     r->c = -1;
1211     r->group = 1;
1212     r->update_every = 1;
1213
1214     return r;
1215 }
1216
1217 RRDR *rrd2rrdr(RRDSET *st, long points, long long after, long long before, int group_method, int aligned)
1218 {
1219     int debug = st->debug;
1220     int absolute_period_requested = -1;
1221
1222     time_t first_entry_t = rrdset_first_entry_t(st);
1223     time_t last_entry_t  = rrdset_last_entry_t(st);
1224
1225     if(before == 0 && after == 0) {
1226         // dump the all the data
1227         before = last_entry_t;
1228         after = first_entry_t;
1229         absolute_period_requested = 0;
1230     }
1231
1232     // allow relative for before (smaller than API_RELATIVE_TIME_MAX)
1233     if(((before < 0)?-before:before) <= API_RELATIVE_TIME_MAX) {
1234         if(abs(before) % st->update_every) {
1235             // make sure it is multiple of st->update_every
1236             if(before < 0) before = before - st->update_every - before % st->update_every;
1237             else           before = before + st->update_every - before % st->update_every;
1238         }
1239         if(before > 0) before = first_entry_t + before;
1240         else           before = last_entry_t  + before;
1241         absolute_period_requested = 0;
1242     }
1243
1244     // allow relative for after (smaller than API_RELATIVE_TIME_MAX)
1245     if(((after < 0)?-after:after) <= API_RELATIVE_TIME_MAX) {
1246         if(after == 0) after = -st->update_every;
1247         if(abs(after) % st->update_every) {
1248             // make sure it is multiple of st->update_every
1249             if(after < 0) after = after - st->update_every - after % st->update_every;
1250             else          after = after + st->update_every - after % st->update_every;
1251         }
1252         after = before + after;
1253         absolute_period_requested = 0;
1254     }
1255
1256     if(absolute_period_requested == -1)
1257         absolute_period_requested = 1;
1258
1259     // make sure they are within our timeframe
1260     if(before > last_entry_t)  before = last_entry_t;
1261     if(before < first_entry_t) before = first_entry_t;
1262
1263     if(after > last_entry_t)  after = last_entry_t;
1264     if(after < first_entry_t) after = first_entry_t;
1265
1266     // check if they are upside down
1267     if(after > before) {
1268         time_t tmp = before;
1269         before = after;
1270         after = tmp;
1271     }
1272
1273     // the duration of the chart
1274     time_t duration = before - after;
1275     long available_points = duration / st->update_every;
1276
1277     if(duration <= 0 || available_points <= 0)
1278         return rrdr_create(st, 1);
1279
1280     // check the wanted points
1281     if(points < 0) points = -points;
1282     if(points > available_points) points = available_points;
1283     if(points == 0) points = available_points;
1284
1285     // calculate proper grouping of source data
1286     long group = available_points / points;
1287     if(group <= 0) group = 1;
1288
1289     // round group to the closest integer
1290     if(available_points % points > points / 2) group++;
1291
1292     time_t after_new  = (aligned) ? (after  - (after  % (group * st->update_every))) : after;
1293     time_t before_new = (aligned) ? (before - (before % (group * st->update_every))) : before;
1294     long points_new   = (before_new - after_new) / st->update_every / group;
1295
1296     // find the starting and ending slots in our round robin db
1297     long    start_at_slot = rrdset_time2slot(st, before_new),
1298             stop_at_slot  = rrdset_time2slot(st, after_new);
1299
1300 #ifdef NETDATA_INTERNAL_CHECKS
1301     if(after_new < first_entry_t) {
1302         error("after_new %u is too small, minimum %u", (uint32_t)after_new, (uint32_t)first_entry_t);
1303     }
1304     if(after_new > last_entry_t) {
1305         error("after_new %u is too big, maximum %u", (uint32_t)after_new, (uint32_t)last_entry_t);
1306     }
1307     if(before_new < first_entry_t) {
1308         error("before_new %u is too small, minimum %u", (uint32_t)before_new, (uint32_t)first_entry_t);
1309     }
1310     if(before_new > last_entry_t) {
1311         error("before_new %u is too big, maximum %u", (uint32_t)before_new, (uint32_t)last_entry_t);
1312     }
1313     if(start_at_slot < 0 || start_at_slot >= st->entries) {
1314         error("start_at_slot is invalid %ld, expected 0 to %ld", start_at_slot, st->entries - 1);
1315     }
1316     if(stop_at_slot < 0 || stop_at_slot >= st->entries) {
1317         error("stop_at_slot is invalid %ld, expected 0 to %ld", stop_at_slot, st->entries - 1);
1318     }
1319     if(points_new > (before_new - after_new) / group / st->update_every + 1) {
1320         error("points_new %ld is more than points %ld", points_new, (before_new - after_new) / group / st->update_every + 1);
1321     }
1322 #endif
1323
1324     //info("RRD2RRDR(): %s: wanted %ld points, got %ld - group=%ld, wanted duration=%u, got %u - wanted %ld - %ld, got %ld - %ld", st->id, points, points_new, group, before - after, before_new - after_new, after, before, after_new, before_new);
1325
1326     after = after_new;
1327     before = before_new;
1328     duration = before - after;
1329     points = points_new;
1330
1331     // Now we have:
1332     // before = the end time of the calculation
1333     // after = the start time of the calculation
1334     // duration = the duration of the calculation
1335     // group = the number of source points to aggregate / group together
1336     // method = the method of grouping source points
1337     // points = the number of points to generate
1338
1339
1340     // -------------------------------------------------------------------------
1341     // initialize our result set
1342
1343     RRDR *r = rrdr_create(st, points);
1344     if(!r) {
1345 #ifdef NETDATA_INTERNAL_CHECKS
1346         error("Cannot create RRDR for %s, after=%u, before=%u, duration=%u, points=%ld", st->id, (uint32_t)after, (uint32_t)before, (uint32_t)duration, points);
1347 #endif
1348         return NULL;
1349     }
1350     if(!r->d) {
1351 #ifdef NETDATA_INTERNAL_CHECKS
1352         error("Returning empty RRDR (no dimensions in RRDSET) for %s, after=%u, before=%u, duration=%u, points=%ld", st->id, (uint32_t)after, (uint32_t)before, (uint32_t)duration, points);
1353 #endif
1354         return r;
1355     }
1356
1357     if(absolute_period_requested == 1)
1358         r->result_options |= RRDR_RESULT_OPTION_ABSOLUTE;
1359     else
1360         r->result_options |= RRDR_RESULT_OPTION_RELATIVE;
1361
1362     // find how many dimensions we have
1363     long dimensions = r->d;
1364
1365
1366     // -------------------------------------------------------------------------
1367     // checks for debugging
1368
1369     if(debug) debug(D_RRD_STATS, "INFO %s first_t: %u, last_t: %u, all_duration: %u, after: %u, before: %u, duration: %u, points: %ld, group: %ld"
1370             , st->id
1371             , (uint32_t)first_entry_t
1372             , (uint32_t)last_entry_t
1373             , (uint32_t)(last_entry_t - first_entry_t)
1374             , (uint32_t)after
1375             , (uint32_t)before
1376             , (uint32_t)duration
1377             , points
1378             , group
1379             );
1380
1381
1382     // -------------------------------------------------------------------------
1383     // temp arrays for keeping values per dimension
1384
1385     calculated_number   last_values[dimensions]; // keep the last value of each dimension
1386     calculated_number   group_values[dimensions]; // keep sums when grouping
1387     long                group_counts[dimensions]; // keep the number of values added to group_values
1388     uint8_t             group_options[dimensions];
1389     uint8_t             found_non_zero[dimensions];
1390
1391
1392     // initialize them
1393     RRDDIM *rd;
1394     long c;
1395     for( rd = st->dimensions, c = 0 ; rd && c < dimensions ; rd = rd->next, c++) {
1396         last_values[c] = 0;
1397         group_values[c] = (group_method == GROUP_MAX || group_method == GROUP_MIN)?NAN:0;
1398         group_counts[c] = 0;
1399         group_options[c] = 0;
1400         found_non_zero[c] = 0;
1401     }
1402
1403
1404     // -------------------------------------------------------------------------
1405     // the main loop
1406
1407     time_t  now = rrdset_slot2time(st, start_at_slot),
1408             dt = st->update_every,
1409             group_start_t = 0;
1410
1411     if(unlikely(debug)) debug(D_RRD_STATS, "BEGIN %s after_t: %u (stop_at_t: %ld), before_t: %u (start_at_t: %ld), start_t(now): %u, current_entry: %ld, entries: %ld"
1412             , st->id
1413             , (uint32_t)after
1414             , stop_at_slot
1415             , (uint32_t)before
1416             , start_at_slot
1417             , (uint32_t)now
1418             , st->current_entry
1419             , st->entries
1420             );
1421
1422     r->group = group;
1423     r->update_every = group * st->update_every;
1424     r->before = now;
1425     r->after = now;
1426
1427     //info("RRD2RRDR(): %s: STARTING", st->id);
1428
1429     long slot = start_at_slot, counter = 0, stop_now = 0, added = 0, group_count = 0, add_this = 0;
1430     for(; !stop_now ; now -= dt, slot--, counter++) {
1431         if(unlikely(slot < 0)) slot = st->entries - 1;
1432         if(unlikely(slot == stop_at_slot)) stop_now = counter;
1433
1434         if(unlikely(debug)) debug(D_RRD_STATS, "ROW %s slot: %ld, entries_counter: %ld, group_count: %ld, added: %ld, now: %ld, %s %s"
1435                 , st->id
1436                 , slot
1437                 , counter
1438                 , group_count + 1
1439                 , added
1440                 , now
1441                 , (group_count + 1 == group)?"PRINT":"  -  "
1442                 , (now >= after && now <= before)?"RANGE":"  -  "
1443                 );
1444
1445         // make sure we return data in the proper time range
1446         if(unlikely(now > before)) continue;
1447         if(unlikely(now < after)) break;
1448
1449         if(unlikely(group_count == 0)) {
1450             group_start_t = now;
1451         }
1452         group_count++;
1453
1454         if(unlikely(group_count == group)) {
1455             if(unlikely(added >= points)) break;
1456             add_this = 1;
1457         }
1458
1459         // do the calculations
1460         for(rd = st->dimensions, c = 0 ; rd && c < dimensions ; rd = rd->next, c++) {
1461             storage_number n = rd->values[slot];
1462             if(unlikely(!does_storage_number_exist(n))) continue;
1463
1464             group_counts[c]++;
1465
1466             calculated_number value = unpack_storage_number(n);
1467             if(likely(value != 0.0)) {
1468                 group_options[c] |= RRDR_NONZERO;
1469                 found_non_zero[c] = 1;
1470             }
1471
1472             if(unlikely(did_storage_number_reset(n)))
1473                 group_options[c] |= RRDR_RESET;
1474
1475             switch(group_method) {
1476                 case GROUP_MIN:
1477                     if(unlikely(isnan(group_values[c])) ||
1478                             fabsl(value) < fabsl(group_values[c]))
1479                         group_values[c] = value;
1480                     break;
1481
1482                 case GROUP_MAX:
1483                     if(unlikely(isnan(group_values[c])) ||
1484                             fabsl(value) > fabsl(group_values[c]))
1485                         group_values[c] = value;
1486                     break;
1487
1488                 default:
1489                 case GROUP_SUM:
1490                 case GROUP_AVERAGE:
1491                 case GROUP_UNDEFINED:
1492                     group_values[c] += value;
1493                     break;
1494
1495                 case GROUP_INCREMENTAL_SUM:
1496                     if(unlikely(slot == start_at_slot))
1497                         last_values[c] = value;
1498
1499                     group_values[c] += last_values[c] - value;
1500                     last_values[c] = value;
1501                     break;
1502             }
1503         }
1504
1505         // added it
1506         if(unlikely(add_this)) {
1507             if(unlikely(!rrdr_line_init(r, group_start_t))) break;
1508
1509             r->after = now;
1510
1511             calculated_number *cn = rrdr_line_values(r);
1512             uint8_t *co = rrdr_line_options(r);
1513
1514             for(rd = st->dimensions, c = 0 ; rd && c < dimensions ; rd = rd->next, c++) {
1515
1516                 // update the dimension options
1517                 if(likely(found_non_zero[c])) r->od[c] |= RRDR_NONZERO;
1518
1519                 // store the specific point options
1520                 co[c] = group_options[c];
1521
1522                 // store the value
1523                 if(unlikely(group_counts[c] == 0)) {
1524                     cn[c] = 0.0;
1525                     co[c] |= RRDR_EMPTY;
1526                     group_values[c] = (group_method == GROUP_MAX || group_method == GROUP_MIN)?NAN:0;
1527                 }
1528                 else {
1529                     switch(group_method) {
1530                         case GROUP_MIN:
1531                         case GROUP_MAX:
1532                             if(unlikely(isnan(group_values[c])))
1533                                 cn[c] = 0;
1534                             else {
1535                                 cn[c] = group_values[c];
1536                                 group_values[c] = NAN;
1537                             }
1538                             break;
1539
1540                         case GROUP_SUM:
1541                         case GROUP_INCREMENTAL_SUM:
1542                             cn[c] = group_values[c];
1543                             group_values[c] = 0;
1544                             break;
1545
1546                         default:
1547                         case GROUP_AVERAGE:
1548                         case GROUP_UNDEFINED:
1549                             cn[c] = group_values[c] / group_counts[c];
1550                             group_values[c] = 0;
1551                             break;
1552                     }
1553
1554                     if(cn[c] < r->min) r->min = cn[c];
1555                     if(cn[c] > r->max) r->max = cn[c];
1556                 }
1557
1558                 // reset for the next loop
1559                 group_counts[c] = 0;
1560                 group_options[c] = 0;
1561             }
1562
1563             added++;
1564             group_count = 0;
1565             add_this = 0;
1566         }
1567     }
1568
1569     rrdr_done(r);
1570     //info("RRD2RRDR(): %s: END %ld loops made, %ld points generated", st->id, counter, rrdr_rows(r));
1571     //error("SHIFT: %s: wanted %ld points, got %ld", st->id, points, rrdr_rows(r));
1572     return r;
1573 }
1574
1575 int rrd2value(RRDSET *st, BUFFER *wb, calculated_number *n, const char *dimensions, long points, long long after, long long before, int group_method, uint32_t options, time_t *db_after, time_t *db_before, int *value_is_null)
1576 {
1577     RRDR *r = rrd2rrdr(st, points, after, before, group_method, !(options & RRDR_OPTION_NOT_ALIGNED));
1578     if(!r) {
1579         if(value_is_null) *value_is_null = 1;
1580         return 500;
1581     }
1582
1583     if(rrdr_rows(r) == 0) {
1584         rrdr_free(r);
1585
1586         if(db_after)  *db_after  = 0;
1587         if(db_before) *db_before = 0;
1588         if(value_is_null) *value_is_null = 1;
1589
1590         return 400;
1591     }
1592
1593     if(r->result_options & RRDR_RESULT_OPTION_RELATIVE)
1594         buffer_no_cacheable(wb);
1595     else if(r->result_options & RRDR_RESULT_OPTION_ABSOLUTE)
1596         buffer_cacheable(wb);
1597
1598     options = rrdr_check_options(r, options, dimensions);
1599
1600     if(dimensions)
1601         rrdr_disable_not_selected_dimensions(r, dimensions);
1602
1603     if(db_after)  *db_after  = r->after;
1604     if(db_before) *db_before = r->before;
1605
1606     long i = (options & RRDR_OPTION_REVERSED)?rrdr_rows(r) - 1:0;
1607     *n = rrdr2value(r, i, options, value_is_null);
1608
1609     rrdr_free(r);
1610     return 200;
1611 }
1612
1613 int rrd2format(RRDSET *st, BUFFER *wb, BUFFER *dimensions, uint32_t format, long points, long long after, long long before, int group_method, uint32_t options, time_t *latest_timestamp)
1614 {
1615     RRDR *r = rrd2rrdr(st, points, after, before, group_method, !(options & RRDR_OPTION_NOT_ALIGNED));
1616     if(!r) {
1617         buffer_strcat(wb, "Cannot generate output with these parameters on this chart.");
1618         return 500;
1619     }
1620
1621     if(r->result_options & RRDR_RESULT_OPTION_RELATIVE)
1622         buffer_no_cacheable(wb);
1623     else if(r->result_options & RRDR_RESULT_OPTION_ABSOLUTE)
1624         buffer_cacheable(wb);
1625
1626     options = rrdr_check_options(r, options, (dimensions)?buffer_tostring(dimensions):NULL);
1627
1628     if(dimensions)
1629         rrdr_disable_not_selected_dimensions(r, buffer_tostring(dimensions));
1630
1631     if(latest_timestamp && rrdr_rows(r) > 0)
1632         *latest_timestamp = r->before;
1633
1634     switch(format) {
1635     case DATASOURCE_SSV:
1636         if(options & RRDR_OPTION_JSON_WRAP) {
1637             wb->contenttype = CT_APPLICATION_JSON;
1638             rrdr_json_wrapper_begin(r, wb, format, options, 1);
1639             rrdr2ssv(r, wb, options, "", " ", "");
1640             rrdr_json_wrapper_end(r, wb, format, options, 1);
1641         }
1642         else {
1643             wb->contenttype = CT_TEXT_PLAIN;
1644             rrdr2ssv(r, wb, options, "", " ", "");
1645         }
1646         break;
1647
1648     case DATASOURCE_SSV_COMMA:
1649         if(options & RRDR_OPTION_JSON_WRAP) {
1650             wb->contenttype = CT_APPLICATION_JSON;
1651             rrdr_json_wrapper_begin(r, wb, format, options, 1);
1652             rrdr2ssv(r, wb, options, "", ",", "");
1653             rrdr_json_wrapper_end(r, wb, format, options, 1);
1654         }
1655         else {
1656             wb->contenttype = CT_TEXT_PLAIN;
1657             rrdr2ssv(r, wb, options, "", ",", "");
1658         }
1659         break;
1660
1661     case DATASOURCE_JS_ARRAY:
1662         if(options & RRDR_OPTION_JSON_WRAP) {
1663             wb->contenttype = CT_APPLICATION_JSON;
1664             rrdr_json_wrapper_begin(r, wb, format, options, 0);
1665             rrdr2ssv(r, wb, options, "[", ",", "]");
1666             rrdr_json_wrapper_end(r, wb, format, options, 0);
1667         }
1668         else {
1669             wb->contenttype = CT_APPLICATION_JSON;
1670             rrdr2ssv(r, wb, options, "[", ",", "]");
1671         }
1672         break;
1673
1674     case DATASOURCE_CSV:
1675         if(options & RRDR_OPTION_JSON_WRAP) {
1676             wb->contenttype = CT_APPLICATION_JSON;
1677             rrdr_json_wrapper_begin(r, wb, format, options, 1);
1678             rrdr2csv(r, wb, options, "", ",", "\\n", "");
1679             rrdr_json_wrapper_end(r, wb, format, options, 1);
1680         }
1681         else {
1682             wb->contenttype = CT_TEXT_PLAIN;
1683             rrdr2csv(r, wb, options, "", ",", "\r\n", "");
1684         }
1685         break;
1686
1687     case DATASOURCE_CSV_JSON_ARRAY:
1688         wb->contenttype = CT_APPLICATION_JSON;
1689         if(options & RRDR_OPTION_JSON_WRAP) {
1690             rrdr_json_wrapper_begin(r, wb, format, options, 0);
1691             buffer_strcat(wb, "[\n");
1692             rrdr2csv(r, wb, options + RRDR_OPTION_LABEL_QUOTES, "[", ",", "]", ",\n");
1693             buffer_strcat(wb, "\n]");
1694             rrdr_json_wrapper_end(r, wb, format, options, 0);
1695         }
1696         else {
1697             wb->contenttype = CT_TEXT_PLAIN;
1698             buffer_strcat(wb, "[\n");
1699             rrdr2csv(r, wb, options + RRDR_OPTION_LABEL_QUOTES, "[", ",", "]", ",\n");
1700             buffer_strcat(wb, "\n]");
1701         }
1702         break;
1703
1704     case DATASOURCE_TSV:
1705         if(options & RRDR_OPTION_JSON_WRAP) {
1706             wb->contenttype = CT_APPLICATION_JSON;
1707             rrdr_json_wrapper_begin(r, wb, format, options, 1);
1708             rrdr2csv(r, wb, options, "", "\t", "\\n", "");
1709             rrdr_json_wrapper_end(r, wb, format, options, 1);
1710         }
1711         else {
1712             wb->contenttype = CT_TEXT_PLAIN;
1713             rrdr2csv(r, wb, options, "", "\t", "\r\n", "");
1714         }
1715         break;
1716
1717     case DATASOURCE_HTML:
1718         if(options & RRDR_OPTION_JSON_WRAP) {
1719             wb->contenttype = CT_APPLICATION_JSON;
1720             rrdr_json_wrapper_begin(r, wb, format, options, 1);
1721             buffer_strcat(wb, "<html>\\n<center>\\n<table border=\\\"0\\\" cellpadding=\\\"5\\\" cellspacing=\\\"5\\\">\\n");
1722             rrdr2csv(r, wb, options, "<tr><td>", "</td><td>", "</td></tr>\\n", "");
1723             buffer_strcat(wb, "</table>\\n</center>\\n</html>\\n");
1724             rrdr_json_wrapper_end(r, wb, format, options, 1);
1725         }
1726         else {
1727             wb->contenttype = CT_TEXT_HTML;
1728             buffer_strcat(wb, "<html>\n<center>\n<table border=\"0\" cellpadding=\"5\" cellspacing=\"5\">\n");
1729             rrdr2csv(r, wb, options, "<tr><td>", "</td><td>", "</td></tr>\n", "");
1730             buffer_strcat(wb, "</table>\n</center>\n</html>\n");
1731         }
1732         break;
1733
1734     case DATASOURCE_DATATABLE_JSONP:
1735         wb->contenttype = CT_APPLICATION_X_JAVASCRIPT;
1736
1737         if(options & RRDR_OPTION_JSON_WRAP)
1738             rrdr_json_wrapper_begin(r, wb, format, options, 0);
1739
1740         rrdr2json(r, wb, options, 1);
1741
1742         if(options & RRDR_OPTION_JSON_WRAP)
1743             rrdr_json_wrapper_end(r, wb, format, options, 0);
1744         break;
1745
1746     case DATASOURCE_DATATABLE_JSON:
1747         wb->contenttype = CT_APPLICATION_JSON;
1748
1749         if(options & RRDR_OPTION_JSON_WRAP)
1750             rrdr_json_wrapper_begin(r, wb, format, options, 0);
1751
1752         rrdr2json(r, wb, options, 1);
1753
1754         if(options & RRDR_OPTION_JSON_WRAP)
1755             rrdr_json_wrapper_end(r, wb, format, options, 0);
1756         break;
1757
1758     case DATASOURCE_JSONP:
1759         wb->contenttype = CT_APPLICATION_X_JAVASCRIPT;
1760         if(options & RRDR_OPTION_JSON_WRAP)
1761             rrdr_json_wrapper_begin(r, wb, format, options, 0);
1762
1763         rrdr2json(r, wb, options, 0);
1764
1765         if(options & RRDR_OPTION_JSON_WRAP)
1766             rrdr_json_wrapper_end(r, wb, format, options, 0);
1767         break;
1768
1769     case DATASOURCE_JSON:
1770     default:
1771         wb->contenttype = CT_APPLICATION_JSON;
1772
1773         if(options & RRDR_OPTION_JSON_WRAP)
1774             rrdr_json_wrapper_begin(r, wb, format, options, 0);
1775
1776         rrdr2json(r, wb, options, 0);
1777
1778         if(options & RRDR_OPTION_JSON_WRAP)
1779             rrdr_json_wrapper_end(r, wb, format, options, 0);
1780         break;
1781     }
1782
1783     rrdr_free(r);
1784     return 200;
1785 }
1786
1787 time_t rrd_stats_json(int type, RRDSET *st, BUFFER *wb, long points, long group, int group_method, time_t after, time_t before, int only_non_zero)
1788 {
1789     int c;
1790     pthread_rwlock_rdlock(&st->rwlock);
1791
1792
1793     // -------------------------------------------------------------------------
1794     // switch from JSON to google JSON
1795
1796     char kq[2] = "\"";
1797     char sq[2] = "\"";
1798     switch(type) {
1799         case DATASOURCE_DATATABLE_JSON:
1800         case DATASOURCE_DATATABLE_JSONP:
1801             kq[0] = '\0';
1802             sq[0] = '\'';
1803             break;
1804
1805         case DATASOURCE_JSON:
1806         default:
1807             break;
1808     }
1809
1810
1811     // -------------------------------------------------------------------------
1812     // validate the parameters
1813
1814     if(points < 1) points = 1;
1815     if(group < 1) group = 1;
1816
1817     if(before == 0 || before > rrdset_last_entry_t(st)) before = rrdset_last_entry_t(st);
1818     if(after  == 0 || after < rrdset_first_entry_t(st)) after = rrdset_first_entry_t(st);
1819
1820     // ---
1821
1822     // our return value (the last timestamp printed)
1823     // this is required to detect re-transmit in google JSONP
1824     time_t last_timestamp = 0;
1825
1826
1827     // -------------------------------------------------------------------------
1828     // find how many dimensions we have
1829
1830     int dimensions = 0;
1831     RRDDIM *rd;
1832     for( rd = st->dimensions ; rd ; rd = rd->next) dimensions++;
1833     if(!dimensions) {
1834         pthread_rwlock_unlock(&st->rwlock);
1835         buffer_strcat(wb, "No dimensions yet.");
1836         return 0;
1837     }
1838
1839
1840     // -------------------------------------------------------------------------
1841     // prepare various strings, to speed up the loop
1842
1843     char overflow_annotation[201]; snprintfz(overflow_annotation, 200, ",{%sv%s:%sRESET OR OVERFLOW%s},{%sv%s:%sThe counters have been wrapped.%s}", kq, kq, sq, sq, kq, kq, sq, sq);
1844     char normal_annotation[201];   snprintfz(normal_annotation,   200, ",{%sv%s:null},{%sv%s:null}", kq, kq, kq, kq);
1845     char pre_date[51];             snprintfz(pre_date,             50, "        {%sc%s:[{%sv%s:%s", kq, kq, kq, kq, sq);
1846     char post_date[21];            snprintfz(post_date,            20, "%s}", sq);
1847     char pre_value[21];            snprintfz(pre_value,            20, ",{%sv%s:", kq, kq);
1848     char post_value[21];           strcpy(post_value,                  "}");
1849
1850
1851     // -------------------------------------------------------------------------
1852     // checks for debugging
1853
1854     if(st->debug) {
1855         debug(D_RRD_STATS, "%s first_entry_t = %ld, last_entry_t = %ld, duration = %ld, after = %ld, before = %ld, duration = %ld, entries_to_show = %ld, group = %ld"
1856             , st->id
1857             , rrdset_first_entry_t(st)
1858             , rrdset_last_entry_t(st)
1859             , rrdset_last_entry_t(st) - rrdset_first_entry_t(st)
1860             , after
1861             , before
1862             , before - after
1863             , points
1864             , group
1865             );
1866
1867         if(before < after)
1868             debug(D_RRD_STATS, "WARNING: %s The newest value in the database (%ld) is earlier than the oldest (%ld)", st->name, before, after);
1869
1870         if((before - after) > st->entries * st->update_every)
1871             debug(D_RRD_STATS, "WARNING: %s The time difference between the oldest and the newest entries (%ld) is higher than the capacity of the database (%ld)", st->name, before - after, st->entries * st->update_every);
1872     }
1873
1874
1875     // -------------------------------------------------------------------------
1876     // temp arrays for keeping values per dimension
1877
1878     calculated_number group_values[dimensions]; // keep sums when grouping
1879     int               print_hidden[dimensions]; // keep hidden flags
1880     int               found_non_zero[dimensions];
1881     int               found_non_existing[dimensions];
1882
1883     // initialize them
1884     for( rd = st->dimensions, c = 0 ; rd && c < dimensions ; rd = rd->next, c++) {
1885         group_values[c] = 0;
1886         print_hidden[c] = (rd->flags & RRDDIM_FLAG_HIDDEN)?1:0;
1887         found_non_zero[c] = 0;
1888         found_non_existing[c] = 0;
1889     }
1890
1891
1892     // error("OLD: points=%d after=%d before=%d group=%d, duration=%d", entries_to_show, before - (st->update_every * group * entries_to_show), before, group, before - after + 1);
1893     // rrd2array(st, entries_to_show, before - (st->update_every * group * entries_to_show), before, group_method, only_non_zero);
1894     // rrd2rrdr(st, entries_to_show, before - (st->update_every * group * entries_to_show), before, group_method);
1895
1896     // -------------------------------------------------------------------------
1897     // remove dimensions that contain only zeros
1898
1899     int max_loop = 1;
1900     if(only_non_zero) max_loop = 2;
1901
1902     for(; max_loop ; max_loop--) {
1903
1904         // -------------------------------------------------------------------------
1905         // print the JSON header
1906
1907         buffer_sprintf(wb, "{\n %scols%s:\n [\n", kq, kq);
1908         buffer_sprintf(wb, "        {%sid%s:%s%s,%slabel%s:%stime%s,%spattern%s:%s%s,%stype%s:%sdatetime%s},\n", kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq);
1909         buffer_sprintf(wb, "        {%sid%s:%s%s,%slabel%s:%s%s,%spattern%s:%s%s,%stype%s:%sstring%s,%sp%s:{%srole%s:%sannotation%s}},\n", kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, kq, kq, sq, sq);
1910         buffer_sprintf(wb, "        {%sid%s:%s%s,%slabel%s:%s%s,%spattern%s:%s%s,%stype%s:%sstring%s,%sp%s:{%srole%s:%sannotationText%s}}", kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, kq, kq, sq, sq);
1911
1912         // print the header for each dimension
1913         // and update the print_hidden array for the dimensions that should be hidden
1914         int pc = 0;
1915         for( rd = st->dimensions, c = 0 ; rd && c < dimensions ; rd = rd->next, c++) {
1916             if(!print_hidden[c]) {
1917                 pc++;
1918                 buffer_sprintf(wb, ",\n     {%sid%s:%s%s,%slabel%s:%s%s%s,%spattern%s:%s%s,%stype%s:%snumber%s}", kq, kq, sq, sq, kq, kq, sq, rd->name, sq, kq, kq, sq, sq, kq, kq, sq, sq);
1919             }
1920         }
1921         if(!pc) {
1922             buffer_sprintf(wb, ",\n     {%sid%s:%s%s,%slabel%s:%s%s%s,%spattern%s:%s%s,%stype%s:%snumber%s}", kq, kq, sq, sq, kq, kq, sq, "no data", sq, kq, kq, sq, sq, kq, kq, sq, sq);
1923         }
1924
1925         // print the begin of row data
1926         buffer_sprintf(wb, "\n  ],\n    %srows%s:\n [\n", kq, kq);
1927
1928
1929         // -------------------------------------------------------------------------
1930         // the main loop
1931
1932         int annotate_reset = 0;
1933         int annotation_count = 0;
1934
1935         long    t = rrdset_time2slot(st, before),
1936                 stop_at_t = rrdset_time2slot(st, after),
1937                 stop_now = 0;
1938
1939         t -= t % group;
1940
1941         time_t  now = rrdset_slot2time(st, t),
1942                 dt = st->update_every;
1943
1944         long count = 0, printed = 0, group_count = 0;
1945         last_timestamp = 0;
1946
1947         if(st->debug) debug(D_RRD_STATS, "%s: REQUEST after:%u before:%u, points:%ld, group:%ld, CHART cur:%ld first: %u last:%u, CALC start_t:%ld, stop_t:%ld"
1948                     , st->id
1949                     , (uint32_t)after
1950                     , (uint32_t)before
1951                     , points
1952                     , group
1953                     , st->current_entry
1954                     , (uint32_t)rrdset_first_entry_t(st)
1955                     , (uint32_t)rrdset_last_entry_t(st)
1956                     , t
1957                     , stop_at_t
1958                     );
1959
1960         long counter = 0;
1961         for(; !stop_now ; now -= dt, t--, counter++) {
1962             if(t < 0) t = st->entries - 1;
1963             if(t == stop_at_t) stop_now = counter;
1964
1965             int print_this = 0;
1966
1967             if(st->debug) debug(D_RRD_STATS, "%s t = %ld, count = %ld, group_count = %ld, printed = %ld, now = %ld, %s %s"
1968                     , st->id
1969                     , t
1970                     , count + 1
1971                     , group_count + 1
1972                     , printed
1973                     , now
1974                     , (group_count + 1 == group)?"PRINT":"  -  "
1975                     , (now >= after && now <= before)?"RANGE":"  -  "
1976                     );
1977
1978
1979             // make sure we return data in the proper time range
1980             if(now > before) continue;
1981             if(now < after) break;
1982
1983             //if(rrdset_slot2time(st, t) != now)
1984             //  error("%s: slot=%ld, now=%ld, slot2time=%ld, diff=%ld, last_entry_t=%ld, rrdset_last_slot=%ld", st->id, t, now, rrdset_slot2time(st,t), now - rrdset_slot2time(st,t), rrdset_last_entry_t(st), rrdset_last_slot(st));
1985
1986             count++;
1987             group_count++;
1988
1989             // check if we have to print this now
1990             if(group_count == group) {
1991                 if(printed >= points) {
1992                     // debug(D_RRD_STATS, "Already printed all rows. Stopping.");
1993                     break;
1994                 }
1995
1996                 // generate the local date time
1997                 struct tm tmbuf, *tm = localtime_r(&now, &tmbuf);
1998                 if(!tm) { error("localtime() failed."); continue; }
1999                 if(now > last_timestamp) last_timestamp = now;
2000
2001                 if(printed) buffer_strcat(wb, "]},\n");
2002                 buffer_strcat(wb, pre_date);
2003                 buffer_jsdate(wb, tm->tm_year + 1900, tm->tm_mon, tm->tm_mday, tm->tm_hour, tm->tm_min, tm->tm_sec);
2004                 buffer_strcat(wb, post_date);
2005
2006                 print_this = 1;
2007             }
2008
2009             // do the calculations
2010             for(rd = st->dimensions, c = 0 ; rd && c < dimensions ; rd = rd->next, c++) {
2011                 storage_number n = rd->values[t];
2012                 calculated_number value = unpack_storage_number(n);
2013
2014                 if(!does_storage_number_exist(n)) {
2015                     value = 0.0;
2016                     found_non_existing[c]++;
2017                 }
2018                 if(did_storage_number_reset(n)) annotate_reset = 1;
2019
2020                 switch(group_method) {
2021                     case GROUP_MAX:
2022                         if(abs(value) > abs(group_values[c])) group_values[c] = value;
2023                         break;
2024
2025                     case GROUP_SUM:
2026                         group_values[c] += value;
2027                         break;
2028
2029                     default:
2030                     case GROUP_AVERAGE:
2031                         group_values[c] += value;
2032                         if(print_this) group_values[c] /= ( group_count - found_non_existing[c] );
2033                         break;
2034                 }
2035             }
2036
2037             if(print_this) {
2038                 if(annotate_reset) {
2039                     annotation_count++;
2040                     buffer_strcat(wb, overflow_annotation);
2041                     annotate_reset = 0;
2042                 }
2043                 else
2044                     buffer_strcat(wb, normal_annotation);
2045
2046                 pc = 0;
2047                 for(c = 0 ; c < dimensions ; c++) {
2048                     if(found_non_existing[c] == group_count) {
2049                         // all entries are non-existing
2050                         pc++;
2051                         buffer_strcat(wb, pre_value);
2052                         buffer_strcat(wb, "null");
2053                         buffer_strcat(wb, post_value);
2054                     }
2055                     else if(!print_hidden[c]) {
2056                         pc++;
2057                         buffer_strcat(wb, pre_value);
2058                         buffer_rrd_value(wb, group_values[c]);
2059                         buffer_strcat(wb, post_value);
2060
2061                         if(group_values[c]) found_non_zero[c]++;
2062                     }
2063
2064                     // reset them for the next loop
2065                     group_values[c] = 0;
2066                     found_non_existing[c] = 0;
2067                 }
2068
2069                 // if all dimensions are hidden, print a null
2070                 if(!pc) {
2071                     buffer_strcat(wb, pre_value);
2072                     buffer_strcat(wb, "null");
2073                     buffer_strcat(wb, post_value);
2074                 }
2075
2076                 printed++;
2077                 group_count = 0;
2078             }
2079         }
2080
2081         if(printed) buffer_strcat(wb, "]}");
2082         buffer_strcat(wb, "\n   ]\n}\n");
2083
2084         if(only_non_zero && max_loop > 1) {
2085             int changed = 0;
2086             for(rd = st->dimensions, c = 0 ; rd && c < dimensions ; rd = rd->next, c++) {
2087                 group_values[c] = 0;
2088                 found_non_existing[c] = 0;
2089
2090                 if(!print_hidden[c] && !found_non_zero[c]) {
2091                     changed = 1;
2092                     print_hidden[c] = 1;
2093                 }
2094             }
2095
2096             if(changed) buffer_flush(wb);
2097             else break;
2098         }
2099         else break;
2100
2101     } // max_loop
2102
2103     debug(D_RRD_STATS, "RRD_STATS_JSON: %s total %lu bytes", st->name, wb->len);
2104
2105     pthread_rwlock_unlock(&st->rwlock);
2106     return last_timestamp;
2107 }