src/rrd2json.c

   1 #ifdef HAVE_CONFIG_H
   2 #include <config.h>
   3 #endif
   4 #include <pthread.h>
   5 #include <stdlib.h>
   6 #include <string.h>
   7 #include <stdint.h>
   8
   9 #include "log.h"
  10 #include "common.h"
  11 #include "rrd2json.h"
  12
  13 #define HOSTNAME_MAX 1024
  14 char *hostname = "unknown";
  15
  16 void rrd_stats_api_v1_chart(RRDSET *st, BUFFER *wb)
  17 {
  18         pthread_rwlock_rdlock(&st->rwlock);
  19
  20         buffer_sprintf(wb,
  21                 "\t\t{\n"
  22                 "\t\t\t\"id\": \"%s\",\n"
  23                 "\t\t\t\"name\": \"%s\",\n"
  24                 "\t\t\t\"type\": \"%s\",\n"
  25                 "\t\t\t\"family\": \"%s\",\n"
  26                 "\t\t\t\"title\": \"%s\",\n"
  27                 "\t\t\t\"priority\": %ld,\n"
  28                 "\t\t\t\"enabled\": %s,\n"
  29                 "\t\t\t\"units\": \"%s\",\n"
  30                 "\t\t\t\"data_url\": \"/api/v1/data?chart=%s\",\n"
  31                 "\t\t\t\"chart_type\": \"%s\",\n"
  32                 "\t\t\t\"duration\": %ld,\n"
  33                 "\t\t\t\"first_entry\": %lu,\n"
  34                 "\t\t\t\"last_entry\": %lu,\n"
  35                 "\t\t\t\"update_every\": %d,\n"
  36                 "\t\t\t\"dimensions\": {\n"
  37                 , st->id
  38                 , st->name
  39                 , st->type
  40                 , st->family
  41                 , st->title
  42                 , st->priority
  43                 , st->enabled?"true":"false"
  44                 , st->units
  45                 , st->name
  46                 , rrdset_type_name(st->chart_type)
  47                 , st->entries * st->update_every
  48                 , rrdset_first_entry_t(st)
  49                 , rrdset_last_entry_t(st)
  50                 , st->update_every
  51                 );
  52
  53         unsigned long memory = st->memsize;
  54
  55         int c = 0;
  56         RRDDIM *rd;
  57         for(rd = st->dimensions; rd ; rd = rd->next) {
  58                 if(rd->flags & RRDDIM_FLAG_HIDDEN) continue;
  59
  60                 memory += rd->memsize;
  61
  62                 buffer_sprintf(wb,
  63                         "%s"
  64                         "\t\t\t\t\"%s\": { \"name\": \"%s\" }"
  65                         , c?",\n":""
  66                         , rd->id
  67                         , rd->name
  68                         );
  69
  70                 c++;
  71         }
  72
  73         buffer_sprintf(wb,
  74                 "\n\t\t\t}\n"
  75                 "\t\t}"
  76                 );
  77
  78         pthread_rwlock_unlock(&st->rwlock);
  79 }
  80
  81 void rrd_stats_api_v1_charts(BUFFER *wb)
  82 {
  83         long c;
  84         RRDSET *st;
  85
  86         buffer_sprintf(wb, "{\n"
  87                    "\t\"hostname\": \"%s\""
  88                 ",\n\t\"update_every\": %d"
  89                 ",\n\t\"history\": %d"
  90                 ",\n\t\"charts\": {"
  91                 , hostname
  92                 , rrd_update_every
  93                 , rrd_default_history_entries
  94                 );
  95
  96         pthread_rwlock_rdlock(&rrdset_root_rwlock);
  97         for(st = rrdset_root, c = 0; st ; st = st->next) {
  98                 if(st->enabled) {
  99                         if(c) buffer_strcat(wb, ",");
 100                         buffer_strcat(wb, "\n\t\t\"");
 101                         buffer_strcat(wb, st->id);
 102                         buffer_strcat(wb, "\": ");
 103                         rrd_stats_api_v1_chart(st, wb);
 104                         c++;
 105                 }
 106         }
 107         pthread_rwlock_unlock(&rrdset_root_rwlock);
 108
 109         buffer_strcat(wb, "\n\t}\n}\n");
 110 }
 111
 112
 113 unsigned long rrd_stats_one_json(RRDSET *st, char *options, BUFFER *wb)
 114 {
 115         time_t now = time(NULL);
 116
 117         pthread_rwlock_rdlock(&st->rwlock);
 118
 119         buffer_sprintf(wb,
 120                 "\t\t{\n"
 121                 "\t\t\t\"id\": \"%s\",\n"
 122                 "\t\t\t\"name\": \"%s\",\n"
 123                 "\t\t\t\"type\": \"%s\",\n"
 124                 "\t\t\t\"family\": \"%s\",\n"
 125                 "\t\t\t\"title\": \"%s\",\n"
 126                 "\t\t\t\"priority\": %ld,\n"
 127                 "\t\t\t\"enabled\": %d,\n"
 128                 "\t\t\t\"units\": \"%s\",\n"
 129                 "\t\t\t\"url\": \"/data/%s/%s\",\n"
 130                 "\t\t\t\"chart_type\": \"%s\",\n"
 131                 "\t\t\t\"counter\": %ld,\n"
 132                 "\t\t\t\"entries\": %ld,\n"
 133                 "\t\t\t\"first_entry_t\": %lu,\n"
 134                 "\t\t\t\"last_entry\": %ld,\n"
 135                 "\t\t\t\"last_entry_t\": %lu,\n"
 136                 "\t\t\t\"last_entry_secs_ago\": %lu,\n"
 137                 "\t\t\t\"update_every\": %d,\n"
 138                 "\t\t\t\"isdetail\": %d,\n"
 139                 "\t\t\t\"usec_since_last_update\": %llu,\n"
 140                 "\t\t\t\"collected_total\": " TOTAL_NUMBER_FORMAT ",\n"
 141                 "\t\t\t\"last_collected_total\": " TOTAL_NUMBER_FORMAT ",\n"
 142                 "\t\t\t\"dimensions\": [\n"
 143                 , st->id
 144                 , st->name
 145                 , st->type
 146                 , st->family
 147                 , st->title
 148                 , st->priority
 149                 , st->enabled
 150                 , st->units
 151                 , st->name, options?options:""
 152                 , rrdset_type_name(st->chart_type)
 153                 , st->counter
 154                 , st->entries
 155                 , rrdset_first_entry_t(st)
 156                 , rrdset_last_slot(st)
 157                 , rrdset_last_entry_t(st)
 158                 , (now < rrdset_last_entry_t(st)) ? (time_t)0 : now - rrdset_last_entry_t(st)
 159                 , st->update_every
 160                 , st->isdetail
 161                 , st->usec_since_last_update
 162                 , st->collected_total
 163                 , st->last_collected_total
 164                 );
 165
 166         unsigned long memory = st->memsize;
 167
 168         RRDDIM *rd;
 169         for(rd = st->dimensions; rd ; rd = rd->next) {
 170
 171                 memory += rd->memsize;
 172
 173                 buffer_sprintf(wb,
 174                         "\t\t\t\t{\n"
 175                         "\t\t\t\t\t\"id\": \"%s\",\n"
 176                         "\t\t\t\t\t\"name\": \"%s\",\n"
 177                         "\t\t\t\t\t\"entries\": %ld,\n"
 178                         "\t\t\t\t\t\"isHidden\": %d,\n"
 179                         "\t\t\t\t\t\"algorithm\": \"%s\",\n"
 180                         "\t\t\t\t\t\"multiplier\": %ld,\n"
 181                         "\t\t\t\t\t\"divisor\": %ld,\n"
 182                         "\t\t\t\t\t\"last_entry_t\": %lu,\n"
 183                         "\t\t\t\t\t\"collected_value\": " COLLECTED_NUMBER_FORMAT ",\n"
 184                         "\t\t\t\t\t\"calculated_value\": " CALCULATED_NUMBER_FORMAT ",\n"
 185                         "\t\t\t\t\t\"last_collected_value\": " COLLECTED_NUMBER_FORMAT ",\n"
 186                         "\t\t\t\t\t\"last_calculated_value\": " CALCULATED_NUMBER_FORMAT ",\n"
 187                         "\t\t\t\t\t\"memory\": %lu\n"
 188                         "\t\t\t\t}%s\n"
 189                         , rd->id
 190                         , rd->name
 191                         , rd->entries
 192                         , (rd->flags & RRDDIM_FLAG_HIDDEN)?1:0
 193                         , rrddim_algorithm_name(rd->algorithm)
 194                         , rd->multiplier
 195                         , rd->divisor
 196                         , rd->last_collected_time.tv_sec
 197                         , rd->collected_value
 198                         , rd->calculated_value
 199                         , rd->last_collected_value
 200                         , rd->last_calculated_value
 201                         , rd->memsize
 202                         , rd->next?",":""
 203                         );
 204         }
 205
 206         buffer_sprintf(wb,
 207                 "\t\t\t],\n"
 208                 "\t\t\t\"memory\" : %lu\n"
 209                 "\t\t}"
 210                 , memory
 211                 );
 212
 213         pthread_rwlock_unlock(&st->rwlock);
 214         return memory;
 215 }
 216
 217 #define RRD_GRAPH_JSON_HEADER "{\n\t\"charts\": [\n"
 218 #define RRD_GRAPH_JSON_FOOTER "\n\t]\n}\n"
 219
 220 void rrd_stats_graph_json(RRDSET *st, char *options, BUFFER *wb)
 221 {
 222         buffer_strcat(wb, RRD_GRAPH_JSON_HEADER);
 223         rrd_stats_one_json(st, options, wb);
 224         buffer_strcat(wb, RRD_GRAPH_JSON_FOOTER);
 225 }
 226
 227 void rrd_stats_all_json(BUFFER *wb)
 228 {
 229         unsigned long memory = 0;
 230         long c;
 231         RRDSET *st;
 232
 233         buffer_strcat(wb, RRD_GRAPH_JSON_HEADER);
 234
 235         pthread_rwlock_rdlock(&rrdset_root_rwlock);
 236         for(st = rrdset_root, c = 0; st ; st = st->next) {
 237                 if(st->enabled) {
 238                         if(c) buffer_strcat(wb, ",\n");
 239                         memory += rrd_stats_one_json(st, NULL, wb);
 240                         c++;
 241                 }
 242         }
 243         pthread_rwlock_unlock(&rrdset_root_rwlock);
 244
 245         buffer_sprintf(wb, "\n\t],\n"
 246                 "\t\"hostname\": \"%s\",\n"
 247                 "\t\"update_every\": %d,\n"
 248                 "\t\"history\": %d,\n"
 249                 "\t\"memory\": %lu\n"
 250                 "}\n"
 251                 , hostname
 252                 , rrd_update_every
 253                 , rrd_default_history_entries
 254                 , memory
 255                 );
 256 }
 257
 258
 259
 260 // ----------------------------------------------------------------------------
 261
 262 // RRDR dimension options
 263 #define RRDR_EMPTY      0x01 // the dimension contains / the value is empty (null)
 264 #define RRDR_RESET      0x02 // the dimension contains / the value is reset
 265 #define RRDR_HIDDEN     0x04 // the dimension contains / the value is hidden
 266 #define RRDR_NONZERO    0x08 // the dimension contains / the value is non-zero
 267
 268 // RRDR result options
 269 #define RRDR_RESULT_OPTION_ABSOLUTE 0x00000001
 270 #define RRDR_RESULT_OPTION_RELATIVE 0x00000002
 271
 272 typedef struct rrdresult {
 273         RRDSET *st;                     // the chart this result refers to
 274
 275         uint32_t result_options;        // RRDR_RESULT_OPTION_*
 276
 277         int d;                                  // the number of dimensions
 278         long n;                                 // the number of values in the arrays
 279         long rows;                          // the number of rows used
 280
 281         uint8_t *od;                    // the options for the dimensions
 282
 283         time_t *t;                              // array of n timestamps
 284         calculated_number *v;   // array n x d values
 285         uint8_t *o;                             // array n x d options
 286
 287         long c;                                 // current line ( -1 ~ n ), ( -1 = none, use rrdr_rows() to get number of rows )
 288
 289         long group;                             // how many collected values were grouped for each row
 290         long update_every;              // what is the suggested update frequency in seconds
 291
 292         calculated_number min;
 293         calculated_number max;
 294
 295         time_t before;
 296         time_t after;
 297
 298         int has_st_lock;                // if st is read locked by us
 299 } RRDR;
 300
 301 #define rrdr_rows(r) ((r)->rows)
 302
 303 /*
 304 static void rrdr_dump(RRDR *r)
 305 {
 306         long c, i;
 307         RRDDIM *d;
 308
 309         fprintf(stderr, "\nCHART %s (%s)\n", r->st->id, r->st->name);
 310
 311         for(c = 0, d = r->st->dimensions; d ;c++, d = d->next) {
 312                 fprintf(stderr, "DIMENSION %s (%s), %s%s%s%s\n"
 313                                 , d->id
 314                                 , d->name
 315                                 , (r->od[c] & RRDR_EMPTY)?"EMPTY ":""
 316                                 , (r->od[c] & RRDR_RESET)?"RESET ":""
 317                                 , (r->od[c] & RRDR_HIDDEN)?"HIDDEN ":""
 318                                 , (r->od[c] & RRDR_NONZERO)?"NONZERO ":""
 319                                 );
 320         }
 321
 322         if(r->rows <= 0) {
 323                 fprintf(stderr, "RRDR does not have any values in it.\n");
 324                 return;
 325         }
 326
 327         fprintf(stderr, "RRDR includes %d values in it:\n", r->rows);
 328
 329         // for each line in the array
 330         for(i = 0; i < r->rows ;i++) {
 331                 calculated_number *cn = &r->v[ i * r->d ];
 332                 uint8_t *co = &r->o[ i * r->d ];
 333
 334                 // print the id and the timestamp of the line
 335                 fprintf(stderr, "%ld %ld ", i + 1, r->t[i]);
 336
 337                 // for each dimension
 338                 for(c = 0, d = r->st->dimensions; d ;c++, d = d->next) {
 339                         if(unlikely(r->od[c] & RRDR_HIDDEN)) continue;
 340                         if(unlikely(!(r->od[c] & RRDR_NONZERO))) continue;
 341
 342                         if(co[c] & RRDR_EMPTY)
 343                                 fprintf(stderr, "null ");
 344                         else
 345                                 fprintf(stderr, CALCULATED_NUMBER_FORMAT " %s%s%s%s "
 346                                         , cn[c]
 347                                         , (co[c] & RRDR_EMPTY)?"E":" "
 348                                         , (co[c] & RRDR_RESET)?"R":" "
 349                                         , (co[c] & RRDR_HIDDEN)?"H":" "
 350                                         , (co[c] & RRDR_NONZERO)?"N":" "
 351                                         );
 352                 }
 353
 354                 fprintf(stderr, "\n");
 355         }
 356 }
 357 */
 358
 359 void rrdr_disable_not_selected_dimensions(RRDR *r, const char *dims)
 360 {
 361         char b[strlen(dims) + 1];
 362         char *o = b, *tok;
 363         strcpy(o, dims);
 364
 365         long c;
 366         RRDDIM *d;
 367
 368         // disable all of them
 369         for(c = 0, d = r->st->dimensions; d ;c++, d = d->next)
 370                 r->od[c] |= RRDR_HIDDEN;
 371
 372         while(o && *o && (tok = mystrsep(&o, ", |"))) {
 373                 if(!*tok) continue;
 374
 375                 // find it and enable it
 376                 for(c = 0, d = r->st->dimensions; d ;c++, d = d->next) {
 377                         if(!strcmp(d->name, tok)) {
 378                                 r->od[c] &= ~RRDR_HIDDEN;
 379
 380                                 // since the user needs this dimension
 381                                 // make it appear as NONZERO, to return it
 382                                 // even if the dimension has only zeros
 383                                 r->od[c] |= RRDR_NONZERO;
 384                         }
 385                 }
 386         }
 387 }
 388
 389 void rrdr_buffer_print_format(BUFFER *wb, uint32_t format)
 390 {
 391         switch(format) {
 392         case DATASOURCE_JSON:
 393                 buffer_strcat(wb, DATASOURCE_FORMAT_JSON);
 394                 break;
 395
 396         case DATASOURCE_DATATABLE_JSON:
 397                 buffer_strcat(wb, DATASOURCE_FORMAT_DATATABLE_JSON);
 398                 break;
 399
 400         case DATASOURCE_DATATABLE_JSONP:
 401                 buffer_strcat(wb, DATASOURCE_FORMAT_DATATABLE_JSONP);
 402                 break;
 403
 404         case DATASOURCE_JSONP:
 405                 buffer_strcat(wb, DATASOURCE_FORMAT_JSONP);
 406                 break;
 407
 408         case DATASOURCE_SSV:
 409                 buffer_strcat(wb, DATASOURCE_FORMAT_SSV);
 410                 break;
 411
 412         case DATASOURCE_CSV:
 413                 buffer_strcat(wb, DATASOURCE_FORMAT_CSV);
 414                 break;
 415
 416         case DATASOURCE_TSV:
 417                 buffer_strcat(wb, DATASOURCE_FORMAT_TSV);
 418                 break;
 419
 420         case DATASOURCE_HTML:
 421                 buffer_strcat(wb, DATASOURCE_FORMAT_HTML);
 422                 break;
 423
 424         case DATASOURCE_JS_ARRAY:
 425                 buffer_strcat(wb, DATASOURCE_FORMAT_JS_ARRAY);
 426                 break;
 427
 428         case DATASOURCE_SSV_COMMA:
 429                 buffer_strcat(wb, DATASOURCE_FORMAT_SSV_COMMA);
 430                 break;
 431
 432         default:
 433                 buffer_strcat(wb, "unknown");
 434                 break;
 435         }
 436 }
 437
 438 uint32_t rrdr_check_options(RRDR *r, uint32_t options, const char *dims)
 439 {
 440         if(options & RRDR_OPTION_NONZERO) {
 441                 long i;
 442
 443                 if(dims && *dims) {
 444                         // the caller wants specific dimensions
 445                         // disable NONZERO option
 446                         // to make sure we don't accidentally prevent
 447                         // the specific dimensions from being returned
 448                         i = 0;
 449                 }
 450                 else {
 451                         // find how many dimensions are not zero
 452                         long c;
 453                         RRDDIM *rd;
 454                         for(c = 0, i = 0, rd = r->st->dimensions; rd && c < r->d ; c++, rd = rd->next) {
 455                                 if(unlikely(r->od[c] & RRDR_HIDDEN)) continue;
 456                                 if(unlikely(!(r->od[c] & RRDR_NONZERO))) continue;
 457                                 i++;
 458                         }
 459                 }
 460
 461                 // if with nonzero we get i = 0 (no dimensions will be returned)
 462                 // disable nonzero to show all dimensions
 463                 if(!i) options &= ~RRDR_OPTION_NONZERO;
 464         }
 465
 466         return options;
 467 }
 468
 469 void rrdr_json_wrapper_begin(RRDR *r, BUFFER *wb, uint32_t format, uint32_t options, int string_value)
 470 {
 471         long rows = rrdr_rows(r);
 472         long c, i;
 473         RRDDIM *rd;
 474
 475         //info("JSONWRAPPER(): %s: BEGIN", r->st->id);
 476         char kq[2] = "",                                        // key quote
 477                 sq[2] = "";                                             // string quote
 478
 479         if( options & RRDR_OPTION_GOOGLE_JSON ) {
 480                 kq[0] = '\0';
 481                 sq[0] = '\'';
 482         }
 483         else {
 484                 kq[0] = '"';
 485                 sq[0] = '"';
 486         }
 487
 488         buffer_sprintf(wb, "{\n"
 489                         "       %sapi%s: 1,\n"
 490                         "       %sid%s: %s%s%s,\n"
 491                         "       %sname%s: %s%s%s,\n"
 492                         "       %sview_update_every%s: %d,\n"
 493                         "       %supdate_every%s: %d,\n"
 494                         "       %sfirst_entry%s: %u,\n"
 495                         "       %slast_entry%s: %u,\n"
 496                         "       %sbefore%s: %u,\n"
 497                         "       %safter%s: %u,\n"
 498                         "       %sdimension_names%s: ["
 499                         , kq, kq
 500                         , kq, kq, sq, r->st->id, sq
 501                         , kq, kq, sq, r->st->name, sq
 502                         , kq, kq, r->update_every
 503                         , kq, kq, r->st->update_every
 504                         , kq, kq, rrdset_first_entry_t(r->st)
 505                         , kq, kq, rrdset_last_entry_t(r->st)
 506                         , kq, kq, r->before
 507                         , kq, kq, r->after
 508                         , kq, kq);
 509
 510         for(c = 0, i = 0, rd = r->st->dimensions; rd && c < r->d ;c++, rd = rd->next) {
 511                 if(unlikely(r->od[c] & RRDR_HIDDEN)) continue;
 512                 if(unlikely((options & RRDR_OPTION_NONZERO) && !(r->od[c] & RRDR_NONZERO))) continue;
 513
 514                 if(i) buffer_strcat(wb, ", ");
 515                 buffer_strcat(wb, sq);
 516                 buffer_strcat(wb, rd->name);
 517                 buffer_strcat(wb, sq);
 518                 i++;
 519         }
 520         if(!i) {
 521 #ifdef NETDATA_INTERNAL_CHECKS
 522                 error("RRDR is empty for %s (RRDR has %d dimensions, options is 0x%08x)", r->st->id, r->d, options);
 523 #endif
 524                 rows = 0;
 525                 buffer_strcat(wb, sq);
 526                 buffer_strcat(wb, "no data");
 527                 buffer_strcat(wb, sq);
 528         }
 529
 530         buffer_sprintf(wb, "],\n"
 531                         "       %sdimension_ids%s: ["
 532                         , kq, kq);
 533
 534         for(c = 0, i = 0, rd = r->st->dimensions; rd && c < r->d ;c++, rd = rd->next) {
 535                 if(unlikely(r->od[c] & RRDR_HIDDEN)) continue;
 536                 if(unlikely((options & RRDR_OPTION_NONZERO) && !(r->od[c] & RRDR_NONZERO))) continue;
 537
 538                 if(i) buffer_strcat(wb, ", ");
 539                 buffer_strcat(wb, sq);
 540                 buffer_strcat(wb, rd->id);
 541                 buffer_strcat(wb, sq);
 542                 i++;
 543         }
 544         if(!i) {
 545                 rows = 0;
 546                 buffer_strcat(wb, sq);
 547                 buffer_strcat(wb, "no data");
 548                 buffer_strcat(wb, sq);
 549         }
 550
 551         buffer_sprintf(wb, "],\n"
 552                         "       %slatest_values%s: ["
 553                         , kq, kq);
 554
 555         for(c = 0, i = 0, rd = r->st->dimensions; rd && c < r->d ;c++, rd = rd->next) {
 556                 if(unlikely(r->od[c] & RRDR_HIDDEN)) continue;
 557                 if(unlikely((options & RRDR_OPTION_NONZERO) && !(r->od[c] & RRDR_NONZERO))) continue;
 558
 559                 if(i) buffer_strcat(wb, ", ");
 560                 i++;
 561
 562                 storage_number n = rd->values[rrdset_last_slot(r->st)];
 563
 564                 if(!does_storage_number_exist(n))
 565                         buffer_strcat(wb, "null");
 566                 else
 567                         buffer_rrd_value(wb, unpack_storage_number(n));
 568         }
 569         if(!i) {
 570                 rows = 0;
 571                 buffer_strcat(wb, "null");
 572         }
 573
 574         buffer_sprintf(wb, "],\n"
 575                         "       %sview_latest_values%s: ["
 576                         , kq, kq);
 577
 578         i = 0;
 579         if(rows) {
 580                 for(c = 0, i = 0, rd = r->st->dimensions; rd && c < r->d ;c++, rd = rd->next) {
 581                         if(unlikely(r->od[c] & RRDR_HIDDEN)) continue;
 582                         if(unlikely((options & RRDR_OPTION_NONZERO) && !(r->od[c] & RRDR_NONZERO))) continue;
 583
 584                         if(i) buffer_strcat(wb, ", ");
 585                         i++;
 586
 587                         calculated_number *cn = &r->v[ (0) * r->d ];
 588                         uint8_t *co = &r->o[ (0) * r->d ];
 589
 590                         if(co[c] & RRDR_EMPTY)
 591                                 buffer_strcat(wb, "null");
 592                         else
 593                                 buffer_rrd_value(wb, cn[c]);
 594                 }
 595         }
 596         if(!i) {
 597                 rows = 0;
 598                 buffer_strcat(wb, "null");
 599         }
 600
 601         buffer_sprintf(wb, "],\n"
 602                         "       %sdimensions%s: %d,\n"
 603                         "       %spoints%s: %d,\n"
 604                         "       %sformat%s: %s"
 605                         , kq, kq, i
 606                         , kq, kq, rows
 607                         , kq, kq, sq
 608                         );
 609
 610         rrdr_buffer_print_format(wb, format);
 611
 612         buffer_sprintf(wb, "%s,\n"
 613                         "       %sresult%s: "
 614                         , sq
 615                         , kq, kq
 616                         );
 617
 618         if(string_value) buffer_strcat(wb, sq);
 619         //info("JSONWRAPPER(): %s: END", r->st->id);
 620 }
 621
 622 void rrdr_json_wrapper_end(RRDR *r, BUFFER *wb, uint32_t format, uint32_t options, int string_value)
 623 {
 624         if(r) {;}
 625         if(format) {;}
 626
 627         char kq[2] = "",                                        // key quote
 628                 sq[2] = "";                                             // string quote
 629
 630         if( options & RRDR_OPTION_GOOGLE_JSON ) {
 631                 kq[0] = '\0';
 632                 sq[0] = '\'';
 633         }
 634         else {
 635                 kq[0] = '"';
 636                 sq[0] = '"';
 637         }
 638
 639         if(string_value) buffer_strcat(wb, sq);
 640
 641         buffer_sprintf(wb, ",\n %smin%s: ", kq, kq);
 642         buffer_rrd_value(wb, r->min);
 643         buffer_sprintf(wb, ",\n %smax%s: ", kq, kq);
 644         buffer_rrd_value(wb, r->max);
 645         buffer_strcat(wb, "\n}\n");
 646 }
 647
 648 #define JSON_DATES_JS 1
 649 #define JSON_DATES_TIMESTAMP 2
 650
 651 static void rrdr2json(RRDR *r, BUFFER *wb, uint32_t options, int datatable)
 652 {
 653         //info("RRD2JSON(): %s: BEGIN", r->st->id);
 654         int row_annotations = 0, dates, dates_with_new = 0;
 655         char kq[2] = "",                                        // key quote
 656                 sq[2] = "",                                             // string quote
 657                 pre_label[101] = "",                    // before each label
 658                 post_label[101] = "",                   // after each label
 659                 pre_date[101] = "",                             // the beginning of line, to the date
 660                 post_date[101] = "",                    // closing the date
 661                 pre_value[101] = "",                    // before each value
 662                 post_value[101] = "",                   // after each value
 663                 post_line[101] = "",                    // at the end of each row
 664                 normal_annotation[201] = "",    // default row annotation
 665                 overflow_annotation[201] = "",  // overflow row annotation
 666                 data_begin[101] = "",                   // between labels and values
 667                 finish[101] = "";                               // at the end of everything
 668
 669         if(datatable) {
 670                 dates = JSON_DATES_JS;
 671                 if( options & RRDR_OPTION_GOOGLE_JSON ) {
 672                         kq[0] = '\0';
 673                         sq[0] = '\'';
 674                 }
 675                 else {
 676                         kq[0] = '"';
 677                         sq[0] = '"';
 678                 }
 679                 row_annotations = 1;
 680                 snprintf(pre_date,   100, "             {%sc%s:[{%sv%s:%s", kq, kq, kq, kq, sq);
 681                 snprintf(post_date,  100, "%s}", sq);
 682                 snprintf(pre_label,  100, ",\n          {%sid%s:%s%s,%slabel%s:%s", kq, kq, sq, sq, kq, kq, sq);
 683                 snprintf(post_label, 100, "%s,%spattern%s:%s%s,%stype%s:%snumber%s}", sq, kq, kq, sq, sq, kq, kq, sq, sq);
 684                 snprintf(pre_value,  100, ",{%sv%s:", kq, kq);
 685                 snprintf(post_value, 100, "}");
 686                 snprintf(post_line,  100, "]}");
 687                 snprintf(data_begin, 100, "\n   ],\n    %srows%s:\n     [\n", kq, kq);
 688                 snprintf(finish,     100, "\n   ]\n}");
 689
 690                 snprintf(overflow_annotation, 200, ",{%sv%s:%sRESET OR OVERFLOW%s},{%sv%s:%sThe counters have been wrapped.%s}", kq, kq, sq, sq, kq, kq, sq, sq);
 691                 snprintf(normal_annotation,   200, ",{%sv%s:null},{%sv%s:null}", kq, kq, kq, kq);
 692
 693                 buffer_sprintf(wb, "{\n %scols%s:\n     [\n", kq, kq, kq, kq);
 694                 buffer_sprintf(wb, "            {%sid%s:%s%s,%slabel%s:%stime%s,%spattern%s:%s%s,%stype%s:%sdatetime%s},\n", kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq);
 695                 buffer_sprintf(wb, "            {%sid%s:%s%s,%slabel%s:%s%s,%spattern%s:%s%s,%stype%s:%sstring%s,%sp%s:{%srole%s:%sannotation%s}},\n", kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, kq, kq, sq, sq);
 696                 buffer_sprintf(wb, "            {%sid%s:%s%s,%slabel%s:%s%s,%spattern%s:%s%s,%stype%s:%sstring%s,%sp%s:{%srole%s:%sannotationText%s}}", kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, kq, kq, sq, sq);
 697
 698                 // remove the valueobjects flag
 699                 // google wants its own keys
 700                 if(options & RRDR_OPTION_OBJECTSROWS)
 701                         options &= ~RRDR_OPTION_OBJECTSROWS;
 702         }
 703         else {
 704                 kq[0] = '"';
 705                 sq[0] = '"';
 706                 if((options & RRDR_OPTION_SECONDS) || (options & RRDR_OPTION_MILLISECONDS)) {
 707                         dates = JSON_DATES_TIMESTAMP;
 708                         dates_with_new = 0;
 709                 }
 710                 else {
 711                         dates = JSON_DATES_JS;
 712                         dates_with_new = 1;
 713                 }
 714                 if( options & RRDR_OPTION_OBJECTSROWS )
 715                         snprintf(pre_date,   100, "             { ");
 716                 else
 717                         snprintf(pre_date,   100, "             [ ");
 718                 snprintf(pre_label,  100, ", \"");
 719                 snprintf(post_label, 100, "\"");
 720                 snprintf(pre_value,  100, ", ");
 721                 if( options & RRDR_OPTION_OBJECTSROWS )
 722                         snprintf(post_line,  100, "}");
 723                 else
 724                         snprintf(post_line,  100, "]");
 725                 snprintf(data_begin, 100, "],\n %sdata%s:\n     [\n", kq, kq);
 726                 snprintf(finish,     100, "\n   ]\n}");
 727
 728                 buffer_sprintf(wb, "{\n %slabels%s: [", kq, kq);
 729                 buffer_sprintf(wb, "%stime%s", sq, sq);
 730         }
 731
 732         // -------------------------------------------------------------------------
 733         // print the JSON header
 734
 735         long c, i;
 736         RRDDIM *rd;
 737
 738         // print the header lines
 739         for(c = 0, i = 0, rd = r->st->dimensions; rd && c < r->d ;c++, rd = rd->next) {
 740                 if(unlikely(r->od[c] & RRDR_HIDDEN)) continue;
 741                 if(unlikely((options & RRDR_OPTION_NONZERO) && !(r->od[c] & RRDR_NONZERO))) continue;
 742
 743                 buffer_strcat(wb, pre_label);
 744                 buffer_strcat(wb, rd->name);
 745                 buffer_strcat(wb, post_label);
 746                 i++;
 747         }
 748         if(!i) {
 749                 buffer_strcat(wb, pre_label);
 750                 buffer_strcat(wb, "no data");
 751                 buffer_strcat(wb, post_label);
 752         }
 753
 754         // print the begin of row data
 755         buffer_strcat(wb, data_begin);
 756
 757         // if all dimensions are hidden, print a null
 758         if(!i) {
 759                 buffer_strcat(wb, finish);
 760                 return;
 761         }
 762
 763         long start = 0, end = rrdr_rows(r), step = 1;
 764         if((options & RRDR_OPTION_REVERSED)) {
 765                 start = rrdr_rows(r) - 1;
 766                 end = -1;
 767                 step = -1;
 768         }
 769
 770         // for each line in the array
 771         calculated_number total = 1;
 772         for(i = start; i != end ;i += step) {
 773                 calculated_number *cn = &r->v[ i * r->d ];
 774                 uint8_t *co = &r->o[ i * r->d ];
 775
 776                 time_t now = r->t[i];
 777
 778                 if(dates == JSON_DATES_JS) {
 779                         // generate the local date time
 780                         struct tm tmbuf, *tm = localtime_r(&now, &tmbuf);
 781                         if(!tm) { error("localtime_r() failed."); continue; }
 782
 783                         if(likely(i != start)) buffer_strcat(wb, ",\n");
 784                         buffer_strcat(wb, pre_date);
 785
 786                         if( options & RRDR_OPTION_OBJECTSROWS )
 787                                 buffer_sprintf(wb, "%stime%s: ", kq, kq);
 788
 789                         if(dates_with_new)
 790                                 buffer_strcat(wb, "new ");
 791
 792                         buffer_jsdate(wb, tm->tm_year + 1900, tm->tm_mon, tm->tm_mday, tm->tm_hour, tm->tm_min, tm->tm_sec);
 793
 794                         buffer_strcat(wb, post_date);
 795
 796                         if(row_annotations) {
 797                                 // google supports one annotation per row
 798                                 int annotation_found = 0;
 799                                 for(c = 0, rd = r->st->dimensions; rd ;c++, rd = rd->next) {
 800                                         if(co[c] & RRDR_RESET) {
 801                                                 buffer_strcat(wb, overflow_annotation);
 802                                                 annotation_found = 1;
 803                                                 break;
 804                                         }
 805                                 }
 806                                 if(!annotation_found)
 807                                         buffer_strcat(wb, normal_annotation);
 808                         }
 809                 }
 810                 else {
 811                         // print the timestamp of the line
 812                         if(likely(i != start)) buffer_strcat(wb, ",\n");
 813                         buffer_strcat(wb, pre_date);
 814
 815                         if( options & RRDR_OPTION_OBJECTSROWS )
 816                                 buffer_sprintf(wb, "%stime%s: ", kq, kq);
 817
 818                         buffer_rrd_value(wb, (calculated_number)r->t[i]);
 819                         // in ms
 820                         if(options & RRDR_OPTION_MILLISECONDS) buffer_strcat(wb, "000");
 821
 822                         buffer_strcat(wb, post_date);
 823                 }
 824
 825                 if(unlikely(options & RRDR_OPTION_PERCENTAGE)) {
 826                         total = 0;
 827                         for(c = 0, rd = r->st->dimensions; rd && c < r->d ;c++, rd = rd->next) {
 828                                 calculated_number n = cn[c];
 829
 830                                 if(likely((options & RRDR_OPTION_ABSOLUTE) && n < 0))
 831                                         n = -n;
 832
 833                                 total += n;
 834                         }
 835                         // prevent a division by zero
 836                         if(total == 0) total = 1;
 837                 }
 838
 839                 // for each dimension
 840                 for(c = 0, rd = r->st->dimensions; rd && c < r->d ;c++, rd = rd->next) {
 841                         if(unlikely(r->od[c] & RRDR_HIDDEN)) continue;
 842                         if(unlikely((options & RRDR_OPTION_NONZERO) && !(r->od[c] & RRDR_NONZERO))) continue;
 843
 844                         calculated_number n = cn[c];
 845
 846                         buffer_strcat(wb, pre_value);
 847
 848                         if( options & RRDR_OPTION_OBJECTSROWS )
 849                                 buffer_sprintf(wb, "%s%s%s: ", kq, rd->name, kq);
 850
 851                         if(co[c] & RRDR_EMPTY) {
 852                                 if(options & RRDR_OPTION_NULL2ZERO)
 853                                         buffer_strcat(wb, "0");
 854                                 else
 855                                         buffer_strcat(wb, "null");
 856                         }
 857                         else {
 858                                 if(unlikely((options & RRDR_OPTION_ABSOLUTE) && n < 0))
 859                                         n = -n;
 860
 861                                 if(unlikely(options & RRDR_OPTION_PERCENTAGE))
 862                                         n = n * 100 / total;
 863
 864                                 buffer_rrd_value(wb, n);
 865                         }
 866
 867                         buffer_strcat(wb, post_value);
 868                 }
 869
 870                 buffer_strcat(wb, post_line);
 871         }
 872
 873         buffer_strcat(wb, finish);
 874         //info("RRD2JSON(): %s: END", r->st->id);
 875 }
 876
 877 static void rrdr2csv(RRDR *r, BUFFER *wb, uint32_t options, const char *startline, const char *separator, const char *endline, const char *betweenlines)
 878 {
 879         //info("RRD2CSV(): %s: BEGIN", r->st->id);
 880         long c, i;
 881         RRDDIM *d;
 882
 883         // print the csv header
 884         for(c = 0, i = 0, d = r->st->dimensions; d && c < r->d ;c++, d = d->next) {
 885                 if(unlikely(r->od[c] & RRDR_HIDDEN)) continue;
 886                 if(unlikely((options & RRDR_OPTION_NONZERO) && !(r->od[c] & RRDR_NONZERO))) continue;
 887
 888                 if(!i) {
 889                         buffer_strcat(wb, startline);
 890                         if(options & RRDR_OPTION_LABEL_QUOTES) buffer_strcat(wb, "\"");
 891                         buffer_strcat(wb, "time");
 892                         if(options & RRDR_OPTION_LABEL_QUOTES) buffer_strcat(wb, "\"");
 893                 }
 894                 buffer_strcat(wb, separator);
 895                 if(options & RRDR_OPTION_LABEL_QUOTES) buffer_strcat(wb, "\"");
 896                 buffer_strcat(wb, d->name);
 897                 if(options & RRDR_OPTION_LABEL_QUOTES) buffer_strcat(wb, "\"");
 898                 i++;
 899         }
 900         buffer_strcat(wb, endline);
 901
 902         if(!i) {
 903                 // no dimensions present
 904                 return;
 905         }
 906
 907         long start = 0, end = rrdr_rows(r), step = 1;
 908         if((options & RRDR_OPTION_REVERSED)) {
 909                 start = rrdr_rows(r) - 1;
 910                 end = -1;
 911                 step = -1;
 912         }
 913
 914         // for each line in the array
 915         calculated_number total = 1;
 916         for(i = start; i != end ;i += step) {
 917                 calculated_number *cn = &r->v[ i * r->d ];
 918                 uint8_t *co = &r->o[ i * r->d ];
 919
 920                 buffer_strcat(wb, betweenlines);
 921                 buffer_strcat(wb, startline);
 922
 923                 time_t now = r->t[i];
 924
 925                 if((options & RRDR_OPTION_SECONDS) || (options & RRDR_OPTION_MILLISECONDS)) {
 926                         // print the timestamp of the line
 927                         buffer_rrd_value(wb, (calculated_number)now);
 928                         // in ms
 929                         if(options & RRDR_OPTION_MILLISECONDS) buffer_strcat(wb, "000");
 930                 }
 931                 else {
 932                         // generate the local date time
 933                         struct tm tmbuf, *tm = localtime_r(&now, &tmbuf);
 934                         if(!tm) { error("localtime() failed."); continue; }
 935                         buffer_date(wb, tm->tm_year + 1900, tm->tm_mon, tm->tm_mday, tm->tm_hour, tm->tm_min, tm->tm_sec);
 936                 }
 937
 938                 if(unlikely(options & RRDR_OPTION_PERCENTAGE)) {
 939                         total = 0;
 940                         for(c = 0, d = r->st->dimensions; d && c < r->d ;c++, d = d->next) {
 941                                 calculated_number n = cn[c];
 942
 943                                 if(likely((options & RRDR_OPTION_ABSOLUTE) && n < 0))
 944                                         n = -n;
 945
 946                                 total += n;
 947                         }
 948                         // prevent a division by zero
 949                         if(total == 0) total = 1;
 950                 }
 951
 952                 // for each dimension
 953                 for(c = 0, d = r->st->dimensions; d && c < r->d ;c++, d = d->next) {
 954                         if(unlikely(r->od[c] & RRDR_HIDDEN)) continue;
 955                         if(unlikely((options & RRDR_OPTION_NONZERO) && !(r->od[c] & RRDR_NONZERO))) continue;
 956
 957                         buffer_strcat(wb, separator);
 958
 959                         calculated_number n = cn[c];
 960
 961                         if(co[c] & RRDR_EMPTY) {
 962                                 if(options & RRDR_OPTION_NULL2ZERO)
 963                                         buffer_strcat(wb, "0");
 964                                 else
 965                                         buffer_strcat(wb, "null");
 966                         }
 967                         else {
 968                                 if(unlikely((options & RRDR_OPTION_ABSOLUTE) && n < 0))
 969                                         n = -n;
 970
 971                                 if(unlikely(options & RRDR_OPTION_PERCENTAGE))
 972                                         n = n * 100 / total;
 973
 974                                 buffer_rrd_value(wb, n);
 975                         }
 976                 }
 977
 978                 buffer_strcat(wb, endline);
 979         }
 980         //info("RRD2CSV(): %s: END", r->st->id);
 981 }
 982
 983 static void rrdr2ssv(RRDR *r, BUFFER *wb, uint32_t options, const char *prefix, const char *separator, const char *suffix)
 984 {
 985         //info("RRD2SSV(): %s: BEGIN", r->st->id);
 986         long c, i;
 987         RRDDIM *d;
 988
 989         buffer_strcat(wb, prefix);
 990         long start = 0, end = rrdr_rows(r), step = 1;
 991         if((options & RRDR_OPTION_REVERSED)) {
 992                 start = rrdr_rows(r) - 1;
 993                 end = -1;
 994                 step = -1;
 995         }
 996
 997         // for each line in the array
 998         calculated_number total = 1;
 999         for(i = start; i != end ;i += step) {
1000
1001                 calculated_number *cn = &r->v[ i * r->d ];
1002                 uint8_t *co = &r->o[ i * r->d ];
1003
1004                 calculated_number sum = 0, min = 0, max = 0, v;
1005                 int all_null = 1, init = 1;
1006
1007                 if(unlikely(options & RRDR_OPTION_PERCENTAGE)) {
1008                         total = 0;
1009                         for(c = 0, d = r->st->dimensions; d && c < r->d ;c++, d = d->next) {
1010                                 calculated_number n = cn[c];
1011
1012                                 if(likely((options & RRDR_OPTION_ABSOLUTE) && n < 0))
1013                                         n = -n;
1014
1015                                 total += n;
1016                         }
1017                         // prevent a division by zero
1018                         if(total == 0) total = 1;
1019                 }
1020
1021                 // for each dimension
1022                 for(c = 0, d = r->st->dimensions; d && c < r->d ;c++, d = d->next) {
1023                         if(unlikely(r->od[c] & RRDR_HIDDEN)) continue;
1024                         if(unlikely((options & RRDR_OPTION_NONZERO) && !(r->od[c] & RRDR_NONZERO))) continue;
1025
1026                         calculated_number n = cn[c];
1027
1028                         if(likely((options & RRDR_OPTION_ABSOLUTE) && n < 0))
1029                                 n = -n;
1030
1031                         if(unlikely(options & RRDR_OPTION_PERCENTAGE))
1032                                 n = n * 100 / total;
1033
1034                         if(unlikely(init)) {
1035                                 if(n > 0) {
1036                                         min = 0;
1037                                         max = n;
1038                                 }
1039                                 else {
1040                                         min = n;
1041                                         max = 0;
1042                                 }
1043                                 init = 0;
1044                         }
1045
1046                         if(likely(!(co[c] & RRDR_EMPTY))) {
1047                                 all_null = 0;
1048                                 sum += n;
1049                         }
1050
1051                         if(n < min) min = n;
1052                         if(n > max) max = n;
1053                 }
1054
1055                 if(likely(i != start))
1056                         buffer_strcat(wb, separator);
1057
1058                 if(all_null) {
1059                         if(options & RRDR_OPTION_NULL2ZERO)
1060                                 buffer_strcat(wb, "0");
1061                         else
1062                                 buffer_strcat(wb, "null");
1063                 }
1064                 else {
1065                         if(options & RRDR_OPTION_MIN2MAX)
1066                                 v = max - min;
1067                         else
1068                                 v = sum;
1069
1070                         if(likely(i != start)) {
1071                                 if(r->min > v) r->min = v;
1072                                 if(r->max < v) r->max = v;
1073                         }
1074                         else {
1075                                 r->min = v;
1076                                 r->max = v;
1077                         }
1078
1079                         buffer_rrd_value(wb, v);
1080                 }
1081         }
1082         buffer_strcat(wb, suffix);
1083         //info("RRD2SSV(): %s: END", r->st->id);
1084 }
1085
1086 inline static calculated_number *rrdr_line_values(RRDR *r)
1087 {
1088         return &r->v[ r->c * r->d ];
1089 }
1090
1091 inline static uint8_t *rrdr_line_options(RRDR *r)
1092 {
1093         return &r->o[ r->c * r->d ];
1094 }
1095
1096 inline static int rrdr_line_init(RRDR *r, time_t t)
1097 {
1098         r->c++;
1099
1100         if(unlikely(r->c >= r->n)) {
1101                 error("requested to step above RRDR size for chart %s", r->st->name);
1102                 r->c = r->n - 1;
1103         }
1104
1105         // save the time
1106         r->t[r->c] = t;
1107
1108         return 1;
1109 }
1110
1111 inline static void rrdr_lock_rrdset(RRDR *r) {
1112         if(unlikely(!r)) {
1113                 error("NULL value given!");
1114                 return;
1115         }
1116
1117         pthread_rwlock_rdlock(&r->st->rwlock);
1118         r->has_st_lock = 1;
1119 }
1120
1121 inline static void rrdr_unlock_rrdset(RRDR *r) {
1122         if(unlikely(!r)) {
1123                 error("NULL value given!");
1124                 return;
1125         }
1126
1127         if(likely(r->has_st_lock)) {
1128                 pthread_rwlock_unlock(&r->st->rwlock);
1129                 r->has_st_lock = 0;
1130         }
1131 }
1132
1133 inline static void rrdr_free(RRDR *r)
1134 {
1135         if(unlikely(!r)) {
1136                 error("NULL value given!");
1137                 return;
1138         }
1139
1140         rrdr_unlock_rrdset(r);
1141         if(likely(r->t)) free(r->t);
1142         if(likely(r->v)) free(r->v);
1143         if(likely(r->o)) free(r->o);
1144         if(likely(r->od)) free(r->od);
1145         free(r);
1146 }
1147
1148 inline void rrdr_done(RRDR *r)
1149 {
1150         r->rows = r->c + 1;
1151         r->c = 0;
1152 }
1153
1154 static RRDR *rrdr_create(RRDSET *st, long n)
1155 {
1156         if(unlikely(!st)) {
1157                 error("NULL value given!");
1158                 return NULL;
1159         }
1160
1161         RRDR *r = calloc(1, sizeof(RRDR));
1162         if(unlikely(!r)) goto cleanup;
1163
1164         r->st = st;
1165
1166         rrdr_lock_rrdset(r);
1167
1168         RRDDIM *rd;
1169         for(rd = st->dimensions ; rd ; rd = rd->next) r->d++;
1170
1171         r->n = n;
1172
1173         r->t = malloc(n * sizeof(time_t));
1174         if(unlikely(!r->t)) goto cleanup;
1175
1176         r->v = malloc(n * r->d * sizeof(calculated_number));
1177         if(unlikely(!r->v)) goto cleanup;
1178
1179         r->o = malloc(n * r->d * sizeof(uint8_t));
1180         if(unlikely(!r->o)) goto cleanup;
1181
1182         r->od = malloc(r->d * sizeof(uint8_t));
1183         if(unlikely(!r->od)) goto cleanup;
1184
1185         // set the hidden flag on hidden dimensions
1186         int c;
1187         for(c = 0, rd = st->dimensions ; rd ; c++, rd = rd->next) {
1188                 if(unlikely(rd->flags & RRDDIM_FLAG_HIDDEN)) r->od[c] = RRDR_HIDDEN;
1189                 else r->od[c] = 0;
1190         }
1191
1192         r->c = -1;
1193
1194         r->group = 1;
1195         r->update_every = 1;
1196
1197         return r;
1198
1199 cleanup:
1200         error("Cannot allocate RRDR memory for %d entries", n);
1201         if(likely(r)) rrdr_free(r);
1202         return NULL;
1203 }
1204
1205 RRDR *rrd2rrdr(RRDSET *st, long points, long long after, long long before, int group_method)
1206 {
1207         int debug = st->debug;
1208         int absolute_period_requested = -1;
1209
1210         time_t first_entry_t = rrdset_first_entry_t(st);
1211         time_t last_entry_t = rrdset_last_entry_t(st);
1212
1213         if(before == 0 && after == 0) {
1214                 before = last_entry_t;
1215                 after = first_entry_t;
1216                 absolute_period_requested = 0;
1217         }
1218
1219         // allow relative for before and after
1220         if(before <= st->update_every * st->entries) {
1221                 before = last_entry_t + before;
1222                 absolute_period_requested = 0;
1223         }
1224
1225         if(after <= st->update_every * st->entries) {
1226                 after = last_entry_t + after;
1227                 absolute_period_requested = 0;
1228         }
1229
1230         if(absolute_period_requested == -1)
1231                 absolute_period_requested = 1;
1232
1233         // make sure they are within our timeframe
1234         if(before > last_entry_t) before = last_entry_t;
1235         if(before < first_entry_t) before = first_entry_t;
1236
1237         if(after > last_entry_t) after = last_entry_t;
1238         if(after < first_entry_t) after = first_entry_t;
1239
1240         // check if they are upside down
1241         if(after > before) {
1242                 time_t tmp = before;
1243                 before = after;
1244                 after = tmp;
1245         }
1246
1247         // the duration of the chart
1248         time_t duration = before - after;
1249         long available_points = duration / st->update_every;
1250
1251         if(duration <= 0 || available_points <= 0)
1252                 return rrdr_create(st, 1);
1253
1254         // check the wanted points
1255         if(points < 0) points = -points;
1256         if(points > available_points) points = available_points;
1257         if(points == 0) points = available_points;
1258
1259         // calculate proper grouping of source data
1260         long group = available_points / points;
1261         if(group <= 0) group = 1;
1262
1263         // round group to the closest integer
1264         if(available_points % points > points / 2) group++;
1265
1266         time_t after_new = after - (after % (group * st->update_every));
1267         time_t before_new = before - (before % (group * st->update_every));
1268         long points_new = (before_new - after_new) / st->update_every / group;
1269
1270         // find the starting and ending slots in our round robin db
1271         long    start_at_slot = rrdset_time2slot(st, before_new),
1272                         stop_at_slot = rrdset_time2slot(st, after_new);
1273
1274 #ifdef NETDATA_INTERNAL_CHECKS
1275         if(after_new < first_entry_t) {
1276                 error("after_new %u is too small, minimum %u", after_new, first_entry_t);
1277         }
1278         if(after_new > last_entry_t) {
1279                 error("after_new %u is too big, maximum %u", after_new, last_entry_t);
1280         }
1281         if(before_new < first_entry_t) {
1282                 error("before_new %u is too small, minimum %u", before_new, first_entry_t);
1283         }
1284         if(before_new > last_entry_t) {
1285                 error("before_new %u is too big, maximum %u", before_new, last_entry_t);
1286         }
1287         if(start_at_slot < 0 || start_at_slot >= st->entries) {
1288                 error("start_at_slot is invalid %ld, expected %ld to %ld", start_at_slot, 0, st->entries - 1);
1289         }
1290         if(stop_at_slot < 0 || stop_at_slot >= st->entries) {
1291                 error("stop_at_slot is invalid %ld, expected %ld to %ld", stop_at_slot, 0, st->entries - 1);
1292         }
1293         if(points_new > (before_new - after_new) / group / st->update_every + 1) {
1294                 error("points_new %ld is more than points %ld", points_new, (before_new - after_new) / group / st->update_every + 1);
1295         }
1296 #endif
1297
1298         //info("RRD2RRDR(): %s: wanted %ld points, got %ld - group=%ld, wanted duration=%u, got %u - wanted %ld - %ld, got %ld - %ld", st->id, points, points_new, group, before - after, before_new - after_new, after, before, after_new, before_new);
1299
1300         after = after_new;
1301         before = before_new;
1302         duration = before - after;
1303         points = points_new;
1304
1305         // Now we have:
1306         // before = the end time of the calculation
1307         // after = the start time of the calculation
1308         // duration = the duration of the calculation
1309         // group = the number of source points to aggregate / group together
1310         // method = the method of grouping source points
1311         // points = the number of points to generate
1312
1313
1314         // -------------------------------------------------------------------------
1315         // initialize our result set
1316
1317         RRDR *r = rrdr_create(st, points);
1318         if(!r) {
1319 #ifdef NETDATA_INTERNAL_CHECKS
1320                 error("Cannot create RRDR for %s, after=%u, before=%u, duration=%u, points=%d", st->id, after, before, duration, points);
1321 #endif
1322                 return NULL;
1323         }
1324         if(!r->d) {
1325 #ifdef NETDATA_INTERNAL_CHECKS
1326                 error("Returning empty RRDR (no dimensions in RRDSET) for %s, after=%u, before=%u, duration=%u, points=%d", st->id, after, before, duration, points);
1327 #endif
1328                 return r;
1329         }
1330
1331         if(absolute_period_requested == 1)
1332                 r->result_options |= RRDR_RESULT_OPTION_ABSOLUTE;
1333         else
1334                 r->result_options |= RRDR_RESULT_OPTION_RELATIVE;
1335
1336         // find how many dimensions we have
1337         long dimensions = r->d;
1338
1339
1340         // -------------------------------------------------------------------------
1341         // checks for debugging
1342
1343         if(debug) debug(D_RRD_STATS, "INFO %s first_t: %lu, last_t: %lu, all_duration: %lu, after: %lu, before: %lu, duration: %lu, points: %ld, group: %ld"
1344                         , st->id
1345                         , first_entry_t
1346                         , last_entry_t
1347                         , last_entry_t - first_entry_t
1348                         , after
1349                         , before
1350                         , duration
1351                         , points
1352                         , group
1353                         );
1354
1355
1356         // -------------------------------------------------------------------------
1357         // temp arrays for keeping values per dimension
1358
1359         calculated_number       group_values[dimensions]; // keep sums when grouping
1360         long                            group_counts[dimensions]; // keep the number of values added to group_values
1361         uint8_t                         group_options[dimensions];
1362         uint8_t                         found_non_zero[dimensions];
1363
1364
1365         // initialize them
1366         RRDDIM *rd;
1367         long c;
1368         for( rd = st->dimensions, c = 0 ; rd && c < dimensions ; rd = rd->next, c++) {
1369                 group_values[c] = 0;
1370                 group_counts[c] = 0;
1371                 group_options[c] = 0;
1372                 found_non_zero[c] = 0;
1373         }
1374
1375
1376         // -------------------------------------------------------------------------
1377         // the main loop
1378
1379         time_t  now = rrdset_slot2time(st, start_at_slot),
1380                         dt = st->update_every,
1381                         group_start_t = 0;
1382
1383         if(unlikely(debug)) debug(D_RRD_STATS, "BEGIN %s after_t: %lu (stop_at_t: %ld), before_t: %lu (start_at_t: %ld), start_t(now): %lu, current_entry: %ld, entries: %ld"
1384                         , st->id
1385                         , after
1386                         , stop_at_slot
1387                         , before
1388                         , start_at_slot
1389                         , now
1390                         , st->current_entry
1391                         , st->entries
1392                         );
1393
1394         r->group = group;
1395         r->update_every = group * st->update_every;
1396         r->before = now;
1397         r->after = now;
1398
1399         //info("RRD2RRDR(): %s: STARTING", st->id);
1400
1401         long slot = start_at_slot, counter = 0, stop_now = 0, added = 0, group_count = 0, add_this = 0;
1402         for(; !stop_now ; now -= dt, slot--, counter++) {
1403                 if(unlikely(slot < 0)) slot = st->entries - 1;
1404                 if(unlikely(slot == stop_at_slot)) stop_now = counter;
1405
1406                 if(unlikely(debug)) debug(D_RRD_STATS, "ROW %s slot: %ld, entries_counter: %ld, group_count: %ld, added: %ld, now: %lu, %s %s"
1407                                 , st->id
1408                                 , slot
1409                                 , counter
1410                                 , group_count + 1
1411                                 , added
1412                                 , now
1413                                 , (group_count + 1 == group)?"PRINT":"  -  "
1414                                 , (now >= after && now <= before)?"RANGE":"  -  "
1415                                 );
1416
1417                 // make sure we return data in the proper time range
1418                 if(unlikely(now > before)) continue;
1419                 if(unlikely(now < after)) break;
1420
1421                 if(unlikely(group_count == 0)) {
1422                         group_start_t = now;
1423                 }
1424                 group_count++;
1425
1426                 if(unlikely(group_count == group)) {
1427                         if(unlikely(added >= points)) break;
1428                         add_this = 1;
1429                 }
1430
1431                 // do the calculations
1432                 for(rd = st->dimensions, c = 0 ; rd && c < dimensions ; rd = rd->next, c++) {
1433                         storage_number n = rd->values[slot];
1434                         if(unlikely(!does_storage_number_exist(n))) continue;
1435
1436                         group_counts[c]++;
1437
1438                         calculated_number value = unpack_storage_number(n);
1439                         if(likely(value != 0.0)) {
1440                                 group_options[c] |= RRDR_NONZERO;
1441                                 found_non_zero[c] = 1;
1442                         }
1443
1444                         if(unlikely(did_storage_number_reset(n)))
1445                                 group_options[c] |= RRDR_RESET;
1446
1447                         switch(group_method) {
1448                                 case GROUP_MAX:
1449                                         if(unlikely(abs(value) > abs(group_values[c])))
1450                                                 group_values[c] = value;
1451                                         break;
1452
1453                                 default:
1454                                 case GROUP_SUM:
1455                                 case GROUP_AVERAGE:
1456                                         group_values[c] += value;
1457                                         break;
1458                         }
1459                 }
1460
1461                 // added it
1462                 if(unlikely(add_this)) {
1463                         if(unlikely(!rrdr_line_init(r, group_start_t))) break;
1464
1465                         r->after = now;
1466
1467                         calculated_number *cn = rrdr_line_values(r);
1468                         uint8_t *co = rrdr_line_options(r);
1469
1470                         for(rd = st->dimensions, c = 0 ; rd && c < dimensions ; rd = rd->next, c++) {
1471
1472                                 // update the dimension options
1473                                 if(likely(found_non_zero[c])) r->od[c] |= RRDR_NONZERO;
1474
1475                                 // store the specific point options
1476                                 co[c] = group_options[c];
1477
1478                                 // store the value
1479                                 if(unlikely(group_counts[c] == 0)) {
1480                                         cn[c] = 0.0;
1481                                         co[c] |= RRDR_EMPTY;
1482                                 }
1483                                 else if(unlikely(group_method == GROUP_AVERAGE)) {
1484                                         cn[c] = group_values[c] / group_counts[c];
1485                                 }
1486                                 else {
1487                                         cn[c] = group_values[c];
1488                                 }
1489
1490                                 if(cn[c] < r->min) r->min = cn[c];
1491                                 if(cn[c] > r->max) r->max = cn[c];
1492
1493                                 // reset them for the next loop
1494                                 group_values[c] = 0;
1495                                 group_counts[c] = 0;
1496                                 group_options[c] = 0;
1497                         }
1498
1499                         added++;
1500                         group_count = 0;
1501                         add_this = 0;
1502                 }
1503         }
1504
1505         rrdr_done(r);
1506         //info("RRD2RRDR(): %s: END %ld loops made, %ld points generated", st->id, counter, rrdr_rows(r));
1507         //error("SHIFT: %s: wanted %ld points, got %ld", st->id, points, rrdr_rows(r));
1508         return r;
1509 }
1510
1511 int rrd2format(RRDSET *st, BUFFER *wb, BUFFER *dimensions, uint32_t format, long points, long long after, long long before, int group_method, uint32_t options, time_t *latest_timestamp)
1512 {
1513         RRDR *r = rrd2rrdr(st, points, after, before, group_method);
1514         if(!r) {
1515                 buffer_strcat(wb, "Cannot generate output with these parameters on this chart.");
1516                 return 500;
1517         }
1518
1519         if(r->result_options & RRDR_RESULT_OPTION_RELATIVE)
1520                 wb->options |= WB_CONTENT_NO_CACHEABLE;
1521         else if(r->result_options & RRDR_RESULT_OPTION_ABSOLUTE)
1522                 wb->options |= WB_CONTENT_CACHEABLE;
1523
1524         options = rrdr_check_options(r, options, (dimensions)?buffer_tostring(dimensions):NULL);
1525
1526         if(dimensions)
1527                 rrdr_disable_not_selected_dimensions(r, buffer_tostring(dimensions));
1528
1529         if(latest_timestamp && rrdr_rows(r) > 0)
1530                 *latest_timestamp = r->before;
1531
1532         switch(format) {
1533         case DATASOURCE_SSV:
1534                 if(options & RRDR_OPTION_JSON_WRAP) {
1535                         wb->contenttype = CT_APPLICATION_JSON;
1536                         rrdr_json_wrapper_begin(r, wb, format, options, 1);
1537                         rrdr2ssv(r, wb, options, "", " ", "");
1538                         rrdr_json_wrapper_end(r, wb, format, options, 1);
1539                 }
1540                 else {
1541                         wb->contenttype = CT_TEXT_PLAIN;
1542                         rrdr2ssv(r, wb, options, "", " ", "");
1543                 }
1544                 break;
1545
1546         case DATASOURCE_SSV_COMMA:
1547                 if(options & RRDR_OPTION_JSON_WRAP) {
1548                         wb->contenttype = CT_APPLICATION_JSON;
1549                         rrdr_json_wrapper_begin(r, wb, format, options, 1);
1550                         rrdr2ssv(r, wb, options, "", ",", "");
1551                         rrdr_json_wrapper_end(r, wb, format, options, 1);
1552                 }
1553                 else {
1554                         wb->contenttype = CT_TEXT_PLAIN;
1555                         rrdr2ssv(r, wb, options, "", ",", "");
1556                 }
1557                 break;
1558
1559         case DATASOURCE_JS_ARRAY:
1560                 if(options & RRDR_OPTION_JSON_WRAP) {
1561                         wb->contenttype = CT_APPLICATION_JSON;
1562                         rrdr_json_wrapper_begin(r, wb, format, options, 0);
1563                         rrdr2ssv(r, wb, options, "[", ",", "]");
1564                         rrdr_json_wrapper_end(r, wb, format, options, 0);
1565                 }
1566                 else {
1567                         wb->contenttype = CT_APPLICATION_JSON;
1568                         rrdr2ssv(r, wb, options, "[", ",", "]");
1569                 }
1570                 break;
1571
1572         case DATASOURCE_CSV:
1573                 if(options & RRDR_OPTION_JSON_WRAP) {
1574                         wb->contenttype = CT_APPLICATION_JSON;
1575                         rrdr_json_wrapper_begin(r, wb, format, options, 1);
1576                         rrdr2csv(r, wb, options, "", ",", "\\n", "");
1577                         rrdr_json_wrapper_end(r, wb, format, options, 1);
1578                 }
1579                 else {
1580                         wb->contenttype = CT_TEXT_PLAIN;
1581                         rrdr2csv(r, wb, options, "", ",", "\r\n", "");
1582                 }
1583                 break;
1584
1585         case DATASOURCE_CSV_JSON_ARRAY:
1586                 wb->contenttype = CT_APPLICATION_JSON;
1587                 if(options & RRDR_OPTION_JSON_WRAP) {
1588                         rrdr_json_wrapper_begin(r, wb, format, options, 0);
1589                         buffer_strcat(wb, "[\n");
1590                         rrdr2csv(r, wb, options + RRDR_OPTION_LABEL_QUOTES, "[", ",", "]", ",\n");
1591                         buffer_strcat(wb, "\n]");
1592                         rrdr_json_wrapper_end(r, wb, format, options, 0);
1593                 }
1594                 else {
1595                         wb->contenttype = CT_TEXT_PLAIN;
1596                         buffer_strcat(wb, "[\n");
1597                         rrdr2csv(r, wb, options + RRDR_OPTION_LABEL_QUOTES, "[", ",", "]", ",\n");
1598                         buffer_strcat(wb, "\n]");
1599                 }
1600                 break;
1601
1602         case DATASOURCE_TSV:
1603                 if(options & RRDR_OPTION_JSON_WRAP) {
1604                         wb->contenttype = CT_APPLICATION_JSON;
1605                         rrdr_json_wrapper_begin(r, wb, format, options, 1);
1606                         rrdr2csv(r, wb, options, "", "\t", "\\n", "");
1607                         rrdr_json_wrapper_end(r, wb, format, options, 1);
1608                 }
1609                 else {
1610                         wb->contenttype = CT_TEXT_PLAIN;
1611                         rrdr2csv(r, wb, options, "", "\t", "\r\n", "");
1612                 }
1613                 break;
1614
1615         case DATASOURCE_HTML:
1616                 if(options & RRDR_OPTION_JSON_WRAP) {
1617                         wb->contenttype = CT_APPLICATION_JSON;
1618                         rrdr_json_wrapper_begin(r, wb, format, options, 1);
1619                         buffer_strcat(wb, "<html>\\n<center>\\n<table border=\\\"0\\\" cellpadding=\\\"5\\\" cellspacing=\\\"5\\\">\\n");
1620                         rrdr2csv(r, wb, options, "<tr><td>", "</td><td>", "</td></tr>\\n", "");
1621                         buffer_strcat(wb, "</table>\\n</center>\\n</html>\\n");
1622                         rrdr_json_wrapper_end(r, wb, format, options, 1);
1623                 }
1624                 else {
1625                         wb->contenttype = CT_TEXT_HTML;
1626                         buffer_strcat(wb, "<html>\n<center>\n<table border=\"0\" cellpadding=\"5\" cellspacing=\"5\">\n");
1627                         rrdr2csv(r, wb, options, "<tr><td>", "</td><td>", "</td></tr>\n", "");
1628                         buffer_strcat(wb, "</table>\n</center>\n</html>\n");
1629                 }
1630                 break;
1631
1632         case DATASOURCE_DATATABLE_JSONP:
1633                 wb->contenttype = CT_APPLICATION_X_JAVASCRIPT;
1634
1635                 if(options & RRDR_OPTION_JSON_WRAP)
1636                         rrdr_json_wrapper_begin(r, wb, format, options, 0);
1637
1638                 rrdr2json(r, wb, options, 1);
1639
1640                 if(options & RRDR_OPTION_JSON_WRAP)
1641                         rrdr_json_wrapper_end(r, wb, format, options, 0);
1642                 break;
1643
1644         case DATASOURCE_DATATABLE_JSON:
1645                 wb->contenttype = CT_APPLICATION_JSON;
1646
1647                 if(options & RRDR_OPTION_JSON_WRAP)
1648                         rrdr_json_wrapper_begin(r, wb, format, options, 0);
1649
1650                 rrdr2json(r, wb, options, 1);
1651
1652                 if(options & RRDR_OPTION_JSON_WRAP)
1653                         rrdr_json_wrapper_end(r, wb, format, options, 0);
1654                 break;
1655
1656         case DATASOURCE_JSONP:
1657                 wb->contenttype = CT_APPLICATION_X_JAVASCRIPT;
1658                 if(options & RRDR_OPTION_JSON_WRAP)
1659                         rrdr_json_wrapper_begin(r, wb, format, options, 0);
1660
1661                 rrdr2json(r, wb, options, 0);
1662
1663                 if(options & RRDR_OPTION_JSON_WRAP)
1664                         rrdr_json_wrapper_end(r, wb, format, options, 0);
1665                 break;
1666
1667         case DATASOURCE_JSON:
1668         default:
1669                 wb->contenttype = CT_APPLICATION_JSON;
1670
1671                 if(options & RRDR_OPTION_JSON_WRAP)
1672                         rrdr_json_wrapper_begin(r, wb, format, options, 0);
1673
1674                 rrdr2json(r, wb, options, 0);
1675
1676                 if(options & RRDR_OPTION_JSON_WRAP)
1677                         rrdr_json_wrapper_end(r, wb, format, options, 0);
1678                 break;
1679         }
1680
1681         rrdr_free(r);
1682         return 200;
1683 }
1684
1685 time_t rrd_stats_json(int type, RRDSET *st, BUFFER *wb, long points, long group, int group_method, time_t after, time_t before, int only_non_zero)
1686 {
1687         int c;
1688         pthread_rwlock_rdlock(&st->rwlock);
1689
1690
1691         // -------------------------------------------------------------------------
1692         // switch from JSON to google JSON
1693
1694         char kq[2] = "\"";
1695         char sq[2] = "\"";
1696         switch(type) {
1697                 case DATASOURCE_DATATABLE_JSON:
1698                 case DATASOURCE_DATATABLE_JSONP:
1699                         kq[0] = '\0';
1700                         sq[0] = '\'';
1701                         break;
1702
1703                 case DATASOURCE_JSON:
1704                 default:
1705                         break;
1706         }
1707
1708
1709         // -------------------------------------------------------------------------
1710         // validate the parameters
1711
1712         if(points < 1) points = 1;
1713         if(group < 1) group = 1;
1714
1715         if(before == 0 || before > rrdset_last_entry_t(st)) before = rrdset_last_entry_t(st);
1716         if(after  == 0 || after < rrdset_first_entry_t(st)) after = rrdset_first_entry_t(st);
1717
1718         // ---
1719
1720         // our return value (the last timestamp printed)
1721         // this is required to detect re-transmit in google JSONP
1722         time_t last_timestamp = 0;
1723
1724
1725         // -------------------------------------------------------------------------
1726         // find how many dimensions we have
1727
1728         int dimensions = 0;
1729         RRDDIM *rd;
1730         for( rd = st->dimensions ; rd ; rd = rd->next) dimensions++;
1731         if(!dimensions) {
1732                 pthread_rwlock_unlock(&st->rwlock);
1733                 buffer_strcat(wb, "No dimensions yet.");
1734                 return 0;
1735         }
1736
1737
1738         // -------------------------------------------------------------------------
1739         // prepare various strings, to speed up the loop
1740
1741         char overflow_annotation[201]; snprintf(overflow_annotation, 200, ",{%sv%s:%sRESET OR OVERFLOW%s},{%sv%s:%sThe counters have been wrapped.%s}", kq, kq, sq, sq, kq, kq, sq, sq);
1742         char normal_annotation[201];   snprintf(normal_annotation,   200, ",{%sv%s:null},{%sv%s:null}", kq, kq, kq, kq);
1743         char pre_date[51];             snprintf(pre_date,             50, "             {%sc%s:[{%sv%s:%s", kq, kq, kq, kq, sq);
1744         char post_date[21];            snprintf(post_date,            20, "%s}", sq);
1745         char pre_value[21];            snprintf(pre_value,            20, ",{%sv%s:", kq, kq);
1746         char post_value[21];           snprintf(post_value,           20, "}");
1747
1748
1749         // -------------------------------------------------------------------------
1750         // checks for debugging
1751
1752         if(st->debug) {
1753                 debug(D_RRD_STATS, "%s first_entry_t = %lu, last_entry_t = %lu, duration = %lu, after = %lu, before = %lu, duration = %lu, entries_to_show = %lu, group = %lu"
1754                         , st->id
1755                         , rrdset_first_entry_t(st)
1756                         , rrdset_last_entry_t(st)
1757                         , rrdset_last_entry_t(st) - rrdset_first_entry_t(st)
1758                         , after
1759                         , before
1760                         , before - after
1761                         , points
1762                         , group
1763                         );
1764
1765                 if(before < after)
1766                         debug(D_RRD_STATS, "WARNING: %s The newest value in the database (%lu) is earlier than the oldest (%lu)", st->name, before, after);
1767
1768                 if((before - after) > st->entries * st->update_every)
1769                         debug(D_RRD_STATS, "WARNING: %s The time difference between the oldest and the newest entries (%lu) is higher than the capacity of the database (%lu)", st->name, before - after, st->entries * st->update_every);
1770         }
1771
1772
1773         // -------------------------------------------------------------------------
1774         // temp arrays for keeping values per dimension
1775
1776         calculated_number group_values[dimensions]; // keep sums when grouping
1777         int               print_hidden[dimensions]; // keep hidden flags
1778         int               found_non_zero[dimensions];
1779         int               found_non_existing[dimensions];
1780
1781         // initialize them
1782         for( rd = st->dimensions, c = 0 ; rd && c < dimensions ; rd = rd->next, c++) {
1783                 group_values[c] = 0;
1784                 print_hidden[c] = (rd->flags & RRDDIM_FLAG_HIDDEN)?1:0;
1785                 found_non_zero[c] = 0;
1786                 found_non_existing[c] = 0;
1787         }
1788
1789
1790         // error("OLD: points=%d after=%d before=%d group=%d, duration=%d", entries_to_show, before - (st->update_every * group * entries_to_show), before, group, before - after + 1);
1791         // rrd2array(st, entries_to_show, before - (st->update_every * group * entries_to_show), before, group_method, only_non_zero);
1792         // rrd2rrdr(st, entries_to_show, before - (st->update_every * group * entries_to_show), before, group_method);
1793
1794         // -------------------------------------------------------------------------
1795         // remove dimensions that contain only zeros
1796
1797         int max_loop = 1;
1798         if(only_non_zero) max_loop = 2;
1799
1800         for(; max_loop ; max_loop--) {
1801
1802                 // -------------------------------------------------------------------------
1803                 // print the JSON header
1804
1805                 buffer_sprintf(wb, "{\n %scols%s:\n     [\n", kq, kq);
1806                 buffer_sprintf(wb, "            {%sid%s:%s%s,%slabel%s:%stime%s,%spattern%s:%s%s,%stype%s:%sdatetime%s},\n", kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq);
1807                 buffer_sprintf(wb, "            {%sid%s:%s%s,%slabel%s:%s%s,%spattern%s:%s%s,%stype%s:%sstring%s,%sp%s:{%srole%s:%sannotation%s}},\n", kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, kq, kq, sq, sq);
1808                 buffer_sprintf(wb, "            {%sid%s:%s%s,%slabel%s:%s%s,%spattern%s:%s%s,%stype%s:%sstring%s,%sp%s:{%srole%s:%sannotationText%s}}", kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, kq, kq, sq, sq);
1809
1810                 // print the header for each dimension
1811                 // and update the print_hidden array for the dimensions that should be hidden
1812                 int pc = 0;
1813                 for( rd = st->dimensions, c = 0 ; rd && c < dimensions ; rd = rd->next, c++) {
1814                         if(!print_hidden[c]) {
1815                                 pc++;
1816                                 buffer_sprintf(wb, ",\n         {%sid%s:%s%s,%slabel%s:%s%s%s,%spattern%s:%s%s,%stype%s:%snumber%s}", kq, kq, sq, sq, kq, kq, sq, rd->name, sq, kq, kq, sq, sq, kq, kq, sq, sq);
1817                         }
1818                 }
1819                 if(!pc) {
1820                         buffer_sprintf(wb, ",\n         {%sid%s:%s%s,%slabel%s:%s%s%s,%spattern%s:%s%s,%stype%s:%snumber%s}", kq, kq, sq, sq, kq, kq, sq, "no data", sq, kq, kq, sq, sq, kq, kq, sq, sq);
1821                 }
1822
1823                 // print the begin of row data
1824                 buffer_sprintf(wb, "\n  ],\n    %srows%s:\n     [\n", kq, kq);
1825
1826
1827                 // -------------------------------------------------------------------------
1828                 // the main loop
1829
1830                 int annotate_reset = 0;
1831                 int annotation_count = 0;
1832
1833                 long    t = rrdset_time2slot(st, before),
1834                                 stop_at_t = rrdset_time2slot(st, after),
1835                                 stop_now = 0;
1836
1837                 t -= t % group;
1838
1839                 time_t  now = rrdset_slot2time(st, t),
1840                                 dt = st->update_every;
1841
1842                 long count = 0, printed = 0, group_count = 0;
1843                 last_timestamp = 0;
1844
1845                 if(st->debug) debug(D_RRD_STATS, "%s: REQUEST after:%lu before:%lu, points:%d, group:%d, CHART cur:%ld first: %lu last:%lu, CALC start_t:%ld, stop_t:%ld"
1846                                         , st->id
1847                                         , after
1848                                         , before
1849                                         , points
1850                                         , group
1851                                         , st->current_entry
1852                                         , rrdset_first_entry_t(st)
1853                                         , rrdset_last_entry_t(st)
1854                                         , t
1855                                         , stop_at_t
1856                                         );
1857
1858                 long counter = 0;
1859                 for(; !stop_now ; now -= dt, t--, counter++) {
1860                         if(t < 0) t = st->entries - 1;
1861                         if(t == stop_at_t) stop_now = counter;
1862
1863                         int print_this = 0;
1864
1865                         if(st->debug) debug(D_RRD_STATS, "%s t = %ld, count = %ld, group_count = %ld, printed = %ld, now = %lu, %s %s"
1866                                         , st->id
1867                                         , t
1868                                         , count + 1
1869                                         , group_count + 1
1870                                         , printed
1871                                         , now
1872                                         , (group_count + 1 == group)?"PRINT":"  -  "
1873                                         , (now >= after && now <= before)?"RANGE":"  -  "
1874                                         );
1875
1876
1877                         // make sure we return data in the proper time range
1878                         if(now > before) continue;
1879                         if(now < after) break;
1880
1881                         //if(rrdset_slot2time(st, t) != now)
1882                         //      error("%s: slot=%ld, now=%ld, slot2time=%ld, diff=%ld, last_entry_t=%ld, rrdset_last_slot=%ld", st->id, t, now, rrdset_slot2time(st,t), now - rrdset_slot2time(st,t), rrdset_last_entry_t(st), rrdset_last_slot(st));
1883
1884                         count++;
1885                         group_count++;
1886
1887                         // check if we have to print this now
1888                         if(group_count == group) {
1889                                 if(printed >= points) {
1890                                         // debug(D_RRD_STATS, "Already printed all rows. Stopping.");
1891                                         break;
1892                                 }
1893
1894                                 // generate the local date time
1895                                 struct tm tmbuf, *tm = localtime_r(&now, &tmbuf);
1896                                 if(!tm) { error("localtime() failed."); continue; }
1897                                 if(now > last_timestamp) last_timestamp = now;
1898
1899                                 if(printed) buffer_strcat(wb, "]},\n");
1900                                 buffer_strcat(wb, pre_date);
1901                                 buffer_jsdate(wb, tm->tm_year + 1900, tm->tm_mon, tm->tm_mday, tm->tm_hour, tm->tm_min, tm->tm_sec);
1902                                 buffer_strcat(wb, post_date);
1903
1904                                 print_this = 1;
1905                         }
1906
1907                         // do the calculations
1908                         for(rd = st->dimensions, c = 0 ; rd && c < dimensions ; rd = rd->next, c++) {
1909                                 storage_number n = rd->values[t];
1910                                 calculated_number value = unpack_storage_number(n);
1911
1912                                 if(!does_storage_number_exist(n)) {
1913                                         value = 0.0;
1914                                         found_non_existing[c]++;
1915                                 }
1916                                 if(did_storage_number_reset(n)) annotate_reset = 1;
1917
1918                                 switch(group_method) {
1919                                         case GROUP_MAX:
1920                                                 if(abs(value) > abs(group_values[c])) group_values[c] = value;
1921                                                 break;
1922
1923                                         case GROUP_SUM:
1924                                                 group_values[c] += value;
1925                                                 break;
1926
1927                                         default:
1928                                         case GROUP_AVERAGE:
1929                                                 group_values[c] += value;
1930                                                 if(print_this) group_values[c] /= ( group_count - found_non_existing[c] );
1931                                                 break;
1932                                 }
1933                         }
1934
1935                         if(print_this) {
1936                                 if(annotate_reset) {
1937                                         annotation_count++;
1938                                         buffer_strcat(wb, overflow_annotation);
1939                                         annotate_reset = 0;
1940                                 }
1941                                 else
1942                                         buffer_strcat(wb, normal_annotation);
1943
1944                                 pc = 0;
1945                                 for(c = 0 ; c < dimensions ; c++) {
1946                                         if(found_non_existing[c] == group_count) {
1947                                                 // all entries are non-existing
1948                                                 pc++;
1949                                                 buffer_strcat(wb, pre_value);
1950                                                 buffer_strcat(wb, "null");
1951                                                 buffer_strcat(wb, post_value);
1952                                         }
1953                                         else if(!print_hidden[c]) {
1954                                                 pc++;
1955                                                 buffer_strcat(wb, pre_value);
1956                                                 buffer_rrd_value(wb, group_values[c]);
1957                                                 buffer_strcat(wb, post_value);
1958
1959                                                 if(group_values[c]) found_non_zero[c]++;
1960                                         }
1961
1962                                         // reset them for the next loop
1963                                         group_values[c] = 0;
1964                                         found_non_existing[c] = 0;
1965                                 }
1966
1967                                 // if all dimensions are hidden, print a null
1968                                 if(!pc) {
1969                                         buffer_strcat(wb, pre_value);
1970                                         buffer_strcat(wb, "null");
1971                                         buffer_strcat(wb, post_value);
1972                                 }
1973
1974                                 printed++;
1975                                 group_count = 0;
1976                         }
1977                 }
1978
1979                 if(printed) buffer_strcat(wb, "]}");
1980                 buffer_strcat(wb, "\n   ]\n}\n");
1981
1982                 if(only_non_zero && max_loop > 1) {
1983                         int changed = 0;
1984                         for(rd = st->dimensions, c = 0 ; rd && c < dimensions ; rd = rd->next, c++) {
1985                                 group_values[c] = 0;
1986                                 found_non_existing[c] = 0;
1987
1988                                 if(!print_hidden[c] && !found_non_zero[c]) {
1989                                         changed = 1;
1990                                         print_hidden[c] = 1;
1991                                 }
1992                         }
1993
1994                         if(changed) buffer_flush(wb);
1995                         else break;
1996                 }
1997                 else break;
1998
1999         } // max_loop
2000
2001         debug(D_RRD_STATS, "RRD_STATS_JSON: %s total %ld bytes", st->name, wb->len);
2002
2003         pthread_rwlock_unlock(&st->rwlock);
2004         return last_timestamp;
2005 }