src/rrd2json.c

   1 #ifdef HAVE_CONFIG_H
   2 #include <config.h>
   3 #endif
   4 #include <pthread.h>
   5 #include <stdlib.h>
   6 #include <string.h>
   7 #include <stdint.h>
   8
   9 #include "log.h"
  10 #include "common.h"
  11 #include "rrd2json.h"
  12
  13 #define HOSTNAME_MAX 1024
  14 char *hostname = "unknown";
  15
  16 void rrd_stats_api_v1_chart(RRDSET *st, BUFFER *wb)
  17 {
  18         pthread_rwlock_rdlock(&st->rwlock);
  19
  20         buffer_sprintf(wb,
  21                 "\t\t{\n"
  22                 "\t\t\t\"id\": \"%s\",\n"
  23                 "\t\t\t\"name\": \"%s\",\n"
  24                 "\t\t\t\"type\": \"%s\",\n"
  25                 "\t\t\t\"family\": \"%s\",\n"
  26                 "\t\t\t\"context\": \"%s\",\n"
  27                 "\t\t\t\"title\": \"%s\",\n"
  28                 "\t\t\t\"priority\": %ld,\n"
  29                 "\t\t\t\"enabled\": %s,\n"
  30                 "\t\t\t\"units\": \"%s\",\n"
  31                 "\t\t\t\"data_url\": \"/api/v1/data?chart=%s\",\n"
  32                 "\t\t\t\"chart_type\": \"%s\",\n"
  33                 "\t\t\t\"duration\": %ld,\n"
  34                 "\t\t\t\"first_entry\": %lu,\n"
  35                 "\t\t\t\"last_entry\": %lu,\n"
  36                 "\t\t\t\"update_every\": %d,\n"
  37                 "\t\t\t\"dimensions\": {\n"
  38                 , st->id
  39                 , st->name
  40                 , st->type
  41                 , st->family
  42                 , st->context
  43                 , st->title
  44                 , st->priority
  45                 , st->enabled?"true":"false"
  46                 , st->units
  47                 , st->name
  48                 , rrdset_type_name(st->chart_type)
  49                 , st->entries * st->update_every
  50                 , rrdset_first_entry_t(st)
  51                 , rrdset_last_entry_t(st)
  52                 , st->update_every
  53                 );
  54
  55         unsigned long memory = st->memsize;
  56
  57         int c = 0;
  58         RRDDIM *rd;
  59         for(rd = st->dimensions; rd ; rd = rd->next) {
  60                 if(rd->flags & RRDDIM_FLAG_HIDDEN) continue;
  61
  62                 memory += rd->memsize;
  63
  64                 buffer_sprintf(wb,
  65                         "%s"
  66                         "\t\t\t\t\"%s\": { \"name\": \"%s\" }"
  67                         , c?",\n":""
  68                         , rd->id
  69                         , rd->name
  70                         );
  71
  72                 c++;
  73         }
  74
  75         buffer_sprintf(wb,
  76                 "\n\t\t\t}\n"
  77                 "\t\t}"
  78                 );
  79
  80         pthread_rwlock_unlock(&st->rwlock);
  81 }
  82
  83 void rrd_stats_api_v1_charts(BUFFER *wb)
  84 {
  85         long c;
  86         RRDSET *st;
  87
  88         buffer_sprintf(wb, "{\n"
  89                    "\t\"hostname\": \"%s\""
  90                 ",\n\t\"update_every\": %d"
  91                 ",\n\t\"history\": %d"
  92                 ",\n\t\"charts\": {"
  93                 , hostname
  94                 , rrd_update_every
  95                 , rrd_default_history_entries
  96                 );
  97
  98         pthread_rwlock_rdlock(&rrdset_root_rwlock);
  99         for(st = rrdset_root, c = 0; st ; st = st->next) {
 100                 if(st->enabled) {
 101                         if(c) buffer_strcat(wb, ",");
 102                         buffer_strcat(wb, "\n\t\t\"");
 103                         buffer_strcat(wb, st->id);
 104                         buffer_strcat(wb, "\": ");
 105                         rrd_stats_api_v1_chart(st, wb);
 106                         c++;
 107                 }
 108         }
 109         pthread_rwlock_unlock(&rrdset_root_rwlock);
 110
 111         buffer_strcat(wb, "\n\t}\n}\n");
 112 }
 113
 114
 115 unsigned long rrd_stats_one_json(RRDSET *st, char *options, BUFFER *wb)
 116 {
 117         time_t now = time(NULL);
 118
 119         pthread_rwlock_rdlock(&st->rwlock);
 120
 121         buffer_sprintf(wb,
 122                 "\t\t{\n"
 123                 "\t\t\t\"id\": \"%s\",\n"
 124                 "\t\t\t\"name\": \"%s\",\n"
 125                 "\t\t\t\"type\": \"%s\",\n"
 126                 "\t\t\t\"family\": \"%s\",\n"
 127                 "\t\t\t\"context\": \"%s\",\n"
 128                 "\t\t\t\"title\": \"%s\",\n"
 129                 "\t\t\t\"priority\": %ld,\n"
 130                 "\t\t\t\"enabled\": %d,\n"
 131                 "\t\t\t\"units\": \"%s\",\n"
 132                 "\t\t\t\"url\": \"/data/%s/%s\",\n"
 133                 "\t\t\t\"chart_type\": \"%s\",\n"
 134                 "\t\t\t\"counter\": %ld,\n"
 135                 "\t\t\t\"entries\": %ld,\n"
 136                 "\t\t\t\"first_entry_t\": %lu,\n"
 137                 "\t\t\t\"last_entry\": %ld,\n"
 138                 "\t\t\t\"last_entry_t\": %lu,\n"
 139                 "\t\t\t\"last_entry_secs_ago\": %lu,\n"
 140                 "\t\t\t\"update_every\": %d,\n"
 141                 "\t\t\t\"isdetail\": %d,\n"
 142                 "\t\t\t\"usec_since_last_update\": %llu,\n"
 143                 "\t\t\t\"collected_total\": " TOTAL_NUMBER_FORMAT ",\n"
 144                 "\t\t\t\"last_collected_total\": " TOTAL_NUMBER_FORMAT ",\n"
 145                 "\t\t\t\"dimensions\": [\n"
 146                 , st->id
 147                 , st->name
 148                 , st->type
 149                 , st->family
 150                 , st->context
 151                 , st->title
 152                 , st->priority
 153                 , st->enabled
 154                 , st->units
 155                 , st->name, options?options:""
 156                 , rrdset_type_name(st->chart_type)
 157                 , st->counter
 158                 , st->entries
 159                 , rrdset_first_entry_t(st)
 160                 , rrdset_last_slot(st)
 161                 , rrdset_last_entry_t(st)
 162                 , (now < rrdset_last_entry_t(st)) ? (time_t)0 : now - rrdset_last_entry_t(st)
 163                 , st->update_every
 164                 , st->isdetail
 165                 , st->usec_since_last_update
 166                 , st->collected_total
 167                 , st->last_collected_total
 168                 );
 169
 170         unsigned long memory = st->memsize;
 171
 172         RRDDIM *rd;
 173         for(rd = st->dimensions; rd ; rd = rd->next) {
 174
 175                 memory += rd->memsize;
 176
 177                 buffer_sprintf(wb,
 178                         "\t\t\t\t{\n"
 179                         "\t\t\t\t\t\"id\": \"%s\",\n"
 180                         "\t\t\t\t\t\"name\": \"%s\",\n"
 181                         "\t\t\t\t\t\"entries\": %ld,\n"
 182                         "\t\t\t\t\t\"isHidden\": %d,\n"
 183                         "\t\t\t\t\t\"algorithm\": \"%s\",\n"
 184                         "\t\t\t\t\t\"multiplier\": %ld,\n"
 185                         "\t\t\t\t\t\"divisor\": %ld,\n"
 186                         "\t\t\t\t\t\"last_entry_t\": %lu,\n"
 187                         "\t\t\t\t\t\"collected_value\": " COLLECTED_NUMBER_FORMAT ",\n"
 188                         "\t\t\t\t\t\"calculated_value\": " CALCULATED_NUMBER_FORMAT ",\n"
 189                         "\t\t\t\t\t\"last_collected_value\": " COLLECTED_NUMBER_FORMAT ",\n"
 190                         "\t\t\t\t\t\"last_calculated_value\": " CALCULATED_NUMBER_FORMAT ",\n"
 191                         "\t\t\t\t\t\"memory\": %lu\n"
 192                         "\t\t\t\t}%s\n"
 193                         , rd->id
 194                         , rd->name
 195                         , rd->entries
 196                         , (rd->flags & RRDDIM_FLAG_HIDDEN)?1:0
 197                         , rrddim_algorithm_name(rd->algorithm)
 198                         , rd->multiplier
 199                         , rd->divisor
 200                         , rd->last_collected_time.tv_sec
 201                         , rd->collected_value
 202                         , rd->calculated_value
 203                         , rd->last_collected_value
 204                         , rd->last_calculated_value
 205                         , rd->memsize
 206                         , rd->next?",":""
 207                         );
 208         }
 209
 210         buffer_sprintf(wb,
 211                 "\t\t\t],\n"
 212                 "\t\t\t\"memory\" : %lu\n"
 213                 "\t\t}"
 214                 , memory
 215                 );
 216
 217         pthread_rwlock_unlock(&st->rwlock);
 218         return memory;
 219 }
 220
 221 #define RRD_GRAPH_JSON_HEADER "{\n\t\"charts\": [\n"
 222 #define RRD_GRAPH_JSON_FOOTER "\n\t]\n}\n"
 223
 224 void rrd_stats_graph_json(RRDSET *st, char *options, BUFFER *wb)
 225 {
 226         buffer_strcat(wb, RRD_GRAPH_JSON_HEADER);
 227         rrd_stats_one_json(st, options, wb);
 228         buffer_strcat(wb, RRD_GRAPH_JSON_FOOTER);
 229 }
 230
 231 void rrd_stats_all_json(BUFFER *wb)
 232 {
 233         unsigned long memory = 0;
 234         long c;
 235         RRDSET *st;
 236
 237         buffer_strcat(wb, RRD_GRAPH_JSON_HEADER);
 238
 239         pthread_rwlock_rdlock(&rrdset_root_rwlock);
 240         for(st = rrdset_root, c = 0; st ; st = st->next) {
 241                 if(st->enabled) {
 242                         if(c) buffer_strcat(wb, ",\n");
 243                         memory += rrd_stats_one_json(st, NULL, wb);
 244                         c++;
 245                 }
 246         }
 247         pthread_rwlock_unlock(&rrdset_root_rwlock);
 248
 249         buffer_sprintf(wb, "\n\t],\n"
 250                 "\t\"hostname\": \"%s\",\n"
 251                 "\t\"update_every\": %d,\n"
 252                 "\t\"history\": %d,\n"
 253                 "\t\"memory\": %lu\n"
 254                 "}\n"
 255                 , hostname
 256                 , rrd_update_every
 257                 , rrd_default_history_entries
 258                 , memory
 259                 );
 260 }
 261
 262
 263
 264 // ----------------------------------------------------------------------------
 265
 266 // RRDR dimension options
 267 #define RRDR_EMPTY      0x01 // the dimension contains / the value is empty (null)
 268 #define RRDR_RESET      0x02 // the dimension contains / the value is reset
 269 #define RRDR_HIDDEN     0x04 // the dimension contains / the value is hidden
 270 #define RRDR_NONZERO    0x08 // the dimension contains / the value is non-zero
 271
 272 // RRDR result options
 273 #define RRDR_RESULT_OPTION_ABSOLUTE 0x00000001
 274 #define RRDR_RESULT_OPTION_RELATIVE 0x00000002
 275
 276 typedef struct rrdresult {
 277         RRDSET *st;                     // the chart this result refers to
 278
 279         uint32_t result_options;        // RRDR_RESULT_OPTION_*
 280
 281         int d;                                  // the number of dimensions
 282         long n;                                 // the number of values in the arrays
 283         long rows;                          // the number of rows used
 284
 285         uint8_t *od;                    // the options for the dimensions
 286
 287         time_t *t;                              // array of n timestamps
 288         calculated_number *v;   // array n x d values
 289         uint8_t *o;                             // array n x d options
 290
 291         long c;                                 // current line ( -1 ~ n ), ( -1 = none, use rrdr_rows() to get number of rows )
 292
 293         long group;                             // how many collected values were grouped for each row
 294         long update_every;              // what is the suggested update frequency in seconds
 295
 296         calculated_number min;
 297         calculated_number max;
 298
 299         time_t before;
 300         time_t after;
 301
 302         int has_st_lock;                // if st is read locked by us
 303 } RRDR;
 304
 305 #define rrdr_rows(r) ((r)->rows)
 306
 307 /*
 308 static void rrdr_dump(RRDR *r)
 309 {
 310         long c, i;
 311         RRDDIM *d;
 312
 313         fprintf(stderr, "\nCHART %s (%s)\n", r->st->id, r->st->name);
 314
 315         for(c = 0, d = r->st->dimensions; d ;c++, d = d->next) {
 316                 fprintf(stderr, "DIMENSION %s (%s), %s%s%s%s\n"
 317                                 , d->id
 318                                 , d->name
 319                                 , (r->od[c] & RRDR_EMPTY)?"EMPTY ":""
 320                                 , (r->od[c] & RRDR_RESET)?"RESET ":""
 321                                 , (r->od[c] & RRDR_HIDDEN)?"HIDDEN ":""
 322                                 , (r->od[c] & RRDR_NONZERO)?"NONZERO ":""
 323                                 );
 324         }
 325
 326         if(r->rows <= 0) {
 327                 fprintf(stderr, "RRDR does not have any values in it.\n");
 328                 return;
 329         }
 330
 331         fprintf(stderr, "RRDR includes %d values in it:\n", r->rows);
 332
 333         // for each line in the array
 334         for(i = 0; i < r->rows ;i++) {
 335                 calculated_number *cn = &r->v[ i * r->d ];
 336                 uint8_t *co = &r->o[ i * r->d ];
 337
 338                 // print the id and the timestamp of the line
 339                 fprintf(stderr, "%ld %ld ", i + 1, r->t[i]);
 340
 341                 // for each dimension
 342                 for(c = 0, d = r->st->dimensions; d ;c++, d = d->next) {
 343                         if(unlikely(r->od[c] & RRDR_HIDDEN)) continue;
 344                         if(unlikely(!(r->od[c] & RRDR_NONZERO))) continue;
 345
 346                         if(co[c] & RRDR_EMPTY)
 347                                 fprintf(stderr, "null ");
 348                         else
 349                                 fprintf(stderr, CALCULATED_NUMBER_FORMAT " %s%s%s%s "
 350                                         , cn[c]
 351                                         , (co[c] & RRDR_EMPTY)?"E":" "
 352                                         , (co[c] & RRDR_RESET)?"R":" "
 353                                         , (co[c] & RRDR_HIDDEN)?"H":" "
 354                                         , (co[c] & RRDR_NONZERO)?"N":" "
 355                                         );
 356                 }
 357
 358                 fprintf(stderr, "\n");
 359         }
 360 }
 361 */
 362
 363 void rrdr_disable_not_selected_dimensions(RRDR *r, const char *dims)
 364 {
 365         char b[strlen(dims) + 1];
 366         char *o = b, *tok;
 367         strcpy(o, dims);
 368
 369         long c;
 370         RRDDIM *d;
 371
 372         // disable all of them
 373         for(c = 0, d = r->st->dimensions; d ;c++, d = d->next)
 374                 r->od[c] |= RRDR_HIDDEN;
 375
 376         while(o && *o && (tok = mystrsep(&o, ", |"))) {
 377                 if(!*tok) continue;
 378
 379                 // find it and enable it
 380                 for(c = 0, d = r->st->dimensions; d ;c++, d = d->next) {
 381                         if(!strcmp(d->name, tok)) {
 382                                 r->od[c] &= ~RRDR_HIDDEN;
 383
 384                                 // since the user needs this dimension
 385                                 // make it appear as NONZERO, to return it
 386                                 // even if the dimension has only zeros
 387                                 r->od[c] |= RRDR_NONZERO;
 388                         }
 389                 }
 390         }
 391 }
 392
 393 void rrdr_buffer_print_format(BUFFER *wb, uint32_t format)
 394 {
 395         switch(format) {
 396         case DATASOURCE_JSON:
 397                 buffer_strcat(wb, DATASOURCE_FORMAT_JSON);
 398                 break;
 399
 400         case DATASOURCE_DATATABLE_JSON:
 401                 buffer_strcat(wb, DATASOURCE_FORMAT_DATATABLE_JSON);
 402                 break;
 403
 404         case DATASOURCE_DATATABLE_JSONP:
 405                 buffer_strcat(wb, DATASOURCE_FORMAT_DATATABLE_JSONP);
 406                 break;
 407
 408         case DATASOURCE_JSONP:
 409                 buffer_strcat(wb, DATASOURCE_FORMAT_JSONP);
 410                 break;
 411
 412         case DATASOURCE_SSV:
 413                 buffer_strcat(wb, DATASOURCE_FORMAT_SSV);
 414                 break;
 415
 416         case DATASOURCE_CSV:
 417                 buffer_strcat(wb, DATASOURCE_FORMAT_CSV);
 418                 break;
 419
 420         case DATASOURCE_TSV:
 421                 buffer_strcat(wb, DATASOURCE_FORMAT_TSV);
 422                 break;
 423
 424         case DATASOURCE_HTML:
 425                 buffer_strcat(wb, DATASOURCE_FORMAT_HTML);
 426                 break;
 427
 428         case DATASOURCE_JS_ARRAY:
 429                 buffer_strcat(wb, DATASOURCE_FORMAT_JS_ARRAY);
 430                 break;
 431
 432         case DATASOURCE_SSV_COMMA:
 433                 buffer_strcat(wb, DATASOURCE_FORMAT_SSV_COMMA);
 434                 break;
 435
 436         default:
 437                 buffer_strcat(wb, "unknown");
 438                 break;
 439         }
 440 }
 441
 442 uint32_t rrdr_check_options(RRDR *r, uint32_t options, const char *dims)
 443 {
 444         if(options & RRDR_OPTION_NONZERO) {
 445                 long i;
 446
 447                 if(dims && *dims) {
 448                         // the caller wants specific dimensions
 449                         // disable NONZERO option
 450                         // to make sure we don't accidentally prevent
 451                         // the specific dimensions from being returned
 452                         i = 0;
 453                 }
 454                 else {
 455                         // find how many dimensions are not zero
 456                         long c;
 457                         RRDDIM *rd;
 458                         for(c = 0, i = 0, rd = r->st->dimensions; rd && c < r->d ; c++, rd = rd->next) {
 459                                 if(unlikely(r->od[c] & RRDR_HIDDEN)) continue;
 460                                 if(unlikely(!(r->od[c] & RRDR_NONZERO))) continue;
 461                                 i++;
 462                         }
 463                 }
 464
 465                 // if with nonzero we get i = 0 (no dimensions will be returned)
 466                 // disable nonzero to show all dimensions
 467                 if(!i) options &= ~RRDR_OPTION_NONZERO;
 468         }
 469
 470         return options;
 471 }
 472
 473 void rrdr_json_wrapper_begin(RRDR *r, BUFFER *wb, uint32_t format, uint32_t options, int string_value)
 474 {
 475         long rows = rrdr_rows(r);
 476         long c, i;
 477         RRDDIM *rd;
 478
 479         //info("JSONWRAPPER(): %s: BEGIN", r->st->id);
 480         char kq[2] = "",                                        // key quote
 481                 sq[2] = "";                                             // string quote
 482
 483         if( options & RRDR_OPTION_GOOGLE_JSON ) {
 484                 kq[0] = '\0';
 485                 sq[0] = '\'';
 486         }
 487         else {
 488                 kq[0] = '"';
 489                 sq[0] = '"';
 490         }
 491
 492         buffer_sprintf(wb, "{\n"
 493                         "       %sapi%s: 1,\n"
 494                         "       %sid%s: %s%s%s,\n"
 495                         "       %sname%s: %s%s%s,\n"
 496                         "       %sview_update_every%s: %d,\n"
 497                         "       %supdate_every%s: %d,\n"
 498                         "       %sfirst_entry%s: %u,\n"
 499                         "       %slast_entry%s: %u,\n"
 500                         "       %sbefore%s: %u,\n"
 501                         "       %safter%s: %u,\n"
 502                         "       %sdimension_names%s: ["
 503                         , kq, kq
 504                         , kq, kq, sq, r->st->id, sq
 505                         , kq, kq, sq, r->st->name, sq
 506                         , kq, kq, r->update_every
 507                         , kq, kq, r->st->update_every
 508                         , kq, kq, rrdset_first_entry_t(r->st)
 509                         , kq, kq, rrdset_last_entry_t(r->st)
 510                         , kq, kq, r->before
 511                         , kq, kq, r->after
 512                         , kq, kq);
 513
 514         for(c = 0, i = 0, rd = r->st->dimensions; rd && c < r->d ;c++, rd = rd->next) {
 515                 if(unlikely(r->od[c] & RRDR_HIDDEN)) continue;
 516                 if(unlikely((options & RRDR_OPTION_NONZERO) && !(r->od[c] & RRDR_NONZERO))) continue;
 517
 518                 if(i) buffer_strcat(wb, ", ");
 519                 buffer_strcat(wb, sq);
 520                 buffer_strcat(wb, rd->name);
 521                 buffer_strcat(wb, sq);
 522                 i++;
 523         }
 524         if(!i) {
 525 #ifdef NETDATA_INTERNAL_CHECKS
 526                 error("RRDR is empty for %s (RRDR has %d dimensions, options is 0x%08x)", r->st->id, r->d, options);
 527 #endif
 528                 rows = 0;
 529                 buffer_strcat(wb, sq);
 530                 buffer_strcat(wb, "no data");
 531                 buffer_strcat(wb, sq);
 532         }
 533
 534         buffer_sprintf(wb, "],\n"
 535                         "       %sdimension_ids%s: ["
 536                         , kq, kq);
 537
 538         for(c = 0, i = 0, rd = r->st->dimensions; rd && c < r->d ;c++, rd = rd->next) {
 539                 if(unlikely(r->od[c] & RRDR_HIDDEN)) continue;
 540                 if(unlikely((options & RRDR_OPTION_NONZERO) && !(r->od[c] & RRDR_NONZERO))) continue;
 541
 542                 if(i) buffer_strcat(wb, ", ");
 543                 buffer_strcat(wb, sq);
 544                 buffer_strcat(wb, rd->id);
 545                 buffer_strcat(wb, sq);
 546                 i++;
 547         }
 548         if(!i) {
 549                 rows = 0;
 550                 buffer_strcat(wb, sq);
 551                 buffer_strcat(wb, "no data");
 552                 buffer_strcat(wb, sq);
 553         }
 554
 555         buffer_sprintf(wb, "],\n"
 556                         "       %slatest_values%s: ["
 557                         , kq, kq);
 558
 559         for(c = 0, i = 0, rd = r->st->dimensions; rd && c < r->d ;c++, rd = rd->next) {
 560                 if(unlikely(r->od[c] & RRDR_HIDDEN)) continue;
 561                 if(unlikely((options & RRDR_OPTION_NONZERO) && !(r->od[c] & RRDR_NONZERO))) continue;
 562
 563                 if(i) buffer_strcat(wb, ", ");
 564                 i++;
 565
 566                 storage_number n = rd->values[rrdset_last_slot(r->st)];
 567
 568                 if(!does_storage_number_exist(n))
 569                         buffer_strcat(wb, "null");
 570                 else
 571                         buffer_rrd_value(wb, unpack_storage_number(n));
 572         }
 573         if(!i) {
 574                 rows = 0;
 575                 buffer_strcat(wb, "null");
 576         }
 577
 578         buffer_sprintf(wb, "],\n"
 579                         "       %sview_latest_values%s: ["
 580                         , kq, kq);
 581
 582         i = 0;
 583         if(rows) {
 584                 for(c = 0, i = 0, rd = r->st->dimensions; rd && c < r->d ;c++, rd = rd->next) {
 585                         if(unlikely(r->od[c] & RRDR_HIDDEN)) continue;
 586                         if(unlikely((options & RRDR_OPTION_NONZERO) && !(r->od[c] & RRDR_NONZERO))) continue;
 587
 588                         if(i) buffer_strcat(wb, ", ");
 589                         i++;
 590
 591                         calculated_number *cn = &r->v[ (0) * r->d ];
 592                         uint8_t *co = &r->o[ (0) * r->d ];
 593
 594                         if(co[c] & RRDR_EMPTY)
 595                                 buffer_strcat(wb, "null");
 596                         else
 597                                 buffer_rrd_value(wb, cn[c]);
 598                 }
 599         }
 600         if(!i) {
 601                 rows = 0;
 602                 buffer_strcat(wb, "null");
 603         }
 604
 605         buffer_sprintf(wb, "],\n"
 606                         "       %sdimensions%s: %d,\n"
 607                         "       %spoints%s: %d,\n"
 608                         "       %sformat%s: %s"
 609                         , kq, kq, i
 610                         , kq, kq, rows
 611                         , kq, kq, sq
 612                         );
 613
 614         rrdr_buffer_print_format(wb, format);
 615
 616         buffer_sprintf(wb, "%s,\n"
 617                         "       %sresult%s: "
 618                         , sq
 619                         , kq, kq
 620                         );
 621
 622         if(string_value) buffer_strcat(wb, sq);
 623         //info("JSONWRAPPER(): %s: END", r->st->id);
 624 }
 625
 626 void rrdr_json_wrapper_end(RRDR *r, BUFFER *wb, uint32_t format, uint32_t options, int string_value)
 627 {
 628         if(r) {;}
 629         if(format) {;}
 630
 631         char kq[2] = "",                                        // key quote
 632                 sq[2] = "";                                             // string quote
 633
 634         if( options & RRDR_OPTION_GOOGLE_JSON ) {
 635                 kq[0] = '\0';
 636                 sq[0] = '\'';
 637         }
 638         else {
 639                 kq[0] = '"';
 640                 sq[0] = '"';
 641         }
 642
 643         if(string_value) buffer_strcat(wb, sq);
 644
 645         buffer_sprintf(wb, ",\n %smin%s: ", kq, kq);
 646         buffer_rrd_value(wb, r->min);
 647         buffer_sprintf(wb, ",\n %smax%s: ", kq, kq);
 648         buffer_rrd_value(wb, r->max);
 649         buffer_strcat(wb, "\n}\n");
 650 }
 651
 652 #define JSON_DATES_JS 1
 653 #define JSON_DATES_TIMESTAMP 2
 654
 655 static void rrdr2json(RRDR *r, BUFFER *wb, uint32_t options, int datatable)
 656 {
 657         //info("RRD2JSON(): %s: BEGIN", r->st->id);
 658         int row_annotations = 0, dates, dates_with_new = 0;
 659         char kq[2] = "",                                        // key quote
 660                 sq[2] = "",                                             // string quote
 661                 pre_label[101] = "",                    // before each label
 662                 post_label[101] = "",                   // after each label
 663                 pre_date[101] = "",                             // the beginning of line, to the date
 664                 post_date[101] = "",                    // closing the date
 665                 pre_value[101] = "",                    // before each value
 666                 post_value[101] = "",                   // after each value
 667                 post_line[101] = "",                    // at the end of each row
 668                 normal_annotation[201] = "",    // default row annotation
 669                 overflow_annotation[201] = "",  // overflow row annotation
 670                 data_begin[101] = "",                   // between labels and values
 671                 finish[101] = "";                               // at the end of everything
 672
 673         if(datatable) {
 674                 dates = JSON_DATES_JS;
 675                 if( options & RRDR_OPTION_GOOGLE_JSON ) {
 676                         kq[0] = '\0';
 677                         sq[0] = '\'';
 678                 }
 679                 else {
 680                         kq[0] = '"';
 681                         sq[0] = '"';
 682                 }
 683                 row_annotations = 1;
 684                 snprintf(pre_date,   100, "             {%sc%s:[{%sv%s:%s", kq, kq, kq, kq, sq);
 685                 snprintf(post_date,  100, "%s}", sq);
 686                 snprintf(pre_label,  100, ",\n          {%sid%s:%s%s,%slabel%s:%s", kq, kq, sq, sq, kq, kq, sq);
 687                 snprintf(post_label, 100, "%s,%spattern%s:%s%s,%stype%s:%snumber%s}", sq, kq, kq, sq, sq, kq, kq, sq, sq);
 688                 snprintf(pre_value,  100, ",{%sv%s:", kq, kq);
 689                 snprintf(post_value, 100, "}");
 690                 snprintf(post_line,  100, "]}");
 691                 snprintf(data_begin, 100, "\n   ],\n    %srows%s:\n     [\n", kq, kq);
 692                 snprintf(finish,     100, "\n   ]\n}");
 693
 694                 snprintf(overflow_annotation, 200, ",{%sv%s:%sRESET OR OVERFLOW%s},{%sv%s:%sThe counters have been wrapped.%s}", kq, kq, sq, sq, kq, kq, sq, sq);
 695                 snprintf(normal_annotation,   200, ",{%sv%s:null},{%sv%s:null}", kq, kq, kq, kq);
 696
 697                 buffer_sprintf(wb, "{\n %scols%s:\n     [\n", kq, kq, kq, kq);
 698                 buffer_sprintf(wb, "            {%sid%s:%s%s,%slabel%s:%stime%s,%spattern%s:%s%s,%stype%s:%sdatetime%s},\n", kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq);
 699                 buffer_sprintf(wb, "            {%sid%s:%s%s,%slabel%s:%s%s,%spattern%s:%s%s,%stype%s:%sstring%s,%sp%s:{%srole%s:%sannotation%s}},\n", kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, kq, kq, sq, sq);
 700                 buffer_sprintf(wb, "            {%sid%s:%s%s,%slabel%s:%s%s,%spattern%s:%s%s,%stype%s:%sstring%s,%sp%s:{%srole%s:%sannotationText%s}}", kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, kq, kq, sq, sq);
 701
 702                 // remove the valueobjects flag
 703                 // google wants its own keys
 704                 if(options & RRDR_OPTION_OBJECTSROWS)
 705                         options &= ~RRDR_OPTION_OBJECTSROWS;
 706         }
 707         else {
 708                 kq[0] = '"';
 709                 sq[0] = '"';
 710                 if((options & RRDR_OPTION_SECONDS) || (options & RRDR_OPTION_MILLISECONDS)) {
 711                         dates = JSON_DATES_TIMESTAMP;
 712                         dates_with_new = 0;
 713                 }
 714                 else {
 715                         dates = JSON_DATES_JS;
 716                         dates_with_new = 1;
 717                 }
 718                 if( options & RRDR_OPTION_OBJECTSROWS )
 719                         snprintf(pre_date,   100, "             { ");
 720                 else
 721                         snprintf(pre_date,   100, "             [ ");
 722                 snprintf(pre_label,  100, ", \"");
 723                 snprintf(post_label, 100, "\"");
 724                 snprintf(pre_value,  100, ", ");
 725                 if( options & RRDR_OPTION_OBJECTSROWS )
 726                         snprintf(post_line,  100, "}");
 727                 else
 728                         snprintf(post_line,  100, "]");
 729                 snprintf(data_begin, 100, "],\n %sdata%s:\n     [\n", kq, kq);
 730                 snprintf(finish,     100, "\n   ]\n}");
 731
 732                 buffer_sprintf(wb, "{\n %slabels%s: [", kq, kq);
 733                 buffer_sprintf(wb, "%stime%s", sq, sq);
 734         }
 735
 736         // -------------------------------------------------------------------------
 737         // print the JSON header
 738
 739         long c, i;
 740         RRDDIM *rd;
 741
 742         // print the header lines
 743         for(c = 0, i = 0, rd = r->st->dimensions; rd && c < r->d ;c++, rd = rd->next) {
 744                 if(unlikely(r->od[c] & RRDR_HIDDEN)) continue;
 745                 if(unlikely((options & RRDR_OPTION_NONZERO) && !(r->od[c] & RRDR_NONZERO))) continue;
 746
 747                 buffer_strcat(wb, pre_label);
 748                 buffer_strcat(wb, rd->name);
 749                 buffer_strcat(wb, post_label);
 750                 i++;
 751         }
 752         if(!i) {
 753                 buffer_strcat(wb, pre_label);
 754                 buffer_strcat(wb, "no data");
 755                 buffer_strcat(wb, post_label);
 756         }
 757
 758         // print the begin of row data
 759         buffer_strcat(wb, data_begin);
 760
 761         // if all dimensions are hidden, print a null
 762         if(!i) {
 763                 buffer_strcat(wb, finish);
 764                 return;
 765         }
 766
 767         long start = 0, end = rrdr_rows(r), step = 1;
 768         if((options & RRDR_OPTION_REVERSED)) {
 769                 start = rrdr_rows(r) - 1;
 770                 end = -1;
 771                 step = -1;
 772         }
 773
 774         // for each line in the array
 775         calculated_number total = 1;
 776         for(i = start; i != end ;i += step) {
 777                 calculated_number *cn = &r->v[ i * r->d ];
 778                 uint8_t *co = &r->o[ i * r->d ];
 779
 780                 time_t now = r->t[i];
 781
 782                 if(dates == JSON_DATES_JS) {
 783                         // generate the local date time
 784                         struct tm tmbuf, *tm = localtime_r(&now, &tmbuf);
 785                         if(!tm) { error("localtime_r() failed."); continue; }
 786
 787                         if(likely(i != start)) buffer_strcat(wb, ",\n");
 788                         buffer_strcat(wb, pre_date);
 789
 790                         if( options & RRDR_OPTION_OBJECTSROWS )
 791                                 buffer_sprintf(wb, "%stime%s: ", kq, kq);
 792
 793                         if(dates_with_new)
 794                                 buffer_strcat(wb, "new ");
 795
 796                         buffer_jsdate(wb, tm->tm_year + 1900, tm->tm_mon, tm->tm_mday, tm->tm_hour, tm->tm_min, tm->tm_sec);
 797
 798                         buffer_strcat(wb, post_date);
 799
 800                         if(row_annotations) {
 801                                 // google supports one annotation per row
 802                                 int annotation_found = 0;
 803                                 for(c = 0, rd = r->st->dimensions; rd ;c++, rd = rd->next) {
 804                                         if(co[c] & RRDR_RESET) {
 805                                                 buffer_strcat(wb, overflow_annotation);
 806                                                 annotation_found = 1;
 807                                                 break;
 808                                         }
 809                                 }
 810                                 if(!annotation_found)
 811                                         buffer_strcat(wb, normal_annotation);
 812                         }
 813                 }
 814                 else {
 815                         // print the timestamp of the line
 816                         if(likely(i != start)) buffer_strcat(wb, ",\n");
 817                         buffer_strcat(wb, pre_date);
 818
 819                         if( options & RRDR_OPTION_OBJECTSROWS )
 820                                 buffer_sprintf(wb, "%stime%s: ", kq, kq);
 821
 822                         buffer_rrd_value(wb, (calculated_number)r->t[i]);
 823                         // in ms
 824                         if(options & RRDR_OPTION_MILLISECONDS) buffer_strcat(wb, "000");
 825
 826                         buffer_strcat(wb, post_date);
 827                 }
 828
 829                 if(unlikely(options & RRDR_OPTION_PERCENTAGE)) {
 830                         total = 0;
 831                         for(c = 0, rd = r->st->dimensions; rd && c < r->d ;c++, rd = rd->next) {
 832                                 calculated_number n = cn[c];
 833
 834                                 if(likely((options & RRDR_OPTION_ABSOLUTE) && n < 0))
 835                                         n = -n;
 836
 837                                 total += n;
 838                         }
 839                         // prevent a division by zero
 840                         if(total == 0) total = 1;
 841                 }
 842
 843                 // for each dimension
 844                 for(c = 0, rd = r->st->dimensions; rd && c < r->d ;c++, rd = rd->next) {
 845                         if(unlikely(r->od[c] & RRDR_HIDDEN)) continue;
 846                         if(unlikely((options & RRDR_OPTION_NONZERO) && !(r->od[c] & RRDR_NONZERO))) continue;
 847
 848                         calculated_number n = cn[c];
 849
 850                         buffer_strcat(wb, pre_value);
 851
 852                         if( options & RRDR_OPTION_OBJECTSROWS )
 853                                 buffer_sprintf(wb, "%s%s%s: ", kq, rd->name, kq);
 854
 855                         if(co[c] & RRDR_EMPTY) {
 856                                 if(options & RRDR_OPTION_NULL2ZERO)
 857                                         buffer_strcat(wb, "0");
 858                                 else
 859                                         buffer_strcat(wb, "null");
 860                         }
 861                         else {
 862                                 if(unlikely((options & RRDR_OPTION_ABSOLUTE) && n < 0))
 863                                         n = -n;
 864
 865                                 if(unlikely(options & RRDR_OPTION_PERCENTAGE))
 866                                         n = n * 100 / total;
 867
 868                                 buffer_rrd_value(wb, n);
 869                         }
 870
 871                         buffer_strcat(wb, post_value);
 872                 }
 873
 874                 buffer_strcat(wb, post_line);
 875         }
 876
 877         buffer_strcat(wb, finish);
 878         //info("RRD2JSON(): %s: END", r->st->id);
 879 }
 880
 881 static void rrdr2csv(RRDR *r, BUFFER *wb, uint32_t options, const char *startline, const char *separator, const char *endline, const char *betweenlines)
 882 {
 883         //info("RRD2CSV(): %s: BEGIN", r->st->id);
 884         long c, i;
 885         RRDDIM *d;
 886
 887         // print the csv header
 888         for(c = 0, i = 0, d = r->st->dimensions; d && c < r->d ;c++, d = d->next) {
 889                 if(unlikely(r->od[c] & RRDR_HIDDEN)) continue;
 890                 if(unlikely((options & RRDR_OPTION_NONZERO) && !(r->od[c] & RRDR_NONZERO))) continue;
 891
 892                 if(!i) {
 893                         buffer_strcat(wb, startline);
 894                         if(options & RRDR_OPTION_LABEL_QUOTES) buffer_strcat(wb, "\"");
 895                         buffer_strcat(wb, "time");
 896                         if(options & RRDR_OPTION_LABEL_QUOTES) buffer_strcat(wb, "\"");
 897                 }
 898                 buffer_strcat(wb, separator);
 899                 if(options & RRDR_OPTION_LABEL_QUOTES) buffer_strcat(wb, "\"");
 900                 buffer_strcat(wb, d->name);
 901                 if(options & RRDR_OPTION_LABEL_QUOTES) buffer_strcat(wb, "\"");
 902                 i++;
 903         }
 904         buffer_strcat(wb, endline);
 905
 906         if(!i) {
 907                 // no dimensions present
 908                 return;
 909         }
 910
 911         long start = 0, end = rrdr_rows(r), step = 1;
 912         if((options & RRDR_OPTION_REVERSED)) {
 913                 start = rrdr_rows(r) - 1;
 914                 end = -1;
 915                 step = -1;
 916         }
 917
 918         // for each line in the array
 919         calculated_number total = 1;
 920         for(i = start; i != end ;i += step) {
 921                 calculated_number *cn = &r->v[ i * r->d ];
 922                 uint8_t *co = &r->o[ i * r->d ];
 923
 924                 buffer_strcat(wb, betweenlines);
 925                 buffer_strcat(wb, startline);
 926
 927                 time_t now = r->t[i];
 928
 929                 if((options & RRDR_OPTION_SECONDS) || (options & RRDR_OPTION_MILLISECONDS)) {
 930                         // print the timestamp of the line
 931                         buffer_rrd_value(wb, (calculated_number)now);
 932                         // in ms
 933                         if(options & RRDR_OPTION_MILLISECONDS) buffer_strcat(wb, "000");
 934                 }
 935                 else {
 936                         // generate the local date time
 937                         struct tm tmbuf, *tm = localtime_r(&now, &tmbuf);
 938                         if(!tm) { error("localtime() failed."); continue; }
 939                         buffer_date(wb, tm->tm_year + 1900, tm->tm_mon, tm->tm_mday, tm->tm_hour, tm->tm_min, tm->tm_sec);
 940                 }
 941
 942                 if(unlikely(options & RRDR_OPTION_PERCENTAGE)) {
 943                         total = 0;
 944                         for(c = 0, d = r->st->dimensions; d && c < r->d ;c++, d = d->next) {
 945                                 calculated_number n = cn[c];
 946
 947                                 if(likely((options & RRDR_OPTION_ABSOLUTE) && n < 0))
 948                                         n = -n;
 949
 950                                 total += n;
 951                         }
 952                         // prevent a division by zero
 953                         if(total == 0) total = 1;
 954                 }
 955
 956                 // for each dimension
 957                 for(c = 0, d = r->st->dimensions; d && c < r->d ;c++, d = d->next) {
 958                         if(unlikely(r->od[c] & RRDR_HIDDEN)) continue;
 959                         if(unlikely((options & RRDR_OPTION_NONZERO) && !(r->od[c] & RRDR_NONZERO))) continue;
 960
 961                         buffer_strcat(wb, separator);
 962
 963                         calculated_number n = cn[c];
 964
 965                         if(co[c] & RRDR_EMPTY) {
 966                                 if(options & RRDR_OPTION_NULL2ZERO)
 967                                         buffer_strcat(wb, "0");
 968                                 else
 969                                         buffer_strcat(wb, "null");
 970                         }
 971                         else {
 972                                 if(unlikely((options & RRDR_OPTION_ABSOLUTE) && n < 0))
 973                                         n = -n;
 974
 975                                 if(unlikely(options & RRDR_OPTION_PERCENTAGE))
 976                                         n = n * 100 / total;
 977
 978                                 buffer_rrd_value(wb, n);
 979                         }
 980                 }
 981
 982                 buffer_strcat(wb, endline);
 983         }
 984         //info("RRD2CSV(): %s: END", r->st->id);
 985 }
 986
 987 static void rrdr2ssv(RRDR *r, BUFFER *wb, uint32_t options, const char *prefix, const char *separator, const char *suffix)
 988 {
 989         //info("RRD2SSV(): %s: BEGIN", r->st->id);
 990         long c, i;
 991         RRDDIM *d;
 992
 993         buffer_strcat(wb, prefix);
 994         long start = 0, end = rrdr_rows(r), step = 1;
 995         if((options & RRDR_OPTION_REVERSED)) {
 996                 start = rrdr_rows(r) - 1;
 997                 end = -1;
 998                 step = -1;
 999         }
1000
1001         // for each line in the array
1002         calculated_number total = 1;
1003         for(i = start; i != end ;i += step) {
1004
1005                 calculated_number *cn = &r->v[ i * r->d ];
1006                 uint8_t *co = &r->o[ i * r->d ];
1007
1008                 calculated_number sum = 0, min = 0, max = 0, v;
1009                 int all_null = 1, init = 1;
1010
1011                 if(unlikely(options & RRDR_OPTION_PERCENTAGE)) {
1012                         total = 0;
1013                         for(c = 0, d = r->st->dimensions; d && c < r->d ;c++, d = d->next) {
1014                                 calculated_number n = cn[c];
1015
1016                                 if(likely((options & RRDR_OPTION_ABSOLUTE) && n < 0))
1017                                         n = -n;
1018
1019                                 total += n;
1020                         }
1021                         // prevent a division by zero
1022                         if(total == 0) total = 1;
1023                 }
1024
1025                 // for each dimension
1026                 for(c = 0, d = r->st->dimensions; d && c < r->d ;c++, d = d->next) {
1027                         if(unlikely(r->od[c] & RRDR_HIDDEN)) continue;
1028                         if(unlikely((options & RRDR_OPTION_NONZERO) && !(r->od[c] & RRDR_NONZERO))) continue;
1029
1030                         calculated_number n = cn[c];
1031
1032                         if(likely((options & RRDR_OPTION_ABSOLUTE) && n < 0))
1033                                 n = -n;
1034
1035                         if(unlikely(options & RRDR_OPTION_PERCENTAGE))
1036                                 n = n * 100 / total;
1037
1038                         if(unlikely(init)) {
1039                                 if(n > 0) {
1040                                         min = 0;
1041                                         max = n;
1042                                 }
1043                                 else {
1044                                         min = n;
1045                                         max = 0;
1046                                 }
1047                                 init = 0;
1048                         }
1049
1050                         if(likely(!(co[c] & RRDR_EMPTY))) {
1051                                 all_null = 0;
1052                                 sum += n;
1053                         }
1054
1055                         if(n < min) min = n;
1056                         if(n > max) max = n;
1057                 }
1058
1059                 if(likely(i != start))
1060                         buffer_strcat(wb, separator);
1061
1062                 if(all_null) {
1063                         if(options & RRDR_OPTION_NULL2ZERO)
1064                                 buffer_strcat(wb, "0");
1065                         else
1066                                 buffer_strcat(wb, "null");
1067                 }
1068                 else {
1069                         if(options & RRDR_OPTION_MIN2MAX)
1070                                 v = max - min;
1071                         else
1072                                 v = sum;
1073
1074                         if(likely(i != start)) {
1075                                 if(r->min > v) r->min = v;
1076                                 if(r->max < v) r->max = v;
1077                         }
1078                         else {
1079                                 r->min = v;
1080                                 r->max = v;
1081                         }
1082
1083                         buffer_rrd_value(wb, v);
1084                 }
1085         }
1086         buffer_strcat(wb, suffix);
1087         //info("RRD2SSV(): %s: END", r->st->id);
1088 }
1089
1090 inline static calculated_number *rrdr_line_values(RRDR *r)
1091 {
1092         return &r->v[ r->c * r->d ];
1093 }
1094
1095 inline static uint8_t *rrdr_line_options(RRDR *r)
1096 {
1097         return &r->o[ r->c * r->d ];
1098 }
1099
1100 inline static int rrdr_line_init(RRDR *r, time_t t)
1101 {
1102         r->c++;
1103
1104         if(unlikely(r->c >= r->n)) {
1105                 error("requested to step above RRDR size for chart %s", r->st->name);
1106                 r->c = r->n - 1;
1107         }
1108
1109         // save the time
1110         r->t[r->c] = t;
1111
1112         return 1;
1113 }
1114
1115 inline static void rrdr_lock_rrdset(RRDR *r) {
1116         if(unlikely(!r)) {
1117                 error("NULL value given!");
1118                 return;
1119         }
1120
1121         pthread_rwlock_rdlock(&r->st->rwlock);
1122         r->has_st_lock = 1;
1123 }
1124
1125 inline static void rrdr_unlock_rrdset(RRDR *r) {
1126         if(unlikely(!r)) {
1127                 error("NULL value given!");
1128                 return;
1129         }
1130
1131         if(likely(r->has_st_lock)) {
1132                 pthread_rwlock_unlock(&r->st->rwlock);
1133                 r->has_st_lock = 0;
1134         }
1135 }
1136
1137 inline static void rrdr_free(RRDR *r)
1138 {
1139         if(unlikely(!r)) {
1140                 error("NULL value given!");
1141                 return;
1142         }
1143
1144         rrdr_unlock_rrdset(r);
1145         if(likely(r->t)) free(r->t);
1146         if(likely(r->v)) free(r->v);
1147         if(likely(r->o)) free(r->o);
1148         if(likely(r->od)) free(r->od);
1149         free(r);
1150 }
1151
1152 inline void rrdr_done(RRDR *r)
1153 {
1154         r->rows = r->c + 1;
1155         r->c = 0;
1156 }
1157
1158 static RRDR *rrdr_create(RRDSET *st, long n)
1159 {
1160         if(unlikely(!st)) {
1161                 error("NULL value given!");
1162                 return NULL;
1163         }
1164
1165         RRDR *r = calloc(1, sizeof(RRDR));
1166         if(unlikely(!r)) goto cleanup;
1167
1168         r->st = st;
1169
1170         rrdr_lock_rrdset(r);
1171
1172         RRDDIM *rd;
1173         for(rd = st->dimensions ; rd ; rd = rd->next) r->d++;
1174
1175         r->n = n;
1176
1177         r->t = malloc(n * sizeof(time_t));
1178         if(unlikely(!r->t)) goto cleanup;
1179
1180         r->v = malloc(n * r->d * sizeof(calculated_number));
1181         if(unlikely(!r->v)) goto cleanup;
1182
1183         r->o = malloc(n * r->d * sizeof(uint8_t));
1184         if(unlikely(!r->o)) goto cleanup;
1185
1186         r->od = malloc(r->d * sizeof(uint8_t));
1187         if(unlikely(!r->od)) goto cleanup;
1188
1189         // set the hidden flag on hidden dimensions
1190         int c;
1191         for(c = 0, rd = st->dimensions ; rd ; c++, rd = rd->next) {
1192                 if(unlikely(rd->flags & RRDDIM_FLAG_HIDDEN)) r->od[c] = RRDR_HIDDEN;
1193                 else r->od[c] = 0;
1194         }
1195
1196         r->c = -1;
1197
1198         r->group = 1;
1199         r->update_every = 1;
1200
1201         return r;
1202
1203 cleanup:
1204         error("Cannot allocate RRDR memory for %d entries", n);
1205         if(likely(r)) rrdr_free(r);
1206         return NULL;
1207 }
1208
1209 RRDR *rrd2rrdr(RRDSET *st, long points, long long after, long long before, int group_method)
1210 {
1211         int debug = st->debug;
1212         int absolute_period_requested = -1;
1213
1214         time_t first_entry_t = rrdset_first_entry_t(st);
1215         time_t last_entry_t = rrdset_last_entry_t(st);
1216
1217         if(before == 0 && after == 0) {
1218                 before = last_entry_t;
1219                 after = first_entry_t;
1220                 absolute_period_requested = 0;
1221         }
1222
1223         // allow relative for before and after
1224         if(before <= st->update_every * st->entries) {
1225                 before = last_entry_t + before;
1226                 absolute_period_requested = 0;
1227         }
1228
1229         if(after <= st->update_every * st->entries) {
1230                 after = last_entry_t + after;
1231                 absolute_period_requested = 0;
1232         }
1233
1234         if(absolute_period_requested == -1)
1235                 absolute_period_requested = 1;
1236
1237         // make sure they are within our timeframe
1238         if(before > last_entry_t) before = last_entry_t;
1239         if(before < first_entry_t) before = first_entry_t;
1240
1241         if(after > last_entry_t) after = last_entry_t;
1242         if(after < first_entry_t) after = first_entry_t;
1243
1244         // check if they are upside down
1245         if(after > before) {
1246                 time_t tmp = before;
1247                 before = after;
1248                 after = tmp;
1249         }
1250
1251         // the duration of the chart
1252         time_t duration = before - after;
1253         long available_points = duration / st->update_every;
1254
1255         if(duration <= 0 || available_points <= 0)
1256                 return rrdr_create(st, 1);
1257
1258         // check the wanted points
1259         if(points < 0) points = -points;
1260         if(points > available_points) points = available_points;
1261         if(points == 0) points = available_points;
1262
1263         // calculate proper grouping of source data
1264         long group = available_points / points;
1265         if(group <= 0) group = 1;
1266
1267         // round group to the closest integer
1268         if(available_points % points > points / 2) group++;
1269
1270         time_t after_new = after - (after % (group * st->update_every));
1271         time_t before_new = before - (before % (group * st->update_every));
1272         long points_new = (before_new - after_new) / st->update_every / group;
1273
1274         // find the starting and ending slots in our round robin db
1275         long    start_at_slot = rrdset_time2slot(st, before_new),
1276                         stop_at_slot = rrdset_time2slot(st, after_new);
1277
1278 #ifdef NETDATA_INTERNAL_CHECKS
1279         if(after_new < first_entry_t) {
1280                 error("after_new %u is too small, minimum %u", after_new, first_entry_t);
1281         }
1282         if(after_new > last_entry_t) {
1283                 error("after_new %u is too big, maximum %u", after_new, last_entry_t);
1284         }
1285         if(before_new < first_entry_t) {
1286                 error("before_new %u is too small, minimum %u", before_new, first_entry_t);
1287         }
1288         if(before_new > last_entry_t) {
1289                 error("before_new %u is too big, maximum %u", before_new, last_entry_t);
1290         }
1291         if(start_at_slot < 0 || start_at_slot >= st->entries) {
1292                 error("start_at_slot is invalid %ld, expected %ld to %ld", start_at_slot, 0, st->entries - 1);
1293         }
1294         if(stop_at_slot < 0 || stop_at_slot >= st->entries) {
1295                 error("stop_at_slot is invalid %ld, expected %ld to %ld", stop_at_slot, 0, st->entries - 1);
1296         }
1297         if(points_new > (before_new - after_new) / group / st->update_every + 1) {
1298                 error("points_new %ld is more than points %ld", points_new, (before_new - after_new) / group / st->update_every + 1);
1299         }
1300 #endif
1301
1302         //info("RRD2RRDR(): %s: wanted %ld points, got %ld - group=%ld, wanted duration=%u, got %u - wanted %ld - %ld, got %ld - %ld", st->id, points, points_new, group, before - after, before_new - after_new, after, before, after_new, before_new);
1303
1304         after = after_new;
1305         before = before_new;
1306         duration = before - after;
1307         points = points_new;
1308
1309         // Now we have:
1310         // before = the end time of the calculation
1311         // after = the start time of the calculation
1312         // duration = the duration of the calculation
1313         // group = the number of source points to aggregate / group together
1314         // method = the method of grouping source points
1315         // points = the number of points to generate
1316
1317
1318         // -------------------------------------------------------------------------
1319         // initialize our result set
1320
1321         RRDR *r = rrdr_create(st, points);
1322         if(!r) {
1323 #ifdef NETDATA_INTERNAL_CHECKS
1324                 error("Cannot create RRDR for %s, after=%u, before=%u, duration=%u, points=%d", st->id, after, before, duration, points);
1325 #endif
1326                 return NULL;
1327         }
1328         if(!r->d) {
1329 #ifdef NETDATA_INTERNAL_CHECKS
1330                 error("Returning empty RRDR (no dimensions in RRDSET) for %s, after=%u, before=%u, duration=%u, points=%d", st->id, after, before, duration, points);
1331 #endif
1332                 return r;
1333         }
1334
1335         if(absolute_period_requested == 1)
1336                 r->result_options |= RRDR_RESULT_OPTION_ABSOLUTE;
1337         else
1338                 r->result_options |= RRDR_RESULT_OPTION_RELATIVE;
1339
1340         // find how many dimensions we have
1341         long dimensions = r->d;
1342
1343
1344         // -------------------------------------------------------------------------
1345         // checks for debugging
1346
1347         if(debug) debug(D_RRD_STATS, "INFO %s first_t: %lu, last_t: %lu, all_duration: %lu, after: %lu, before: %lu, duration: %lu, points: %ld, group: %ld"
1348                         , st->id
1349                         , first_entry_t
1350                         , last_entry_t
1351                         , last_entry_t - first_entry_t
1352                         , after
1353                         , before
1354                         , duration
1355                         , points
1356                         , group
1357                         );
1358
1359
1360         // -------------------------------------------------------------------------
1361         // temp arrays for keeping values per dimension
1362
1363         calculated_number       group_values[dimensions]; // keep sums when grouping
1364         long                            group_counts[dimensions]; // keep the number of values added to group_values
1365         uint8_t                         group_options[dimensions];
1366         uint8_t                         found_non_zero[dimensions];
1367
1368
1369         // initialize them
1370         RRDDIM *rd;
1371         long c;
1372         for( rd = st->dimensions, c = 0 ; rd && c < dimensions ; rd = rd->next, c++) {
1373                 group_values[c] = 0;
1374                 group_counts[c] = 0;
1375                 group_options[c] = 0;
1376                 found_non_zero[c] = 0;
1377         }
1378
1379
1380         // -------------------------------------------------------------------------
1381         // the main loop
1382
1383         time_t  now = rrdset_slot2time(st, start_at_slot),
1384                         dt = st->update_every,
1385                         group_start_t = 0;
1386
1387         if(unlikely(debug)) debug(D_RRD_STATS, "BEGIN %s after_t: %lu (stop_at_t: %ld), before_t: %lu (start_at_t: %ld), start_t(now): %lu, current_entry: %ld, entries: %ld"
1388                         , st->id
1389                         , after
1390                         , stop_at_slot
1391                         , before
1392                         , start_at_slot
1393                         , now
1394                         , st->current_entry
1395                         , st->entries
1396                         );
1397
1398         r->group = group;
1399         r->update_every = group * st->update_every;
1400         r->before = now;
1401         r->after = now;
1402
1403         //info("RRD2RRDR(): %s: STARTING", st->id);
1404
1405         long slot = start_at_slot, counter = 0, stop_now = 0, added = 0, group_count = 0, add_this = 0;
1406         for(; !stop_now ; now -= dt, slot--, counter++) {
1407                 if(unlikely(slot < 0)) slot = st->entries - 1;
1408                 if(unlikely(slot == stop_at_slot)) stop_now = counter;
1409
1410                 if(unlikely(debug)) debug(D_RRD_STATS, "ROW %s slot: %ld, entries_counter: %ld, group_count: %ld, added: %ld, now: %lu, %s %s"
1411                                 , st->id
1412                                 , slot
1413                                 , counter
1414                                 , group_count + 1
1415                                 , added
1416                                 , now
1417                                 , (group_count + 1 == group)?"PRINT":"  -  "
1418                                 , (now >= after && now <= before)?"RANGE":"  -  "
1419                                 );
1420
1421                 // make sure we return data in the proper time range
1422                 if(unlikely(now > before)) continue;
1423                 if(unlikely(now < after)) break;
1424
1425                 if(unlikely(group_count == 0)) {
1426                         group_start_t = now;
1427                 }
1428                 group_count++;
1429
1430                 if(unlikely(group_count == group)) {
1431                         if(unlikely(added >= points)) break;
1432                         add_this = 1;
1433                 }
1434
1435                 // do the calculations
1436                 for(rd = st->dimensions, c = 0 ; rd && c < dimensions ; rd = rd->next, c++) {
1437                         storage_number n = rd->values[slot];
1438                         if(unlikely(!does_storage_number_exist(n))) continue;
1439
1440                         group_counts[c]++;
1441
1442                         calculated_number value = unpack_storage_number(n);
1443                         if(likely(value != 0.0)) {
1444                                 group_options[c] |= RRDR_NONZERO;
1445                                 found_non_zero[c] = 1;
1446                         }
1447
1448                         if(unlikely(did_storage_number_reset(n)))
1449                                 group_options[c] |= RRDR_RESET;
1450
1451                         switch(group_method) {
1452                                 case GROUP_MAX:
1453                                         if(unlikely(abs(value) > abs(group_values[c])))
1454                                                 group_values[c] = value;
1455                                         break;
1456
1457                                 default:
1458                                 case GROUP_SUM:
1459                                 case GROUP_AVERAGE:
1460                                         group_values[c] += value;
1461                                         break;
1462                         }
1463                 }
1464
1465                 // added it
1466                 if(unlikely(add_this)) {
1467                         if(unlikely(!rrdr_line_init(r, group_start_t))) break;
1468
1469                         r->after = now;
1470
1471                         calculated_number *cn = rrdr_line_values(r);
1472                         uint8_t *co = rrdr_line_options(r);
1473
1474                         for(rd = st->dimensions, c = 0 ; rd && c < dimensions ; rd = rd->next, c++) {
1475
1476                                 // update the dimension options
1477                                 if(likely(found_non_zero[c])) r->od[c] |= RRDR_NONZERO;
1478
1479                                 // store the specific point options
1480                                 co[c] = group_options[c];
1481
1482                                 // store the value
1483                                 if(unlikely(group_counts[c] == 0)) {
1484                                         cn[c] = 0.0;
1485                                         co[c] |= RRDR_EMPTY;
1486                                 }
1487                                 else if(unlikely(group_method == GROUP_AVERAGE)) {
1488                                         cn[c] = group_values[c] / group_counts[c];
1489                                 }
1490                                 else {
1491                                         cn[c] = group_values[c];
1492                                 }
1493
1494                                 if(cn[c] < r->min) r->min = cn[c];
1495                                 if(cn[c] > r->max) r->max = cn[c];
1496
1497                                 // reset them for the next loop
1498                                 group_values[c] = 0;
1499                                 group_counts[c] = 0;
1500                                 group_options[c] = 0;
1501                         }
1502
1503                         added++;
1504                         group_count = 0;
1505                         add_this = 0;
1506                 }
1507         }
1508
1509         rrdr_done(r);
1510         //info("RRD2RRDR(): %s: END %ld loops made, %ld points generated", st->id, counter, rrdr_rows(r));
1511         //error("SHIFT: %s: wanted %ld points, got %ld", st->id, points, rrdr_rows(r));
1512         return r;
1513 }
1514
1515 int rrd2format(RRDSET *st, BUFFER *wb, BUFFER *dimensions, uint32_t format, long points, long long after, long long before, int group_method, uint32_t options, time_t *latest_timestamp)
1516 {
1517         RRDR *r = rrd2rrdr(st, points, after, before, group_method);
1518         if(!r) {
1519                 buffer_strcat(wb, "Cannot generate output with these parameters on this chart.");
1520                 return 500;
1521         }
1522
1523         if(r->result_options & RRDR_RESULT_OPTION_RELATIVE)
1524                 wb->options |= WB_CONTENT_NO_CACHEABLE;
1525         else if(r->result_options & RRDR_RESULT_OPTION_ABSOLUTE)
1526                 wb->options |= WB_CONTENT_CACHEABLE;
1527
1528         options = rrdr_check_options(r, options, (dimensions)?buffer_tostring(dimensions):NULL);
1529
1530         if(dimensions)
1531                 rrdr_disable_not_selected_dimensions(r, buffer_tostring(dimensions));
1532
1533         if(latest_timestamp && rrdr_rows(r) > 0)
1534                 *latest_timestamp = r->before;
1535
1536         switch(format) {
1537         case DATASOURCE_SSV:
1538                 if(options & RRDR_OPTION_JSON_WRAP) {
1539                         wb->contenttype = CT_APPLICATION_JSON;
1540                         rrdr_json_wrapper_begin(r, wb, format, options, 1);
1541                         rrdr2ssv(r, wb, options, "", " ", "");
1542                         rrdr_json_wrapper_end(r, wb, format, options, 1);
1543                 }
1544                 else {
1545                         wb->contenttype = CT_TEXT_PLAIN;
1546                         rrdr2ssv(r, wb, options, "", " ", "");
1547                 }
1548                 break;
1549
1550         case DATASOURCE_SSV_COMMA:
1551                 if(options & RRDR_OPTION_JSON_WRAP) {
1552                         wb->contenttype = CT_APPLICATION_JSON;
1553                         rrdr_json_wrapper_begin(r, wb, format, options, 1);
1554                         rrdr2ssv(r, wb, options, "", ",", "");
1555                         rrdr_json_wrapper_end(r, wb, format, options, 1);
1556                 }
1557                 else {
1558                         wb->contenttype = CT_TEXT_PLAIN;
1559                         rrdr2ssv(r, wb, options, "", ",", "");
1560                 }
1561                 break;
1562
1563         case DATASOURCE_JS_ARRAY:
1564                 if(options & RRDR_OPTION_JSON_WRAP) {
1565                         wb->contenttype = CT_APPLICATION_JSON;
1566                         rrdr_json_wrapper_begin(r, wb, format, options, 0);
1567                         rrdr2ssv(r, wb, options, "[", ",", "]");
1568                         rrdr_json_wrapper_end(r, wb, format, options, 0);
1569                 }
1570                 else {
1571                         wb->contenttype = CT_APPLICATION_JSON;
1572                         rrdr2ssv(r, wb, options, "[", ",", "]");
1573                 }
1574                 break;
1575
1576         case DATASOURCE_CSV:
1577                 if(options & RRDR_OPTION_JSON_WRAP) {
1578                         wb->contenttype = CT_APPLICATION_JSON;
1579                         rrdr_json_wrapper_begin(r, wb, format, options, 1);
1580                         rrdr2csv(r, wb, options, "", ",", "\\n", "");
1581                         rrdr_json_wrapper_end(r, wb, format, options, 1);
1582                 }
1583                 else {
1584                         wb->contenttype = CT_TEXT_PLAIN;
1585                         rrdr2csv(r, wb, options, "", ",", "\r\n", "");
1586                 }
1587                 break;
1588
1589         case DATASOURCE_CSV_JSON_ARRAY:
1590                 wb->contenttype = CT_APPLICATION_JSON;
1591                 if(options & RRDR_OPTION_JSON_WRAP) {
1592                         rrdr_json_wrapper_begin(r, wb, format, options, 0);
1593                         buffer_strcat(wb, "[\n");
1594                         rrdr2csv(r, wb, options + RRDR_OPTION_LABEL_QUOTES, "[", ",", "]", ",\n");
1595                         buffer_strcat(wb, "\n]");
1596                         rrdr_json_wrapper_end(r, wb, format, options, 0);
1597                 }
1598                 else {
1599                         wb->contenttype = CT_TEXT_PLAIN;
1600                         buffer_strcat(wb, "[\n");
1601                         rrdr2csv(r, wb, options + RRDR_OPTION_LABEL_QUOTES, "[", ",", "]", ",\n");
1602                         buffer_strcat(wb, "\n]");
1603                 }
1604                 break;
1605
1606         case DATASOURCE_TSV:
1607                 if(options & RRDR_OPTION_JSON_WRAP) {
1608                         wb->contenttype = CT_APPLICATION_JSON;
1609                         rrdr_json_wrapper_begin(r, wb, format, options, 1);
1610                         rrdr2csv(r, wb, options, "", "\t", "\\n", "");
1611                         rrdr_json_wrapper_end(r, wb, format, options, 1);
1612                 }
1613                 else {
1614                         wb->contenttype = CT_TEXT_PLAIN;
1615                         rrdr2csv(r, wb, options, "", "\t", "\r\n", "");
1616                 }
1617                 break;
1618
1619         case DATASOURCE_HTML:
1620                 if(options & RRDR_OPTION_JSON_WRAP) {
1621                         wb->contenttype = CT_APPLICATION_JSON;
1622                         rrdr_json_wrapper_begin(r, wb, format, options, 1);
1623                         buffer_strcat(wb, "<html>\\n<center>\\n<table border=\\\"0\\\" cellpadding=\\\"5\\\" cellspacing=\\\"5\\\">\\n");
1624                         rrdr2csv(r, wb, options, "<tr><td>", "</td><td>", "</td></tr>\\n", "");
1625                         buffer_strcat(wb, "</table>\\n</center>\\n</html>\\n");
1626                         rrdr_json_wrapper_end(r, wb, format, options, 1);
1627                 }
1628                 else {
1629                         wb->contenttype = CT_TEXT_HTML;
1630                         buffer_strcat(wb, "<html>\n<center>\n<table border=\"0\" cellpadding=\"5\" cellspacing=\"5\">\n");
1631                         rrdr2csv(r, wb, options, "<tr><td>", "</td><td>", "</td></tr>\n", "");
1632                         buffer_strcat(wb, "</table>\n</center>\n</html>\n");
1633                 }
1634                 break;
1635
1636         case DATASOURCE_DATATABLE_JSONP:
1637                 wb->contenttype = CT_APPLICATION_X_JAVASCRIPT;
1638
1639                 if(options & RRDR_OPTION_JSON_WRAP)
1640                         rrdr_json_wrapper_begin(r, wb, format, options, 0);
1641
1642                 rrdr2json(r, wb, options, 1);
1643
1644                 if(options & RRDR_OPTION_JSON_WRAP)
1645                         rrdr_json_wrapper_end(r, wb, format, options, 0);
1646                 break;
1647
1648         case DATASOURCE_DATATABLE_JSON:
1649                 wb->contenttype = CT_APPLICATION_JSON;
1650
1651                 if(options & RRDR_OPTION_JSON_WRAP)
1652                         rrdr_json_wrapper_begin(r, wb, format, options, 0);
1653
1654                 rrdr2json(r, wb, options, 1);
1655
1656                 if(options & RRDR_OPTION_JSON_WRAP)
1657                         rrdr_json_wrapper_end(r, wb, format, options, 0);
1658                 break;
1659
1660         case DATASOURCE_JSONP:
1661                 wb->contenttype = CT_APPLICATION_X_JAVASCRIPT;
1662                 if(options & RRDR_OPTION_JSON_WRAP)
1663                         rrdr_json_wrapper_begin(r, wb, format, options, 0);
1664
1665                 rrdr2json(r, wb, options, 0);
1666
1667                 if(options & RRDR_OPTION_JSON_WRAP)
1668                         rrdr_json_wrapper_end(r, wb, format, options, 0);
1669                 break;
1670
1671         case DATASOURCE_JSON:
1672         default:
1673                 wb->contenttype = CT_APPLICATION_JSON;
1674
1675                 if(options & RRDR_OPTION_JSON_WRAP)
1676                         rrdr_json_wrapper_begin(r, wb, format, options, 0);
1677
1678                 rrdr2json(r, wb, options, 0);
1679
1680                 if(options & RRDR_OPTION_JSON_WRAP)
1681                         rrdr_json_wrapper_end(r, wb, format, options, 0);
1682                 break;
1683         }
1684
1685         rrdr_free(r);
1686         return 200;
1687 }
1688
1689 time_t rrd_stats_json(int type, RRDSET *st, BUFFER *wb, long points, long group, int group_method, time_t after, time_t before, int only_non_zero)
1690 {
1691         int c;
1692         pthread_rwlock_rdlock(&st->rwlock);
1693
1694
1695         // -------------------------------------------------------------------------
1696         // switch from JSON to google JSON
1697
1698         char kq[2] = "\"";
1699         char sq[2] = "\"";
1700         switch(type) {
1701                 case DATASOURCE_DATATABLE_JSON:
1702                 case DATASOURCE_DATATABLE_JSONP:
1703                         kq[0] = '\0';
1704                         sq[0] = '\'';
1705                         break;
1706
1707                 case DATASOURCE_JSON:
1708                 default:
1709                         break;
1710         }
1711
1712
1713         // -------------------------------------------------------------------------
1714         // validate the parameters
1715
1716         if(points < 1) points = 1;
1717         if(group < 1) group = 1;
1718
1719         if(before == 0 || before > rrdset_last_entry_t(st)) before = rrdset_last_entry_t(st);
1720         if(after  == 0 || after < rrdset_first_entry_t(st)) after = rrdset_first_entry_t(st);
1721
1722         // ---
1723
1724         // our return value (the last timestamp printed)
1725         // this is required to detect re-transmit in google JSONP
1726         time_t last_timestamp = 0;
1727
1728
1729         // -------------------------------------------------------------------------
1730         // find how many dimensions we have
1731
1732         int dimensions = 0;
1733         RRDDIM *rd;
1734         for( rd = st->dimensions ; rd ; rd = rd->next) dimensions++;
1735         if(!dimensions) {
1736                 pthread_rwlock_unlock(&st->rwlock);
1737                 buffer_strcat(wb, "No dimensions yet.");
1738                 return 0;
1739         }
1740
1741
1742         // -------------------------------------------------------------------------
1743         // prepare various strings, to speed up the loop
1744
1745         char overflow_annotation[201]; snprintf(overflow_annotation, 200, ",{%sv%s:%sRESET OR OVERFLOW%s},{%sv%s:%sThe counters have been wrapped.%s}", kq, kq, sq, sq, kq, kq, sq, sq);
1746         char normal_annotation[201];   snprintf(normal_annotation,   200, ",{%sv%s:null},{%sv%s:null}", kq, kq, kq, kq);
1747         char pre_date[51];             snprintf(pre_date,             50, "             {%sc%s:[{%sv%s:%s", kq, kq, kq, kq, sq);
1748         char post_date[21];            snprintf(post_date,            20, "%s}", sq);
1749         char pre_value[21];            snprintf(pre_value,            20, ",{%sv%s:", kq, kq);
1750         char post_value[21];           snprintf(post_value,           20, "}");
1751
1752
1753         // -------------------------------------------------------------------------
1754         // checks for debugging
1755
1756         if(st->debug) {
1757                 debug(D_RRD_STATS, "%s first_entry_t = %lu, last_entry_t = %lu, duration = %lu, after = %lu, before = %lu, duration = %lu, entries_to_show = %lu, group = %lu"
1758                         , st->id
1759                         , rrdset_first_entry_t(st)
1760                         , rrdset_last_entry_t(st)
1761                         , rrdset_last_entry_t(st) - rrdset_first_entry_t(st)
1762                         , after
1763                         , before
1764                         , before - after
1765                         , points
1766                         , group
1767                         );
1768
1769                 if(before < after)
1770                         debug(D_RRD_STATS, "WARNING: %s The newest value in the database (%lu) is earlier than the oldest (%lu)", st->name, before, after);
1771
1772                 if((before - after) > st->entries * st->update_every)
1773                         debug(D_RRD_STATS, "WARNING: %s The time difference between the oldest and the newest entries (%lu) is higher than the capacity of the database (%lu)", st->name, before - after, st->entries * st->update_every);
1774         }
1775
1776
1777         // -------------------------------------------------------------------------
1778         // temp arrays for keeping values per dimension
1779
1780         calculated_number group_values[dimensions]; // keep sums when grouping
1781         int               print_hidden[dimensions]; // keep hidden flags
1782         int               found_non_zero[dimensions];
1783         int               found_non_existing[dimensions];
1784
1785         // initialize them
1786         for( rd = st->dimensions, c = 0 ; rd && c < dimensions ; rd = rd->next, c++) {
1787                 group_values[c] = 0;
1788                 print_hidden[c] = (rd->flags & RRDDIM_FLAG_HIDDEN)?1:0;
1789                 found_non_zero[c] = 0;
1790                 found_non_existing[c] = 0;
1791         }
1792
1793
1794         // error("OLD: points=%d after=%d before=%d group=%d, duration=%d", entries_to_show, before - (st->update_every * group * entries_to_show), before, group, before - after + 1);
1795         // rrd2array(st, entries_to_show, before - (st->update_every * group * entries_to_show), before, group_method, only_non_zero);
1796         // rrd2rrdr(st, entries_to_show, before - (st->update_every * group * entries_to_show), before, group_method);
1797
1798         // -------------------------------------------------------------------------
1799         // remove dimensions that contain only zeros
1800
1801         int max_loop = 1;
1802         if(only_non_zero) max_loop = 2;
1803
1804         for(; max_loop ; max_loop--) {
1805
1806                 // -------------------------------------------------------------------------
1807                 // print the JSON header
1808
1809                 buffer_sprintf(wb, "{\n %scols%s:\n     [\n", kq, kq);
1810                 buffer_sprintf(wb, "            {%sid%s:%s%s,%slabel%s:%stime%s,%spattern%s:%s%s,%stype%s:%sdatetime%s},\n", kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq);
1811                 buffer_sprintf(wb, "            {%sid%s:%s%s,%slabel%s:%s%s,%spattern%s:%s%s,%stype%s:%sstring%s,%sp%s:{%srole%s:%sannotation%s}},\n", kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, kq, kq, sq, sq);
1812                 buffer_sprintf(wb, "            {%sid%s:%s%s,%slabel%s:%s%s,%spattern%s:%s%s,%stype%s:%sstring%s,%sp%s:{%srole%s:%sannotationText%s}}", kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, kq, kq, sq, sq);
1813
1814                 // print the header for each dimension
1815                 // and update the print_hidden array for the dimensions that should be hidden
1816                 int pc = 0;
1817                 for( rd = st->dimensions, c = 0 ; rd && c < dimensions ; rd = rd->next, c++) {
1818                         if(!print_hidden[c]) {
1819                                 pc++;
1820                                 buffer_sprintf(wb, ",\n         {%sid%s:%s%s,%slabel%s:%s%s%s,%spattern%s:%s%s,%stype%s:%snumber%s}", kq, kq, sq, sq, kq, kq, sq, rd->name, sq, kq, kq, sq, sq, kq, kq, sq, sq);
1821                         }
1822                 }
1823                 if(!pc) {
1824                         buffer_sprintf(wb, ",\n         {%sid%s:%s%s,%slabel%s:%s%s%s,%spattern%s:%s%s,%stype%s:%snumber%s}", kq, kq, sq, sq, kq, kq, sq, "no data", sq, kq, kq, sq, sq, kq, kq, sq, sq);
1825                 }
1826
1827                 // print the begin of row data
1828                 buffer_sprintf(wb, "\n  ],\n    %srows%s:\n     [\n", kq, kq);
1829
1830
1831                 // -------------------------------------------------------------------------
1832                 // the main loop
1833
1834                 int annotate_reset = 0;
1835                 int annotation_count = 0;
1836
1837                 long    t = rrdset_time2slot(st, before),
1838                                 stop_at_t = rrdset_time2slot(st, after),
1839                                 stop_now = 0;
1840
1841                 t -= t % group;
1842
1843                 time_t  now = rrdset_slot2time(st, t),
1844                                 dt = st->update_every;
1845
1846                 long count = 0, printed = 0, group_count = 0;
1847                 last_timestamp = 0;
1848
1849                 if(st->debug) debug(D_RRD_STATS, "%s: REQUEST after:%lu before:%lu, points:%d, group:%d, CHART cur:%ld first: %lu last:%lu, CALC start_t:%ld, stop_t:%ld"
1850                                         , st->id
1851                                         , after
1852                                         , before
1853                                         , points
1854                                         , group
1855                                         , st->current_entry
1856                                         , rrdset_first_entry_t(st)
1857                                         , rrdset_last_entry_t(st)
1858                                         , t
1859                                         , stop_at_t
1860                                         );
1861
1862                 long counter = 0;
1863                 for(; !stop_now ; now -= dt, t--, counter++) {
1864                         if(t < 0) t = st->entries - 1;
1865                         if(t == stop_at_t) stop_now = counter;
1866
1867                         int print_this = 0;
1868
1869                         if(st->debug) debug(D_RRD_STATS, "%s t = %ld, count = %ld, group_count = %ld, printed = %ld, now = %lu, %s %s"
1870                                         , st->id
1871                                         , t
1872                                         , count + 1
1873                                         , group_count + 1
1874                                         , printed
1875                                         , now
1876                                         , (group_count + 1 == group)?"PRINT":"  -  "
1877                                         , (now >= after && now <= before)?"RANGE":"  -  "
1878                                         );
1879
1880
1881                         // make sure we return data in the proper time range
1882                         if(now > before) continue;
1883                         if(now < after) break;
1884
1885                         //if(rrdset_slot2time(st, t) != now)
1886                         //      error("%s: slot=%ld, now=%ld, slot2time=%ld, diff=%ld, last_entry_t=%ld, rrdset_last_slot=%ld", st->id, t, now, rrdset_slot2time(st,t), now - rrdset_slot2time(st,t), rrdset_last_entry_t(st), rrdset_last_slot(st));
1887
1888                         count++;
1889                         group_count++;
1890
1891                         // check if we have to print this now
1892                         if(group_count == group) {
1893                                 if(printed >= points) {
1894                                         // debug(D_RRD_STATS, "Already printed all rows. Stopping.");
1895                                         break;
1896                                 }
1897
1898                                 // generate the local date time
1899                                 struct tm tmbuf, *tm = localtime_r(&now, &tmbuf);
1900                                 if(!tm) { error("localtime() failed."); continue; }
1901                                 if(now > last_timestamp) last_timestamp = now;
1902
1903                                 if(printed) buffer_strcat(wb, "]},\n");
1904                                 buffer_strcat(wb, pre_date);
1905                                 buffer_jsdate(wb, tm->tm_year + 1900, tm->tm_mon, tm->tm_mday, tm->tm_hour, tm->tm_min, tm->tm_sec);
1906                                 buffer_strcat(wb, post_date);
1907
1908                                 print_this = 1;
1909                         }
1910
1911                         // do the calculations
1912                         for(rd = st->dimensions, c = 0 ; rd && c < dimensions ; rd = rd->next, c++) {
1913                                 storage_number n = rd->values[t];
1914                                 calculated_number value = unpack_storage_number(n);
1915
1916                                 if(!does_storage_number_exist(n)) {
1917                                         value = 0.0;
1918                                         found_non_existing[c]++;
1919                                 }
1920                                 if(did_storage_number_reset(n)) annotate_reset = 1;
1921
1922                                 switch(group_method) {
1923                                         case GROUP_MAX:
1924                                                 if(abs(value) > abs(group_values[c])) group_values[c] = value;
1925                                                 break;
1926
1927                                         case GROUP_SUM:
1928                                                 group_values[c] += value;
1929                                                 break;
1930
1931                                         default:
1932                                         case GROUP_AVERAGE:
1933                                                 group_values[c] += value;
1934                                                 if(print_this) group_values[c] /= ( group_count - found_non_existing[c] );
1935                                                 break;
1936                                 }
1937                         }
1938
1939                         if(print_this) {
1940                                 if(annotate_reset) {
1941                                         annotation_count++;
1942                                         buffer_strcat(wb, overflow_annotation);
1943                                         annotate_reset = 0;
1944                                 }
1945                                 else
1946                                         buffer_strcat(wb, normal_annotation);
1947
1948                                 pc = 0;
1949                                 for(c = 0 ; c < dimensions ; c++) {
1950                                         if(found_non_existing[c] == group_count) {
1951                                                 // all entries are non-existing
1952                                                 pc++;
1953                                                 buffer_strcat(wb, pre_value);
1954                                                 buffer_strcat(wb, "null");
1955                                                 buffer_strcat(wb, post_value);
1956                                         }
1957                                         else if(!print_hidden[c]) {
1958                                                 pc++;
1959                                                 buffer_strcat(wb, pre_value);
1960                                                 buffer_rrd_value(wb, group_values[c]);
1961                                                 buffer_strcat(wb, post_value);
1962
1963                                                 if(group_values[c]) found_non_zero[c]++;
1964                                         }
1965
1966                                         // reset them for the next loop
1967                                         group_values[c] = 0;
1968                                         found_non_existing[c] = 0;
1969                                 }
1970
1971                                 // if all dimensions are hidden, print a null
1972                                 if(!pc) {
1973                                         buffer_strcat(wb, pre_value);
1974                                         buffer_strcat(wb, "null");
1975                                         buffer_strcat(wb, post_value);
1976                                 }
1977
1978                                 printed++;
1979                                 group_count = 0;
1980                         }
1981                 }
1982
1983                 if(printed) buffer_strcat(wb, "]}");
1984                 buffer_strcat(wb, "\n   ]\n}\n");
1985
1986                 if(only_non_zero && max_loop > 1) {
1987                         int changed = 0;
1988                         for(rd = st->dimensions, c = 0 ; rd && c < dimensions ; rd = rd->next, c++) {
1989                                 group_values[c] = 0;
1990                                 found_non_existing[c] = 0;
1991
1992                                 if(!print_hidden[c] && !found_non_zero[c]) {
1993                                         changed = 1;
1994                                         print_hidden[c] = 1;
1995                                 }
1996                         }
1997
1998                         if(changed) buffer_flush(wb);
1999                         else break;
2000                 }
2001                 else break;
2002
2003         } // max_loop
2004
2005         debug(D_RRD_STATS, "RRD_STATS_JSON: %s total %ld bytes", st->name, wb->len);
2006
2007         pthread_rwlock_unlock(&st->rwlock);
2008         return last_timestamp;
2009 }