src/rrd2json.c

   1 #ifdef HAVE_CONFIG_H
   2 #include <config.h>
   3 #endif
   4 #include <pthread.h>
   5 #include <stdlib.h>
   6 #include <string.h>
   7 #include <stdint.h>
   8 #include <math.h>
   9
  10 #include "log.h"
  11 #include "common.h"
  12 #include "rrd2json.h"
  13
  14 #define HOSTNAME_MAX 1024
  15 char *hostname = "unknown";
  16
  17 void rrd_stats_api_v1_chart(RRDSET *st, BUFFER *wb)
  18 {
  19         pthread_rwlock_rdlock(&st->rwlock);
  20
  21         buffer_sprintf(wb,
  22                 "\t\t{\n"
  23                 "\t\t\t\"id\": \"%s\",\n"
  24                 "\t\t\t\"name\": \"%s\",\n"
  25                 "\t\t\t\"type\": \"%s\",\n"
  26                 "\t\t\t\"family\": \"%s\",\n"
  27                 "\t\t\t\"context\": \"%s\",\n"
  28                 "\t\t\t\"title\": \"%s\",\n"
  29                 "\t\t\t\"priority\": %ld,\n"
  30                 "\t\t\t\"enabled\": %s,\n"
  31                 "\t\t\t\"units\": \"%s\",\n"
  32                 "\t\t\t\"data_url\": \"/api/v1/data?chart=%s\",\n"
  33                 "\t\t\t\"chart_type\": \"%s\",\n"
  34                 "\t\t\t\"duration\": %ld,\n"
  35                 "\t\t\t\"first_entry\": %lu,\n"
  36                 "\t\t\t\"last_entry\": %lu,\n"
  37                 "\t\t\t\"update_every\": %d,\n"
  38                 "\t\t\t\"dimensions\": {\n"
  39                 , st->id
  40                 , st->name
  41                 , st->type
  42                 , st->family
  43                 , st->context
  44                 , st->title
  45                 , st->priority
  46                 , st->enabled?"true":"false"
  47                 , st->units
  48                 , st->name
  49                 , rrdset_type_name(st->chart_type)
  50                 , st->entries * st->update_every
  51                 , rrdset_first_entry_t(st)
  52                 , rrdset_last_entry_t(st)
  53                 , st->update_every
  54                 );
  55
  56         unsigned long memory = st->memsize;
  57
  58         int c = 0;
  59         RRDDIM *rd;
  60         for(rd = st->dimensions; rd ; rd = rd->next) {
  61                 if(rd->flags & RRDDIM_FLAG_HIDDEN) continue;
  62
  63                 memory += rd->memsize;
  64
  65                 buffer_sprintf(wb,
  66                         "%s"
  67                         "\t\t\t\t\"%s\": { \"name\": \"%s\" }"
  68                         , c?",\n":""
  69                         , rd->id
  70                         , rd->name
  71                         );
  72
  73                 c++;
  74         }
  75
  76         buffer_sprintf(wb,
  77                 "\n\t\t\t}\n"
  78                 "\t\t}"
  79                 );
  80
  81         pthread_rwlock_unlock(&st->rwlock);
  82 }
  83
  84 void rrd_stats_api_v1_charts(BUFFER *wb)
  85 {
  86         long c;
  87         RRDSET *st;
  88
  89         buffer_sprintf(wb, "{\n"
  90                    "\t\"hostname\": \"%s\""
  91                 ",\n\t\"update_every\": %d"
  92                 ",\n\t\"history\": %d"
  93                 ",\n\t\"charts\": {"
  94                 , hostname
  95                 , rrd_update_every
  96                 , rrd_default_history_entries
  97                 );
  98
  99         pthread_rwlock_rdlock(&rrdset_root_rwlock);
 100         for(st = rrdset_root, c = 0; st ; st = st->next) {
 101                 if(st->enabled) {
 102                         if(c) buffer_strcat(wb, ",");
 103                         buffer_strcat(wb, "\n\t\t\"");
 104                         buffer_strcat(wb, st->id);
 105                         buffer_strcat(wb, "\": ");
 106                         rrd_stats_api_v1_chart(st, wb);
 107                         c++;
 108                 }
 109         }
 110         pthread_rwlock_unlock(&rrdset_root_rwlock);
 111
 112         buffer_strcat(wb, "\n\t}\n}\n");
 113 }
 114
 115
 116 unsigned long rrd_stats_one_json(RRDSET *st, char *options, BUFFER *wb)
 117 {
 118         time_t now = time(NULL);
 119
 120         pthread_rwlock_rdlock(&st->rwlock);
 121
 122         buffer_sprintf(wb,
 123                 "\t\t{\n"
 124                 "\t\t\t\"id\": \"%s\",\n"
 125                 "\t\t\t\"name\": \"%s\",\n"
 126                 "\t\t\t\"type\": \"%s\",\n"
 127                 "\t\t\t\"family\": \"%s\",\n"
 128                 "\t\t\t\"context\": \"%s\",\n"
 129                 "\t\t\t\"title\": \"%s\",\n"
 130                 "\t\t\t\"priority\": %ld,\n"
 131                 "\t\t\t\"enabled\": %d,\n"
 132                 "\t\t\t\"units\": \"%s\",\n"
 133                 "\t\t\t\"url\": \"/data/%s/%s\",\n"
 134                 "\t\t\t\"chart_type\": \"%s\",\n"
 135                 "\t\t\t\"counter\": %ld,\n"
 136                 "\t\t\t\"entries\": %ld,\n"
 137                 "\t\t\t\"first_entry_t\": %lu,\n"
 138                 "\t\t\t\"last_entry\": %ld,\n"
 139                 "\t\t\t\"last_entry_t\": %lu,\n"
 140                 "\t\t\t\"last_entry_secs_ago\": %lu,\n"
 141                 "\t\t\t\"update_every\": %d,\n"
 142                 "\t\t\t\"isdetail\": %d,\n"
 143                 "\t\t\t\"usec_since_last_update\": %llu,\n"
 144                 "\t\t\t\"collected_total\": " TOTAL_NUMBER_FORMAT ",\n"
 145                 "\t\t\t\"last_collected_total\": " TOTAL_NUMBER_FORMAT ",\n"
 146                 "\t\t\t\"dimensions\": [\n"
 147                 , st->id
 148                 , st->name
 149                 , st->type
 150                 , st->family
 151                 , st->context
 152                 , st->title
 153                 , st->priority
 154                 , st->enabled
 155                 , st->units
 156                 , st->name, options?options:""
 157                 , rrdset_type_name(st->chart_type)
 158                 , st->counter
 159                 , st->entries
 160                 , rrdset_first_entry_t(st)
 161                 , rrdset_last_slot(st)
 162                 , rrdset_last_entry_t(st)
 163                 , (now < rrdset_last_entry_t(st)) ? (time_t)0 : now - rrdset_last_entry_t(st)
 164                 , st->update_every
 165                 , st->isdetail
 166                 , st->usec_since_last_update
 167                 , st->collected_total
 168                 , st->last_collected_total
 169                 );
 170
 171         unsigned long memory = st->memsize;
 172
 173         RRDDIM *rd;
 174         for(rd = st->dimensions; rd ; rd = rd->next) {
 175
 176                 memory += rd->memsize;
 177
 178                 buffer_sprintf(wb,
 179                         "\t\t\t\t{\n"
 180                         "\t\t\t\t\t\"id\": \"%s\",\n"
 181                         "\t\t\t\t\t\"name\": \"%s\",\n"
 182                         "\t\t\t\t\t\"entries\": %ld,\n"
 183                         "\t\t\t\t\t\"isHidden\": %d,\n"
 184                         "\t\t\t\t\t\"algorithm\": \"%s\",\n"
 185                         "\t\t\t\t\t\"multiplier\": %ld,\n"
 186                         "\t\t\t\t\t\"divisor\": %ld,\n"
 187                         "\t\t\t\t\t\"last_entry_t\": %lu,\n"
 188                         "\t\t\t\t\t\"collected_value\": " COLLECTED_NUMBER_FORMAT ",\n"
 189                         "\t\t\t\t\t\"calculated_value\": " CALCULATED_NUMBER_FORMAT ",\n"
 190                         "\t\t\t\t\t\"last_collected_value\": " COLLECTED_NUMBER_FORMAT ",\n"
 191                         "\t\t\t\t\t\"last_calculated_value\": " CALCULATED_NUMBER_FORMAT ",\n"
 192                         "\t\t\t\t\t\"memory\": %lu\n"
 193                         "\t\t\t\t}%s\n"
 194                         , rd->id
 195                         , rd->name
 196                         , rd->entries
 197                         , (rd->flags & RRDDIM_FLAG_HIDDEN)?1:0
 198                         , rrddim_algorithm_name(rd->algorithm)
 199                         , rd->multiplier
 200                         , rd->divisor
 201                         , rd->last_collected_time.tv_sec
 202                         , rd->collected_value
 203                         , rd->calculated_value
 204                         , rd->last_collected_value
 205                         , rd->last_calculated_value
 206                         , rd->memsize
 207                         , rd->next?",":""
 208                         );
 209         }
 210
 211         buffer_sprintf(wb,
 212                 "\t\t\t],\n"
 213                 "\t\t\t\"memory\" : %lu\n"
 214                 "\t\t}"
 215                 , memory
 216                 );
 217
 218         pthread_rwlock_unlock(&st->rwlock);
 219         return memory;
 220 }
 221
 222 #define RRD_GRAPH_JSON_HEADER "{\n\t\"charts\": [\n"
 223 #define RRD_GRAPH_JSON_FOOTER "\n\t]\n}\n"
 224
 225 void rrd_stats_graph_json(RRDSET *st, char *options, BUFFER *wb)
 226 {
 227         buffer_strcat(wb, RRD_GRAPH_JSON_HEADER);
 228         rrd_stats_one_json(st, options, wb);
 229         buffer_strcat(wb, RRD_GRAPH_JSON_FOOTER);
 230 }
 231
 232 void rrd_stats_all_json(BUFFER *wb)
 233 {
 234         unsigned long memory = 0;
 235         long c;
 236         RRDSET *st;
 237
 238         buffer_strcat(wb, RRD_GRAPH_JSON_HEADER);
 239
 240         pthread_rwlock_rdlock(&rrdset_root_rwlock);
 241         for(st = rrdset_root, c = 0; st ; st = st->next) {
 242                 if(st->enabled) {
 243                         if(c) buffer_strcat(wb, ",\n");
 244                         memory += rrd_stats_one_json(st, NULL, wb);
 245                         c++;
 246                 }
 247         }
 248         pthread_rwlock_unlock(&rrdset_root_rwlock);
 249
 250         buffer_sprintf(wb, "\n\t],\n"
 251                 "\t\"hostname\": \"%s\",\n"
 252                 "\t\"update_every\": %d,\n"
 253                 "\t\"history\": %d,\n"
 254                 "\t\"memory\": %lu\n"
 255                 "}\n"
 256                 , hostname
 257                 , rrd_update_every
 258                 , rrd_default_history_entries
 259                 , memory
 260                 );
 261 }
 262
 263
 264
 265 // ----------------------------------------------------------------------------
 266
 267 // RRDR dimension options
 268 #define RRDR_EMPTY      0x01 // the dimension contains / the value is empty (null)
 269 #define RRDR_RESET      0x02 // the dimension contains / the value is reset
 270 #define RRDR_HIDDEN     0x04 // the dimension contains / the value is hidden
 271 #define RRDR_NONZERO    0x08 // the dimension contains / the value is non-zero
 272
 273 // RRDR result options
 274 #define RRDR_RESULT_OPTION_ABSOLUTE 0x00000001
 275 #define RRDR_RESULT_OPTION_RELATIVE 0x00000002
 276
 277 typedef struct rrdresult {
 278         RRDSET *st;                     // the chart this result refers to
 279
 280         uint32_t result_options;        // RRDR_RESULT_OPTION_*
 281
 282         int d;                                  // the number of dimensions
 283         long n;                                 // the number of values in the arrays
 284         long rows;                          // the number of rows used
 285
 286         uint8_t *od;                    // the options for the dimensions
 287
 288         time_t *t;                              // array of n timestamps
 289         calculated_number *v;   // array n x d values
 290         uint8_t *o;                             // array n x d options
 291
 292         long c;                                 // current line ( -1 ~ n ), ( -1 = none, use rrdr_rows() to get number of rows )
 293
 294         long group;                             // how many collected values were grouped for each row
 295         long update_every;              // what is the suggested update frequency in seconds
 296
 297         calculated_number min;
 298         calculated_number max;
 299
 300         time_t before;
 301         time_t after;
 302
 303         int has_st_lock;                // if st is read locked by us
 304 } RRDR;
 305
 306 #define rrdr_rows(r) ((r)->rows)
 307
 308 /*
 309 static void rrdr_dump(RRDR *r)
 310 {
 311         long c, i;
 312         RRDDIM *d;
 313
 314         fprintf(stderr, "\nCHART %s (%s)\n", r->st->id, r->st->name);
 315
 316         for(c = 0, d = r->st->dimensions; d ;c++, d = d->next) {
 317                 fprintf(stderr, "DIMENSION %s (%s), %s%s%s%s\n"
 318                                 , d->id
 319                                 , d->name
 320                                 , (r->od[c] & RRDR_EMPTY)?"EMPTY ":""
 321                                 , (r->od[c] & RRDR_RESET)?"RESET ":""
 322                                 , (r->od[c] & RRDR_HIDDEN)?"HIDDEN ":""
 323                                 , (r->od[c] & RRDR_NONZERO)?"NONZERO ":""
 324                                 );
 325         }
 326
 327         if(r->rows <= 0) {
 328                 fprintf(stderr, "RRDR does not have any values in it.\n");
 329                 return;
 330         }
 331
 332         fprintf(stderr, "RRDR includes %d values in it:\n", r->rows);
 333
 334         // for each line in the array
 335         for(i = 0; i < r->rows ;i++) {
 336                 calculated_number *cn = &r->v[ i * r->d ];
 337                 uint8_t *co = &r->o[ i * r->d ];
 338
 339                 // print the id and the timestamp of the line
 340                 fprintf(stderr, "%ld %ld ", i + 1, r->t[i]);
 341
 342                 // for each dimension
 343                 for(c = 0, d = r->st->dimensions; d ;c++, d = d->next) {
 344                         if(unlikely(r->od[c] & RRDR_HIDDEN)) continue;
 345                         if(unlikely(!(r->od[c] & RRDR_NONZERO))) continue;
 346
 347                         if(co[c] & RRDR_EMPTY)
 348                                 fprintf(stderr, "null ");
 349                         else
 350                                 fprintf(stderr, CALCULATED_NUMBER_FORMAT " %s%s%s%s "
 351                                         , cn[c]
 352                                         , (co[c] & RRDR_EMPTY)?"E":" "
 353                                         , (co[c] & RRDR_RESET)?"R":" "
 354                                         , (co[c] & RRDR_HIDDEN)?"H":" "
 355                                         , (co[c] & RRDR_NONZERO)?"N":" "
 356                                         );
 357                 }
 358
 359                 fprintf(stderr, "\n");
 360         }
 361 }
 362 */
 363
 364 void rrdr_disable_not_selected_dimensions(RRDR *r, const char *dims)
 365 {
 366         char b[strlen(dims) + 1];
 367         char *o = b, *tok;
 368         strcpy(o, dims);
 369
 370         long c;
 371         RRDDIM *d;
 372
 373         // disable all of them
 374         for(c = 0, d = r->st->dimensions; d ;c++, d = d->next)
 375                 r->od[c] |= RRDR_HIDDEN;
 376
 377         while(o && *o && (tok = mystrsep(&o, ", |"))) {
 378                 if(!*tok) continue;
 379
 380                 // find it and enable it
 381                 for(c = 0, d = r->st->dimensions; d ;c++, d = d->next) {
 382                         if(!strcmp(d->name, tok)) {
 383                                 r->od[c] &= ~RRDR_HIDDEN;
 384
 385                                 // since the user needs this dimension
 386                                 // make it appear as NONZERO, to return it
 387                                 // even if the dimension has only zeros
 388                                 r->od[c] |= RRDR_NONZERO;
 389                         }
 390                 }
 391         }
 392 }
 393
 394 void rrdr_buffer_print_format(BUFFER *wb, uint32_t format)
 395 {
 396         switch(format) {
 397         case DATASOURCE_JSON:
 398                 buffer_strcat(wb, DATASOURCE_FORMAT_JSON);
 399                 break;
 400
 401         case DATASOURCE_DATATABLE_JSON:
 402                 buffer_strcat(wb, DATASOURCE_FORMAT_DATATABLE_JSON);
 403                 break;
 404
 405         case DATASOURCE_DATATABLE_JSONP:
 406                 buffer_strcat(wb, DATASOURCE_FORMAT_DATATABLE_JSONP);
 407                 break;
 408
 409         case DATASOURCE_JSONP:
 410                 buffer_strcat(wb, DATASOURCE_FORMAT_JSONP);
 411                 break;
 412
 413         case DATASOURCE_SSV:
 414                 buffer_strcat(wb, DATASOURCE_FORMAT_SSV);
 415                 break;
 416
 417         case DATASOURCE_CSV:
 418                 buffer_strcat(wb, DATASOURCE_FORMAT_CSV);
 419                 break;
 420
 421         case DATASOURCE_TSV:
 422                 buffer_strcat(wb, DATASOURCE_FORMAT_TSV);
 423                 break;
 424
 425         case DATASOURCE_HTML:
 426                 buffer_strcat(wb, DATASOURCE_FORMAT_HTML);
 427                 break;
 428
 429         case DATASOURCE_JS_ARRAY:
 430                 buffer_strcat(wb, DATASOURCE_FORMAT_JS_ARRAY);
 431                 break;
 432
 433         case DATASOURCE_SSV_COMMA:
 434                 buffer_strcat(wb, DATASOURCE_FORMAT_SSV_COMMA);
 435                 break;
 436
 437         default:
 438                 buffer_strcat(wb, "unknown");
 439                 break;
 440         }
 441 }
 442
 443 uint32_t rrdr_check_options(RRDR *r, uint32_t options, const char *dims)
 444 {
 445         if(options & RRDR_OPTION_NONZERO) {
 446                 long i;
 447
 448                 if(dims && *dims) {
 449                         // the caller wants specific dimensions
 450                         // disable NONZERO option
 451                         // to make sure we don't accidentally prevent
 452                         // the specific dimensions from being returned
 453                         i = 0;
 454                 }
 455                 else {
 456                         // find how many dimensions are not zero
 457                         long c;
 458                         RRDDIM *rd;
 459                         for(c = 0, i = 0, rd = r->st->dimensions; rd && c < r->d ; c++, rd = rd->next) {
 460                                 if(unlikely(r->od[c] & RRDR_HIDDEN)) continue;
 461                                 if(unlikely(!(r->od[c] & RRDR_NONZERO))) continue;
 462                                 i++;
 463                         }
 464                 }
 465
 466                 // if with nonzero we get i = 0 (no dimensions will be returned)
 467                 // disable nonzero to show all dimensions
 468                 if(!i) options &= ~RRDR_OPTION_NONZERO;
 469         }
 470
 471         return options;
 472 }
 473
 474 void rrdr_json_wrapper_begin(RRDR *r, BUFFER *wb, uint32_t format, uint32_t options, int string_value)
 475 {
 476         long rows = rrdr_rows(r);
 477         long c, i;
 478         RRDDIM *rd;
 479
 480         //info("JSONWRAPPER(): %s: BEGIN", r->st->id);
 481         char kq[2] = "",                                        // key quote
 482                 sq[2] = "";                                             // string quote
 483
 484         if( options & RRDR_OPTION_GOOGLE_JSON ) {
 485                 kq[0] = '\0';
 486                 sq[0] = '\'';
 487         }
 488         else {
 489                 kq[0] = '"';
 490                 sq[0] = '"';
 491         }
 492
 493         buffer_sprintf(wb, "{\n"
 494                         "       %sapi%s: 1,\n"
 495                         "       %sid%s: %s%s%s,\n"
 496                         "       %sname%s: %s%s%s,\n"
 497                         "       %sview_update_every%s: %d,\n"
 498                         "       %supdate_every%s: %d,\n"
 499                         "       %sfirst_entry%s: %u,\n"
 500                         "       %slast_entry%s: %u,\n"
 501                         "       %sbefore%s: %u,\n"
 502                         "       %safter%s: %u,\n"
 503                         "       %sdimension_names%s: ["
 504                         , kq, kq
 505                         , kq, kq, sq, r->st->id, sq
 506                         , kq, kq, sq, r->st->name, sq
 507                         , kq, kq, r->update_every
 508                         , kq, kq, r->st->update_every
 509                         , kq, kq, rrdset_first_entry_t(r->st)
 510                         , kq, kq, rrdset_last_entry_t(r->st)
 511                         , kq, kq, r->before
 512                         , kq, kq, r->after
 513                         , kq, kq);
 514
 515         for(c = 0, i = 0, rd = r->st->dimensions; rd && c < r->d ;c++, rd = rd->next) {
 516                 if(unlikely(r->od[c] & RRDR_HIDDEN)) continue;
 517                 if(unlikely((options & RRDR_OPTION_NONZERO) && !(r->od[c] & RRDR_NONZERO))) continue;
 518
 519                 if(i) buffer_strcat(wb, ", ");
 520                 buffer_strcat(wb, sq);
 521                 buffer_strcat(wb, rd->name);
 522                 buffer_strcat(wb, sq);
 523                 i++;
 524         }
 525         if(!i) {
 526 #ifdef NETDATA_INTERNAL_CHECKS
 527                 error("RRDR is empty for %s (RRDR has %d dimensions, options is 0x%08x)", r->st->id, r->d, options);
 528 #endif
 529                 rows = 0;
 530                 buffer_strcat(wb, sq);
 531                 buffer_strcat(wb, "no data");
 532                 buffer_strcat(wb, sq);
 533         }
 534
 535         buffer_sprintf(wb, "],\n"
 536                         "       %sdimension_ids%s: ["
 537                         , kq, kq);
 538
 539         for(c = 0, i = 0, rd = r->st->dimensions; rd && c < r->d ;c++, rd = rd->next) {
 540                 if(unlikely(r->od[c] & RRDR_HIDDEN)) continue;
 541                 if(unlikely((options & RRDR_OPTION_NONZERO) && !(r->od[c] & RRDR_NONZERO))) continue;
 542
 543                 if(i) buffer_strcat(wb, ", ");
 544                 buffer_strcat(wb, sq);
 545                 buffer_strcat(wb, rd->id);
 546                 buffer_strcat(wb, sq);
 547                 i++;
 548         }
 549         if(!i) {
 550                 rows = 0;
 551                 buffer_strcat(wb, sq);
 552                 buffer_strcat(wb, "no data");
 553                 buffer_strcat(wb, sq);
 554         }
 555
 556         buffer_sprintf(wb, "],\n"
 557                         "       %slatest_values%s: ["
 558                         , kq, kq);
 559
 560         for(c = 0, i = 0, rd = r->st->dimensions; rd && c < r->d ;c++, rd = rd->next) {
 561                 if(unlikely(r->od[c] & RRDR_HIDDEN)) continue;
 562                 if(unlikely((options & RRDR_OPTION_NONZERO) && !(r->od[c] & RRDR_NONZERO))) continue;
 563
 564                 if(i) buffer_strcat(wb, ", ");
 565                 i++;
 566
 567                 storage_number n = rd->values[rrdset_last_slot(r->st)];
 568
 569                 if(!does_storage_number_exist(n))
 570                         buffer_strcat(wb, "null");
 571                 else
 572                         buffer_rrd_value(wb, unpack_storage_number(n));
 573         }
 574         if(!i) {
 575                 rows = 0;
 576                 buffer_strcat(wb, "null");
 577         }
 578
 579         buffer_sprintf(wb, "],\n"
 580                         "       %sview_latest_values%s: ["
 581                         , kq, kq);
 582
 583         i = 0;
 584         if(rows) {
 585                 for(c = 0, i = 0, rd = r->st->dimensions; rd && c < r->d ;c++, rd = rd->next) {
 586                         if(unlikely(r->od[c] & RRDR_HIDDEN)) continue;
 587                         if(unlikely((options & RRDR_OPTION_NONZERO) && !(r->od[c] & RRDR_NONZERO))) continue;
 588
 589                         if(i) buffer_strcat(wb, ", ");
 590                         i++;
 591
 592                         calculated_number *cn = &r->v[ (0) * r->d ];
 593                         uint8_t *co = &r->o[ (0) * r->d ];
 594
 595                         if(co[c] & RRDR_EMPTY)
 596                                 buffer_strcat(wb, "null");
 597                         else
 598                                 buffer_rrd_value(wb, cn[c]);
 599                 }
 600         }
 601         if(!i) {
 602                 rows = 0;
 603                 buffer_strcat(wb, "null");
 604         }
 605
 606         buffer_sprintf(wb, "],\n"
 607                         "       %sdimensions%s: %d,\n"
 608                         "       %spoints%s: %d,\n"
 609                         "       %sformat%s: %s"
 610                         , kq, kq, i
 611                         , kq, kq, rows
 612                         , kq, kq, sq
 613                         );
 614
 615         rrdr_buffer_print_format(wb, format);
 616
 617         buffer_sprintf(wb, "%s,\n"
 618                         "       %sresult%s: "
 619                         , sq
 620                         , kq, kq
 621                         );
 622
 623         if(string_value) buffer_strcat(wb, sq);
 624         //info("JSONWRAPPER(): %s: END", r->st->id);
 625 }
 626
 627 void rrdr_json_wrapper_end(RRDR *r, BUFFER *wb, uint32_t format, uint32_t options, int string_value)
 628 {
 629         if(r) {;}
 630         if(format) {;}
 631
 632         char kq[2] = "",                                        // key quote
 633                 sq[2] = "";                                             // string quote
 634
 635         if( options & RRDR_OPTION_GOOGLE_JSON ) {
 636                 kq[0] = '\0';
 637                 sq[0] = '\'';
 638         }
 639         else {
 640                 kq[0] = '"';
 641                 sq[0] = '"';
 642         }
 643
 644         if(string_value) buffer_strcat(wb, sq);
 645
 646         buffer_sprintf(wb, ",\n %smin%s: ", kq, kq);
 647         buffer_rrd_value(wb, r->min);
 648         buffer_sprintf(wb, ",\n %smax%s: ", kq, kq);
 649         buffer_rrd_value(wb, r->max);
 650         buffer_strcat(wb, "\n}\n");
 651 }
 652
 653 #define JSON_DATES_JS 1
 654 #define JSON_DATES_TIMESTAMP 2
 655
 656 static void rrdr2json(RRDR *r, BUFFER *wb, uint32_t options, int datatable)
 657 {
 658         //info("RRD2JSON(): %s: BEGIN", r->st->id);
 659         int row_annotations = 0, dates, dates_with_new = 0;
 660         char kq[2] = "",                                        // key quote
 661                 sq[2] = "",                                             // string quote
 662                 pre_label[101] = "",                    // before each label
 663                 post_label[101] = "",                   // after each label
 664                 pre_date[101] = "",                             // the beginning of line, to the date
 665                 post_date[101] = "",                    // closing the date
 666                 pre_value[101] = "",                    // before each value
 667                 post_value[101] = "",                   // after each value
 668                 post_line[101] = "",                    // at the end of each row
 669                 normal_annotation[201] = "",    // default row annotation
 670                 overflow_annotation[201] = "",  // overflow row annotation
 671                 data_begin[101] = "",                   // between labels and values
 672                 finish[101] = "";                               // at the end of everything
 673
 674         if(datatable) {
 675                 dates = JSON_DATES_JS;
 676                 if( options & RRDR_OPTION_GOOGLE_JSON ) {
 677                         kq[0] = '\0';
 678                         sq[0] = '\'';
 679                 }
 680                 else {
 681                         kq[0] = '"';
 682                         sq[0] = '"';
 683                 }
 684                 row_annotations = 1;
 685                 snprintfz(pre_date,   100, "            {%sc%s:[{%sv%s:%s", kq, kq, kq, kq, sq);
 686                 snprintfz(post_date,  100, "%s}", sq);
 687                 snprintfz(pre_label,  100, ",\n         {%sid%s:%s%s,%slabel%s:%s", kq, kq, sq, sq, kq, kq, sq);
 688                 snprintfz(post_label, 100, "%s,%spattern%s:%s%s,%stype%s:%snumber%s}", sq, kq, kq, sq, sq, kq, kq, sq, sq);
 689                 snprintfz(pre_value,  100, ",{%sv%s:", kq, kq);
 690                 strcpy(post_value,         "}");
 691                 strcpy(post_line,          "]}");
 692                 snprintfz(data_begin, 100, "\n  ],\n    %srows%s:\n     [\n", kq, kq);
 693                 strcpy(finish,             "\n  ]\n}");
 694
 695                 snprintfz(overflow_annotation, 200, ",{%sv%s:%sRESET OR OVERFLOW%s},{%sv%s:%sThe counters have been wrapped.%s}", kq, kq, sq, sq, kq, kq, sq, sq);
 696                 snprintfz(normal_annotation,   200, ",{%sv%s:null},{%sv%s:null}", kq, kq, kq, kq);
 697
 698                 buffer_sprintf(wb, "{\n %scols%s:\n     [\n", kq, kq, kq, kq);
 699                 buffer_sprintf(wb, "            {%sid%s:%s%s,%slabel%s:%stime%s,%spattern%s:%s%s,%stype%s:%sdatetime%s},\n", kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq);
 700                 buffer_sprintf(wb, "            {%sid%s:%s%s,%slabel%s:%s%s,%spattern%s:%s%s,%stype%s:%sstring%s,%sp%s:{%srole%s:%sannotation%s}},\n", kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, kq, kq, sq, sq);
 701                 buffer_sprintf(wb, "            {%sid%s:%s%s,%slabel%s:%s%s,%spattern%s:%s%s,%stype%s:%sstring%s,%sp%s:{%srole%s:%sannotationText%s}}", kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, kq, kq, sq, sq);
 702
 703                 // remove the valueobjects flag
 704                 // google wants its own keys
 705                 if(options & RRDR_OPTION_OBJECTSROWS)
 706                         options &= ~RRDR_OPTION_OBJECTSROWS;
 707         }
 708         else {
 709                 kq[0] = '"';
 710                 sq[0] = '"';
 711                 if((options & RRDR_OPTION_SECONDS) || (options & RRDR_OPTION_MILLISECONDS)) {
 712                         dates = JSON_DATES_TIMESTAMP;
 713                         dates_with_new = 0;
 714                 }
 715                 else {
 716                         dates = JSON_DATES_JS;
 717                         dates_with_new = 1;
 718                 }
 719                 if( options & RRDR_OPTION_OBJECTSROWS )
 720                         strcpy(pre_date, "              { ");
 721                 else
 722                         strcpy(pre_date, "              [ ");
 723                 strcpy(pre_label,  ", \"");
 724                 strcpy(post_label, "\"");
 725                 strcpy(pre_value,  ", ");
 726                 if( options & RRDR_OPTION_OBJECTSROWS )
 727                         strcpy(post_line, "}");
 728                 else
 729                         strcpy(post_line, "]");
 730                 snprintfz(data_begin, 100, "],\n        %sdata%s:\n     [\n", kq, kq);
 731                 strcpy(finish,             "\n  ]\n}");
 732
 733                 buffer_sprintf(wb, "{\n %slabels%s: [", kq, kq);
 734                 buffer_sprintf(wb, "%stime%s", sq, sq);
 735         }
 736
 737         // -------------------------------------------------------------------------
 738         // print the JSON header
 739
 740         long c, i;
 741         RRDDIM *rd;
 742
 743         // print the header lines
 744         for(c = 0, i = 0, rd = r->st->dimensions; rd && c < r->d ;c++, rd = rd->next) {
 745                 if(unlikely(r->od[c] & RRDR_HIDDEN)) continue;
 746                 if(unlikely((options & RRDR_OPTION_NONZERO) && !(r->od[c] & RRDR_NONZERO))) continue;
 747
 748                 buffer_strcat(wb, pre_label);
 749                 buffer_strcat(wb, rd->name);
 750                 buffer_strcat(wb, post_label);
 751                 i++;
 752         }
 753         if(!i) {
 754                 buffer_strcat(wb, pre_label);
 755                 buffer_strcat(wb, "no data");
 756                 buffer_strcat(wb, post_label);
 757         }
 758
 759         // print the begin of row data
 760         buffer_strcat(wb, data_begin);
 761
 762         // if all dimensions are hidden, print a null
 763         if(!i) {
 764                 buffer_strcat(wb, finish);
 765                 return;
 766         }
 767
 768         long start = 0, end = rrdr_rows(r), step = 1;
 769         if((options & RRDR_OPTION_REVERSED)) {
 770                 start = rrdr_rows(r) - 1;
 771                 end = -1;
 772                 step = -1;
 773         }
 774
 775         // for each line in the array
 776         calculated_number total = 1;
 777         for(i = start; i != end ;i += step) {
 778                 calculated_number *cn = &r->v[ i * r->d ];
 779                 uint8_t *co = &r->o[ i * r->d ];
 780
 781                 time_t now = r->t[i];
 782
 783                 if(dates == JSON_DATES_JS) {
 784                         // generate the local date time
 785                         struct tm tmbuf, *tm = localtime_r(&now, &tmbuf);
 786                         if(!tm) { error("localtime_r() failed."); continue; }
 787
 788                         if(likely(i != start)) buffer_strcat(wb, ",\n");
 789                         buffer_strcat(wb, pre_date);
 790
 791                         if( options & RRDR_OPTION_OBJECTSROWS )
 792                                 buffer_sprintf(wb, "%stime%s: ", kq, kq);
 793
 794                         if(dates_with_new)
 795                                 buffer_strcat(wb, "new ");
 796
 797                         buffer_jsdate(wb, tm->tm_year + 1900, tm->tm_mon, tm->tm_mday, tm->tm_hour, tm->tm_min, tm->tm_sec);
 798
 799                         buffer_strcat(wb, post_date);
 800
 801                         if(row_annotations) {
 802                                 // google supports one annotation per row
 803                                 int annotation_found = 0;
 804                                 for(c = 0, rd = r->st->dimensions; rd ;c++, rd = rd->next) {
 805                                         if(co[c] & RRDR_RESET) {
 806                                                 buffer_strcat(wb, overflow_annotation);
 807                                                 annotation_found = 1;
 808                                                 break;
 809                                         }
 810                                 }
 811                                 if(!annotation_found)
 812                                         buffer_strcat(wb, normal_annotation);
 813                         }
 814                 }
 815                 else {
 816                         // print the timestamp of the line
 817                         if(likely(i != start)) buffer_strcat(wb, ",\n");
 818                         buffer_strcat(wb, pre_date);
 819
 820                         if( options & RRDR_OPTION_OBJECTSROWS )
 821                                 buffer_sprintf(wb, "%stime%s: ", kq, kq);
 822
 823                         buffer_rrd_value(wb, (calculated_number)r->t[i]);
 824                         // in ms
 825                         if(options & RRDR_OPTION_MILLISECONDS) buffer_strcat(wb, "000");
 826
 827                         buffer_strcat(wb, post_date);
 828                 }
 829
 830                 if(unlikely(options & RRDR_OPTION_PERCENTAGE)) {
 831                         total = 0;
 832                         for(c = 0, rd = r->st->dimensions; rd && c < r->d ;c++, rd = rd->next) {
 833                                 calculated_number n = cn[c];
 834
 835                                 if(likely((options & RRDR_OPTION_ABSOLUTE) && n < 0))
 836                                         n = -n;
 837
 838                                 total += n;
 839                         }
 840                         // prevent a division by zero
 841                         if(total == 0) total = 1;
 842                 }
 843
 844                 // for each dimension
 845                 for(c = 0, rd = r->st->dimensions; rd && c < r->d ;c++, rd = rd->next) {
 846                         if(unlikely(r->od[c] & RRDR_HIDDEN)) continue;
 847                         if(unlikely((options & RRDR_OPTION_NONZERO) && !(r->od[c] & RRDR_NONZERO))) continue;
 848
 849                         calculated_number n = cn[c];
 850
 851                         buffer_strcat(wb, pre_value);
 852
 853                         if( options & RRDR_OPTION_OBJECTSROWS )
 854                                 buffer_sprintf(wb, "%s%s%s: ", kq, rd->name, kq);
 855
 856                         if(co[c] & RRDR_EMPTY) {
 857                                 if(options & RRDR_OPTION_NULL2ZERO)
 858                                         buffer_strcat(wb, "0");
 859                                 else
 860                                         buffer_strcat(wb, "null");
 861                         }
 862                         else {
 863                                 if(unlikely((options & RRDR_OPTION_ABSOLUTE) && n < 0))
 864                                         n = -n;
 865
 866                                 if(unlikely(options & RRDR_OPTION_PERCENTAGE))
 867                                         n = n * 100 / total;
 868
 869                                 buffer_rrd_value(wb, n);
 870                         }
 871
 872                         buffer_strcat(wb, post_value);
 873                 }
 874
 875                 buffer_strcat(wb, post_line);
 876         }
 877
 878         buffer_strcat(wb, finish);
 879         //info("RRD2JSON(): %s: END", r->st->id);
 880 }
 881
 882 static void rrdr2csv(RRDR *r, BUFFER *wb, uint32_t options, const char *startline, const char *separator, const char *endline, const char *betweenlines)
 883 {
 884         //info("RRD2CSV(): %s: BEGIN", r->st->id);
 885         long c, i;
 886         RRDDIM *d;
 887
 888         // print the csv header
 889         for(c = 0, i = 0, d = r->st->dimensions; d && c < r->d ;c++, d = d->next) {
 890                 if(unlikely(r->od[c] & RRDR_HIDDEN)) continue;
 891                 if(unlikely((options & RRDR_OPTION_NONZERO) && !(r->od[c] & RRDR_NONZERO))) continue;
 892
 893                 if(!i) {
 894                         buffer_strcat(wb, startline);
 895                         if(options & RRDR_OPTION_LABEL_QUOTES) buffer_strcat(wb, "\"");
 896                         buffer_strcat(wb, "time");
 897                         if(options & RRDR_OPTION_LABEL_QUOTES) buffer_strcat(wb, "\"");
 898                 }
 899                 buffer_strcat(wb, separator);
 900                 if(options & RRDR_OPTION_LABEL_QUOTES) buffer_strcat(wb, "\"");
 901                 buffer_strcat(wb, d->name);
 902                 if(options & RRDR_OPTION_LABEL_QUOTES) buffer_strcat(wb, "\"");
 903                 i++;
 904         }
 905         buffer_strcat(wb, endline);
 906
 907         if(!i) {
 908                 // no dimensions present
 909                 return;
 910         }
 911
 912         long start = 0, end = rrdr_rows(r), step = 1;
 913         if((options & RRDR_OPTION_REVERSED)) {
 914                 start = rrdr_rows(r) - 1;
 915                 end = -1;
 916                 step = -1;
 917         }
 918
 919         // for each line in the array
 920         calculated_number total = 1;
 921         for(i = start; i != end ;i += step) {
 922                 calculated_number *cn = &r->v[ i * r->d ];
 923                 uint8_t *co = &r->o[ i * r->d ];
 924
 925                 buffer_strcat(wb, betweenlines);
 926                 buffer_strcat(wb, startline);
 927
 928                 time_t now = r->t[i];
 929
 930                 if((options & RRDR_OPTION_SECONDS) || (options & RRDR_OPTION_MILLISECONDS)) {
 931                         // print the timestamp of the line
 932                         buffer_rrd_value(wb, (calculated_number)now);
 933                         // in ms
 934                         if(options & RRDR_OPTION_MILLISECONDS) buffer_strcat(wb, "000");
 935                 }
 936                 else {
 937                         // generate the local date time
 938                         struct tm tmbuf, *tm = localtime_r(&now, &tmbuf);
 939                         if(!tm) { error("localtime() failed."); continue; }
 940                         buffer_date(wb, tm->tm_year + 1900, tm->tm_mon + 1, tm->tm_mday, tm->tm_hour, tm->tm_min, tm->tm_sec);
 941                 }
 942
 943                 if(unlikely(options & RRDR_OPTION_PERCENTAGE)) {
 944                         total = 0;
 945                         for(c = 0, d = r->st->dimensions; d && c < r->d ;c++, d = d->next) {
 946                                 calculated_number n = cn[c];
 947
 948                                 if(likely((options & RRDR_OPTION_ABSOLUTE) && n < 0))
 949                                         n = -n;
 950
 951                                 total += n;
 952                         }
 953                         // prevent a division by zero
 954                         if(total == 0) total = 1;
 955                 }
 956
 957                 // for each dimension
 958                 for(c = 0, d = r->st->dimensions; d && c < r->d ;c++, d = d->next) {
 959                         if(unlikely(r->od[c] & RRDR_HIDDEN)) continue;
 960                         if(unlikely((options & RRDR_OPTION_NONZERO) && !(r->od[c] & RRDR_NONZERO))) continue;
 961
 962                         buffer_strcat(wb, separator);
 963
 964                         calculated_number n = cn[c];
 965
 966                         if(co[c] & RRDR_EMPTY) {
 967                                 if(options & RRDR_OPTION_NULL2ZERO)
 968                                         buffer_strcat(wb, "0");
 969                                 else
 970                                         buffer_strcat(wb, "null");
 971                         }
 972                         else {
 973                                 if(unlikely((options & RRDR_OPTION_ABSOLUTE) && n < 0))
 974                                         n = -n;
 975
 976                                 if(unlikely(options & RRDR_OPTION_PERCENTAGE))
 977                                         n = n * 100 / total;
 978
 979                                 buffer_rrd_value(wb, n);
 980                         }
 981                 }
 982
 983                 buffer_strcat(wb, endline);
 984         }
 985         //info("RRD2CSV(): %s: END", r->st->id);
 986 }
 987
 988 static void rrdr2ssv(RRDR *r, BUFFER *wb, uint32_t options, const char *prefix, const char *separator, const char *suffix)
 989 {
 990         //info("RRD2SSV(): %s: BEGIN", r->st->id);
 991         long c, i;
 992         RRDDIM *d;
 993
 994         buffer_strcat(wb, prefix);
 995         long start = 0, end = rrdr_rows(r), step = 1;
 996         if((options & RRDR_OPTION_REVERSED)) {
 997                 start = rrdr_rows(r) - 1;
 998                 end = -1;
 999                 step = -1;
1000         }
1001
1002         // for each line in the array
1003         calculated_number total = 1;
1004         for(i = start; i != end ;i += step) {
1005
1006                 calculated_number *cn = &r->v[ i * r->d ];
1007                 uint8_t *co = &r->o[ i * r->d ];
1008
1009                 calculated_number sum = 0, min = 0, max = 0, v;
1010                 int all_null = 1, init = 1;
1011
1012                 if(unlikely(options & RRDR_OPTION_PERCENTAGE)) {
1013                         total = 0;
1014                         for(c = 0, d = r->st->dimensions; d && c < r->d ;c++, d = d->next) {
1015                                 calculated_number n = cn[c];
1016
1017                                 if(likely((options & RRDR_OPTION_ABSOLUTE) && n < 0))
1018                                         n = -n;
1019
1020                                 total += n;
1021                         }
1022                         // prevent a division by zero
1023                         if(total == 0) total = 1;
1024                 }
1025
1026                 // for each dimension
1027                 for(c = 0, d = r->st->dimensions; d && c < r->d ;c++, d = d->next) {
1028                         if(unlikely(r->od[c] & RRDR_HIDDEN)) continue;
1029                         if(unlikely((options & RRDR_OPTION_NONZERO) && !(r->od[c] & RRDR_NONZERO))) continue;
1030
1031                         calculated_number n = cn[c];
1032
1033                         if(likely((options & RRDR_OPTION_ABSOLUTE) && n < 0))
1034                                 n = -n;
1035
1036                         if(unlikely(options & RRDR_OPTION_PERCENTAGE))
1037                                 n = n * 100 / total;
1038
1039                         if(unlikely(init)) {
1040                                 if(n > 0) {
1041                                         min = 0;
1042                                         max = n;
1043                                 }
1044                                 else {
1045                                         min = n;
1046                                         max = 0;
1047                                 }
1048                                 init = 0;
1049                         }
1050
1051                         if(likely(!(co[c] & RRDR_EMPTY))) {
1052                                 all_null = 0;
1053                                 sum += n;
1054                         }
1055
1056                         if(n < min) min = n;
1057                         if(n > max) max = n;
1058                 }
1059
1060                 if(likely(i != start))
1061                         buffer_strcat(wb, separator);
1062
1063                 if(all_null) {
1064                         if(options & RRDR_OPTION_NULL2ZERO)
1065                                 buffer_strcat(wb, "0");
1066                         else
1067                                 buffer_strcat(wb, "null");
1068                 }
1069                 else {
1070                         if(options & RRDR_OPTION_MIN2MAX)
1071                                 v = max - min;
1072                         else
1073                                 v = sum;
1074
1075                         if(likely(i != start)) {
1076                                 if(r->min > v) r->min = v;
1077                                 if(r->max < v) r->max = v;
1078                         }
1079                         else {
1080                                 r->min = v;
1081                                 r->max = v;
1082                         }
1083
1084                         buffer_rrd_value(wb, v);
1085                 }
1086         }
1087         buffer_strcat(wb, suffix);
1088         //info("RRD2SSV(): %s: END", r->st->id);
1089 }
1090
1091 inline static calculated_number *rrdr_line_values(RRDR *r)
1092 {
1093         return &r->v[ r->c * r->d ];
1094 }
1095
1096 inline static uint8_t *rrdr_line_options(RRDR *r)
1097 {
1098         return &r->o[ r->c * r->d ];
1099 }
1100
1101 inline static int rrdr_line_init(RRDR *r, time_t t)
1102 {
1103         r->c++;
1104
1105         if(unlikely(r->c >= r->n)) {
1106                 error("requested to step above RRDR size for chart %s", r->st->name);
1107                 r->c = r->n - 1;
1108         }
1109
1110         // save the time
1111         r->t[r->c] = t;
1112
1113         return 1;
1114 }
1115
1116 inline static void rrdr_lock_rrdset(RRDR *r) {
1117         if(unlikely(!r)) {
1118                 error("NULL value given!");
1119                 return;
1120         }
1121
1122         pthread_rwlock_rdlock(&r->st->rwlock);
1123         r->has_st_lock = 1;
1124 }
1125
1126 inline static void rrdr_unlock_rrdset(RRDR *r) {
1127         if(unlikely(!r)) {
1128                 error("NULL value given!");
1129                 return;
1130         }
1131
1132         if(likely(r->has_st_lock)) {
1133                 pthread_rwlock_unlock(&r->st->rwlock);
1134                 r->has_st_lock = 0;
1135         }
1136 }
1137
1138 inline static void rrdr_free(RRDR *r)
1139 {
1140         if(unlikely(!r)) {
1141                 error("NULL value given!");
1142                 return;
1143         }
1144
1145         rrdr_unlock_rrdset(r);
1146         if(likely(r->t)) free(r->t);
1147         if(likely(r->v)) free(r->v);
1148         if(likely(r->o)) free(r->o);
1149         if(likely(r->od)) free(r->od);
1150         free(r);
1151 }
1152
1153 inline void rrdr_done(RRDR *r)
1154 {
1155         r->rows = r->c + 1;
1156         r->c = 0;
1157 }
1158
1159 static RRDR *rrdr_create(RRDSET *st, long n)
1160 {
1161         if(unlikely(!st)) {
1162                 error("NULL value given!");
1163                 return NULL;
1164         }
1165
1166         RRDR *r = calloc(1, sizeof(RRDR));
1167         if(unlikely(!r)) goto cleanup;
1168
1169         r->st = st;
1170
1171         rrdr_lock_rrdset(r);
1172
1173         RRDDIM *rd;
1174         for(rd = st->dimensions ; rd ; rd = rd->next) r->d++;
1175
1176         r->n = n;
1177
1178         r->t = malloc(n * sizeof(time_t));
1179         if(unlikely(!r->t)) goto cleanup;
1180
1181         r->v = malloc(n * r->d * sizeof(calculated_number));
1182         if(unlikely(!r->v)) goto cleanup;
1183
1184         r->o = malloc(n * r->d * sizeof(uint8_t));
1185         if(unlikely(!r->o)) goto cleanup;
1186
1187         r->od = malloc(r->d * sizeof(uint8_t));
1188         if(unlikely(!r->od)) goto cleanup;
1189
1190         // set the hidden flag on hidden dimensions
1191         int c;
1192         for(c = 0, rd = st->dimensions ; rd ; c++, rd = rd->next) {
1193                 if(unlikely(rd->flags & RRDDIM_FLAG_HIDDEN)) r->od[c] = RRDR_HIDDEN;
1194                 else r->od[c] = 0;
1195         }
1196
1197         r->c = -1;
1198
1199         r->group = 1;
1200         r->update_every = 1;
1201
1202         return r;
1203
1204 cleanup:
1205         error("Cannot allocate RRDR memory for %d entries", n);
1206         if(likely(r)) rrdr_free(r);
1207         return NULL;
1208 }
1209
1210 RRDR *rrd2rrdr(RRDSET *st, long points, long long after, long long before, int group_method)
1211 {
1212         int debug = st->debug;
1213         int absolute_period_requested = -1;
1214
1215         time_t first_entry_t = rrdset_first_entry_t(st);
1216         time_t last_entry_t = rrdset_last_entry_t(st);
1217
1218         if(before == 0 && after == 0) {
1219                 before = last_entry_t;
1220                 after = first_entry_t;
1221                 absolute_period_requested = 0;
1222         }
1223
1224         // allow relative for before and after
1225         if(before <= st->update_every * st->entries) {
1226                 before = last_entry_t + before;
1227                 absolute_period_requested = 0;
1228         }
1229
1230         if(after <= st->update_every * st->entries) {
1231                 after = last_entry_t + after;
1232                 absolute_period_requested = 0;
1233         }
1234
1235         if(absolute_period_requested == -1)
1236                 absolute_period_requested = 1;
1237
1238         // make sure they are within our timeframe
1239         if(before > last_entry_t) before = last_entry_t;
1240         if(before < first_entry_t) before = first_entry_t;
1241
1242         if(after > last_entry_t) after = last_entry_t;
1243         if(after < first_entry_t) after = first_entry_t;
1244
1245         // check if they are upside down
1246         if(after > before) {
1247                 time_t tmp = before;
1248                 before = after;
1249                 after = tmp;
1250         }
1251
1252         // the duration of the chart
1253         time_t duration = before - after;
1254         long available_points = duration / st->update_every;
1255
1256         if(duration <= 0 || available_points <= 0)
1257                 return rrdr_create(st, 1);
1258
1259         // check the wanted points
1260         if(points < 0) points = -points;
1261         if(points > available_points) points = available_points;
1262         if(points == 0) points = available_points;
1263
1264         // calculate proper grouping of source data
1265         long group = available_points / points;
1266         if(group <= 0) group = 1;
1267
1268         // round group to the closest integer
1269         if(available_points % points > points / 2) group++;
1270
1271         time_t after_new = after - (after % (group * st->update_every));
1272         time_t before_new = before - (before % (group * st->update_every));
1273         long points_new = (before_new - after_new) / st->update_every / group;
1274
1275         // find the starting and ending slots in our round robin db
1276         long    start_at_slot = rrdset_time2slot(st, before_new),
1277                         stop_at_slot = rrdset_time2slot(st, after_new);
1278
1279 #ifdef NETDATA_INTERNAL_CHECKS
1280         if(after_new < first_entry_t) {
1281                 error("after_new %u is too small, minimum %u", after_new, first_entry_t);
1282         }
1283         if(after_new > last_entry_t) {
1284                 error("after_new %u is too big, maximum %u", after_new, last_entry_t);
1285         }
1286         if(before_new < first_entry_t) {
1287                 error("before_new %u is too small, minimum %u", before_new, first_entry_t);
1288         }
1289         if(before_new > last_entry_t) {
1290                 error("before_new %u is too big, maximum %u", before_new, last_entry_t);
1291         }
1292         if(start_at_slot < 0 || start_at_slot >= st->entries) {
1293                 error("start_at_slot is invalid %ld, expected %ld to %ld", start_at_slot, 0, st->entries - 1);
1294         }
1295         if(stop_at_slot < 0 || stop_at_slot >= st->entries) {
1296                 error("stop_at_slot is invalid %ld, expected %ld to %ld", stop_at_slot, 0, st->entries - 1);
1297         }
1298         if(points_new > (before_new - after_new) / group / st->update_every + 1) {
1299                 error("points_new %ld is more than points %ld", points_new, (before_new - after_new) / group / st->update_every + 1);
1300         }
1301 #endif
1302
1303         //info("RRD2RRDR(): %s: wanted %ld points, got %ld - group=%ld, wanted duration=%u, got %u - wanted %ld - %ld, got %ld - %ld", st->id, points, points_new, group, before - after, before_new - after_new, after, before, after_new, before_new);
1304
1305         after = after_new;
1306         before = before_new;
1307         duration = before - after;
1308         points = points_new;
1309
1310         // Now we have:
1311         // before = the end time of the calculation
1312         // after = the start time of the calculation
1313         // duration = the duration of the calculation
1314         // group = the number of source points to aggregate / group together
1315         // method = the method of grouping source points
1316         // points = the number of points to generate
1317
1318
1319         // -------------------------------------------------------------------------
1320         // initialize our result set
1321
1322         RRDR *r = rrdr_create(st, points);
1323         if(!r) {
1324 #ifdef NETDATA_INTERNAL_CHECKS
1325                 error("Cannot create RRDR for %s, after=%u, before=%u, duration=%u, points=%d", st->id, after, before, duration, points);
1326 #endif
1327                 return NULL;
1328         }
1329         if(!r->d) {
1330 #ifdef NETDATA_INTERNAL_CHECKS
1331                 error("Returning empty RRDR (no dimensions in RRDSET) for %s, after=%u, before=%u, duration=%u, points=%d", st->id, after, before, duration, points);
1332 #endif
1333                 return r;
1334         }
1335
1336         if(absolute_period_requested == 1)
1337                 r->result_options |= RRDR_RESULT_OPTION_ABSOLUTE;
1338         else
1339                 r->result_options |= RRDR_RESULT_OPTION_RELATIVE;
1340
1341         // find how many dimensions we have
1342         long dimensions = r->d;
1343
1344
1345         // -------------------------------------------------------------------------
1346         // checks for debugging
1347
1348         if(debug) debug(D_RRD_STATS, "INFO %s first_t: %lu, last_t: %lu, all_duration: %lu, after: %lu, before: %lu, duration: %lu, points: %ld, group: %ld"
1349                         , st->id
1350                         , first_entry_t
1351                         , last_entry_t
1352                         , last_entry_t - first_entry_t
1353                         , after
1354                         , before
1355                         , duration
1356                         , points
1357                         , group
1358                         );
1359
1360
1361         // -------------------------------------------------------------------------
1362         // temp arrays for keeping values per dimension
1363
1364         calculated_number       group_values[dimensions]; // keep sums when grouping
1365         long                            group_counts[dimensions]; // keep the number of values added to group_values
1366         uint8_t                         group_options[dimensions];
1367         uint8_t                         found_non_zero[dimensions];
1368
1369
1370         // initialize them
1371         RRDDIM *rd;
1372         long c;
1373         for( rd = st->dimensions, c = 0 ; rd && c < dimensions ; rd = rd->next, c++) {
1374                 group_values[c] = 0;
1375                 group_counts[c] = 0;
1376                 group_options[c] = 0;
1377                 found_non_zero[c] = 0;
1378         }
1379
1380
1381         // -------------------------------------------------------------------------
1382         // the main loop
1383
1384         time_t  now = rrdset_slot2time(st, start_at_slot),
1385                         dt = st->update_every,
1386                         group_start_t = 0;
1387
1388         if(unlikely(debug)) debug(D_RRD_STATS, "BEGIN %s after_t: %lu (stop_at_t: %ld), before_t: %lu (start_at_t: %ld), start_t(now): %lu, current_entry: %ld, entries: %ld"
1389                         , st->id
1390                         , after
1391                         , stop_at_slot
1392                         , before
1393                         , start_at_slot
1394                         , now
1395                         , st->current_entry
1396                         , st->entries
1397                         );
1398
1399         r->group = group;
1400         r->update_every = group * st->update_every;
1401         r->before = now;
1402         r->after = now;
1403
1404         //info("RRD2RRDR(): %s: STARTING", st->id);
1405
1406         long slot = start_at_slot, counter = 0, stop_now = 0, added = 0, group_count = 0, add_this = 0;
1407         for(; !stop_now ; now -= dt, slot--, counter++) {
1408                 if(unlikely(slot < 0)) slot = st->entries - 1;
1409                 if(unlikely(slot == stop_at_slot)) stop_now = counter;
1410
1411                 if(unlikely(debug)) debug(D_RRD_STATS, "ROW %s slot: %ld, entries_counter: %ld, group_count: %ld, added: %ld, now: %lu, %s %s"
1412                                 , st->id
1413                                 , slot
1414                                 , counter
1415                                 , group_count + 1
1416                                 , added
1417                                 , now
1418                                 , (group_count + 1 == group)?"PRINT":"  -  "
1419                                 , (now >= after && now <= before)?"RANGE":"  -  "
1420                                 );
1421
1422                 // make sure we return data in the proper time range
1423                 if(unlikely(now > before)) continue;
1424                 if(unlikely(now < after)) break;
1425
1426                 if(unlikely(group_count == 0)) {
1427                         group_start_t = now;
1428                 }
1429                 group_count++;
1430
1431                 if(unlikely(group_count == group)) {
1432                         if(unlikely(added >= points)) break;
1433                         add_this = 1;
1434                 }
1435
1436                 // do the calculations
1437                 for(rd = st->dimensions, c = 0 ; rd && c < dimensions ; rd = rd->next, c++) {
1438                         storage_number n = rd->values[slot];
1439                         if(unlikely(!does_storage_number_exist(n))) continue;
1440
1441                         group_counts[c]++;
1442
1443                         calculated_number value = unpack_storage_number(n);
1444                         if(likely(value != 0.0)) {
1445                                 group_options[c] |= RRDR_NONZERO;
1446                                 found_non_zero[c] = 1;
1447                         }
1448
1449                         if(unlikely(did_storage_number_reset(n)))
1450                                 group_options[c] |= RRDR_RESET;
1451
1452                         switch(group_method) {
1453                                 case GROUP_MAX:
1454                                         if(unlikely(fabsl(value) > fabsl(group_values[c])))
1455                                                 group_values[c] = value;
1456                                         break;
1457
1458                                 default:
1459                                 case GROUP_SUM:
1460                                 case GROUP_AVERAGE:
1461                                         group_values[c] += value;
1462                                         break;
1463                         }
1464                 }
1465
1466                 // added it
1467                 if(unlikely(add_this)) {
1468                         if(unlikely(!rrdr_line_init(r, group_start_t))) break;
1469
1470                         r->after = now;
1471
1472                         calculated_number *cn = rrdr_line_values(r);
1473                         uint8_t *co = rrdr_line_options(r);
1474
1475                         for(rd = st->dimensions, c = 0 ; rd && c < dimensions ; rd = rd->next, c++) {
1476
1477                                 // update the dimension options
1478                                 if(likely(found_non_zero[c])) r->od[c] |= RRDR_NONZERO;
1479
1480                                 // store the specific point options
1481                                 co[c] = group_options[c];
1482
1483                                 // store the value
1484                                 if(unlikely(group_counts[c] == 0)) {
1485                                         cn[c] = 0.0;
1486                                         co[c] |= RRDR_EMPTY;
1487                                 }
1488                                 else if(unlikely(group_method == GROUP_AVERAGE)) {
1489                                         cn[c] = group_values[c] / group_counts[c];
1490                                 }
1491                                 else {
1492                                         cn[c] = group_values[c];
1493                                 }
1494
1495                                 if(cn[c] < r->min) r->min = cn[c];
1496                                 if(cn[c] > r->max) r->max = cn[c];
1497
1498                                 // reset them for the next loop
1499                                 group_values[c] = 0;
1500                                 group_counts[c] = 0;
1501                                 group_options[c] = 0;
1502                         }
1503
1504                         added++;
1505                         group_count = 0;
1506                         add_this = 0;
1507                 }
1508         }
1509
1510         rrdr_done(r);
1511         //info("RRD2RRDR(): %s: END %ld loops made, %ld points generated", st->id, counter, rrdr_rows(r));
1512         //error("SHIFT: %s: wanted %ld points, got %ld", st->id, points, rrdr_rows(r));
1513         return r;
1514 }
1515
1516 int rrd2format(RRDSET *st, BUFFER *wb, BUFFER *dimensions, uint32_t format, long points, long long after, long long before, int group_method, uint32_t options, time_t *latest_timestamp)
1517 {
1518         RRDR *r = rrd2rrdr(st, points, after, before, group_method);
1519         if(!r) {
1520                 buffer_strcat(wb, "Cannot generate output with these parameters on this chart.");
1521                 return 500;
1522         }
1523
1524         if(r->result_options & RRDR_RESULT_OPTION_RELATIVE)
1525                 wb->options |= WB_CONTENT_NO_CACHEABLE;
1526         else if(r->result_options & RRDR_RESULT_OPTION_ABSOLUTE)
1527                 wb->options |= WB_CONTENT_CACHEABLE;
1528
1529         options = rrdr_check_options(r, options, (dimensions)?buffer_tostring(dimensions):NULL);
1530
1531         if(dimensions)
1532                 rrdr_disable_not_selected_dimensions(r, buffer_tostring(dimensions));
1533
1534         if(latest_timestamp && rrdr_rows(r) > 0)
1535                 *latest_timestamp = r->before;
1536
1537         switch(format) {
1538         case DATASOURCE_SSV:
1539                 if(options & RRDR_OPTION_JSON_WRAP) {
1540                         wb->contenttype = CT_APPLICATION_JSON;
1541                         rrdr_json_wrapper_begin(r, wb, format, options, 1);
1542                         rrdr2ssv(r, wb, options, "", " ", "");
1543                         rrdr_json_wrapper_end(r, wb, format, options, 1);
1544                 }
1545                 else {
1546                         wb->contenttype = CT_TEXT_PLAIN;
1547                         rrdr2ssv(r, wb, options, "", " ", "");
1548                 }
1549                 break;
1550
1551         case DATASOURCE_SSV_COMMA:
1552                 if(options & RRDR_OPTION_JSON_WRAP) {
1553                         wb->contenttype = CT_APPLICATION_JSON;
1554                         rrdr_json_wrapper_begin(r, wb, format, options, 1);
1555                         rrdr2ssv(r, wb, options, "", ",", "");
1556                         rrdr_json_wrapper_end(r, wb, format, options, 1);
1557                 }
1558                 else {
1559                         wb->contenttype = CT_TEXT_PLAIN;
1560                         rrdr2ssv(r, wb, options, "", ",", "");
1561                 }
1562                 break;
1563
1564         case DATASOURCE_JS_ARRAY:
1565                 if(options & RRDR_OPTION_JSON_WRAP) {
1566                         wb->contenttype = CT_APPLICATION_JSON;
1567                         rrdr_json_wrapper_begin(r, wb, format, options, 0);
1568                         rrdr2ssv(r, wb, options, "[", ",", "]");
1569                         rrdr_json_wrapper_end(r, wb, format, options, 0);
1570                 }
1571                 else {
1572                         wb->contenttype = CT_APPLICATION_JSON;
1573                         rrdr2ssv(r, wb, options, "[", ",", "]");
1574                 }
1575                 break;
1576
1577         case DATASOURCE_CSV:
1578                 if(options & RRDR_OPTION_JSON_WRAP) {
1579                         wb->contenttype = CT_APPLICATION_JSON;
1580                         rrdr_json_wrapper_begin(r, wb, format, options, 1);
1581                         rrdr2csv(r, wb, options, "", ",", "\\n", "");
1582                         rrdr_json_wrapper_end(r, wb, format, options, 1);
1583                 }
1584                 else {
1585                         wb->contenttype = CT_TEXT_PLAIN;
1586                         rrdr2csv(r, wb, options, "", ",", "\r\n", "");
1587                 }
1588                 break;
1589
1590         case DATASOURCE_CSV_JSON_ARRAY:
1591                 wb->contenttype = CT_APPLICATION_JSON;
1592                 if(options & RRDR_OPTION_JSON_WRAP) {
1593                         rrdr_json_wrapper_begin(r, wb, format, options, 0);
1594                         buffer_strcat(wb, "[\n");
1595                         rrdr2csv(r, wb, options + RRDR_OPTION_LABEL_QUOTES, "[", ",", "]", ",\n");
1596                         buffer_strcat(wb, "\n]");
1597                         rrdr_json_wrapper_end(r, wb, format, options, 0);
1598                 }
1599                 else {
1600                         wb->contenttype = CT_TEXT_PLAIN;
1601                         buffer_strcat(wb, "[\n");
1602                         rrdr2csv(r, wb, options + RRDR_OPTION_LABEL_QUOTES, "[", ",", "]", ",\n");
1603                         buffer_strcat(wb, "\n]");
1604                 }
1605                 break;
1606
1607         case DATASOURCE_TSV:
1608                 if(options & RRDR_OPTION_JSON_WRAP) {
1609                         wb->contenttype = CT_APPLICATION_JSON;
1610                         rrdr_json_wrapper_begin(r, wb, format, options, 1);
1611                         rrdr2csv(r, wb, options, "", "\t", "\\n", "");
1612                         rrdr_json_wrapper_end(r, wb, format, options, 1);
1613                 }
1614                 else {
1615                         wb->contenttype = CT_TEXT_PLAIN;
1616                         rrdr2csv(r, wb, options, "", "\t", "\r\n", "");
1617                 }
1618                 break;
1619
1620         case DATASOURCE_HTML:
1621                 if(options & RRDR_OPTION_JSON_WRAP) {
1622                         wb->contenttype = CT_APPLICATION_JSON;
1623                         rrdr_json_wrapper_begin(r, wb, format, options, 1);
1624                         buffer_strcat(wb, "<html>\\n<center>\\n<table border=\\\"0\\\" cellpadding=\\\"5\\\" cellspacing=\\\"5\\\">\\n");
1625                         rrdr2csv(r, wb, options, "<tr><td>", "</td><td>", "</td></tr>\\n", "");
1626                         buffer_strcat(wb, "</table>\\n</center>\\n</html>\\n");
1627                         rrdr_json_wrapper_end(r, wb, format, options, 1);
1628                 }
1629                 else {
1630                         wb->contenttype = CT_TEXT_HTML;
1631                         buffer_strcat(wb, "<html>\n<center>\n<table border=\"0\" cellpadding=\"5\" cellspacing=\"5\">\n");
1632                         rrdr2csv(r, wb, options, "<tr><td>", "</td><td>", "</td></tr>\n", "");
1633                         buffer_strcat(wb, "</table>\n</center>\n</html>\n");
1634                 }
1635                 break;
1636
1637         case DATASOURCE_DATATABLE_JSONP:
1638                 wb->contenttype = CT_APPLICATION_X_JAVASCRIPT;
1639
1640                 if(options & RRDR_OPTION_JSON_WRAP)
1641                         rrdr_json_wrapper_begin(r, wb, format, options, 0);
1642
1643                 rrdr2json(r, wb, options, 1);
1644
1645                 if(options & RRDR_OPTION_JSON_WRAP)
1646                         rrdr_json_wrapper_end(r, wb, format, options, 0);
1647                 break;
1648
1649         case DATASOURCE_DATATABLE_JSON:
1650                 wb->contenttype = CT_APPLICATION_JSON;
1651
1652                 if(options & RRDR_OPTION_JSON_WRAP)
1653                         rrdr_json_wrapper_begin(r, wb, format, options, 0);
1654
1655                 rrdr2json(r, wb, options, 1);
1656
1657                 if(options & RRDR_OPTION_JSON_WRAP)
1658                         rrdr_json_wrapper_end(r, wb, format, options, 0);
1659                 break;
1660
1661         case DATASOURCE_JSONP:
1662                 wb->contenttype = CT_APPLICATION_X_JAVASCRIPT;
1663                 if(options & RRDR_OPTION_JSON_WRAP)
1664                         rrdr_json_wrapper_begin(r, wb, format, options, 0);
1665
1666                 rrdr2json(r, wb, options, 0);
1667
1668                 if(options & RRDR_OPTION_JSON_WRAP)
1669                         rrdr_json_wrapper_end(r, wb, format, options, 0);
1670                 break;
1671
1672         case DATASOURCE_JSON:
1673         default:
1674                 wb->contenttype = CT_APPLICATION_JSON;
1675
1676                 if(options & RRDR_OPTION_JSON_WRAP)
1677                         rrdr_json_wrapper_begin(r, wb, format, options, 0);
1678
1679                 rrdr2json(r, wb, options, 0);
1680
1681                 if(options & RRDR_OPTION_JSON_WRAP)
1682                         rrdr_json_wrapper_end(r, wb, format, options, 0);
1683                 break;
1684         }
1685
1686         rrdr_free(r);
1687         return 200;
1688 }
1689
1690 time_t rrd_stats_json(int type, RRDSET *st, BUFFER *wb, long points, long group, int group_method, time_t after, time_t before, int only_non_zero)
1691 {
1692         int c;
1693         pthread_rwlock_rdlock(&st->rwlock);
1694
1695
1696         // -------------------------------------------------------------------------
1697         // switch from JSON to google JSON
1698
1699         char kq[2] = "\"";
1700         char sq[2] = "\"";
1701         switch(type) {
1702                 case DATASOURCE_DATATABLE_JSON:
1703                 case DATASOURCE_DATATABLE_JSONP:
1704                         kq[0] = '\0';
1705                         sq[0] = '\'';
1706                         break;
1707
1708                 case DATASOURCE_JSON:
1709                 default:
1710                         break;
1711         }
1712
1713
1714         // -------------------------------------------------------------------------
1715         // validate the parameters
1716
1717         if(points < 1) points = 1;
1718         if(group < 1) group = 1;
1719
1720         if(before == 0 || before > rrdset_last_entry_t(st)) before = rrdset_last_entry_t(st);
1721         if(after  == 0 || after < rrdset_first_entry_t(st)) after = rrdset_first_entry_t(st);
1722
1723         // ---
1724
1725         // our return value (the last timestamp printed)
1726         // this is required to detect re-transmit in google JSONP
1727         time_t last_timestamp = 0;
1728
1729
1730         // -------------------------------------------------------------------------
1731         // find how many dimensions we have
1732
1733         int dimensions = 0;
1734         RRDDIM *rd;
1735         for( rd = st->dimensions ; rd ; rd = rd->next) dimensions++;
1736         if(!dimensions) {
1737                 pthread_rwlock_unlock(&st->rwlock);
1738                 buffer_strcat(wb, "No dimensions yet.");
1739                 return 0;
1740         }
1741
1742
1743         // -------------------------------------------------------------------------
1744         // prepare various strings, to speed up the loop
1745
1746         char overflow_annotation[201]; snprintfz(overflow_annotation, 200, ",{%sv%s:%sRESET OR OVERFLOW%s},{%sv%s:%sThe counters have been wrapped.%s}", kq, kq, sq, sq, kq, kq, sq, sq);
1747         char normal_annotation[201];   snprintfz(normal_annotation,   200, ",{%sv%s:null},{%sv%s:null}", kq, kq, kq, kq);
1748         char pre_date[51];             snprintfz(pre_date,             50, "            {%sc%s:[{%sv%s:%s", kq, kq, kq, kq, sq);
1749         char post_date[21];            snprintfz(post_date,            20, "%s}", sq);
1750         char pre_value[21];            snprintfz(pre_value,            20, ",{%sv%s:", kq, kq);
1751         char post_value[21];           strcpy(post_value,                  "}");
1752
1753
1754         // -------------------------------------------------------------------------
1755         // checks for debugging
1756
1757         if(st->debug) {
1758                 debug(D_RRD_STATS, "%s first_entry_t = %lu, last_entry_t = %lu, duration = %lu, after = %lu, before = %lu, duration = %lu, entries_to_show = %lu, group = %lu"
1759                         , st->id
1760                         , rrdset_first_entry_t(st)
1761                         , rrdset_last_entry_t(st)
1762                         , rrdset_last_entry_t(st) - rrdset_first_entry_t(st)
1763                         , after
1764                         , before
1765                         , before - after
1766                         , points
1767                         , group
1768                         );
1769
1770                 if(before < after)
1771                         debug(D_RRD_STATS, "WARNING: %s The newest value in the database (%lu) is earlier than the oldest (%lu)", st->name, before, after);
1772
1773                 if((before - after) > st->entries * st->update_every)
1774                         debug(D_RRD_STATS, "WARNING: %s The time difference between the oldest and the newest entries (%lu) is higher than the capacity of the database (%lu)", st->name, before - after, st->entries * st->update_every);
1775         }
1776
1777
1778         // -------------------------------------------------------------------------
1779         // temp arrays for keeping values per dimension
1780
1781         calculated_number group_values[dimensions]; // keep sums when grouping
1782         int               print_hidden[dimensions]; // keep hidden flags
1783         int               found_non_zero[dimensions];
1784         int               found_non_existing[dimensions];
1785
1786         // initialize them
1787         for( rd = st->dimensions, c = 0 ; rd && c < dimensions ; rd = rd->next, c++) {
1788                 group_values[c] = 0;
1789                 print_hidden[c] = (rd->flags & RRDDIM_FLAG_HIDDEN)?1:0;
1790                 found_non_zero[c] = 0;
1791                 found_non_existing[c] = 0;
1792         }
1793
1794
1795         // error("OLD: points=%d after=%d before=%d group=%d, duration=%d", entries_to_show, before - (st->update_every * group * entries_to_show), before, group, before - after + 1);
1796         // rrd2array(st, entries_to_show, before - (st->update_every * group * entries_to_show), before, group_method, only_non_zero);
1797         // rrd2rrdr(st, entries_to_show, before - (st->update_every * group * entries_to_show), before, group_method);
1798
1799         // -------------------------------------------------------------------------
1800         // remove dimensions that contain only zeros
1801
1802         int max_loop = 1;
1803         if(only_non_zero) max_loop = 2;
1804
1805         for(; max_loop ; max_loop--) {
1806
1807                 // -------------------------------------------------------------------------
1808                 // print the JSON header
1809
1810                 buffer_sprintf(wb, "{\n %scols%s:\n     [\n", kq, kq);
1811                 buffer_sprintf(wb, "            {%sid%s:%s%s,%slabel%s:%stime%s,%spattern%s:%s%s,%stype%s:%sdatetime%s},\n", kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq);
1812                 buffer_sprintf(wb, "            {%sid%s:%s%s,%slabel%s:%s%s,%spattern%s:%s%s,%stype%s:%sstring%s,%sp%s:{%srole%s:%sannotation%s}},\n", kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, kq, kq, sq, sq);
1813                 buffer_sprintf(wb, "            {%sid%s:%s%s,%slabel%s:%s%s,%spattern%s:%s%s,%stype%s:%sstring%s,%sp%s:{%srole%s:%sannotationText%s}}", kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, kq, kq, sq, sq);
1814
1815                 // print the header for each dimension
1816                 // and update the print_hidden array for the dimensions that should be hidden
1817                 int pc = 0;
1818                 for( rd = st->dimensions, c = 0 ; rd && c < dimensions ; rd = rd->next, c++) {
1819                         if(!print_hidden[c]) {
1820                                 pc++;
1821                                 buffer_sprintf(wb, ",\n         {%sid%s:%s%s,%slabel%s:%s%s%s,%spattern%s:%s%s,%stype%s:%snumber%s}", kq, kq, sq, sq, kq, kq, sq, rd->name, sq, kq, kq, sq, sq, kq, kq, sq, sq);
1822                         }
1823                 }
1824                 if(!pc) {
1825                         buffer_sprintf(wb, ",\n         {%sid%s:%s%s,%slabel%s:%s%s%s,%spattern%s:%s%s,%stype%s:%snumber%s}", kq, kq, sq, sq, kq, kq, sq, "no data", sq, kq, kq, sq, sq, kq, kq, sq, sq);
1826                 }
1827
1828                 // print the begin of row data
1829                 buffer_sprintf(wb, "\n  ],\n    %srows%s:\n     [\n", kq, kq);
1830
1831
1832                 // -------------------------------------------------------------------------
1833                 // the main loop
1834
1835                 int annotate_reset = 0;
1836                 int annotation_count = 0;
1837
1838                 long    t = rrdset_time2slot(st, before),
1839                                 stop_at_t = rrdset_time2slot(st, after),
1840                                 stop_now = 0;
1841
1842                 t -= t % group;
1843
1844                 time_t  now = rrdset_slot2time(st, t),
1845                                 dt = st->update_every;
1846
1847                 long count = 0, printed = 0, group_count = 0;
1848                 last_timestamp = 0;
1849
1850                 if(st->debug) debug(D_RRD_STATS, "%s: REQUEST after:%lu before:%lu, points:%d, group:%d, CHART cur:%ld first: %lu last:%lu, CALC start_t:%ld, stop_t:%ld"
1851                                         , st->id
1852                                         , after
1853                                         , before
1854                                         , points
1855                                         , group
1856                                         , st->current_entry
1857                                         , rrdset_first_entry_t(st)
1858                                         , rrdset_last_entry_t(st)
1859                                         , t
1860                                         , stop_at_t
1861                                         );
1862
1863                 long counter = 0;
1864                 for(; !stop_now ; now -= dt, t--, counter++) {
1865                         if(t < 0) t = st->entries - 1;
1866                         if(t == stop_at_t) stop_now = counter;
1867
1868                         int print_this = 0;
1869
1870                         if(st->debug) debug(D_RRD_STATS, "%s t = %ld, count = %ld, group_count = %ld, printed = %ld, now = %lu, %s %s"
1871                                         , st->id
1872                                         , t
1873                                         , count + 1
1874                                         , group_count + 1
1875                                         , printed
1876                                         , now
1877                                         , (group_count + 1 == group)?"PRINT":"  -  "
1878                                         , (now >= after && now <= before)?"RANGE":"  -  "
1879                                         );
1880
1881
1882                         // make sure we return data in the proper time range
1883                         if(now > before) continue;
1884                         if(now < after) break;
1885
1886                         //if(rrdset_slot2time(st, t) != now)
1887                         //      error("%s: slot=%ld, now=%ld, slot2time=%ld, diff=%ld, last_entry_t=%ld, rrdset_last_slot=%ld", st->id, t, now, rrdset_slot2time(st,t), now - rrdset_slot2time(st,t), rrdset_last_entry_t(st), rrdset_last_slot(st));
1888
1889                         count++;
1890                         group_count++;
1891
1892                         // check if we have to print this now
1893                         if(group_count == group) {
1894                                 if(printed >= points) {
1895                                         // debug(D_RRD_STATS, "Already printed all rows. Stopping.");
1896                                         break;
1897                                 }
1898
1899                                 // generate the local date time
1900                                 struct tm tmbuf, *tm = localtime_r(&now, &tmbuf);
1901                                 if(!tm) { error("localtime() failed."); continue; }
1902                                 if(now > last_timestamp) last_timestamp = now;
1903
1904                                 if(printed) buffer_strcat(wb, "]},\n");
1905                                 buffer_strcat(wb, pre_date);
1906                                 buffer_jsdate(wb, tm->tm_year + 1900, tm->tm_mon, tm->tm_mday, tm->tm_hour, tm->tm_min, tm->tm_sec);
1907                                 buffer_strcat(wb, post_date);
1908
1909                                 print_this = 1;
1910                         }
1911
1912                         // do the calculations
1913                         for(rd = st->dimensions, c = 0 ; rd && c < dimensions ; rd = rd->next, c++) {
1914                                 storage_number n = rd->values[t];
1915                                 calculated_number value = unpack_storage_number(n);
1916
1917                                 if(!does_storage_number_exist(n)) {
1918                                         value = 0.0;
1919                                         found_non_existing[c]++;
1920                                 }
1921                                 if(did_storage_number_reset(n)) annotate_reset = 1;
1922
1923                                 switch(group_method) {
1924                                         case GROUP_MAX:
1925                                                 if(abs(value) > abs(group_values[c])) group_values[c] = value;
1926                                                 break;
1927
1928                                         case GROUP_SUM:
1929                                                 group_values[c] += value;
1930                                                 break;
1931
1932                                         default:
1933                                         case GROUP_AVERAGE:
1934                                                 group_values[c] += value;
1935                                                 if(print_this) group_values[c] /= ( group_count - found_non_existing[c] );
1936                                                 break;
1937                                 }
1938                         }
1939
1940                         if(print_this) {
1941                                 if(annotate_reset) {
1942                                         annotation_count++;
1943                                         buffer_strcat(wb, overflow_annotation);
1944                                         annotate_reset = 0;
1945                                 }
1946                                 else
1947                                         buffer_strcat(wb, normal_annotation);
1948
1949                                 pc = 0;
1950                                 for(c = 0 ; c < dimensions ; c++) {
1951                                         if(found_non_existing[c] == group_count) {
1952                                                 // all entries are non-existing
1953                                                 pc++;
1954                                                 buffer_strcat(wb, pre_value);
1955                                                 buffer_strcat(wb, "null");
1956                                                 buffer_strcat(wb, post_value);
1957                                         }
1958                                         else if(!print_hidden[c]) {
1959                                                 pc++;
1960                                                 buffer_strcat(wb, pre_value);
1961                                                 buffer_rrd_value(wb, group_values[c]);
1962                                                 buffer_strcat(wb, post_value);
1963
1964                                                 if(group_values[c]) found_non_zero[c]++;
1965                                         }
1966
1967                                         // reset them for the next loop
1968                                         group_values[c] = 0;
1969                                         found_non_existing[c] = 0;
1970                                 }
1971
1972                                 // if all dimensions are hidden, print a null
1973                                 if(!pc) {
1974                                         buffer_strcat(wb, pre_value);
1975                                         buffer_strcat(wb, "null");
1976                                         buffer_strcat(wb, post_value);
1977                                 }
1978
1979                                 printed++;
1980                                 group_count = 0;
1981                         }
1982                 }
1983
1984                 if(printed) buffer_strcat(wb, "]}");
1985                 buffer_strcat(wb, "\n   ]\n}\n");
1986
1987                 if(only_non_zero && max_loop > 1) {
1988                         int changed = 0;
1989                         for(rd = st->dimensions, c = 0 ; rd && c < dimensions ; rd = rd->next, c++) {
1990                                 group_values[c] = 0;
1991                                 found_non_existing[c] = 0;
1992
1993                                 if(!print_hidden[c] && !found_non_zero[c]) {
1994                                         changed = 1;
1995                                         print_hidden[c] = 1;
1996                                 }
1997                         }
1998
1999                         if(changed) buffer_flush(wb);
2000                         else break;
2001                 }
2002                 else break;
2003
2004         } // max_loop
2005
2006         debug(D_RRD_STATS, "RRD_STATS_JSON: %s total %ld bytes", st->name, wb->len);
2007
2008         pthread_rwlock_unlock(&st->rwlock);
2009         return last_timestamp;
2010 }