src/rrd2json.c

   1 #ifdef HAVE_CONFIG_H
   2 #include <config.h>
   3 #endif
   4 #include <pthread.h>
   5 #include <stdlib.h>
   6 #include <string.h>
   7 #include <stdint.h>
   8 #include <math.h>
   9
  10 #include "log.h"
  11 #include "common.h"
  12 #include "rrd2json.h"
  13
  14 #define HOSTNAME_MAX 1024
  15 char *hostname = "unknown";
  16
  17 void rrd_stats_api_v1_chart(RRDSET *st, BUFFER *wb)
  18 {
  19         pthread_rwlock_rdlock(&st->rwlock);
  20
  21         buffer_sprintf(wb,
  22                 "\t\t{\n"
  23                 "\t\t\t\"id\": \"%s\",\n"
  24                 "\t\t\t\"name\": \"%s\",\n"
  25                 "\t\t\t\"type\": \"%s\",\n"
  26                 "\t\t\t\"family\": \"%s\",\n"
  27                 "\t\t\t\"context\": \"%s\",\n"
  28                 "\t\t\t\"title\": \"%s\",\n"
  29                 "\t\t\t\"priority\": %ld,\n"
  30                 "\t\t\t\"enabled\": %s,\n"
  31                 "\t\t\t\"units\": \"%s\",\n"
  32                 "\t\t\t\"data_url\": \"/api/v1/data?chart=%s\",\n"
  33                 "\t\t\t\"chart_type\": \"%s\",\n"
  34                 "\t\t\t\"duration\": %ld,\n"
  35                 "\t\t\t\"first_entry\": %lu,\n"
  36                 "\t\t\t\"last_entry\": %lu,\n"
  37                 "\t\t\t\"update_every\": %d,\n"
  38                 "\t\t\t\"dimensions\": {\n"
  39                 , st->id
  40                 , st->name
  41                 , st->type
  42                 , st->family
  43                 , st->context
  44                 , st->title
  45                 , st->priority
  46                 , st->enabled?"true":"false"
  47                 , st->units
  48                 , st->name
  49                 , rrdset_type_name(st->chart_type)
  50                 , st->entries * st->update_every
  51                 , rrdset_first_entry_t(st)
  52                 , rrdset_last_entry_t(st)
  53                 , st->update_every
  54                 );
  55
  56         unsigned long memory = st->memsize;
  57
  58         int c = 0;
  59         RRDDIM *rd;
  60         for(rd = st->dimensions; rd ; rd = rd->next) {
  61                 if(rd->flags & RRDDIM_FLAG_HIDDEN) continue;
  62
  63                 memory += rd->memsize;
  64
  65                 buffer_sprintf(wb,
  66                         "%s"
  67                         "\t\t\t\t\"%s\": { \"name\": \"%s\" }"
  68                         , c?",\n":""
  69                         , rd->id
  70                         , rd->name
  71                         );
  72
  73                 c++;
  74         }
  75
  76         buffer_sprintf(wb,
  77                 "\n\t\t\t}\n"
  78                 "\t\t}"
  79                 );
  80
  81         pthread_rwlock_unlock(&st->rwlock);
  82 }
  83
  84 void rrd_stats_api_v1_charts(BUFFER *wb)
  85 {
  86         long c;
  87         RRDSET *st;
  88
  89         buffer_sprintf(wb, "{\n"
  90                    "\t\"hostname\": \"%s\""
  91                 ",\n\t\"update_every\": %d"
  92                 ",\n\t\"history\": %d"
  93                 ",\n\t\"charts\": {"
  94                 , hostname
  95                 , rrd_update_every
  96                 , rrd_default_history_entries
  97                 );
  98
  99         pthread_rwlock_rdlock(&rrdset_root_rwlock);
 100         for(st = rrdset_root, c = 0; st ; st = st->next) {
 101                 if(st->enabled) {
 102                         if(c) buffer_strcat(wb, ",");
 103                         buffer_strcat(wb, "\n\t\t\"");
 104                         buffer_strcat(wb, st->id);
 105                         buffer_strcat(wb, "\": ");
 106                         rrd_stats_api_v1_chart(st, wb);
 107                         c++;
 108                 }
 109         }
 110         pthread_rwlock_unlock(&rrdset_root_rwlock);
 111
 112         buffer_strcat(wb, "\n\t}\n}\n");
 113 }
 114
 115
 116 unsigned long rrd_stats_one_json(RRDSET *st, char *options, BUFFER *wb)
 117 {
 118         time_t now = time(NULL);
 119
 120         pthread_rwlock_rdlock(&st->rwlock);
 121
 122         buffer_sprintf(wb,
 123                 "\t\t{\n"
 124                 "\t\t\t\"id\": \"%s\",\n"
 125                 "\t\t\t\"name\": \"%s\",\n"
 126                 "\t\t\t\"type\": \"%s\",\n"
 127                 "\t\t\t\"family\": \"%s\",\n"
 128                 "\t\t\t\"context\": \"%s\",\n"
 129                 "\t\t\t\"title\": \"%s\",\n"
 130                 "\t\t\t\"priority\": %ld,\n"
 131                 "\t\t\t\"enabled\": %d,\n"
 132                 "\t\t\t\"units\": \"%s\",\n"
 133                 "\t\t\t\"url\": \"/data/%s/%s\",\n"
 134                 "\t\t\t\"chart_type\": \"%s\",\n"
 135                 "\t\t\t\"counter\": %ld,\n"
 136                 "\t\t\t\"entries\": %ld,\n"
 137                 "\t\t\t\"first_entry_t\": %lu,\n"
 138                 "\t\t\t\"last_entry\": %ld,\n"
 139                 "\t\t\t\"last_entry_t\": %lu,\n"
 140                 "\t\t\t\"last_entry_secs_ago\": %lu,\n"
 141                 "\t\t\t\"update_every\": %d,\n"
 142                 "\t\t\t\"isdetail\": %d,\n"
 143                 "\t\t\t\"usec_since_last_update\": %llu,\n"
 144                 "\t\t\t\"collected_total\": " TOTAL_NUMBER_FORMAT ",\n"
 145                 "\t\t\t\"last_collected_total\": " TOTAL_NUMBER_FORMAT ",\n"
 146                 "\t\t\t\"dimensions\": [\n"
 147                 , st->id
 148                 , st->name
 149                 , st->type
 150                 , st->family
 151                 , st->context
 152                 , st->title
 153                 , st->priority
 154                 , st->enabled
 155                 , st->units
 156                 , st->name, options?options:""
 157                 , rrdset_type_name(st->chart_type)
 158                 , st->counter
 159                 , st->entries
 160                 , rrdset_first_entry_t(st)
 161                 , rrdset_last_slot(st)
 162                 , rrdset_last_entry_t(st)
 163                 , (now < rrdset_last_entry_t(st)) ? (time_t)0 : now - rrdset_last_entry_t(st)
 164                 , st->update_every
 165                 , st->isdetail
 166                 , st->usec_since_last_update
 167                 , st->collected_total
 168                 , st->last_collected_total
 169                 );
 170
 171         unsigned long memory = st->memsize;
 172
 173         RRDDIM *rd;
 174         for(rd = st->dimensions; rd ; rd = rd->next) {
 175
 176                 memory += rd->memsize;
 177
 178                 buffer_sprintf(wb,
 179                         "\t\t\t\t{\n"
 180                         "\t\t\t\t\t\"id\": \"%s\",\n"
 181                         "\t\t\t\t\t\"name\": \"%s\",\n"
 182                         "\t\t\t\t\t\"entries\": %ld,\n"
 183                         "\t\t\t\t\t\"isHidden\": %d,\n"
 184                         "\t\t\t\t\t\"algorithm\": \"%s\",\n"
 185                         "\t\t\t\t\t\"multiplier\": %ld,\n"
 186                         "\t\t\t\t\t\"divisor\": %ld,\n"
 187                         "\t\t\t\t\t\"last_entry_t\": %lu,\n"
 188                         "\t\t\t\t\t\"collected_value\": " COLLECTED_NUMBER_FORMAT ",\n"
 189                         "\t\t\t\t\t\"calculated_value\": " CALCULATED_NUMBER_FORMAT ",\n"
 190                         "\t\t\t\t\t\"last_collected_value\": " COLLECTED_NUMBER_FORMAT ",\n"
 191                         "\t\t\t\t\t\"last_calculated_value\": " CALCULATED_NUMBER_FORMAT ",\n"
 192                         "\t\t\t\t\t\"memory\": %lu\n"
 193                         "\t\t\t\t}%s\n"
 194                         , rd->id
 195                         , rd->name
 196                         , rd->entries
 197                         , (rd->flags & RRDDIM_FLAG_HIDDEN)?1:0
 198                         , rrddim_algorithm_name(rd->algorithm)
 199                         , rd->multiplier
 200                         , rd->divisor
 201                         , rd->last_collected_time.tv_sec
 202                         , rd->collected_value
 203                         , rd->calculated_value
 204                         , rd->last_collected_value
 205                         , rd->last_calculated_value
 206                         , rd->memsize
 207                         , rd->next?",":""
 208                         );
 209         }
 210
 211         buffer_sprintf(wb,
 212                 "\t\t\t],\n"
 213                 "\t\t\t\"memory\" : %lu\n"
 214                 "\t\t}"
 215                 , memory
 216                 );
 217
 218         pthread_rwlock_unlock(&st->rwlock);
 219         return memory;
 220 }
 221
 222 #define RRD_GRAPH_JSON_HEADER "{\n\t\"charts\": [\n"
 223 #define RRD_GRAPH_JSON_FOOTER "\n\t]\n}\n"
 224
 225 void rrd_stats_graph_json(RRDSET *st, char *options, BUFFER *wb)
 226 {
 227         buffer_strcat(wb, RRD_GRAPH_JSON_HEADER);
 228         rrd_stats_one_json(st, options, wb);
 229         buffer_strcat(wb, RRD_GRAPH_JSON_FOOTER);
 230 }
 231
 232 void rrd_stats_all_json(BUFFER *wb)
 233 {
 234         unsigned long memory = 0;
 235         long c;
 236         RRDSET *st;
 237
 238         buffer_strcat(wb, RRD_GRAPH_JSON_HEADER);
 239
 240         pthread_rwlock_rdlock(&rrdset_root_rwlock);
 241         for(st = rrdset_root, c = 0; st ; st = st->next) {
 242                 if(st->enabled) {
 243                         if(c) buffer_strcat(wb, ",\n");
 244                         memory += rrd_stats_one_json(st, NULL, wb);
 245                         c++;
 246                 }
 247         }
 248         pthread_rwlock_unlock(&rrdset_root_rwlock);
 249
 250         buffer_sprintf(wb, "\n\t],\n"
 251                 "\t\"hostname\": \"%s\",\n"
 252                 "\t\"update_every\": %d,\n"
 253                 "\t\"history\": %d,\n"
 254                 "\t\"memory\": %lu\n"
 255                 "}\n"
 256                 , hostname
 257                 , rrd_update_every
 258                 , rrd_default_history_entries
 259                 , memory
 260                 );
 261 }
 262
 263
 264
 265 // ----------------------------------------------------------------------------
 266
 267 // RRDR dimension options
 268 #define RRDR_EMPTY      0x01 // the dimension contains / the value is empty (null)
 269 #define RRDR_RESET      0x02 // the dimension contains / the value is reset
 270 #define RRDR_HIDDEN     0x04 // the dimension contains / the value is hidden
 271 #define RRDR_NONZERO    0x08 // the dimension contains / the value is non-zero
 272
 273 // RRDR result options
 274 #define RRDR_RESULT_OPTION_ABSOLUTE 0x00000001
 275 #define RRDR_RESULT_OPTION_RELATIVE 0x00000002
 276
 277 typedef struct rrdresult {
 278         RRDSET *st;                     // the chart this result refers to
 279
 280         uint32_t result_options;        // RRDR_RESULT_OPTION_*
 281
 282         int d;                                  // the number of dimensions
 283         long n;                                 // the number of values in the arrays
 284         long rows;                          // the number of rows used
 285
 286         uint8_t *od;                    // the options for the dimensions
 287
 288         time_t *t;                              // array of n timestamps
 289         calculated_number *v;   // array n x d values
 290         uint8_t *o;                             // array n x d options
 291
 292         long c;                                 // current line ( -1 ~ n ), ( -1 = none, use rrdr_rows() to get number of rows )
 293
 294         long group;                             // how many collected values were grouped for each row
 295         int update_every;               // what is the suggested update frequency in seconds
 296
 297         calculated_number min;
 298         calculated_number max;
 299
 300         time_t before;
 301         time_t after;
 302
 303         int has_st_lock;                // if st is read locked by us
 304 } RRDR;
 305
 306 #define rrdr_rows(r) ((r)->rows)
 307
 308 /*
 309 static void rrdr_dump(RRDR *r)
 310 {
 311         long c, i;
 312         RRDDIM *d;
 313
 314         fprintf(stderr, "\nCHART %s (%s)\n", r->st->id, r->st->name);
 315
 316         for(c = 0, d = r->st->dimensions; d ;c++, d = d->next) {
 317                 fprintf(stderr, "DIMENSION %s (%s), %s%s%s%s\n"
 318                                 , d->id
 319                                 , d->name
 320                                 , (r->od[c] & RRDR_EMPTY)?"EMPTY ":""
 321                                 , (r->od[c] & RRDR_RESET)?"RESET ":""
 322                                 , (r->od[c] & RRDR_HIDDEN)?"HIDDEN ":""
 323                                 , (r->od[c] & RRDR_NONZERO)?"NONZERO ":""
 324                                 );
 325         }
 326
 327         if(r->rows <= 0) {
 328                 fprintf(stderr, "RRDR does not have any values in it.\n");
 329                 return;
 330         }
 331
 332         fprintf(stderr, "RRDR includes %d values in it:\n", r->rows);
 333
 334         // for each line in the array
 335         for(i = 0; i < r->rows ;i++) {
 336                 calculated_number *cn = &r->v[ i * r->d ];
 337                 uint8_t *co = &r->o[ i * r->d ];
 338
 339                 // print the id and the timestamp of the line
 340                 fprintf(stderr, "%ld %ld ", i + 1, r->t[i]);
 341
 342                 // for each dimension
 343                 for(c = 0, d = r->st->dimensions; d ;c++, d = d->next) {
 344                         if(unlikely(r->od[c] & RRDR_HIDDEN)) continue;
 345                         if(unlikely(!(r->od[c] & RRDR_NONZERO))) continue;
 346
 347                         if(co[c] & RRDR_EMPTY)
 348                                 fprintf(stderr, "null ");
 349                         else
 350                                 fprintf(stderr, CALCULATED_NUMBER_FORMAT " %s%s%s%s "
 351                                         , cn[c]
 352                                         , (co[c] & RRDR_EMPTY)?"E":" "
 353                                         , (co[c] & RRDR_RESET)?"R":" "
 354                                         , (co[c] & RRDR_HIDDEN)?"H":" "
 355                                         , (co[c] & RRDR_NONZERO)?"N":" "
 356                                         );
 357                 }
 358
 359                 fprintf(stderr, "\n");
 360         }
 361 }
 362 */
 363
 364 void rrdr_disable_not_selected_dimensions(RRDR *r, const char *dims)
 365 {
 366         char b[strlen(dims) + 1];
 367         char *o = b, *tok;
 368         strcpy(o, dims);
 369
 370         long c;
 371         RRDDIM *d;
 372
 373         // disable all of them
 374         for(c = 0, d = r->st->dimensions; d ;c++, d = d->next)
 375                 r->od[c] |= RRDR_HIDDEN;
 376
 377         while(o && *o && (tok = mystrsep(&o, ",|"))) {
 378                 if(!*tok) continue;
 379
 380                 uint32_t hash = simple_hash(tok);
 381
 382                 // find it and enable it
 383                 for(c = 0, d = r->st->dimensions; d ;c++, d = d->next) {
 384                         if(unlikely((hash == d->hash && !strcmp(d->id, tok)) || !strcmp(d->name, tok))) {
 385                                 r->od[c] &= ~RRDR_HIDDEN;
 386
 387                                 // since the user needs this dimension
 388                                 // make it appear as NONZERO, to return it
 389                                 // even if the dimension has only zeros
 390                                 r->od[c] |= RRDR_NONZERO;
 391                         }
 392                 }
 393         }
 394 }
 395
 396 void rrdr_buffer_print_format(BUFFER *wb, uint32_t format)
 397 {
 398         switch(format) {
 399         case DATASOURCE_JSON:
 400                 buffer_strcat(wb, DATASOURCE_FORMAT_JSON);
 401                 break;
 402
 403         case DATASOURCE_DATATABLE_JSON:
 404                 buffer_strcat(wb, DATASOURCE_FORMAT_DATATABLE_JSON);
 405                 break;
 406
 407         case DATASOURCE_DATATABLE_JSONP:
 408                 buffer_strcat(wb, DATASOURCE_FORMAT_DATATABLE_JSONP);
 409                 break;
 410
 411         case DATASOURCE_JSONP:
 412                 buffer_strcat(wb, DATASOURCE_FORMAT_JSONP);
 413                 break;
 414
 415         case DATASOURCE_SSV:
 416                 buffer_strcat(wb, DATASOURCE_FORMAT_SSV);
 417                 break;
 418
 419         case DATASOURCE_CSV:
 420                 buffer_strcat(wb, DATASOURCE_FORMAT_CSV);
 421                 break;
 422
 423         case DATASOURCE_TSV:
 424                 buffer_strcat(wb, DATASOURCE_FORMAT_TSV);
 425                 break;
 426
 427         case DATASOURCE_HTML:
 428                 buffer_strcat(wb, DATASOURCE_FORMAT_HTML);
 429                 break;
 430
 431         case DATASOURCE_JS_ARRAY:
 432                 buffer_strcat(wb, DATASOURCE_FORMAT_JS_ARRAY);
 433                 break;
 434
 435         case DATASOURCE_SSV_COMMA:
 436                 buffer_strcat(wb, DATASOURCE_FORMAT_SSV_COMMA);
 437                 break;
 438
 439         default:
 440                 buffer_strcat(wb, "unknown");
 441                 break;
 442         }
 443 }
 444
 445 uint32_t rrdr_check_options(RRDR *r, uint32_t options, const char *dims)
 446 {
 447         if(options & RRDR_OPTION_NONZERO) {
 448                 long i;
 449
 450                 if(dims && *dims) {
 451                         // the caller wants specific dimensions
 452                         // disable NONZERO option
 453                         // to make sure we don't accidentally prevent
 454                         // the specific dimensions from being returned
 455                         i = 0;
 456                 }
 457                 else {
 458                         // find how many dimensions are not zero
 459                         long c;
 460                         RRDDIM *rd;
 461                         for(c = 0, i = 0, rd = r->st->dimensions; rd && c < r->d ; c++, rd = rd->next) {
 462                                 if(unlikely(r->od[c] & RRDR_HIDDEN)) continue;
 463                                 if(unlikely(!(r->od[c] & RRDR_NONZERO))) continue;
 464                                 i++;
 465                         }
 466                 }
 467
 468                 // if with nonzero we get i = 0 (no dimensions will be returned)
 469                 // disable nonzero to show all dimensions
 470                 if(!i) options &= ~RRDR_OPTION_NONZERO;
 471         }
 472
 473         return options;
 474 }
 475
 476 void rrdr_json_wrapper_begin(RRDR *r, BUFFER *wb, uint32_t format, uint32_t options, int string_value)
 477 {
 478         long rows = rrdr_rows(r);
 479         long c, i;
 480         RRDDIM *rd;
 481
 482         //info("JSONWRAPPER(): %s: BEGIN", r->st->id);
 483         char kq[2] = "",                                        // key quote
 484                 sq[2] = "";                                             // string quote
 485
 486         if( options & RRDR_OPTION_GOOGLE_JSON ) {
 487                 kq[0] = '\0';
 488                 sq[0] = '\'';
 489         }
 490         else {
 491                 kq[0] = '"';
 492                 sq[0] = '"';
 493         }
 494
 495         buffer_sprintf(wb, "{\n"
 496                         "       %sapi%s: 1,\n"
 497                         "       %sid%s: %s%s%s,\n"
 498                         "       %sname%s: %s%s%s,\n"
 499                         "       %sview_update_every%s: %d,\n"
 500                         "       %supdate_every%s: %d,\n"
 501                         "       %sfirst_entry%s: %u,\n"
 502                         "       %slast_entry%s: %u,\n"
 503                         "       %sbefore%s: %u,\n"
 504                         "       %safter%s: %u,\n"
 505                         "       %sdimension_names%s: ["
 506                         , kq, kq
 507                         , kq, kq, sq, r->st->id, sq
 508                         , kq, kq, sq, r->st->name, sq
 509                         , kq, kq, r->update_every
 510                         , kq, kq, r->st->update_every
 511                         , kq, kq, (uint32_t)rrdset_first_entry_t(r->st)
 512                         , kq, kq, (uint32_t)rrdset_last_entry_t(r->st)
 513                         , kq, kq, (uint32_t)r->before
 514                         , kq, kq, (uint32_t)r->after
 515                         , kq, kq);
 516
 517         for(c = 0, i = 0, rd = r->st->dimensions; rd && c < r->d ;c++, rd = rd->next) {
 518                 if(unlikely(r->od[c] & RRDR_HIDDEN)) continue;
 519                 if(unlikely((options & RRDR_OPTION_NONZERO) && !(r->od[c] & RRDR_NONZERO))) continue;
 520
 521                 if(i) buffer_strcat(wb, ", ");
 522                 buffer_strcat(wb, sq);
 523                 buffer_strcat(wb, rd->name);
 524                 buffer_strcat(wb, sq);
 525                 i++;
 526         }
 527         if(!i) {
 528 #ifdef NETDATA_INTERNAL_CHECKS
 529                 error("RRDR is empty for %s (RRDR has %d dimensions, options is 0x%08x)", r->st->id, r->d, options);
 530 #endif
 531                 rows = 0;
 532                 buffer_strcat(wb, sq);
 533                 buffer_strcat(wb, "no data");
 534                 buffer_strcat(wb, sq);
 535         }
 536
 537         buffer_sprintf(wb, "],\n"
 538                         "       %sdimension_ids%s: ["
 539                         , kq, kq);
 540
 541         for(c = 0, i = 0, rd = r->st->dimensions; rd && c < r->d ;c++, rd = rd->next) {
 542                 if(unlikely(r->od[c] & RRDR_HIDDEN)) continue;
 543                 if(unlikely((options & RRDR_OPTION_NONZERO) && !(r->od[c] & RRDR_NONZERO))) continue;
 544
 545                 if(i) buffer_strcat(wb, ", ");
 546                 buffer_strcat(wb, sq);
 547                 buffer_strcat(wb, rd->id);
 548                 buffer_strcat(wb, sq);
 549                 i++;
 550         }
 551         if(!i) {
 552                 rows = 0;
 553                 buffer_strcat(wb, sq);
 554                 buffer_strcat(wb, "no data");
 555                 buffer_strcat(wb, sq);
 556         }
 557
 558         buffer_sprintf(wb, "],\n"
 559                         "       %slatest_values%s: ["
 560                         , kq, kq);
 561
 562         for(c = 0, i = 0, rd = r->st->dimensions; rd && c < r->d ;c++, rd = rd->next) {
 563                 if(unlikely(r->od[c] & RRDR_HIDDEN)) continue;
 564                 if(unlikely((options & RRDR_OPTION_NONZERO) && !(r->od[c] & RRDR_NONZERO))) continue;
 565
 566                 if(i) buffer_strcat(wb, ", ");
 567                 i++;
 568
 569                 storage_number n = rd->values[rrdset_last_slot(r->st)];
 570
 571                 if(!does_storage_number_exist(n))
 572                         buffer_strcat(wb, "null");
 573                 else
 574                         buffer_rrd_value(wb, unpack_storage_number(n));
 575         }
 576         if(!i) {
 577                 rows = 0;
 578                 buffer_strcat(wb, "null");
 579         }
 580
 581         buffer_sprintf(wb, "],\n"
 582                         "       %sview_latest_values%s: ["
 583                         , kq, kq);
 584
 585         i = 0;
 586         if(rows) {
 587                 for(c = 0, i = 0, rd = r->st->dimensions; rd && c < r->d ;c++, rd = rd->next) {
 588                         if(unlikely(r->od[c] & RRDR_HIDDEN)) continue;
 589                         if(unlikely((options & RRDR_OPTION_NONZERO) && !(r->od[c] & RRDR_NONZERO))) continue;
 590
 591                         if(i) buffer_strcat(wb, ", ");
 592                         i++;
 593
 594                         calculated_number *cn = &r->v[ (0) * r->d ];
 595                         uint8_t *co = &r->o[ (0) * r->d ];
 596
 597                         if(co[c] & RRDR_EMPTY)
 598                                 buffer_strcat(wb, "null");
 599                         else
 600                                 buffer_rrd_value(wb, cn[c]);
 601                 }
 602         }
 603         if(!i) {
 604                 rows = 0;
 605                 buffer_strcat(wb, "null");
 606         }
 607
 608         buffer_sprintf(wb, "],\n"
 609                         "       %sdimensions%s: %ld,\n"
 610                         "       %spoints%s: %ld,\n"
 611                         "       %sformat%s: %s"
 612                         , kq, kq, i
 613                         , kq, kq, rows
 614                         , kq, kq, sq
 615                         );
 616
 617         rrdr_buffer_print_format(wb, format);
 618
 619         buffer_sprintf(wb, "%s,\n"
 620                         "       %sresult%s: "
 621                         , sq
 622                         , kq, kq
 623                         );
 624
 625         if(string_value) buffer_strcat(wb, sq);
 626         //info("JSONWRAPPER(): %s: END", r->st->id);
 627 }
 628
 629 void rrdr_json_wrapper_end(RRDR *r, BUFFER *wb, uint32_t format, uint32_t options, int string_value)
 630 {
 631         if(r) {;}
 632         if(format) {;}
 633
 634         char kq[2] = "",                                        // key quote
 635                 sq[2] = "";                                             // string quote
 636
 637         if( options & RRDR_OPTION_GOOGLE_JSON ) {
 638                 kq[0] = '\0';
 639                 sq[0] = '\'';
 640         }
 641         else {
 642                 kq[0] = '"';
 643                 sq[0] = '"';
 644         }
 645
 646         if(string_value) buffer_strcat(wb, sq);
 647
 648         buffer_sprintf(wb, ",\n %smin%s: ", kq, kq);
 649         buffer_rrd_value(wb, r->min);
 650         buffer_sprintf(wb, ",\n %smax%s: ", kq, kq);
 651         buffer_rrd_value(wb, r->max);
 652         buffer_strcat(wb, "\n}\n");
 653 }
 654
 655 #define JSON_DATES_JS 1
 656 #define JSON_DATES_TIMESTAMP 2
 657
 658 static void rrdr2json(RRDR *r, BUFFER *wb, uint32_t options, int datatable)
 659 {
 660         //info("RRD2JSON(): %s: BEGIN", r->st->id);
 661         int row_annotations = 0, dates, dates_with_new = 0;
 662         char kq[2] = "",                                        // key quote
 663                 sq[2] = "",                                             // string quote
 664                 pre_label[101] = "",                    // before each label
 665                 post_label[101] = "",                   // after each label
 666                 pre_date[101] = "",                             // the beginning of line, to the date
 667                 post_date[101] = "",                    // closing the date
 668                 pre_value[101] = "",                    // before each value
 669                 post_value[101] = "",                   // after each value
 670                 post_line[101] = "",                    // at the end of each row
 671                 normal_annotation[201] = "",    // default row annotation
 672                 overflow_annotation[201] = "",  // overflow row annotation
 673                 data_begin[101] = "",                   // between labels and values
 674                 finish[101] = "";                               // at the end of everything
 675
 676         if(datatable) {
 677                 dates = JSON_DATES_JS;
 678                 if( options & RRDR_OPTION_GOOGLE_JSON ) {
 679                         kq[0] = '\0';
 680                         sq[0] = '\'';
 681                 }
 682                 else {
 683                         kq[0] = '"';
 684                         sq[0] = '"';
 685                 }
 686                 row_annotations = 1;
 687                 snprintfz(pre_date,   100, "            {%sc%s:[{%sv%s:%s", kq, kq, kq, kq, sq);
 688                 snprintfz(post_date,  100, "%s}", sq);
 689                 snprintfz(pre_label,  100, ",\n         {%sid%s:%s%s,%slabel%s:%s", kq, kq, sq, sq, kq, kq, sq);
 690                 snprintfz(post_label, 100, "%s,%spattern%s:%s%s,%stype%s:%snumber%s}", sq, kq, kq, sq, sq, kq, kq, sq, sq);
 691                 snprintfz(pre_value,  100, ",{%sv%s:", kq, kq);
 692                 strcpy(post_value,         "}");
 693                 strcpy(post_line,          "]}");
 694                 snprintfz(data_begin, 100, "\n  ],\n    %srows%s:\n     [\n", kq, kq);
 695                 strcpy(finish,             "\n  ]\n}");
 696
 697                 snprintfz(overflow_annotation, 200, ",{%sv%s:%sRESET OR OVERFLOW%s},{%sv%s:%sThe counters have been wrapped.%s}", kq, kq, sq, sq, kq, kq, sq, sq);
 698                 snprintfz(normal_annotation,   200, ",{%sv%s:null},{%sv%s:null}", kq, kq, kq, kq);
 699
 700                 buffer_sprintf(wb, "{\n %scols%s:\n     [\n", kq, kq);
 701                 buffer_sprintf(wb, "            {%sid%s:%s%s,%slabel%s:%stime%s,%spattern%s:%s%s,%stype%s:%sdatetime%s},\n", kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq);
 702                 buffer_sprintf(wb, "            {%sid%s:%s%s,%slabel%s:%s%s,%spattern%s:%s%s,%stype%s:%sstring%s,%sp%s:{%srole%s:%sannotation%s}},\n", kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, kq, kq, sq, sq);
 703                 buffer_sprintf(wb, "            {%sid%s:%s%s,%slabel%s:%s%s,%spattern%s:%s%s,%stype%s:%sstring%s,%sp%s:{%srole%s:%sannotationText%s}}", kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, kq, kq, sq, sq);
 704
 705                 // remove the valueobjects flag
 706                 // google wants its own keys
 707                 if(options & RRDR_OPTION_OBJECTSROWS)
 708                         options &= ~RRDR_OPTION_OBJECTSROWS;
 709         }
 710         else {
 711                 kq[0] = '"';
 712                 sq[0] = '"';
 713                 if((options & RRDR_OPTION_SECONDS) || (options & RRDR_OPTION_MILLISECONDS)) {
 714                         dates = JSON_DATES_TIMESTAMP;
 715                         dates_with_new = 0;
 716                 }
 717                 else {
 718                         dates = JSON_DATES_JS;
 719                         dates_with_new = 1;
 720                 }
 721                 if( options & RRDR_OPTION_OBJECTSROWS )
 722                         strcpy(pre_date, "              { ");
 723                 else
 724                         strcpy(pre_date, "              [ ");
 725                 strcpy(pre_label,  ", \"");
 726                 strcpy(post_label, "\"");
 727                 strcpy(pre_value,  ", ");
 728                 if( options & RRDR_OPTION_OBJECTSROWS )
 729                         strcpy(post_line, "}");
 730                 else
 731                         strcpy(post_line, "]");
 732                 snprintfz(data_begin, 100, "],\n        %sdata%s:\n     [\n", kq, kq);
 733                 strcpy(finish,             "\n  ]\n}");
 734
 735                 buffer_sprintf(wb, "{\n %slabels%s: [", kq, kq);
 736                 buffer_sprintf(wb, "%stime%s", sq, sq);
 737         }
 738
 739         // -------------------------------------------------------------------------
 740         // print the JSON header
 741
 742         long c, i;
 743         RRDDIM *rd;
 744
 745         // print the header lines
 746         for(c = 0, i = 0, rd = r->st->dimensions; rd && c < r->d ;c++, rd = rd->next) {
 747                 if(unlikely(r->od[c] & RRDR_HIDDEN)) continue;
 748                 if(unlikely((options & RRDR_OPTION_NONZERO) && !(r->od[c] & RRDR_NONZERO))) continue;
 749
 750                 buffer_strcat(wb, pre_label);
 751                 buffer_strcat(wb, rd->name);
 752                 buffer_strcat(wb, post_label);
 753                 i++;
 754         }
 755         if(!i) {
 756                 buffer_strcat(wb, pre_label);
 757                 buffer_strcat(wb, "no data");
 758                 buffer_strcat(wb, post_label);
 759         }
 760
 761         // print the begin of row data
 762         buffer_strcat(wb, data_begin);
 763
 764         // if all dimensions are hidden, print a null
 765         if(!i) {
 766                 buffer_strcat(wb, finish);
 767                 return;
 768         }
 769
 770         long start = 0, end = rrdr_rows(r), step = 1;
 771         if((options & RRDR_OPTION_REVERSED)) {
 772                 start = rrdr_rows(r) - 1;
 773                 end = -1;
 774                 step = -1;
 775         }
 776
 777         // for each line in the array
 778         calculated_number total = 1;
 779         for(i = start; i != end ;i += step) {
 780                 calculated_number *cn = &r->v[ i * r->d ];
 781                 uint8_t *co = &r->o[ i * r->d ];
 782
 783                 time_t now = r->t[i];
 784
 785                 if(dates == JSON_DATES_JS) {
 786                         // generate the local date time
 787                         struct tm tmbuf, *tm = localtime_r(&now, &tmbuf);
 788                         if(!tm) { error("localtime_r() failed."); continue; }
 789
 790                         if(likely(i != start)) buffer_strcat(wb, ",\n");
 791                         buffer_strcat(wb, pre_date);
 792
 793                         if( options & RRDR_OPTION_OBJECTSROWS )
 794                                 buffer_sprintf(wb, "%stime%s: ", kq, kq);
 795
 796                         if(dates_with_new)
 797                                 buffer_strcat(wb, "new ");
 798
 799                         buffer_jsdate(wb, tm->tm_year + 1900, tm->tm_mon, tm->tm_mday, tm->tm_hour, tm->tm_min, tm->tm_sec);
 800
 801                         buffer_strcat(wb, post_date);
 802
 803                         if(row_annotations) {
 804                                 // google supports one annotation per row
 805                                 int annotation_found = 0;
 806                                 for(c = 0, rd = r->st->dimensions; rd ;c++, rd = rd->next) {
 807                                         if(co[c] & RRDR_RESET) {
 808                                                 buffer_strcat(wb, overflow_annotation);
 809                                                 annotation_found = 1;
 810                                                 break;
 811                                         }
 812                                 }
 813                                 if(!annotation_found)
 814                                         buffer_strcat(wb, normal_annotation);
 815                         }
 816                 }
 817                 else {
 818                         // print the timestamp of the line
 819                         if(likely(i != start)) buffer_strcat(wb, ",\n");
 820                         buffer_strcat(wb, pre_date);
 821
 822                         if( options & RRDR_OPTION_OBJECTSROWS )
 823                                 buffer_sprintf(wb, "%stime%s: ", kq, kq);
 824
 825                         buffer_rrd_value(wb, (calculated_number)r->t[i]);
 826                         // in ms
 827                         if(options & RRDR_OPTION_MILLISECONDS) buffer_strcat(wb, "000");
 828
 829                         buffer_strcat(wb, post_date);
 830                 }
 831
 832                 if(unlikely(options & RRDR_OPTION_PERCENTAGE)) {
 833                         total = 0;
 834                         for(c = 0, rd = r->st->dimensions; rd && c < r->d ;c++, rd = rd->next) {
 835                                 calculated_number n = cn[c];
 836
 837                                 if(likely((options & RRDR_OPTION_ABSOLUTE) && n < 0))
 838                                         n = -n;
 839
 840                                 total += n;
 841                         }
 842                         // prevent a division by zero
 843                         if(total == 0) total = 1;
 844                 }
 845
 846                 // for each dimension
 847                 for(c = 0, rd = r->st->dimensions; rd && c < r->d ;c++, rd = rd->next) {
 848                         if(unlikely(r->od[c] & RRDR_HIDDEN)) continue;
 849                         if(unlikely((options & RRDR_OPTION_NONZERO) && !(r->od[c] & RRDR_NONZERO))) continue;
 850
 851                         calculated_number n = cn[c];
 852
 853                         buffer_strcat(wb, pre_value);
 854
 855                         if( options & RRDR_OPTION_OBJECTSROWS )
 856                                 buffer_sprintf(wb, "%s%s%s: ", kq, rd->name, kq);
 857
 858                         if(co[c] & RRDR_EMPTY) {
 859                                 if(options & RRDR_OPTION_NULL2ZERO)
 860                                         buffer_strcat(wb, "0");
 861                                 else
 862                                         buffer_strcat(wb, "null");
 863                         }
 864                         else {
 865                                 if(unlikely((options & RRDR_OPTION_ABSOLUTE) && n < 0))
 866                                         n = -n;
 867
 868                                 if(unlikely(options & RRDR_OPTION_PERCENTAGE))
 869                                         n = n * 100 / total;
 870
 871                                 buffer_rrd_value(wb, n);
 872                         }
 873
 874                         buffer_strcat(wb, post_value);
 875                 }
 876
 877                 buffer_strcat(wb, post_line);
 878         }
 879
 880         buffer_strcat(wb, finish);
 881         //info("RRD2JSON(): %s: END", r->st->id);
 882 }
 883
 884 static void rrdr2csv(RRDR *r, BUFFER *wb, uint32_t options, const char *startline, const char *separator, const char *endline, const char *betweenlines)
 885 {
 886         //info("RRD2CSV(): %s: BEGIN", r->st->id);
 887         long c, i;
 888         RRDDIM *d;
 889
 890         // print the csv header
 891         for(c = 0, i = 0, d = r->st->dimensions; d && c < r->d ;c++, d = d->next) {
 892                 if(unlikely(r->od[c] & RRDR_HIDDEN)) continue;
 893                 if(unlikely((options & RRDR_OPTION_NONZERO) && !(r->od[c] & RRDR_NONZERO))) continue;
 894
 895                 if(!i) {
 896                         buffer_strcat(wb, startline);
 897                         if(options & RRDR_OPTION_LABEL_QUOTES) buffer_strcat(wb, "\"");
 898                         buffer_strcat(wb, "time");
 899                         if(options & RRDR_OPTION_LABEL_QUOTES) buffer_strcat(wb, "\"");
 900                 }
 901                 buffer_strcat(wb, separator);
 902                 if(options & RRDR_OPTION_LABEL_QUOTES) buffer_strcat(wb, "\"");
 903                 buffer_strcat(wb, d->name);
 904                 if(options & RRDR_OPTION_LABEL_QUOTES) buffer_strcat(wb, "\"");
 905                 i++;
 906         }
 907         buffer_strcat(wb, endline);
 908
 909         if(!i) {
 910                 // no dimensions present
 911                 return;
 912         }
 913
 914         long start = 0, end = rrdr_rows(r), step = 1;
 915         if((options & RRDR_OPTION_REVERSED)) {
 916                 start = rrdr_rows(r) - 1;
 917                 end = -1;
 918                 step = -1;
 919         }
 920
 921         // for each line in the array
 922         calculated_number total = 1;
 923         for(i = start; i != end ;i += step) {
 924                 calculated_number *cn = &r->v[ i * r->d ];
 925                 uint8_t *co = &r->o[ i * r->d ];
 926
 927                 buffer_strcat(wb, betweenlines);
 928                 buffer_strcat(wb, startline);
 929
 930                 time_t now = r->t[i];
 931
 932                 if((options & RRDR_OPTION_SECONDS) || (options & RRDR_OPTION_MILLISECONDS)) {
 933                         // print the timestamp of the line
 934                         buffer_rrd_value(wb, (calculated_number)now);
 935                         // in ms
 936                         if(options & RRDR_OPTION_MILLISECONDS) buffer_strcat(wb, "000");
 937                 }
 938                 else {
 939                         // generate the local date time
 940                         struct tm tmbuf, *tm = localtime_r(&now, &tmbuf);
 941                         if(!tm) { error("localtime() failed."); continue; }
 942                         buffer_date(wb, tm->tm_year + 1900, tm->tm_mon + 1, tm->tm_mday, tm->tm_hour, tm->tm_min, tm->tm_sec);
 943                 }
 944
 945                 if(unlikely(options & RRDR_OPTION_PERCENTAGE)) {
 946                         total = 0;
 947                         for(c = 0, d = r->st->dimensions; d && c < r->d ;c++, d = d->next) {
 948                                 calculated_number n = cn[c];
 949
 950                                 if(likely((options & RRDR_OPTION_ABSOLUTE) && n < 0))
 951                                         n = -n;
 952
 953                                 total += n;
 954                         }
 955                         // prevent a division by zero
 956                         if(total == 0) total = 1;
 957                 }
 958
 959                 // for each dimension
 960                 for(c = 0, d = r->st->dimensions; d && c < r->d ;c++, d = d->next) {
 961                         if(unlikely(r->od[c] & RRDR_HIDDEN)) continue;
 962                         if(unlikely((options & RRDR_OPTION_NONZERO) && !(r->od[c] & RRDR_NONZERO))) continue;
 963
 964                         buffer_strcat(wb, separator);
 965
 966                         calculated_number n = cn[c];
 967
 968                         if(co[c] & RRDR_EMPTY) {
 969                                 if(options & RRDR_OPTION_NULL2ZERO)
 970                                         buffer_strcat(wb, "0");
 971                                 else
 972                                         buffer_strcat(wb, "null");
 973                         }
 974                         else {
 975                                 if(unlikely((options & RRDR_OPTION_ABSOLUTE) && n < 0))
 976                                         n = -n;
 977
 978                                 if(unlikely(options & RRDR_OPTION_PERCENTAGE))
 979                                         n = n * 100 / total;
 980
 981                                 buffer_rrd_value(wb, n);
 982                         }
 983                 }
 984
 985                 buffer_strcat(wb, endline);
 986         }
 987         //info("RRD2CSV(): %s: END", r->st->id);
 988 }
 989
 990 inline static calculated_number rrdr2value(RRDR *r, long i, uint32_t options, int *all_values_are_null) {
 991         long c;
 992         RRDDIM *d;
 993
 994         calculated_number *cn = &r->v[ i * r->d ];
 995         uint8_t *co = &r->o[ i * r->d ];
 996
 997         calculated_number sum = 0, min = 0, max = 0, v;
 998         int all_null = 1, init = 1;
 999
1000         calculated_number total = 1;
1001         if(unlikely(options & RRDR_OPTION_PERCENTAGE)) {
1002                 total = 0;
1003                 for(c = 0, d = r->st->dimensions; d && c < r->d ;c++, d = d->next) {
1004                         calculated_number n = cn[c];
1005
1006                         if(likely((options & RRDR_OPTION_ABSOLUTE) && n < 0))
1007                                 n = -n;
1008
1009                         total += n;
1010                 }
1011                 // prevent a division by zero
1012                 if(total == 0) total = 1;
1013         }
1014
1015         // for each dimension
1016         for(c = 0, d = r->st->dimensions; d && c < r->d ;c++, d = d->next) {
1017                 if(unlikely(r->od[c] & RRDR_HIDDEN)) continue;
1018                 if(unlikely((options & RRDR_OPTION_NONZERO) && !(r->od[c] & RRDR_NONZERO))) continue;
1019
1020                 calculated_number n = cn[c];
1021
1022                 if(likely((options & RRDR_OPTION_ABSOLUTE) && n < 0))
1023                         n = -n;
1024
1025                 if(unlikely(options & RRDR_OPTION_PERCENTAGE))
1026                         n = n * 100 / total;
1027
1028                 if(unlikely(init)) {
1029                         if(n > 0) {
1030                                 min = 0;
1031                                 max = n;
1032                         }
1033                         else {
1034                                 min = n;
1035                                 max = 0;
1036                         }
1037                         init = 0;
1038                 }
1039
1040                 if(likely(!(co[c] & RRDR_EMPTY))) {
1041                         all_null = 0;
1042                         sum += n;
1043                 }
1044
1045                 if(n < min) min = n;
1046                 if(n > max) max = n;
1047         }
1048
1049         if(unlikely(all_null)) {
1050                 if(likely(*all_values_are_null))
1051                         *all_values_are_null = 1;
1052                 return 0;
1053         }
1054         else {
1055                 if(likely(*all_values_are_null))
1056                         *all_values_are_null = 0;
1057         }
1058
1059         if(options & RRDR_OPTION_MIN2MAX)
1060                 v = max - min;
1061         else
1062                 v = sum;
1063
1064         return v;
1065 }
1066
1067 static void rrdr2ssv(RRDR *r, BUFFER *wb, uint32_t options, const char *prefix, const char *separator, const char *suffix)
1068 {
1069         //info("RRD2SSV(): %s: BEGIN", r->st->id);
1070         long i;
1071
1072         buffer_strcat(wb, prefix);
1073         long start = 0, end = rrdr_rows(r), step = 1;
1074         if((options & RRDR_OPTION_REVERSED)) {
1075                 start = rrdr_rows(r) - 1;
1076                 end = -1;
1077                 step = -1;
1078         }
1079
1080         // for each line in the array
1081         for(i = start; i != end ;i += step) {
1082                 int all_values_are_null = 0;
1083                 calculated_number v = rrdr2value(r, i, options, &all_values_are_null);
1084
1085                 if(likely(i != start)) {
1086                         if(r->min > v) r->min = v;
1087                         if(r->max < v) r->max = v;
1088                 }
1089                 else {
1090                         r->min = v;
1091                         r->max = v;
1092                 }
1093
1094                 if(likely(i != start))
1095                         buffer_strcat(wb, separator);
1096
1097                 if(all_values_are_null) {
1098                         if(options & RRDR_OPTION_NULL2ZERO)
1099                                 buffer_strcat(wb, "0");
1100                         else
1101                                 buffer_strcat(wb, "null");
1102                 }
1103                 else
1104                         buffer_rrd_value(wb, v);
1105         }
1106         buffer_strcat(wb, suffix);
1107         //info("RRD2SSV(): %s: END", r->st->id);
1108 }
1109
1110 inline static calculated_number *rrdr_line_values(RRDR *r)
1111 {
1112         return &r->v[ r->c * r->d ];
1113 }
1114
1115 inline static uint8_t *rrdr_line_options(RRDR *r)
1116 {
1117         return &r->o[ r->c * r->d ];
1118 }
1119
1120 inline static int rrdr_line_init(RRDR *r, time_t t)
1121 {
1122         r->c++;
1123
1124         if(unlikely(r->c >= r->n)) {
1125                 error("requested to step above RRDR size for chart %s", r->st->name);
1126                 r->c = r->n - 1;
1127         }
1128
1129         // save the time
1130         r->t[r->c] = t;
1131
1132         return 1;
1133 }
1134
1135 inline static void rrdr_lock_rrdset(RRDR *r) {
1136         if(unlikely(!r)) {
1137                 error("NULL value given!");
1138                 return;
1139         }
1140
1141         pthread_rwlock_rdlock(&r->st->rwlock);
1142         r->has_st_lock = 1;
1143 }
1144
1145 inline static void rrdr_unlock_rrdset(RRDR *r) {
1146         if(unlikely(!r)) {
1147                 error("NULL value given!");
1148                 return;
1149         }
1150
1151         if(likely(r->has_st_lock)) {
1152                 pthread_rwlock_unlock(&r->st->rwlock);
1153                 r->has_st_lock = 0;
1154         }
1155 }
1156
1157 inline static void rrdr_free(RRDR *r)
1158 {
1159         if(unlikely(!r)) {
1160                 error("NULL value given!");
1161                 return;
1162         }
1163
1164         rrdr_unlock_rrdset(r);
1165         if(likely(r->t)) free(r->t);
1166         if(likely(r->v)) free(r->v);
1167         if(likely(r->o)) free(r->o);
1168         if(likely(r->od)) free(r->od);
1169         free(r);
1170 }
1171
1172 inline void rrdr_done(RRDR *r)
1173 {
1174         r->rows = r->c + 1;
1175         r->c = 0;
1176 }
1177
1178 static RRDR *rrdr_create(RRDSET *st, long n)
1179 {
1180         if(unlikely(!st)) {
1181                 error("NULL value given!");
1182                 return NULL;
1183         }
1184
1185         RRDR *r = calloc(1, sizeof(RRDR));
1186         if(unlikely(!r)) goto cleanup;
1187
1188         r->st = st;
1189
1190         rrdr_lock_rrdset(r);
1191
1192         RRDDIM *rd;
1193         for(rd = st->dimensions ; rd ; rd = rd->next) r->d++;
1194
1195         r->n = n;
1196
1197         r->t = malloc(n * sizeof(time_t));
1198         if(unlikely(!r->t)) goto cleanup;
1199
1200         r->v = malloc(n * r->d * sizeof(calculated_number));
1201         if(unlikely(!r->v)) goto cleanup;
1202
1203         r->o = malloc(n * r->d * sizeof(uint8_t));
1204         if(unlikely(!r->o)) goto cleanup;
1205
1206         r->od = malloc(r->d * sizeof(uint8_t));
1207         if(unlikely(!r->od)) goto cleanup;
1208
1209         // set the hidden flag on hidden dimensions
1210         int c;
1211         for(c = 0, rd = st->dimensions ; rd ; c++, rd = rd->next) {
1212                 if(unlikely(rd->flags & RRDDIM_FLAG_HIDDEN)) r->od[c] = RRDR_HIDDEN;
1213                 else r->od[c] = 0;
1214         }
1215
1216         r->c = -1;
1217
1218         r->group = 1;
1219         r->update_every = 1;
1220
1221         return r;
1222
1223 cleanup:
1224         error("Cannot allocate RRDR memory for %ld entries", n);
1225         if(likely(r)) rrdr_free(r);
1226         return NULL;
1227 }
1228
1229 RRDR *rrd2rrdr(RRDSET *st, long points, long long after, long long before, int group_method, int aligned)
1230 {
1231         int debug = st->debug;
1232         int absolute_period_requested = -1;
1233
1234         time_t first_entry_t = rrdset_first_entry_t(st);
1235         time_t last_entry_t  = rrdset_last_entry_t(st);
1236
1237         if(before == 0 && after == 0) {
1238                 before = last_entry_t;
1239                 after = first_entry_t;
1240                 absolute_period_requested = 0;
1241         }
1242
1243         // allow relative for before and after
1244         if(((before < 0)?-before:before) <= (st->update_every * st->entries)) {
1245                 before = last_entry_t + before;
1246                 absolute_period_requested = 0;
1247         }
1248
1249         if(((after < 0)?-after:after) <= (st->update_every * st->entries)) {
1250                 if(after == 0) after = -st->update_every;
1251                 after = before + after;
1252                 absolute_period_requested = 0;
1253         }
1254
1255         if(absolute_period_requested == -1)
1256                 absolute_period_requested = 1;
1257
1258         // make sure they are within our timeframe
1259         if(before > last_entry_t)  before = last_entry_t;
1260         if(before < first_entry_t) before = first_entry_t;
1261
1262         if(after > last_entry_t)  after = last_entry_t;
1263         if(after < first_entry_t) after = first_entry_t;
1264
1265         // check if they are upside down
1266         if(after > before) {
1267                 time_t tmp = before;
1268                 before = after;
1269                 after = tmp;
1270         }
1271
1272         // the duration of the chart
1273         time_t duration = before - after;
1274         long available_points = duration / st->update_every;
1275
1276         if(duration <= 0 || available_points <= 0)
1277                 return rrdr_create(st, 1);
1278
1279         // check the wanted points
1280         if(points < 0) points = -points;
1281         if(points > available_points) points = available_points;
1282         if(points == 0) points = available_points;
1283
1284         // calculate proper grouping of source data
1285         long group = available_points / points;
1286         if(group <= 0) group = 1;
1287
1288         // round group to the closest integer
1289         if(available_points % points > points / 2) group++;
1290
1291         time_t after_new  = (aligned) ? (after  - (after  % (group * st->update_every))) : after;
1292         time_t before_new = (aligned) ? (before - (before % (group * st->update_every))) : before;
1293         long points_new   = (before_new - after_new) / st->update_every / group;
1294
1295         // find the starting and ending slots in our round robin db
1296         long    start_at_slot = rrdset_time2slot(st, before_new),
1297                         stop_at_slot  = rrdset_time2slot(st, after_new);
1298
1299 #ifdef NETDATA_INTERNAL_CHECKS
1300         if(after_new < first_entry_t) {
1301                 error("after_new %u is too small, minimum %u", (uint32_t)after_new, (uint32_t)first_entry_t);
1302         }
1303         if(after_new > last_entry_t) {
1304                 error("after_new %u is too big, maximum %u", (uint32_t)after_new, (uint32_t)last_entry_t);
1305         }
1306         if(before_new < first_entry_t) {
1307                 error("before_new %u is too small, minimum %u", (uint32_t)before_new, (uint32_t)first_entry_t);
1308         }
1309         if(before_new > last_entry_t) {
1310                 error("before_new %u is too big, maximum %u", (uint32_t)before_new, (uint32_t)last_entry_t);
1311         }
1312         if(start_at_slot < 0 || start_at_slot >= st->entries) {
1313                 error("start_at_slot is invalid %ld, expected 0 to %ld", start_at_slot, st->entries - 1);
1314         }
1315         if(stop_at_slot < 0 || stop_at_slot >= st->entries) {
1316                 error("stop_at_slot is invalid %ld, expected 0 to %ld", stop_at_slot, st->entries - 1);
1317         }
1318         if(points_new > (before_new - after_new) / group / st->update_every + 1) {
1319                 error("points_new %ld is more than points %ld", points_new, (before_new - after_new) / group / st->update_every + 1);
1320         }
1321 #endif
1322
1323         //info("RRD2RRDR(): %s: wanted %ld points, got %ld - group=%ld, wanted duration=%u, got %u - wanted %ld - %ld, got %ld - %ld", st->id, points, points_new, group, before - after, before_new - after_new, after, before, after_new, before_new);
1324
1325         after = after_new;
1326         before = before_new;
1327         duration = before - after;
1328         points = points_new;
1329
1330         // Now we have:
1331         // before = the end time of the calculation
1332         // after = the start time of the calculation
1333         // duration = the duration of the calculation
1334         // group = the number of source points to aggregate / group together
1335         // method = the method of grouping source points
1336         // points = the number of points to generate
1337
1338
1339         // -------------------------------------------------------------------------
1340         // initialize our result set
1341
1342         RRDR *r = rrdr_create(st, points);
1343         if(!r) {
1344 #ifdef NETDATA_INTERNAL_CHECKS
1345                 error("Cannot create RRDR for %s, after=%u, before=%u, duration=%u, points=%ld", st->id, (uint32_t)after, (uint32_t)before, (uint32_t)duration, points);
1346 #endif
1347                 return NULL;
1348         }
1349         if(!r->d) {
1350 #ifdef NETDATA_INTERNAL_CHECKS
1351                 error("Returning empty RRDR (no dimensions in RRDSET) for %s, after=%u, before=%u, duration=%u, points=%ld", st->id, (uint32_t)after, (uint32_t)before, (uint32_t)duration, points);
1352 #endif
1353                 return r;
1354         }
1355
1356         if(absolute_period_requested == 1)
1357                 r->result_options |= RRDR_RESULT_OPTION_ABSOLUTE;
1358         else
1359                 r->result_options |= RRDR_RESULT_OPTION_RELATIVE;
1360
1361         // find how many dimensions we have
1362         long dimensions = r->d;
1363
1364
1365         // -------------------------------------------------------------------------
1366         // checks for debugging
1367
1368         if(debug) debug(D_RRD_STATS, "INFO %s first_t: %u, last_t: %u, all_duration: %u, after: %u, before: %u, duration: %u, points: %ld, group: %ld"
1369                         , st->id
1370                         , (uint32_t)first_entry_t
1371                         , (uint32_t)last_entry_t
1372                         , (uint32_t)(last_entry_t - first_entry_t)
1373                         , (uint32_t)after
1374                         , (uint32_t)before
1375                         , (uint32_t)duration
1376                         , points
1377                         , group
1378                         );
1379
1380
1381         // -------------------------------------------------------------------------
1382         // temp arrays for keeping values per dimension
1383
1384         calculated_number       last_values[dimensions]; // keep the last value of each dimension
1385         calculated_number       group_values[dimensions]; // keep sums when grouping
1386         long                            group_counts[dimensions]; // keep the number of values added to group_values
1387         uint8_t                         group_options[dimensions];
1388         uint8_t                         found_non_zero[dimensions];
1389
1390
1391         // initialize them
1392         RRDDIM *rd;
1393         long c;
1394         for( rd = st->dimensions, c = 0 ; rd && c < dimensions ; rd = rd->next, c++) {
1395                 last_values[c] = 0;
1396                 group_values[c] = 0;
1397                 group_counts[c] = 0;
1398                 group_options[c] = 0;
1399                 found_non_zero[c] = 0;
1400         }
1401
1402
1403         // -------------------------------------------------------------------------
1404         // the main loop
1405
1406         time_t  now = rrdset_slot2time(st, start_at_slot),
1407                         dt = st->update_every,
1408                         group_start_t = 0;
1409
1410         if(unlikely(debug)) debug(D_RRD_STATS, "BEGIN %s after_t: %u (stop_at_t: %ld), before_t: %u (start_at_t: %ld), start_t(now): %u, current_entry: %ld, entries: %ld"
1411                         , st->id
1412                         , (uint32_t)after
1413                         , stop_at_slot
1414                         , (uint32_t)before
1415                         , start_at_slot
1416                         , (uint32_t)now
1417                         , st->current_entry
1418                         , st->entries
1419                         );
1420
1421         r->group = group;
1422         r->update_every = group * st->update_every;
1423         r->before = now;
1424         r->after = now;
1425
1426         //info("RRD2RRDR(): %s: STARTING", st->id);
1427
1428         long slot = start_at_slot, counter = 0, stop_now = 0, added = 0, group_count = 0, add_this = 0;
1429         for(; !stop_now ; now -= dt, slot--, counter++) {
1430                 if(unlikely(slot < 0)) slot = st->entries - 1;
1431                 if(unlikely(slot == stop_at_slot)) stop_now = counter;
1432
1433                 if(unlikely(debug)) debug(D_RRD_STATS, "ROW %s slot: %ld, entries_counter: %ld, group_count: %ld, added: %ld, now: %lu, %s %s"
1434                                 , st->id
1435                                 , slot
1436                                 , counter
1437                                 , group_count + 1
1438                                 , added
1439                                 , now
1440                                 , (group_count + 1 == group)?"PRINT":"  -  "
1441                                 , (now >= after && now <= before)?"RANGE":"  -  "
1442                                 );
1443
1444                 // make sure we return data in the proper time range
1445                 if(unlikely(now > before)) continue;
1446                 if(unlikely(now < after)) break;
1447
1448                 if(unlikely(group_count == 0)) {
1449                         group_start_t = now;
1450                 }
1451                 group_count++;
1452
1453                 if(unlikely(group_count == group)) {
1454                         if(unlikely(added >= points)) break;
1455                         add_this = 1;
1456                 }
1457
1458                 // do the calculations
1459                 for(rd = st->dimensions, c = 0 ; rd && c < dimensions ; rd = rd->next, c++) {
1460                         storage_number n = rd->values[slot];
1461                         if(unlikely(!does_storage_number_exist(n))) continue;
1462
1463                         group_counts[c]++;
1464
1465                         calculated_number value = unpack_storage_number(n);
1466                         if(likely(value != 0.0)) {
1467                                 group_options[c] |= RRDR_NONZERO;
1468                                 found_non_zero[c] = 1;
1469                         }
1470
1471                         if(unlikely(did_storage_number_reset(n)))
1472                                 group_options[c] |= RRDR_RESET;
1473
1474                         switch(group_method) {
1475                                 case GROUP_MAX:
1476                                         if(unlikely(fabsl(value) > fabsl(group_values[c])))
1477                                                 group_values[c] = value;
1478                                         break;
1479
1480                                 default:
1481                                 case GROUP_SUM:
1482                                 case GROUP_AVERAGE:
1483                                         group_values[c] += value;
1484                                         break;
1485
1486                                 case GROUP_INCREMENTAL_SUM:
1487                                         if(unlikely(slot == start_at_slot))
1488                                                 last_values[c] = value;
1489
1490                                         group_values[c] += last_values[c] - value;
1491                                         last_values[c] = value;
1492                                         break;
1493                         }
1494                 }
1495
1496                 // added it
1497                 if(unlikely(add_this)) {
1498                         if(unlikely(!rrdr_line_init(r, group_start_t))) break;
1499
1500                         r->after = now;
1501
1502                         calculated_number *cn = rrdr_line_values(r);
1503                         uint8_t *co = rrdr_line_options(r);
1504
1505                         for(rd = st->dimensions, c = 0 ; rd && c < dimensions ; rd = rd->next, c++) {
1506
1507                                 // update the dimension options
1508                                 if(likely(found_non_zero[c])) r->od[c] |= RRDR_NONZERO;
1509
1510                                 // store the specific point options
1511                                 co[c] = group_options[c];
1512
1513                                 // store the value
1514                                 if(unlikely(group_counts[c] == 0)) {
1515                                         cn[c] = 0.0;
1516                                         co[c] |= RRDR_EMPTY;
1517                                 }
1518                                 else if(unlikely(group_method == GROUP_AVERAGE)) {
1519                                         // GROUP_AVERAGE
1520                                         cn[c] = group_values[c] / group_counts[c];
1521                                 }
1522                                 else {
1523                                         // GROUP_SUM
1524                                         // GROUP_MAX
1525                                         // GROUP_INCREMENTAL_SUM
1526                                         cn[c] = group_values[c];
1527                                 }
1528
1529                                 if(cn[c] < r->min) r->min = cn[c];
1530                                 if(cn[c] > r->max) r->max = cn[c];
1531
1532                                 // reset them for the next loop
1533                                 group_values[c] = 0;
1534                                 group_counts[c] = 0;
1535                                 group_options[c] = 0;
1536                         }
1537
1538                         added++;
1539                         group_count = 0;
1540                         add_this = 0;
1541                 }
1542         }
1543
1544         rrdr_done(r);
1545         //info("RRD2RRDR(): %s: END %ld loops made, %ld points generated", st->id, counter, rrdr_rows(r));
1546         //error("SHIFT: %s: wanted %ld points, got %ld", st->id, points, rrdr_rows(r));
1547         return r;
1548 }
1549
1550 int rrd2value(RRDSET *st, BUFFER *wb, calculated_number *n, BUFFER *dimensions, long points, long long after, long long before, int group_method, uint32_t options, time_t *latest_timestamp, int *value_is_null)
1551 {
1552         RRDR *r = rrd2rrdr(st, points, after, before, group_method, !(options & RRDR_OPTION_NOT_ALIGNED));
1553         if(!r) {
1554                 if(value_is_null) *value_is_null = 1;
1555                 return 500;
1556         }
1557
1558         if(rrdr_rows(r) == 0) {
1559                 rrdr_free(r);
1560                 if(value_is_null) *value_is_null = 1;
1561                 return 400;
1562         }
1563
1564         if(r->result_options & RRDR_RESULT_OPTION_RELATIVE)
1565                 wb->options |= WB_CONTENT_NO_CACHEABLE;
1566         else if(r->result_options & RRDR_RESULT_OPTION_ABSOLUTE)
1567                 wb->options |= WB_CONTENT_CACHEABLE;
1568
1569         options = rrdr_check_options(r, options, (dimensions)?buffer_tostring(dimensions):NULL);
1570
1571         if(dimensions)
1572                 rrdr_disable_not_selected_dimensions(r, buffer_tostring(dimensions));
1573
1574         if(latest_timestamp)
1575                 *latest_timestamp = r->before;
1576
1577         long i = (options & RRDR_OPTION_REVERSED)?rrdr_rows(r) - 1:0;
1578         *n = rrdr2value(r, i, options, value_is_null);
1579
1580         rrdr_free(r);
1581         return 200;
1582 }
1583
1584 int rrd2format(RRDSET *st, BUFFER *wb, BUFFER *dimensions, uint32_t format, long points, long long after, long long before, int group_method, uint32_t options, time_t *latest_timestamp)
1585 {
1586         RRDR *r = rrd2rrdr(st, points, after, before, group_method, !(options & RRDR_OPTION_NOT_ALIGNED));
1587         if(!r) {
1588                 buffer_strcat(wb, "Cannot generate output with these parameters on this chart.");
1589                 return 500;
1590         }
1591
1592         if(r->result_options & RRDR_RESULT_OPTION_RELATIVE)
1593                 wb->options |= WB_CONTENT_NO_CACHEABLE;
1594         else if(r->result_options & RRDR_RESULT_OPTION_ABSOLUTE)
1595                 wb->options |= WB_CONTENT_CACHEABLE;
1596
1597         options = rrdr_check_options(r, options, (dimensions)?buffer_tostring(dimensions):NULL);
1598
1599         if(dimensions)
1600                 rrdr_disable_not_selected_dimensions(r, buffer_tostring(dimensions));
1601
1602         if(latest_timestamp && rrdr_rows(r) > 0)
1603                 *latest_timestamp = r->before;
1604
1605         switch(format) {
1606         case DATASOURCE_SSV:
1607                 if(options & RRDR_OPTION_JSON_WRAP) {
1608                         wb->contenttype = CT_APPLICATION_JSON;
1609                         rrdr_json_wrapper_begin(r, wb, format, options, 1);
1610                         rrdr2ssv(r, wb, options, "", " ", "");
1611                         rrdr_json_wrapper_end(r, wb, format, options, 1);
1612                 }
1613                 else {
1614                         wb->contenttype = CT_TEXT_PLAIN;
1615                         rrdr2ssv(r, wb, options, "", " ", "");
1616                 }
1617                 break;
1618
1619         case DATASOURCE_SSV_COMMA:
1620                 if(options & RRDR_OPTION_JSON_WRAP) {
1621                         wb->contenttype = CT_APPLICATION_JSON;
1622                         rrdr_json_wrapper_begin(r, wb, format, options, 1);
1623                         rrdr2ssv(r, wb, options, "", ",", "");
1624                         rrdr_json_wrapper_end(r, wb, format, options, 1);
1625                 }
1626                 else {
1627                         wb->contenttype = CT_TEXT_PLAIN;
1628                         rrdr2ssv(r, wb, options, "", ",", "");
1629                 }
1630                 break;
1631
1632         case DATASOURCE_JS_ARRAY:
1633                 if(options & RRDR_OPTION_JSON_WRAP) {
1634                         wb->contenttype = CT_APPLICATION_JSON;
1635                         rrdr_json_wrapper_begin(r, wb, format, options, 0);
1636                         rrdr2ssv(r, wb, options, "[", ",", "]");
1637                         rrdr_json_wrapper_end(r, wb, format, options, 0);
1638                 }
1639                 else {
1640                         wb->contenttype = CT_APPLICATION_JSON;
1641                         rrdr2ssv(r, wb, options, "[", ",", "]");
1642                 }
1643                 break;
1644
1645         case DATASOURCE_CSV:
1646                 if(options & RRDR_OPTION_JSON_WRAP) {
1647                         wb->contenttype = CT_APPLICATION_JSON;
1648                         rrdr_json_wrapper_begin(r, wb, format, options, 1);
1649                         rrdr2csv(r, wb, options, "", ",", "\\n", "");
1650                         rrdr_json_wrapper_end(r, wb, format, options, 1);
1651                 }
1652                 else {
1653                         wb->contenttype = CT_TEXT_PLAIN;
1654                         rrdr2csv(r, wb, options, "", ",", "\r\n", "");
1655                 }
1656                 break;
1657
1658         case DATASOURCE_CSV_JSON_ARRAY:
1659                 wb->contenttype = CT_APPLICATION_JSON;
1660                 if(options & RRDR_OPTION_JSON_WRAP) {
1661                         rrdr_json_wrapper_begin(r, wb, format, options, 0);
1662                         buffer_strcat(wb, "[\n");
1663                         rrdr2csv(r, wb, options + RRDR_OPTION_LABEL_QUOTES, "[", ",", "]", ",\n");
1664                         buffer_strcat(wb, "\n]");
1665                         rrdr_json_wrapper_end(r, wb, format, options, 0);
1666                 }
1667                 else {
1668                         wb->contenttype = CT_TEXT_PLAIN;
1669                         buffer_strcat(wb, "[\n");
1670                         rrdr2csv(r, wb, options + RRDR_OPTION_LABEL_QUOTES, "[", ",", "]", ",\n");
1671                         buffer_strcat(wb, "\n]");
1672                 }
1673                 break;
1674
1675         case DATASOURCE_TSV:
1676                 if(options & RRDR_OPTION_JSON_WRAP) {
1677                         wb->contenttype = CT_APPLICATION_JSON;
1678                         rrdr_json_wrapper_begin(r, wb, format, options, 1);
1679                         rrdr2csv(r, wb, options, "", "\t", "\\n", "");
1680                         rrdr_json_wrapper_end(r, wb, format, options, 1);
1681                 }
1682                 else {
1683                         wb->contenttype = CT_TEXT_PLAIN;
1684                         rrdr2csv(r, wb, options, "", "\t", "\r\n", "");
1685                 }
1686                 break;
1687
1688         case DATASOURCE_HTML:
1689                 if(options & RRDR_OPTION_JSON_WRAP) {
1690                         wb->contenttype = CT_APPLICATION_JSON;
1691                         rrdr_json_wrapper_begin(r, wb, format, options, 1);
1692                         buffer_strcat(wb, "<html>\\n<center>\\n<table border=\\\"0\\\" cellpadding=\\\"5\\\" cellspacing=\\\"5\\\">\\n");
1693                         rrdr2csv(r, wb, options, "<tr><td>", "</td><td>", "</td></tr>\\n", "");
1694                         buffer_strcat(wb, "</table>\\n</center>\\n</html>\\n");
1695                         rrdr_json_wrapper_end(r, wb, format, options, 1);
1696                 }
1697                 else {
1698                         wb->contenttype = CT_TEXT_HTML;
1699                         buffer_strcat(wb, "<html>\n<center>\n<table border=\"0\" cellpadding=\"5\" cellspacing=\"5\">\n");
1700                         rrdr2csv(r, wb, options, "<tr><td>", "</td><td>", "</td></tr>\n", "");
1701                         buffer_strcat(wb, "</table>\n</center>\n</html>\n");
1702                 }
1703                 break;
1704
1705         case DATASOURCE_DATATABLE_JSONP:
1706                 wb->contenttype = CT_APPLICATION_X_JAVASCRIPT;
1707
1708                 if(options & RRDR_OPTION_JSON_WRAP)
1709                         rrdr_json_wrapper_begin(r, wb, format, options, 0);
1710
1711                 rrdr2json(r, wb, options, 1);
1712
1713                 if(options & RRDR_OPTION_JSON_WRAP)
1714                         rrdr_json_wrapper_end(r, wb, format, options, 0);
1715                 break;
1716
1717         case DATASOURCE_DATATABLE_JSON:
1718                 wb->contenttype = CT_APPLICATION_JSON;
1719
1720                 if(options & RRDR_OPTION_JSON_WRAP)
1721                         rrdr_json_wrapper_begin(r, wb, format, options, 0);
1722
1723                 rrdr2json(r, wb, options, 1);
1724
1725                 if(options & RRDR_OPTION_JSON_WRAP)
1726                         rrdr_json_wrapper_end(r, wb, format, options, 0);
1727                 break;
1728
1729         case DATASOURCE_JSONP:
1730                 wb->contenttype = CT_APPLICATION_X_JAVASCRIPT;
1731                 if(options & RRDR_OPTION_JSON_WRAP)
1732                         rrdr_json_wrapper_begin(r, wb, format, options, 0);
1733
1734                 rrdr2json(r, wb, options, 0);
1735
1736                 if(options & RRDR_OPTION_JSON_WRAP)
1737                         rrdr_json_wrapper_end(r, wb, format, options, 0);
1738                 break;
1739
1740         case DATASOURCE_JSON:
1741         default:
1742                 wb->contenttype = CT_APPLICATION_JSON;
1743
1744                 if(options & RRDR_OPTION_JSON_WRAP)
1745                         rrdr_json_wrapper_begin(r, wb, format, options, 0);
1746
1747                 rrdr2json(r, wb, options, 0);
1748
1749                 if(options & RRDR_OPTION_JSON_WRAP)
1750                         rrdr_json_wrapper_end(r, wb, format, options, 0);
1751                 break;
1752         }
1753
1754         rrdr_free(r);
1755         return 200;
1756 }
1757
1758 time_t rrd_stats_json(int type, RRDSET *st, BUFFER *wb, long points, long group, int group_method, time_t after, time_t before, int only_non_zero)
1759 {
1760         int c;
1761         pthread_rwlock_rdlock(&st->rwlock);
1762
1763
1764         // -------------------------------------------------------------------------
1765         // switch from JSON to google JSON
1766
1767         char kq[2] = "\"";
1768         char sq[2] = "\"";
1769         switch(type) {
1770                 case DATASOURCE_DATATABLE_JSON:
1771                 case DATASOURCE_DATATABLE_JSONP:
1772                         kq[0] = '\0';
1773                         sq[0] = '\'';
1774                         break;
1775
1776                 case DATASOURCE_JSON:
1777                 default:
1778                         break;
1779         }
1780
1781
1782         // -------------------------------------------------------------------------
1783         // validate the parameters
1784
1785         if(points < 1) points = 1;
1786         if(group < 1) group = 1;
1787
1788         if(before == 0 || before > rrdset_last_entry_t(st)) before = rrdset_last_entry_t(st);
1789         if(after  == 0 || after < rrdset_first_entry_t(st)) after = rrdset_first_entry_t(st);
1790
1791         // ---
1792
1793         // our return value (the last timestamp printed)
1794         // this is required to detect re-transmit in google JSONP
1795         time_t last_timestamp = 0;
1796
1797
1798         // -------------------------------------------------------------------------
1799         // find how many dimensions we have
1800
1801         int dimensions = 0;
1802         RRDDIM *rd;
1803         for( rd = st->dimensions ; rd ; rd = rd->next) dimensions++;
1804         if(!dimensions) {
1805                 pthread_rwlock_unlock(&st->rwlock);
1806                 buffer_strcat(wb, "No dimensions yet.");
1807                 return 0;
1808         }
1809
1810
1811         // -------------------------------------------------------------------------
1812         // prepare various strings, to speed up the loop
1813
1814         char overflow_annotation[201]; snprintfz(overflow_annotation, 200, ",{%sv%s:%sRESET OR OVERFLOW%s},{%sv%s:%sThe counters have been wrapped.%s}", kq, kq, sq, sq, kq, kq, sq, sq);
1815         char normal_annotation[201];   snprintfz(normal_annotation,   200, ",{%sv%s:null},{%sv%s:null}", kq, kq, kq, kq);
1816         char pre_date[51];             snprintfz(pre_date,             50, "            {%sc%s:[{%sv%s:%s", kq, kq, kq, kq, sq);
1817         char post_date[21];            snprintfz(post_date,            20, "%s}", sq);
1818         char pre_value[21];            snprintfz(pre_value,            20, ",{%sv%s:", kq, kq);
1819         char post_value[21];           strcpy(post_value,                  "}");
1820
1821
1822         // -------------------------------------------------------------------------
1823         // checks for debugging
1824
1825         if(st->debug) {
1826                 debug(D_RRD_STATS, "%s first_entry_t = %lu, last_entry_t = %lu, duration = %lu, after = %lu, before = %lu, duration = %lu, entries_to_show = %lu, group = %lu"
1827                         , st->id
1828                         , rrdset_first_entry_t(st)
1829                         , rrdset_last_entry_t(st)
1830                         , rrdset_last_entry_t(st) - rrdset_first_entry_t(st)
1831                         , after
1832                         , before
1833                         , before - after
1834                         , points
1835                         , group
1836                         );
1837
1838                 if(before < after)
1839                         debug(D_RRD_STATS, "WARNING: %s The newest value in the database (%lu) is earlier than the oldest (%lu)", st->name, before, after);
1840
1841                 if((before - after) > st->entries * st->update_every)
1842                         debug(D_RRD_STATS, "WARNING: %s The time difference between the oldest and the newest entries (%lu) is higher than the capacity of the database (%lu)", st->name, before - after, st->entries * st->update_every);
1843         }
1844
1845
1846         // -------------------------------------------------------------------------
1847         // temp arrays for keeping values per dimension
1848
1849         calculated_number group_values[dimensions]; // keep sums when grouping
1850         int               print_hidden[dimensions]; // keep hidden flags
1851         int               found_non_zero[dimensions];
1852         int               found_non_existing[dimensions];
1853
1854         // initialize them
1855         for( rd = st->dimensions, c = 0 ; rd && c < dimensions ; rd = rd->next, c++) {
1856                 group_values[c] = 0;
1857                 print_hidden[c] = (rd->flags & RRDDIM_FLAG_HIDDEN)?1:0;
1858                 found_non_zero[c] = 0;
1859                 found_non_existing[c] = 0;
1860         }
1861
1862
1863         // error("OLD: points=%d after=%d before=%d group=%d, duration=%d", entries_to_show, before - (st->update_every * group * entries_to_show), before, group, before - after + 1);
1864         // rrd2array(st, entries_to_show, before - (st->update_every * group * entries_to_show), before, group_method, only_non_zero);
1865         // rrd2rrdr(st, entries_to_show, before - (st->update_every * group * entries_to_show), before, group_method);
1866
1867         // -------------------------------------------------------------------------
1868         // remove dimensions that contain only zeros
1869
1870         int max_loop = 1;
1871         if(only_non_zero) max_loop = 2;
1872
1873         for(; max_loop ; max_loop--) {
1874
1875                 // -------------------------------------------------------------------------
1876                 // print the JSON header
1877
1878                 buffer_sprintf(wb, "{\n %scols%s:\n     [\n", kq, kq);
1879                 buffer_sprintf(wb, "            {%sid%s:%s%s,%slabel%s:%stime%s,%spattern%s:%s%s,%stype%s:%sdatetime%s},\n", kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq);
1880                 buffer_sprintf(wb, "            {%sid%s:%s%s,%slabel%s:%s%s,%spattern%s:%s%s,%stype%s:%sstring%s,%sp%s:{%srole%s:%sannotation%s}},\n", kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, kq, kq, sq, sq);
1881                 buffer_sprintf(wb, "            {%sid%s:%s%s,%slabel%s:%s%s,%spattern%s:%s%s,%stype%s:%sstring%s,%sp%s:{%srole%s:%sannotationText%s}}", kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, kq, kq, sq, sq);
1882
1883                 // print the header for each dimension
1884                 // and update the print_hidden array for the dimensions that should be hidden
1885                 int pc = 0;
1886                 for( rd = st->dimensions, c = 0 ; rd && c < dimensions ; rd = rd->next, c++) {
1887                         if(!print_hidden[c]) {
1888                                 pc++;
1889                                 buffer_sprintf(wb, ",\n         {%sid%s:%s%s,%slabel%s:%s%s%s,%spattern%s:%s%s,%stype%s:%snumber%s}", kq, kq, sq, sq, kq, kq, sq, rd->name, sq, kq, kq, sq, sq, kq, kq, sq, sq);
1890                         }
1891                 }
1892                 if(!pc) {
1893                         buffer_sprintf(wb, ",\n         {%sid%s:%s%s,%slabel%s:%s%s%s,%spattern%s:%s%s,%stype%s:%snumber%s}", kq, kq, sq, sq, kq, kq, sq, "no data", sq, kq, kq, sq, sq, kq, kq, sq, sq);
1894                 }
1895
1896                 // print the begin of row data
1897                 buffer_sprintf(wb, "\n  ],\n    %srows%s:\n     [\n", kq, kq);
1898
1899
1900                 // -------------------------------------------------------------------------
1901                 // the main loop
1902
1903                 int annotate_reset = 0;
1904                 int annotation_count = 0;
1905
1906                 long    t = rrdset_time2slot(st, before),
1907                                 stop_at_t = rrdset_time2slot(st, after),
1908                                 stop_now = 0;
1909
1910                 t -= t % group;
1911
1912                 time_t  now = rrdset_slot2time(st, t),
1913                                 dt = st->update_every;
1914
1915                 long count = 0, printed = 0, group_count = 0;
1916                 last_timestamp = 0;
1917
1918                 if(st->debug) debug(D_RRD_STATS, "%s: REQUEST after:%u before:%u, points:%ld, group:%ld, CHART cur:%ld first: %u last:%u, CALC start_t:%ld, stop_t:%ld"
1919                                         , st->id
1920                                         , (uint32_t)after
1921                                         , (uint32_t)before
1922                                         , points
1923                                         , group
1924                                         , st->current_entry
1925                                         , (uint32_t)rrdset_first_entry_t(st)
1926                                         , (uint32_t)rrdset_last_entry_t(st)
1927                                         , t
1928                                         , stop_at_t
1929                                         );
1930
1931                 long counter = 0;
1932                 for(; !stop_now ; now -= dt, t--, counter++) {
1933                         if(t < 0) t = st->entries - 1;
1934                         if(t == stop_at_t) stop_now = counter;
1935
1936                         int print_this = 0;
1937
1938                         if(st->debug) debug(D_RRD_STATS, "%s t = %ld, count = %ld, group_count = %ld, printed = %ld, now = %lu, %s %s"
1939                                         , st->id
1940                                         , t
1941                                         , count + 1
1942                                         , group_count + 1
1943                                         , printed
1944                                         , now
1945                                         , (group_count + 1 == group)?"PRINT":"  -  "
1946                                         , (now >= after && now <= before)?"RANGE":"  -  "
1947                                         );
1948
1949
1950                         // make sure we return data in the proper time range
1951                         if(now > before) continue;
1952                         if(now < after) break;
1953
1954                         //if(rrdset_slot2time(st, t) != now)
1955                         //      error("%s: slot=%ld, now=%ld, slot2time=%ld, diff=%ld, last_entry_t=%ld, rrdset_last_slot=%ld", st->id, t, now, rrdset_slot2time(st,t), now - rrdset_slot2time(st,t), rrdset_last_entry_t(st), rrdset_last_slot(st));
1956
1957                         count++;
1958                         group_count++;
1959
1960                         // check if we have to print this now
1961                         if(group_count == group) {
1962                                 if(printed >= points) {
1963                                         // debug(D_RRD_STATS, "Already printed all rows. Stopping.");
1964                                         break;
1965                                 }
1966
1967                                 // generate the local date time
1968                                 struct tm tmbuf, *tm = localtime_r(&now, &tmbuf);
1969                                 if(!tm) { error("localtime() failed."); continue; }
1970                                 if(now > last_timestamp) last_timestamp = now;
1971
1972                                 if(printed) buffer_strcat(wb, "]},\n");
1973                                 buffer_strcat(wb, pre_date);
1974                                 buffer_jsdate(wb, tm->tm_year + 1900, tm->tm_mon, tm->tm_mday, tm->tm_hour, tm->tm_min, tm->tm_sec);
1975                                 buffer_strcat(wb, post_date);
1976
1977                                 print_this = 1;
1978                         }
1979
1980                         // do the calculations
1981                         for(rd = st->dimensions, c = 0 ; rd && c < dimensions ; rd = rd->next, c++) {
1982                                 storage_number n = rd->values[t];
1983                                 calculated_number value = unpack_storage_number(n);
1984
1985                                 if(!does_storage_number_exist(n)) {
1986                                         value = 0.0;
1987                                         found_non_existing[c]++;
1988                                 }
1989                                 if(did_storage_number_reset(n)) annotate_reset = 1;
1990
1991                                 switch(group_method) {
1992                                         case GROUP_MAX:
1993                                                 if(abs(value) > abs(group_values[c])) group_values[c] = value;
1994                                                 break;
1995
1996                                         case GROUP_SUM:
1997                                                 group_values[c] += value;
1998                                                 break;
1999
2000                                         default:
2001                                         case GROUP_AVERAGE:
2002                                                 group_values[c] += value;
2003                                                 if(print_this) group_values[c] /= ( group_count - found_non_existing[c] );
2004                                                 break;
2005                                 }
2006                         }
2007
2008                         if(print_this) {
2009                                 if(annotate_reset) {
2010                                         annotation_count++;
2011                                         buffer_strcat(wb, overflow_annotation);
2012                                         annotate_reset = 0;
2013                                 }
2014                                 else
2015                                         buffer_strcat(wb, normal_annotation);
2016
2017                                 pc = 0;
2018                                 for(c = 0 ; c < dimensions ; c++) {
2019                                         if(found_non_existing[c] == group_count) {
2020                                                 // all entries are non-existing
2021                                                 pc++;
2022                                                 buffer_strcat(wb, pre_value);
2023                                                 buffer_strcat(wb, "null");
2024                                                 buffer_strcat(wb, post_value);
2025                                         }
2026                                         else if(!print_hidden[c]) {
2027                                                 pc++;
2028                                                 buffer_strcat(wb, pre_value);
2029                                                 buffer_rrd_value(wb, group_values[c]);
2030                                                 buffer_strcat(wb, post_value);
2031
2032                                                 if(group_values[c]) found_non_zero[c]++;
2033                                         }
2034
2035                                         // reset them for the next loop
2036                                         group_values[c] = 0;
2037                                         found_non_existing[c] = 0;
2038                                 }
2039
2040                                 // if all dimensions are hidden, print a null
2041                                 if(!pc) {
2042                                         buffer_strcat(wb, pre_value);
2043                                         buffer_strcat(wb, "null");
2044                                         buffer_strcat(wb, post_value);
2045                                 }
2046
2047                                 printed++;
2048                                 group_count = 0;
2049                         }
2050                 }
2051
2052                 if(printed) buffer_strcat(wb, "]}");
2053                 buffer_strcat(wb, "\n   ]\n}\n");
2054
2055                 if(only_non_zero && max_loop > 1) {
2056                         int changed = 0;
2057                         for(rd = st->dimensions, c = 0 ; rd && c < dimensions ; rd = rd->next, c++) {
2058                                 group_values[c] = 0;
2059                                 found_non_existing[c] = 0;
2060
2061                                 if(!print_hidden[c] && !found_non_zero[c]) {
2062                                         changed = 1;
2063                                         print_hidden[c] = 1;
2064                                 }
2065                         }
2066
2067                         if(changed) buffer_flush(wb);
2068                         else break;
2069                 }
2070                 else break;
2071
2072         } // max_loop
2073
2074         debug(D_RRD_STATS, "RRD_STATS_JSON: %s total %ld bytes", st->name, wb->len);
2075
2076         pthread_rwlock_unlock(&st->rwlock);
2077         return last_timestamp;
2078 }