src/rrd2json.c

   1 #include "common.h"
   2
   3 #define HOSTNAME_MAX 1024
   4 char *hostname = "unknown";
   5
   6 void rrd_stats_api_v1_chart(RRDSET *st, BUFFER *wb)
   7 {
   8         pthread_rwlock_rdlock(&st->rwlock);
   9
  10         buffer_sprintf(wb,
  11                 "\t\t{\n"
  12                 "\t\t\t\"id\": \"%s\",\n"
  13                 "\t\t\t\"name\": \"%s\",\n"
  14                 "\t\t\t\"type\": \"%s\",\n"
  15                 "\t\t\t\"family\": \"%s\",\n"
  16                 "\t\t\t\"context\": \"%s\",\n"
  17                 "\t\t\t\"title\": \"%s\",\n"
  18                 "\t\t\t\"priority\": %ld,\n"
  19                 "\t\t\t\"enabled\": %s,\n"
  20                 "\t\t\t\"units\": \"%s\",\n"
  21                 "\t\t\t\"data_url\": \"/api/v1/data?chart=%s\",\n"
  22                 "\t\t\t\"chart_type\": \"%s\",\n"
  23                 "\t\t\t\"duration\": %ld,\n"
  24                 "\t\t\t\"first_entry\": %ld,\n"
  25                 "\t\t\t\"last_entry\": %ld,\n"
  26                 "\t\t\t\"update_every\": %d,\n"
  27                 "\t\t\t\"dimensions\": {\n"
  28                 , st->id
  29                 , st->name
  30                 , st->type
  31                 , st->family
  32                 , st->context
  33                 , st->title
  34                 , st->priority
  35                 , st->enabled?"true":"false"
  36                 , st->units
  37                 , st->name
  38                 , rrdset_type_name(st->chart_type)
  39                 , st->entries * st->update_every
  40                 , rrdset_first_entry_t(st)
  41                 , rrdset_last_entry_t(st)
  42                 , st->update_every
  43                 );
  44
  45         unsigned long memory = st->memsize;
  46
  47         int c = 0;
  48         RRDDIM *rd;
  49         for(rd = st->dimensions; rd ; rd = rd->next) {
  50                 if(rd->flags & RRDDIM_FLAG_HIDDEN) continue;
  51
  52                 memory += rd->memsize;
  53
  54                 buffer_sprintf(wb,
  55                         "%s"
  56                         "\t\t\t\t\"%s\": { \"name\": \"%s\" }"
  57                         , c?",\n":""
  58                         , rd->id
  59                         , rd->name
  60                         );
  61
  62                 c++;
  63         }
  64
  65         buffer_sprintf(wb,
  66                 "\n\t\t\t}\n"
  67                 "\t\t}"
  68                 );
  69
  70         pthread_rwlock_unlock(&st->rwlock);
  71 }
  72
  73 void rrd_stats_api_v1_charts(BUFFER *wb)
  74 {
  75         long c;
  76         RRDSET *st;
  77
  78         buffer_sprintf(wb, "{\n"
  79                    "\t\"hostname\": \"%s\""
  80                 ",\n\t\"update_every\": %d"
  81                 ",\n\t\"history\": %d"
  82                 ",\n\t\"charts\": {"
  83                 , hostname
  84                 , rrd_update_every
  85                 , rrd_default_history_entries
  86                 );
  87
  88         pthread_rwlock_rdlock(&localhost.rrdset_root_rwlock);
  89         for(st = localhost.rrdset_root, c = 0; st ; st = st->next) {
  90                 if(st->enabled && st->dimensions) {
  91                         if(c) buffer_strcat(wb, ",");
  92                         buffer_strcat(wb, "\n\t\t\"");
  93                         buffer_strcat(wb, st->id);
  94                         buffer_strcat(wb, "\": ");
  95                         rrd_stats_api_v1_chart(st, wb);
  96                         c++;
  97                 }
  98         }
  99         pthread_rwlock_unlock(&localhost.rrdset_root_rwlock);
 100
 101         buffer_strcat(wb, "\n\t}\n}\n");
 102 }
 103
 104
 105 unsigned long rrd_stats_one_json(RRDSET *st, char *options, BUFFER *wb)
 106 {
 107         time_t now = time(NULL);
 108
 109         pthread_rwlock_rdlock(&st->rwlock);
 110
 111         buffer_sprintf(wb,
 112                 "\t\t{\n"
 113                 "\t\t\t\"id\": \"%s\",\n"
 114                 "\t\t\t\"name\": \"%s\",\n"
 115                 "\t\t\t\"type\": \"%s\",\n"
 116                 "\t\t\t\"family\": \"%s\",\n"
 117                 "\t\t\t\"context\": \"%s\",\n"
 118                 "\t\t\t\"title\": \"%s\",\n"
 119                 "\t\t\t\"priority\": %ld,\n"
 120                 "\t\t\t\"enabled\": %d,\n"
 121                 "\t\t\t\"units\": \"%s\",\n"
 122                 "\t\t\t\"url\": \"/data/%s/%s\",\n"
 123                 "\t\t\t\"chart_type\": \"%s\",\n"
 124                 "\t\t\t\"counter\": %lu,\n"
 125                 "\t\t\t\"entries\": %ld,\n"
 126                 "\t\t\t\"first_entry_t\": %ld,\n"
 127                 "\t\t\t\"last_entry\": %lu,\n"
 128                 "\t\t\t\"last_entry_t\": %ld,\n"
 129                 "\t\t\t\"last_entry_secs_ago\": %ld,\n"
 130                 "\t\t\t\"update_every\": %d,\n"
 131                 "\t\t\t\"isdetail\": %d,\n"
 132                 "\t\t\t\"usec_since_last_update\": %llu,\n"
 133                 "\t\t\t\"collected_total\": " TOTAL_NUMBER_FORMAT ",\n"
 134                 "\t\t\t\"last_collected_total\": " TOTAL_NUMBER_FORMAT ",\n"
 135                 "\t\t\t\"dimensions\": [\n"
 136                 , st->id
 137                 , st->name
 138                 , st->type
 139                 , st->family
 140                 , st->context
 141                 , st->title
 142                 , st->priority
 143                 , st->enabled
 144                 , st->units
 145                 , st->name, options?options:""
 146                 , rrdset_type_name(st->chart_type)
 147                 , st->counter
 148                 , st->entries
 149                 , rrdset_first_entry_t(st)
 150                 , rrdset_last_slot(st)
 151                 , rrdset_last_entry_t(st)
 152                 , (now < rrdset_last_entry_t(st)) ? (time_t)0 : now - rrdset_last_entry_t(st)
 153                 , st->update_every
 154                 , st->isdetail
 155                 , st->usec_since_last_update
 156                 , st->collected_total
 157                 , st->last_collected_total
 158                 );
 159
 160         unsigned long memory = st->memsize;
 161
 162         RRDDIM *rd;
 163         for(rd = st->dimensions; rd ; rd = rd->next) {
 164
 165                 memory += rd->memsize;
 166
 167                 buffer_sprintf(wb,
 168                         "\t\t\t\t{\n"
 169                         "\t\t\t\t\t\"id\": \"%s\",\n"
 170                         "\t\t\t\t\t\"name\": \"%s\",\n"
 171                         "\t\t\t\t\t\"entries\": %ld,\n"
 172                         "\t\t\t\t\t\"isHidden\": %d,\n"
 173                         "\t\t\t\t\t\"algorithm\": \"%s\",\n"
 174                         "\t\t\t\t\t\"multiplier\": %ld,\n"
 175                         "\t\t\t\t\t\"divisor\": %ld,\n"
 176                         "\t\t\t\t\t\"last_entry_t\": %ld,\n"
 177                         "\t\t\t\t\t\"collected_value\": " COLLECTED_NUMBER_FORMAT ",\n"
 178                         "\t\t\t\t\t\"calculated_value\": " CALCULATED_NUMBER_FORMAT ",\n"
 179                         "\t\t\t\t\t\"last_collected_value\": " COLLECTED_NUMBER_FORMAT ",\n"
 180                         "\t\t\t\t\t\"last_calculated_value\": " CALCULATED_NUMBER_FORMAT ",\n"
 181                         "\t\t\t\t\t\"memory\": %lu\n"
 182                         "\t\t\t\t}%s\n"
 183                         , rd->id
 184                         , rd->name
 185                         , rd->entries
 186                         , (rd->flags & RRDDIM_FLAG_HIDDEN)?1:0
 187                         , rrddim_algorithm_name(rd->algorithm)
 188                         , rd->multiplier
 189                         , rd->divisor
 190                         , rd->last_collected_time.tv_sec
 191                         , rd->collected_value
 192                         , rd->calculated_value
 193                         , rd->last_collected_value
 194                         , rd->last_calculated_value
 195                         , rd->memsize
 196                         , rd->next?",":""
 197                         );
 198         }
 199
 200         buffer_sprintf(wb,
 201                 "\t\t\t],\n"
 202                 "\t\t\t\"memory\" : %lu\n"
 203                 "\t\t}"
 204                 , memory
 205                 );
 206
 207         pthread_rwlock_unlock(&st->rwlock);
 208         return memory;
 209 }
 210
 211 #define RRD_GRAPH_JSON_HEADER "{\n\t\"charts\": [\n"
 212 #define RRD_GRAPH_JSON_FOOTER "\n\t]\n}\n"
 213
 214 void rrd_stats_graph_json(RRDSET *st, char *options, BUFFER *wb)
 215 {
 216         buffer_strcat(wb, RRD_GRAPH_JSON_HEADER);
 217         rrd_stats_one_json(st, options, wb);
 218         buffer_strcat(wb, RRD_GRAPH_JSON_FOOTER);
 219 }
 220
 221 void rrd_stats_all_json(BUFFER *wb)
 222 {
 223         unsigned long memory = 0;
 224         long c;
 225         RRDSET *st;
 226
 227         buffer_strcat(wb, RRD_GRAPH_JSON_HEADER);
 228
 229         pthread_rwlock_rdlock(&localhost.rrdset_root_rwlock);
 230         for(st = localhost.rrdset_root, c = 0; st ; st = st->next) {
 231                 if(st->enabled && st->dimensions) {
 232                         if(c) buffer_strcat(wb, ",\n");
 233                         memory += rrd_stats_one_json(st, NULL, wb);
 234                         c++;
 235                 }
 236         }
 237         pthread_rwlock_unlock(&localhost.rrdset_root_rwlock);
 238
 239         buffer_sprintf(wb, "\n\t],\n"
 240                 "\t\"hostname\": \"%s\",\n"
 241                 "\t\"update_every\": %d,\n"
 242                 "\t\"history\": %d,\n"
 243                 "\t\"memory\": %lu\n"
 244                 "}\n"
 245                 , hostname
 246                 , rrd_update_every
 247                 , rrd_default_history_entries
 248                 , memory
 249                 );
 250 }
 251
 252
 253
 254 // ----------------------------------------------------------------------------
 255
 256 // RRDR dimension options
 257 #define RRDR_EMPTY      0x01 // the dimension contains / the value is empty (null)
 258 #define RRDR_RESET      0x02 // the dimension contains / the value is reset
 259 #define RRDR_HIDDEN     0x04 // the dimension contains / the value is hidden
 260 #define RRDR_NONZERO    0x08 // the dimension contains / the value is non-zero
 261
 262 // RRDR result options
 263 #define RRDR_RESULT_OPTION_ABSOLUTE 0x00000001
 264 #define RRDR_RESULT_OPTION_RELATIVE 0x00000002
 265
 266 typedef struct rrdresult {
 267         RRDSET *st;                     // the chart this result refers to
 268
 269         uint32_t result_options;        // RRDR_RESULT_OPTION_*
 270
 271         int d;                                  // the number of dimensions
 272         long n;                                 // the number of values in the arrays
 273         long rows;                          // the number of rows used
 274
 275         uint8_t *od;                    // the options for the dimensions
 276
 277         time_t *t;                              // array of n timestamps
 278         calculated_number *v;   // array n x d values
 279         uint8_t *o;                             // array n x d options
 280
 281         long c;                                 // current line ( -1 ~ n ), ( -1 = none, use rrdr_rows() to get number of rows )
 282
 283         long group;                             // how many collected values were grouped for each row
 284         int update_every;               // what is the suggested update frequency in seconds
 285
 286         calculated_number min;
 287         calculated_number max;
 288
 289         time_t before;
 290         time_t after;
 291
 292         int has_st_lock;                // if st is read locked by us
 293 } RRDR;
 294
 295 #define rrdr_rows(r) ((r)->rows)
 296
 297 /*
 298 static void rrdr_dump(RRDR *r)
 299 {
 300         long c, i;
 301         RRDDIM *d;
 302
 303         fprintf(stderr, "\nCHART %s (%s)\n", r->st->id, r->st->name);
 304
 305         for(c = 0, d = r->st->dimensions; d ;c++, d = d->next) {
 306                 fprintf(stderr, "DIMENSION %s (%s), %s%s%s%s\n"
 307                                 , d->id
 308                                 , d->name
 309                                 , (r->od[c] & RRDR_EMPTY)?"EMPTY ":""
 310                                 , (r->od[c] & RRDR_RESET)?"RESET ":""
 311                                 , (r->od[c] & RRDR_HIDDEN)?"HIDDEN ":""
 312                                 , (r->od[c] & RRDR_NONZERO)?"NONZERO ":""
 313                                 );
 314         }
 315
 316         if(r->rows <= 0) {
 317                 fprintf(stderr, "RRDR does not have any values in it.\n");
 318                 return;
 319         }
 320
 321         fprintf(stderr, "RRDR includes %d values in it:\n", r->rows);
 322
 323         // for each line in the array
 324         for(i = 0; i < r->rows ;i++) {
 325                 calculated_number *cn = &r->v[ i * r->d ];
 326                 uint8_t *co = &r->o[ i * r->d ];
 327
 328                 // print the id and the timestamp of the line
 329                 fprintf(stderr, "%ld %ld ", i + 1, r->t[i]);
 330
 331                 // for each dimension
 332                 for(c = 0, d = r->st->dimensions; d ;c++, d = d->next) {
 333                         if(unlikely(r->od[c] & RRDR_HIDDEN)) continue;
 334                         if(unlikely(!(r->od[c] & RRDR_NONZERO))) continue;
 335
 336                         if(co[c] & RRDR_EMPTY)
 337                                 fprintf(stderr, "null ");
 338                         else
 339                                 fprintf(stderr, CALCULATED_NUMBER_FORMAT " %s%s%s%s "
 340                                         , cn[c]
 341                                         , (co[c] & RRDR_EMPTY)?"E":" "
 342                                         , (co[c] & RRDR_RESET)?"R":" "
 343                                         , (co[c] & RRDR_HIDDEN)?"H":" "
 344                                         , (co[c] & RRDR_NONZERO)?"N":" "
 345                                         );
 346                 }
 347
 348                 fprintf(stderr, "\n");
 349         }
 350 }
 351 */
 352
 353 void rrdr_disable_not_selected_dimensions(RRDR *r, const char *dims)
 354 {
 355         char b[strlen(dims) + 1];
 356         char *o = b, *tok;
 357         strcpy(o, dims);
 358
 359         long c;
 360         RRDDIM *d;
 361
 362         // disable all of them
 363         for(c = 0, d = r->st->dimensions; d ;c++, d = d->next)
 364                 r->od[c] |= RRDR_HIDDEN;
 365
 366         while(o && *o && (tok = mystrsep(&o, ",|"))) {
 367                 if(!*tok) continue;
 368
 369                 uint32_t hash = simple_hash(tok);
 370
 371                 // find it and enable it
 372                 for(c = 0, d = r->st->dimensions; d ;c++, d = d->next) {
 373                         if(unlikely((hash == d->hash && !strcmp(d->id, tok)) || !strcmp(d->name, tok))) {
 374                                 r->od[c] &= ~RRDR_HIDDEN;
 375
 376                                 // since the user needs this dimension
 377                                 // make it appear as NONZERO, to return it
 378                                 // even if the dimension has only zeros
 379                                 r->od[c] |= RRDR_NONZERO;
 380                         }
 381                 }
 382         }
 383 }
 384
 385 void rrdr_buffer_print_format(BUFFER *wb, uint32_t format)
 386 {
 387         switch(format) {
 388         case DATASOURCE_JSON:
 389                 buffer_strcat(wb, DATASOURCE_FORMAT_JSON);
 390                 break;
 391
 392         case DATASOURCE_DATATABLE_JSON:
 393                 buffer_strcat(wb, DATASOURCE_FORMAT_DATATABLE_JSON);
 394                 break;
 395
 396         case DATASOURCE_DATATABLE_JSONP:
 397                 buffer_strcat(wb, DATASOURCE_FORMAT_DATATABLE_JSONP);
 398                 break;
 399
 400         case DATASOURCE_JSONP:
 401                 buffer_strcat(wb, DATASOURCE_FORMAT_JSONP);
 402                 break;
 403
 404         case DATASOURCE_SSV:
 405                 buffer_strcat(wb, DATASOURCE_FORMAT_SSV);
 406                 break;
 407
 408         case DATASOURCE_CSV:
 409                 buffer_strcat(wb, DATASOURCE_FORMAT_CSV);
 410                 break;
 411
 412         case DATASOURCE_TSV:
 413                 buffer_strcat(wb, DATASOURCE_FORMAT_TSV);
 414                 break;
 415
 416         case DATASOURCE_HTML:
 417                 buffer_strcat(wb, DATASOURCE_FORMAT_HTML);
 418                 break;
 419
 420         case DATASOURCE_JS_ARRAY:
 421                 buffer_strcat(wb, DATASOURCE_FORMAT_JS_ARRAY);
 422                 break;
 423
 424         case DATASOURCE_SSV_COMMA:
 425                 buffer_strcat(wb, DATASOURCE_FORMAT_SSV_COMMA);
 426                 break;
 427
 428         default:
 429                 buffer_strcat(wb, "unknown");
 430                 break;
 431         }
 432 }
 433
 434 uint32_t rrdr_check_options(RRDR *r, uint32_t options, const char *dims)
 435 {
 436         if(options & RRDR_OPTION_NONZERO) {
 437                 long i;
 438
 439                 if(dims && *dims) {
 440                         // the caller wants specific dimensions
 441                         // disable NONZERO option
 442                         // to make sure we don't accidentally prevent
 443                         // the specific dimensions from being returned
 444                         i = 0;
 445                 }
 446                 else {
 447                         // find how many dimensions are not zero
 448                         long c;
 449                         RRDDIM *rd;
 450                         for(c = 0, i = 0, rd = r->st->dimensions; rd && c < r->d ; c++, rd = rd->next) {
 451                                 if(unlikely(r->od[c] & RRDR_HIDDEN)) continue;
 452                                 if(unlikely(!(r->od[c] & RRDR_NONZERO))) continue;
 453                                 i++;
 454                         }
 455                 }
 456
 457                 // if with nonzero we get i = 0 (no dimensions will be returned)
 458                 // disable nonzero to show all dimensions
 459                 if(!i) options &= ~RRDR_OPTION_NONZERO;
 460         }
 461
 462         return options;
 463 }
 464
 465 void rrdr_json_wrapper_begin(RRDR *r, BUFFER *wb, uint32_t format, uint32_t options, int string_value)
 466 {
 467         long rows = rrdr_rows(r);
 468         long c, i;
 469         RRDDIM *rd;
 470
 471         //info("JSONWRAPPER(): %s: BEGIN", r->st->id);
 472         char kq[2] = "",                                        // key quote
 473                 sq[2] = "";                                             // string quote
 474
 475         if( options & RRDR_OPTION_GOOGLE_JSON ) {
 476                 kq[0] = '\0';
 477                 sq[0] = '\'';
 478         }
 479         else {
 480                 kq[0] = '"';
 481                 sq[0] = '"';
 482         }
 483
 484         buffer_sprintf(wb, "{\n"
 485                         "       %sapi%s: 1,\n"
 486                         "       %sid%s: %s%s%s,\n"
 487                         "       %sname%s: %s%s%s,\n"
 488                         "       %sview_update_every%s: %d,\n"
 489                         "       %supdate_every%s: %d,\n"
 490                         "       %sfirst_entry%s: %u,\n"
 491                         "       %slast_entry%s: %u,\n"
 492                         "       %sbefore%s: %u,\n"
 493                         "       %safter%s: %u,\n"
 494                         "       %sdimension_names%s: ["
 495                         , kq, kq
 496                         , kq, kq, sq, r->st->id, sq
 497                         , kq, kq, sq, r->st->name, sq
 498                         , kq, kq, r->update_every
 499                         , kq, kq, r->st->update_every
 500                         , kq, kq, (uint32_t)rrdset_first_entry_t(r->st)
 501                         , kq, kq, (uint32_t)rrdset_last_entry_t(r->st)
 502                         , kq, kq, (uint32_t)r->before
 503                         , kq, kq, (uint32_t)r->after
 504                         , kq, kq);
 505
 506         for(c = 0, i = 0, rd = r->st->dimensions; rd && c < r->d ;c++, rd = rd->next) {
 507                 if(unlikely(r->od[c] & RRDR_HIDDEN)) continue;
 508                 if(unlikely((options & RRDR_OPTION_NONZERO) && !(r->od[c] & RRDR_NONZERO))) continue;
 509
 510                 if(i) buffer_strcat(wb, ", ");
 511                 buffer_strcat(wb, sq);
 512                 buffer_strcat(wb, rd->name);
 513                 buffer_strcat(wb, sq);
 514                 i++;
 515         }
 516         if(!i) {
 517 #ifdef NETDATA_INTERNAL_CHECKS
 518                 error("RRDR is empty for %s (RRDR has %d dimensions, options is 0x%08x)", r->st->id, r->d, options);
 519 #endif
 520                 rows = 0;
 521                 buffer_strcat(wb, sq);
 522                 buffer_strcat(wb, "no data");
 523                 buffer_strcat(wb, sq);
 524         }
 525
 526         buffer_sprintf(wb, "],\n"
 527                         "       %sdimension_ids%s: ["
 528                         , kq, kq);
 529
 530         for(c = 0, i = 0, rd = r->st->dimensions; rd && c < r->d ;c++, rd = rd->next) {
 531                 if(unlikely(r->od[c] & RRDR_HIDDEN)) continue;
 532                 if(unlikely((options & RRDR_OPTION_NONZERO) && !(r->od[c] & RRDR_NONZERO))) continue;
 533
 534                 if(i) buffer_strcat(wb, ", ");
 535                 buffer_strcat(wb, sq);
 536                 buffer_strcat(wb, rd->id);
 537                 buffer_strcat(wb, sq);
 538                 i++;
 539         }
 540         if(!i) {
 541                 rows = 0;
 542                 buffer_strcat(wb, sq);
 543                 buffer_strcat(wb, "no data");
 544                 buffer_strcat(wb, sq);
 545         }
 546
 547         buffer_sprintf(wb, "],\n"
 548                         "       %slatest_values%s: ["
 549                         , kq, kq);
 550
 551         for(c = 0, i = 0, rd = r->st->dimensions; rd && c < r->d ;c++, rd = rd->next) {
 552                 if(unlikely(r->od[c] & RRDR_HIDDEN)) continue;
 553                 if(unlikely((options & RRDR_OPTION_NONZERO) && !(r->od[c] & RRDR_NONZERO))) continue;
 554
 555                 if(i) buffer_strcat(wb, ", ");
 556                 i++;
 557
 558                 storage_number n = rd->values[rrdset_last_slot(r->st)];
 559
 560                 if(!does_storage_number_exist(n))
 561                         buffer_strcat(wb, "null");
 562                 else
 563                         buffer_rrd_value(wb, unpack_storage_number(n));
 564         }
 565         if(!i) {
 566                 rows = 0;
 567                 buffer_strcat(wb, "null");
 568         }
 569
 570         buffer_sprintf(wb, "],\n"
 571                         "       %sview_latest_values%s: ["
 572                         , kq, kq);
 573
 574         i = 0;
 575         if(rows) {
 576                 for(c = 0, i = 0, rd = r->st->dimensions; rd && c < r->d ;c++, rd = rd->next) {
 577                         if(unlikely(r->od[c] & RRDR_HIDDEN)) continue;
 578                         if(unlikely((options & RRDR_OPTION_NONZERO) && !(r->od[c] & RRDR_NONZERO))) continue;
 579
 580                         if(i) buffer_strcat(wb, ", ");
 581                         i++;
 582
 583                         calculated_number *cn = &r->v[ (0) * r->d ];
 584                         uint8_t *co = &r->o[ (0) * r->d ];
 585
 586                         if(co[c] & RRDR_EMPTY)
 587                                 buffer_strcat(wb, "null");
 588                         else
 589                                 buffer_rrd_value(wb, cn[c]);
 590                 }
 591         }
 592         if(!i) {
 593                 rows = 0;
 594                 buffer_strcat(wb, "null");
 595         }
 596
 597         buffer_sprintf(wb, "],\n"
 598                         "       %sdimensions%s: %ld,\n"
 599                         "       %spoints%s: %ld,\n"
 600                         "       %sformat%s: %s"
 601                         , kq, kq, i
 602                         , kq, kq, rows
 603                         , kq, kq, sq
 604                         );
 605
 606         rrdr_buffer_print_format(wb, format);
 607
 608         buffer_sprintf(wb, "%s,\n"
 609                         "       %sresult%s: "
 610                         , sq
 611                         , kq, kq
 612                         );
 613
 614         if(string_value) buffer_strcat(wb, sq);
 615         //info("JSONWRAPPER(): %s: END", r->st->id);
 616 }
 617
 618 void rrdr_json_wrapper_end(RRDR *r, BUFFER *wb, uint32_t format, uint32_t options, int string_value)
 619 {
 620         (void)format;
 621
 622         char kq[2] = "",                                        // key quote
 623                 sq[2] = "";                                             // string quote
 624
 625         if( options & RRDR_OPTION_GOOGLE_JSON ) {
 626                 kq[0] = '\0';
 627                 sq[0] = '\'';
 628         }
 629         else {
 630                 kq[0] = '"';
 631                 sq[0] = '"';
 632         }
 633
 634         if(string_value) buffer_strcat(wb, sq);
 635
 636         buffer_sprintf(wb, ",\n %smin%s: ", kq, kq);
 637         buffer_rrd_value(wb, r->min);
 638         buffer_sprintf(wb, ",\n %smax%s: ", kq, kq);
 639         buffer_rrd_value(wb, r->max);
 640         buffer_strcat(wb, "\n}\n");
 641 }
 642
 643 #define JSON_DATES_JS 1
 644 #define JSON_DATES_TIMESTAMP 2
 645
 646 static void rrdr2json(RRDR *r, BUFFER *wb, uint32_t options, int datatable)
 647 {
 648         //info("RRD2JSON(): %s: BEGIN", r->st->id);
 649         int row_annotations = 0, dates, dates_with_new = 0;
 650         char kq[2] = "",                                        // key quote
 651                 sq[2] = "",                                             // string quote
 652                 pre_label[101] = "",                    // before each label
 653                 post_label[101] = "",                   // after each label
 654                 pre_date[101] = "",                             // the beginning of line, to the date
 655                 post_date[101] = "",                    // closing the date
 656                 pre_value[101] = "",                    // before each value
 657                 post_value[101] = "",                   // after each value
 658                 post_line[101] = "",                    // at the end of each row
 659                 normal_annotation[201] = "",    // default row annotation
 660                 overflow_annotation[201] = "",  // overflow row annotation
 661                 data_begin[101] = "",                   // between labels and values
 662                 finish[101] = "";                               // at the end of everything
 663
 664         if(datatable) {
 665                 dates = JSON_DATES_JS;
 666                 if( options & RRDR_OPTION_GOOGLE_JSON ) {
 667                         kq[0] = '\0';
 668                         sq[0] = '\'';
 669                 }
 670                 else {
 671                         kq[0] = '"';
 672                         sq[0] = '"';
 673                 }
 674                 row_annotations = 1;
 675                 snprintfz(pre_date,   100, "            {%sc%s:[{%sv%s:%s", kq, kq, kq, kq, sq);
 676                 snprintfz(post_date,  100, "%s}", sq);
 677                 snprintfz(pre_label,  100, ",\n         {%sid%s:%s%s,%slabel%s:%s", kq, kq, sq, sq, kq, kq, sq);
 678                 snprintfz(post_label, 100, "%s,%spattern%s:%s%s,%stype%s:%snumber%s}", sq, kq, kq, sq, sq, kq, kq, sq, sq);
 679                 snprintfz(pre_value,  100, ",{%sv%s:", kq, kq);
 680                 strcpy(post_value,         "}");
 681                 strcpy(post_line,          "]}");
 682                 snprintfz(data_begin, 100, "\n  ],\n    %srows%s:\n     [\n", kq, kq);
 683                 strcpy(finish,             "\n  ]\n}");
 684
 685                 snprintfz(overflow_annotation, 200, ",{%sv%s:%sRESET OR OVERFLOW%s},{%sv%s:%sThe counters have been wrapped.%s}", kq, kq, sq, sq, kq, kq, sq, sq);
 686                 snprintfz(normal_annotation,   200, ",{%sv%s:null},{%sv%s:null}", kq, kq, kq, kq);
 687
 688                 buffer_sprintf(wb, "{\n %scols%s:\n     [\n", kq, kq);
 689                 buffer_sprintf(wb, "            {%sid%s:%s%s,%slabel%s:%stime%s,%spattern%s:%s%s,%stype%s:%sdatetime%s},\n", kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq);
 690                 buffer_sprintf(wb, "            {%sid%s:%s%s,%slabel%s:%s%s,%spattern%s:%s%s,%stype%s:%sstring%s,%sp%s:{%srole%s:%sannotation%s}},\n", kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, kq, kq, sq, sq);
 691                 buffer_sprintf(wb, "            {%sid%s:%s%s,%slabel%s:%s%s,%spattern%s:%s%s,%stype%s:%sstring%s,%sp%s:{%srole%s:%sannotationText%s}}", kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, kq, kq, sq, sq);
 692
 693                 // remove the valueobjects flag
 694                 // google wants its own keys
 695                 if(options & RRDR_OPTION_OBJECTSROWS)
 696                         options &= ~RRDR_OPTION_OBJECTSROWS;
 697         }
 698         else {
 699                 kq[0] = '"';
 700                 sq[0] = '"';
 701                 if((options & RRDR_OPTION_SECONDS) || (options & RRDR_OPTION_MILLISECONDS)) {
 702                         dates = JSON_DATES_TIMESTAMP;
 703                         dates_with_new = 0;
 704                 }
 705                 else {
 706                         dates = JSON_DATES_JS;
 707                         dates_with_new = 1;
 708                 }
 709                 if( options & RRDR_OPTION_OBJECTSROWS )
 710                         strcpy(pre_date, "              { ");
 711                 else
 712                         strcpy(pre_date, "              [ ");
 713                 strcpy(pre_label,  ", \"");
 714                 strcpy(post_label, "\"");
 715                 strcpy(pre_value,  ", ");
 716                 if( options & RRDR_OPTION_OBJECTSROWS )
 717                         strcpy(post_line, "}");
 718                 else
 719                         strcpy(post_line, "]");
 720                 snprintfz(data_begin, 100, "],\n        %sdata%s:\n     [\n", kq, kq);
 721                 strcpy(finish,             "\n  ]\n}");
 722
 723                 buffer_sprintf(wb, "{\n %slabels%s: [", kq, kq);
 724                 buffer_sprintf(wb, "%stime%s", sq, sq);
 725         }
 726
 727         // -------------------------------------------------------------------------
 728         // print the JSON header
 729
 730         long c, i;
 731         RRDDIM *rd;
 732
 733         // print the header lines
 734         for(c = 0, i = 0, rd = r->st->dimensions; rd && c < r->d ;c++, rd = rd->next) {
 735                 if(unlikely(r->od[c] & RRDR_HIDDEN)) continue;
 736                 if(unlikely((options & RRDR_OPTION_NONZERO) && !(r->od[c] & RRDR_NONZERO))) continue;
 737
 738                 buffer_strcat(wb, pre_label);
 739                 buffer_strcat(wb, rd->name);
 740                 buffer_strcat(wb, post_label);
 741                 i++;
 742         }
 743         if(!i) {
 744                 buffer_strcat(wb, pre_label);
 745                 buffer_strcat(wb, "no data");
 746                 buffer_strcat(wb, post_label);
 747         }
 748
 749         // print the begin of row data
 750         buffer_strcat(wb, data_begin);
 751
 752         // if all dimensions are hidden, print a null
 753         if(!i) {
 754                 buffer_strcat(wb, finish);
 755                 return;
 756         }
 757
 758         long start = 0, end = rrdr_rows(r), step = 1;
 759         if((options & RRDR_OPTION_REVERSED)) {
 760                 start = rrdr_rows(r) - 1;
 761                 end = -1;
 762                 step = -1;
 763         }
 764
 765         // for each line in the array
 766         calculated_number total = 1;
 767         for(i = start; i != end ;i += step) {
 768                 calculated_number *cn = &r->v[ i * r->d ];
 769                 uint8_t *co = &r->o[ i * r->d ];
 770
 771                 time_t now = r->t[i];
 772
 773                 if(dates == JSON_DATES_JS) {
 774                         // generate the local date time
 775                         struct tm tmbuf, *tm = localtime_r(&now, &tmbuf);
 776                         if(!tm) { error("localtime_r() failed."); continue; }
 777
 778                         if(likely(i != start)) buffer_strcat(wb, ",\n");
 779                         buffer_strcat(wb, pre_date);
 780
 781                         if( options & RRDR_OPTION_OBJECTSROWS )
 782                                 buffer_sprintf(wb, "%stime%s: ", kq, kq);
 783
 784                         if(dates_with_new)
 785                                 buffer_strcat(wb, "new ");
 786
 787                         buffer_jsdate(wb, tm->tm_year + 1900, tm->tm_mon, tm->tm_mday, tm->tm_hour, tm->tm_min, tm->tm_sec);
 788
 789                         buffer_strcat(wb, post_date);
 790
 791                         if(row_annotations) {
 792                                 // google supports one annotation per row
 793                                 int annotation_found = 0;
 794                                 for(c = 0, rd = r->st->dimensions; rd ;c++, rd = rd->next) {
 795                                         if(co[c] & RRDR_RESET) {
 796                                                 buffer_strcat(wb, overflow_annotation);
 797                                                 annotation_found = 1;
 798                                                 break;
 799                                         }
 800                                 }
 801                                 if(!annotation_found)
 802                                         buffer_strcat(wb, normal_annotation);
 803                         }
 804                 }
 805                 else {
 806                         // print the timestamp of the line
 807                         if(likely(i != start)) buffer_strcat(wb, ",\n");
 808                         buffer_strcat(wb, pre_date);
 809
 810                         if( options & RRDR_OPTION_OBJECTSROWS )
 811                                 buffer_sprintf(wb, "%stime%s: ", kq, kq);
 812
 813                         buffer_rrd_value(wb, (calculated_number)r->t[i]);
 814                         // in ms
 815                         if(options & RRDR_OPTION_MILLISECONDS) buffer_strcat(wb, "000");
 816
 817                         buffer_strcat(wb, post_date);
 818                 }
 819
 820                 if(unlikely(options & RRDR_OPTION_PERCENTAGE)) {
 821                         total = 0;
 822                         for(c = 0, rd = r->st->dimensions; rd && c < r->d ;c++, rd = rd->next) {
 823                                 calculated_number n = cn[c];
 824
 825                                 if(likely((options & RRDR_OPTION_ABSOLUTE) && n < 0))
 826                                         n = -n;
 827
 828                                 total += n;
 829                         }
 830                         // prevent a division by zero
 831                         if(total == 0) total = 1;
 832                 }
 833
 834                 // for each dimension
 835                 for(c = 0, rd = r->st->dimensions; rd && c < r->d ;c++, rd = rd->next) {
 836                         if(unlikely(r->od[c] & RRDR_HIDDEN)) continue;
 837                         if(unlikely((options & RRDR_OPTION_NONZERO) && !(r->od[c] & RRDR_NONZERO))) continue;
 838
 839                         calculated_number n = cn[c];
 840
 841                         buffer_strcat(wb, pre_value);
 842
 843                         if( options & RRDR_OPTION_OBJECTSROWS )
 844                                 buffer_sprintf(wb, "%s%s%s: ", kq, rd->name, kq);
 845
 846                         if(co[c] & RRDR_EMPTY) {
 847                                 if(options & RRDR_OPTION_NULL2ZERO)
 848                                         buffer_strcat(wb, "0");
 849                                 else
 850                                         buffer_strcat(wb, "null");
 851                         }
 852                         else {
 853                                 if(unlikely((options & RRDR_OPTION_ABSOLUTE) && n < 0))
 854                                         n = -n;
 855
 856                                 if(unlikely(options & RRDR_OPTION_PERCENTAGE))
 857                                         n = n * 100 / total;
 858
 859                                 buffer_rrd_value(wb, n);
 860                         }
 861
 862                         buffer_strcat(wb, post_value);
 863                 }
 864
 865                 buffer_strcat(wb, post_line);
 866         }
 867
 868         buffer_strcat(wb, finish);
 869         //info("RRD2JSON(): %s: END", r->st->id);
 870 }
 871
 872 static void rrdr2csv(RRDR *r, BUFFER *wb, uint32_t options, const char *startline, const char *separator, const char *endline, const char *betweenlines)
 873 {
 874         //info("RRD2CSV(): %s: BEGIN", r->st->id);
 875         long c, i;
 876         RRDDIM *d;
 877
 878         // print the csv header
 879         for(c = 0, i = 0, d = r->st->dimensions; d && c < r->d ;c++, d = d->next) {
 880                 if(unlikely(r->od[c] & RRDR_HIDDEN)) continue;
 881                 if(unlikely((options & RRDR_OPTION_NONZERO) && !(r->od[c] & RRDR_NONZERO))) continue;
 882
 883                 if(!i) {
 884                         buffer_strcat(wb, startline);
 885                         if(options & RRDR_OPTION_LABEL_QUOTES) buffer_strcat(wb, "\"");
 886                         buffer_strcat(wb, "time");
 887                         if(options & RRDR_OPTION_LABEL_QUOTES) buffer_strcat(wb, "\"");
 888                 }
 889                 buffer_strcat(wb, separator);
 890                 if(options & RRDR_OPTION_LABEL_QUOTES) buffer_strcat(wb, "\"");
 891                 buffer_strcat(wb, d->name);
 892                 if(options & RRDR_OPTION_LABEL_QUOTES) buffer_strcat(wb, "\"");
 893                 i++;
 894         }
 895         buffer_strcat(wb, endline);
 896
 897         if(!i) {
 898                 // no dimensions present
 899                 return;
 900         }
 901
 902         long start = 0, end = rrdr_rows(r), step = 1;
 903         if((options & RRDR_OPTION_REVERSED)) {
 904                 start = rrdr_rows(r) - 1;
 905                 end = -1;
 906                 step = -1;
 907         }
 908
 909         // for each line in the array
 910         calculated_number total = 1;
 911         for(i = start; i != end ;i += step) {
 912                 calculated_number *cn = &r->v[ i * r->d ];
 913                 uint8_t *co = &r->o[ i * r->d ];
 914
 915                 buffer_strcat(wb, betweenlines);
 916                 buffer_strcat(wb, startline);
 917
 918                 time_t now = r->t[i];
 919
 920                 if((options & RRDR_OPTION_SECONDS) || (options & RRDR_OPTION_MILLISECONDS)) {
 921                         // print the timestamp of the line
 922                         buffer_rrd_value(wb, (calculated_number)now);
 923                         // in ms
 924                         if(options & RRDR_OPTION_MILLISECONDS) buffer_strcat(wb, "000");
 925                 }
 926                 else {
 927                         // generate the local date time
 928                         struct tm tmbuf, *tm = localtime_r(&now, &tmbuf);
 929                         if(!tm) { error("localtime() failed."); continue; }
 930                         buffer_date(wb, tm->tm_year + 1900, tm->tm_mon + 1, tm->tm_mday, tm->tm_hour, tm->tm_min, tm->tm_sec);
 931                 }
 932
 933                 if(unlikely(options & RRDR_OPTION_PERCENTAGE)) {
 934                         total = 0;
 935                         for(c = 0, d = r->st->dimensions; d && c < r->d ;c++, d = d->next) {
 936                                 calculated_number n = cn[c];
 937
 938                                 if(likely((options & RRDR_OPTION_ABSOLUTE) && n < 0))
 939                                         n = -n;
 940
 941                                 total += n;
 942                         }
 943                         // prevent a division by zero
 944                         if(total == 0) total = 1;
 945                 }
 946
 947                 // for each dimension
 948                 for(c = 0, d = r->st->dimensions; d && c < r->d ;c++, d = d->next) {
 949                         if(unlikely(r->od[c] & RRDR_HIDDEN)) continue;
 950                         if(unlikely((options & RRDR_OPTION_NONZERO) && !(r->od[c] & RRDR_NONZERO))) continue;
 951
 952                         buffer_strcat(wb, separator);
 953
 954                         calculated_number n = cn[c];
 955
 956                         if(co[c] & RRDR_EMPTY) {
 957                                 if(options & RRDR_OPTION_NULL2ZERO)
 958                                         buffer_strcat(wb, "0");
 959                                 else
 960                                         buffer_strcat(wb, "null");
 961                         }
 962                         else {
 963                                 if(unlikely((options & RRDR_OPTION_ABSOLUTE) && n < 0))
 964                                         n = -n;
 965
 966                                 if(unlikely(options & RRDR_OPTION_PERCENTAGE))
 967                                         n = n * 100 / total;
 968
 969                                 buffer_rrd_value(wb, n);
 970                         }
 971                 }
 972
 973                 buffer_strcat(wb, endline);
 974         }
 975         //info("RRD2CSV(): %s: END", r->st->id);
 976 }
 977
 978 inline static calculated_number rrdr2value(RRDR *r, long i, uint32_t options, int *all_values_are_null) {
 979         long c;
 980         RRDDIM *d;
 981
 982         calculated_number *cn = &r->v[ i * r->d ];
 983         uint8_t *co = &r->o[ i * r->d ];
 984
 985         calculated_number sum = 0, min = 0, max = 0, v;
 986         int all_null = 1, init = 1;
 987
 988         calculated_number total = 1;
 989         if(unlikely(options & RRDR_OPTION_PERCENTAGE)) {
 990                 total = 0;
 991                 for(c = 0, d = r->st->dimensions; d && c < r->d ;c++, d = d->next) {
 992                         calculated_number n = cn[c];
 993
 994                         if(likely((options & RRDR_OPTION_ABSOLUTE) && n < 0))
 995                                 n = -n;
 996
 997                         total += n;
 998                 }
 999                 // prevent a division by zero
1000                 if(total == 0) total = 1;
1001         }
1002
1003         // for each dimension
1004         for(c = 0, d = r->st->dimensions; d && c < r->d ;c++, d = d->next) {
1005                 if(unlikely(r->od[c] & RRDR_HIDDEN)) continue;
1006                 if(unlikely((options & RRDR_OPTION_NONZERO) && !(r->od[c] & RRDR_NONZERO))) continue;
1007
1008                 calculated_number n = cn[c];
1009
1010                 if(likely((options & RRDR_OPTION_ABSOLUTE) && n < 0))
1011                         n = -n;
1012
1013                 if(unlikely(options & RRDR_OPTION_PERCENTAGE))
1014                         n = n * 100 / total;
1015
1016                 if(unlikely(init)) {
1017                         if(n > 0) {
1018                                 min = 0;
1019                                 max = n;
1020                         }
1021                         else {
1022                                 min = n;
1023                                 max = 0;
1024                         }
1025                         init = 0;
1026                 }
1027
1028                 if(likely(!(co[c] & RRDR_EMPTY))) {
1029                         all_null = 0;
1030                         sum += n;
1031                 }
1032
1033                 if(n < min) min = n;
1034                 if(n > max) max = n;
1035         }
1036
1037         if(unlikely(all_null)) {
1038                 if(likely(*all_values_are_null))
1039                         *all_values_are_null = 1;
1040                 return 0;
1041         }
1042         else {
1043                 if(likely(*all_values_are_null))
1044                         *all_values_are_null = 0;
1045         }
1046
1047         if(options & RRDR_OPTION_MIN2MAX)
1048                 v = max - min;
1049         else
1050                 v = sum;
1051
1052         return v;
1053 }
1054
1055 static void rrdr2ssv(RRDR *r, BUFFER *wb, uint32_t options, const char *prefix, const char *separator, const char *suffix)
1056 {
1057         //info("RRD2SSV(): %s: BEGIN", r->st->id);
1058         long i;
1059
1060         buffer_strcat(wb, prefix);
1061         long start = 0, end = rrdr_rows(r), step = 1;
1062         if((options & RRDR_OPTION_REVERSED)) {
1063                 start = rrdr_rows(r) - 1;
1064                 end = -1;
1065                 step = -1;
1066         }
1067
1068         // for each line in the array
1069         for(i = start; i != end ;i += step) {
1070                 int all_values_are_null = 0;
1071                 calculated_number v = rrdr2value(r, i, options, &all_values_are_null);
1072
1073                 if(likely(i != start)) {
1074                         if(r->min > v) r->min = v;
1075                         if(r->max < v) r->max = v;
1076                 }
1077                 else {
1078                         r->min = v;
1079                         r->max = v;
1080                 }
1081
1082                 if(likely(i != start))
1083                         buffer_strcat(wb, separator);
1084
1085                 if(all_values_are_null) {
1086                         if(options & RRDR_OPTION_NULL2ZERO)
1087                                 buffer_strcat(wb, "0");
1088                         else
1089                                 buffer_strcat(wb, "null");
1090                 }
1091                 else
1092                         buffer_rrd_value(wb, v);
1093         }
1094         buffer_strcat(wb, suffix);
1095         //info("RRD2SSV(): %s: END", r->st->id);
1096 }
1097
1098 inline static calculated_number *rrdr_line_values(RRDR *r)
1099 {
1100         return &r->v[ r->c * r->d ];
1101 }
1102
1103 inline static uint8_t *rrdr_line_options(RRDR *r)
1104 {
1105         return &r->o[ r->c * r->d ];
1106 }
1107
1108 inline static int rrdr_line_init(RRDR *r, time_t t)
1109 {
1110         r->c++;
1111
1112         if(unlikely(r->c >= r->n)) {
1113                 error("requested to step above RRDR size for chart %s", r->st->name);
1114                 r->c = r->n - 1;
1115         }
1116
1117         // save the time
1118         r->t[r->c] = t;
1119
1120         return 1;
1121 }
1122
1123 inline static void rrdr_lock_rrdset(RRDR *r) {
1124         if(unlikely(!r)) {
1125                 error("NULL value given!");
1126                 return;
1127         }
1128
1129         pthread_rwlock_rdlock(&r->st->rwlock);
1130         r->has_st_lock = 1;
1131 }
1132
1133 inline static void rrdr_unlock_rrdset(RRDR *r) {
1134         if(unlikely(!r)) {
1135                 error("NULL value given!");
1136                 return;
1137         }
1138
1139         if(likely(r->has_st_lock)) {
1140                 pthread_rwlock_unlock(&r->st->rwlock);
1141                 r->has_st_lock = 0;
1142         }
1143 }
1144
1145 inline static void rrdr_free(RRDR *r)
1146 {
1147         if(unlikely(!r)) {
1148                 error("NULL value given!");
1149                 return;
1150         }
1151
1152         rrdr_unlock_rrdset(r);
1153         freez(r->t);
1154         freez(r->v);
1155         freez(r->o);
1156         freez(r->od);
1157         freez(r);
1158 }
1159
1160 inline void rrdr_done(RRDR *r)
1161 {
1162         r->rows = r->c + 1;
1163         r->c = 0;
1164 }
1165
1166 static RRDR *rrdr_create(RRDSET *st, long n)
1167 {
1168         if(unlikely(!st)) {
1169                 error("NULL value given!");
1170                 return NULL;
1171         }
1172
1173         RRDR *r = callocz(1, sizeof(RRDR));
1174         r->st = st;
1175
1176         rrdr_lock_rrdset(r);
1177
1178         RRDDIM *rd;
1179         for(rd = st->dimensions ; rd ; rd = rd->next) r->d++;
1180
1181         r->n = n;
1182
1183         r->t = mallocz(n * sizeof(time_t));
1184         r->v = mallocz(n * r->d * sizeof(calculated_number));
1185         r->o = mallocz(n * r->d * sizeof(uint8_t));
1186         r->od = mallocz(r->d * sizeof(uint8_t));
1187
1188         // set the hidden flag on hidden dimensions
1189         int c;
1190         for(c = 0, rd = st->dimensions ; rd ; c++, rd = rd->next) {
1191                 if(unlikely(rd->flags & RRDDIM_FLAG_HIDDEN)) r->od[c] = RRDR_HIDDEN;
1192                 else r->od[c] = 0;
1193         }
1194
1195         r->c = -1;
1196         r->group = 1;
1197         r->update_every = 1;
1198
1199         return r;
1200 }
1201
1202 RRDR *rrd2rrdr(RRDSET *st, long points, long long after, long long before, int group_method, int aligned)
1203 {
1204         int debug = st->debug;
1205         int absolute_period_requested = -1;
1206
1207         time_t first_entry_t = rrdset_first_entry_t(st);
1208         time_t last_entry_t  = rrdset_last_entry_t(st);
1209
1210         if(before == 0 && after == 0) {
1211                 before = last_entry_t;
1212                 after = first_entry_t;
1213                 absolute_period_requested = 0;
1214         }
1215
1216         // allow relative for before and after
1217         if(((before < 0)?-before:before) <= (st->update_every * st->entries)) {
1218                 before = last_entry_t + before;
1219                 absolute_period_requested = 0;
1220         }
1221
1222         if(((after < 0)?-after:after) <= (st->update_every * st->entries)) {
1223                 if(after == 0) after = -st->update_every;
1224                 after = before + after;
1225                 absolute_period_requested = 0;
1226         }
1227
1228         if(absolute_period_requested == -1)
1229                 absolute_period_requested = 1;
1230
1231         // make sure they are within our timeframe
1232         if(before > last_entry_t)  before = last_entry_t;
1233         if(before < first_entry_t) before = first_entry_t;
1234
1235         if(after > last_entry_t)  after = last_entry_t;
1236         if(after < first_entry_t) after = first_entry_t;
1237
1238         // check if they are upside down
1239         if(after > before) {
1240                 time_t tmp = before;
1241                 before = after;
1242                 after = tmp;
1243         }
1244
1245         // the duration of the chart
1246         time_t duration = before - after;
1247         long available_points = duration / st->update_every;
1248
1249         if(duration <= 0 || available_points <= 0)
1250                 return rrdr_create(st, 1);
1251
1252         // check the wanted points
1253         if(points < 0) points = -points;
1254         if(points > available_points) points = available_points;
1255         if(points == 0) points = available_points;
1256
1257         // calculate proper grouping of source data
1258         long group = available_points / points;
1259         if(group <= 0) group = 1;
1260
1261         // round group to the closest integer
1262         if(available_points % points > points / 2) group++;
1263
1264         time_t after_new  = (aligned) ? (after  - (after  % (group * st->update_every))) : after;
1265         time_t before_new = (aligned) ? (before - (before % (group * st->update_every))) : before;
1266         long points_new   = (before_new - after_new) / st->update_every / group;
1267
1268         // find the starting and ending slots in our round robin db
1269         long    start_at_slot = rrdset_time2slot(st, before_new),
1270                         stop_at_slot  = rrdset_time2slot(st, after_new);
1271
1272 #ifdef NETDATA_INTERNAL_CHECKS
1273         if(after_new < first_entry_t) {
1274                 error("after_new %u is too small, minimum %u", (uint32_t)after_new, (uint32_t)first_entry_t);
1275         }
1276         if(after_new > last_entry_t) {
1277                 error("after_new %u is too big, maximum %u", (uint32_t)after_new, (uint32_t)last_entry_t);
1278         }
1279         if(before_new < first_entry_t) {
1280                 error("before_new %u is too small, minimum %u", (uint32_t)before_new, (uint32_t)first_entry_t);
1281         }
1282         if(before_new > last_entry_t) {
1283                 error("before_new %u is too big, maximum %u", (uint32_t)before_new, (uint32_t)last_entry_t);
1284         }
1285         if(start_at_slot < 0 || start_at_slot >= st->entries) {
1286                 error("start_at_slot is invalid %ld, expected 0 to %ld", start_at_slot, st->entries - 1);
1287         }
1288         if(stop_at_slot < 0 || stop_at_slot >= st->entries) {
1289                 error("stop_at_slot is invalid %ld, expected 0 to %ld", stop_at_slot, st->entries - 1);
1290         }
1291         if(points_new > (before_new - after_new) / group / st->update_every + 1) {
1292                 error("points_new %ld is more than points %ld", points_new, (before_new - after_new) / group / st->update_every + 1);
1293         }
1294 #endif
1295
1296         //info("RRD2RRDR(): %s: wanted %ld points, got %ld - group=%ld, wanted duration=%u, got %u - wanted %ld - %ld, got %ld - %ld", st->id, points, points_new, group, before - after, before_new - after_new, after, before, after_new, before_new);
1297
1298         after = after_new;
1299         before = before_new;
1300         duration = before - after;
1301         points = points_new;
1302
1303         // Now we have:
1304         // before = the end time of the calculation
1305         // after = the start time of the calculation
1306         // duration = the duration of the calculation
1307         // group = the number of source points to aggregate / group together
1308         // method = the method of grouping source points
1309         // points = the number of points to generate
1310
1311
1312         // -------------------------------------------------------------------------
1313         // initialize our result set
1314
1315         RRDR *r = rrdr_create(st, points);
1316         if(!r) {
1317 #ifdef NETDATA_INTERNAL_CHECKS
1318                 error("Cannot create RRDR for %s, after=%u, before=%u, duration=%u, points=%ld", st->id, (uint32_t)after, (uint32_t)before, (uint32_t)duration, points);
1319 #endif
1320                 return NULL;
1321         }
1322         if(!r->d) {
1323 #ifdef NETDATA_INTERNAL_CHECKS
1324                 error("Returning empty RRDR (no dimensions in RRDSET) for %s, after=%u, before=%u, duration=%u, points=%ld", st->id, (uint32_t)after, (uint32_t)before, (uint32_t)duration, points);
1325 #endif
1326                 return r;
1327         }
1328
1329         if(absolute_period_requested == 1)
1330                 r->result_options |= RRDR_RESULT_OPTION_ABSOLUTE;
1331         else
1332                 r->result_options |= RRDR_RESULT_OPTION_RELATIVE;
1333
1334         // find how many dimensions we have
1335         long dimensions = r->d;
1336
1337
1338         // -------------------------------------------------------------------------
1339         // checks for debugging
1340
1341         if(debug) debug(D_RRD_STATS, "INFO %s first_t: %u, last_t: %u, all_duration: %u, after: %u, before: %u, duration: %u, points: %ld, group: %ld"
1342                         , st->id
1343                         , (uint32_t)first_entry_t
1344                         , (uint32_t)last_entry_t
1345                         , (uint32_t)(last_entry_t - first_entry_t)
1346                         , (uint32_t)after
1347                         , (uint32_t)before
1348                         , (uint32_t)duration
1349                         , points
1350                         , group
1351                         );
1352
1353
1354         // -------------------------------------------------------------------------
1355         // temp arrays for keeping values per dimension
1356
1357         calculated_number       last_values[dimensions]; // keep the last value of each dimension
1358         calculated_number       group_values[dimensions]; // keep sums when grouping
1359         long                            group_counts[dimensions]; // keep the number of values added to group_values
1360         uint8_t                         group_options[dimensions];
1361         uint8_t                         found_non_zero[dimensions];
1362
1363
1364         // initialize them
1365         RRDDIM *rd;
1366         long c;
1367         for( rd = st->dimensions, c = 0 ; rd && c < dimensions ; rd = rd->next, c++) {
1368                 last_values[c] = 0;
1369                 group_values[c] = 0;
1370                 group_counts[c] = 0;
1371                 group_options[c] = 0;
1372                 found_non_zero[c] = 0;
1373         }
1374
1375
1376         // -------------------------------------------------------------------------
1377         // the main loop
1378
1379         time_t  now = rrdset_slot2time(st, start_at_slot),
1380                         dt = st->update_every,
1381                         group_start_t = 0;
1382
1383         if(unlikely(debug)) debug(D_RRD_STATS, "BEGIN %s after_t: %u (stop_at_t: %ld), before_t: %u (start_at_t: %ld), start_t(now): %u, current_entry: %ld, entries: %ld"
1384                         , st->id
1385                         , (uint32_t)after
1386                         , stop_at_slot
1387                         , (uint32_t)before
1388                         , start_at_slot
1389                         , (uint32_t)now
1390                         , st->current_entry
1391                         , st->entries
1392                         );
1393
1394         r->group = group;
1395         r->update_every = group * st->update_every;
1396         r->before = now;
1397         r->after = now;
1398
1399         //info("RRD2RRDR(): %s: STARTING", st->id);
1400
1401         long slot = start_at_slot, counter = 0, stop_now = 0, added = 0, group_count = 0, add_this = 0;
1402         for(; !stop_now ; now -= dt, slot--, counter++) {
1403                 if(unlikely(slot < 0)) slot = st->entries - 1;
1404                 if(unlikely(slot == stop_at_slot)) stop_now = counter;
1405
1406                 if(unlikely(debug)) debug(D_RRD_STATS, "ROW %s slot: %ld, entries_counter: %ld, group_count: %ld, added: %ld, now: %ld, %s %s"
1407                                 , st->id
1408                                 , slot
1409                                 , counter
1410                                 , group_count + 1
1411                                 , added
1412                                 , now
1413                                 , (group_count + 1 == group)?"PRINT":"  -  "
1414                                 , (now >= after && now <= before)?"RANGE":"  -  "
1415                                 );
1416
1417                 // make sure we return data in the proper time range
1418                 if(unlikely(now > before)) continue;
1419                 if(unlikely(now < after)) break;
1420
1421                 if(unlikely(group_count == 0)) {
1422                         group_start_t = now;
1423                 }
1424                 group_count++;
1425
1426                 if(unlikely(group_count == group)) {
1427                         if(unlikely(added >= points)) break;
1428                         add_this = 1;
1429                 }
1430
1431                 // do the calculations
1432                 for(rd = st->dimensions, c = 0 ; rd && c < dimensions ; rd = rd->next, c++) {
1433                         storage_number n = rd->values[slot];
1434                         if(unlikely(!does_storage_number_exist(n))) continue;
1435
1436                         group_counts[c]++;
1437
1438                         calculated_number value = unpack_storage_number(n);
1439                         if(likely(value != 0.0)) {
1440                                 group_options[c] |= RRDR_NONZERO;
1441                                 found_non_zero[c] = 1;
1442                         }
1443
1444                         if(unlikely(did_storage_number_reset(n)))
1445                                 group_options[c] |= RRDR_RESET;
1446
1447                         switch(group_method) {
1448                                 case GROUP_MAX:
1449                                         if(unlikely(fabsl(value) > fabsl(group_values[c])))
1450                                                 group_values[c] = value;
1451                                         break;
1452
1453                                 default:
1454                                 case GROUP_SUM:
1455                                 case GROUP_AVERAGE:
1456                                         group_values[c] += value;
1457                                         break;
1458
1459                                 case GROUP_INCREMENTAL_SUM:
1460                                         if(unlikely(slot == start_at_slot))
1461                                                 last_values[c] = value;
1462
1463                                         group_values[c] += last_values[c] - value;
1464                                         last_values[c] = value;
1465                                         break;
1466                         }
1467                 }
1468
1469                 // added it
1470                 if(unlikely(add_this)) {
1471                         if(unlikely(!rrdr_line_init(r, group_start_t))) break;
1472
1473                         r->after = now;
1474
1475                         calculated_number *cn = rrdr_line_values(r);
1476                         uint8_t *co = rrdr_line_options(r);
1477
1478                         for(rd = st->dimensions, c = 0 ; rd && c < dimensions ; rd = rd->next, c++) {
1479
1480                                 // update the dimension options
1481                                 if(likely(found_non_zero[c])) r->od[c] |= RRDR_NONZERO;
1482
1483                                 // store the specific point options
1484                                 co[c] = group_options[c];
1485
1486                                 // store the value
1487                                 if(unlikely(group_counts[c] == 0)) {
1488                                         cn[c] = 0.0;
1489                                         co[c] |= RRDR_EMPTY;
1490                                 }
1491                                 else if(unlikely(group_method == GROUP_AVERAGE)) {
1492                                         // GROUP_AVERAGE
1493                                         cn[c] = group_values[c] / group_counts[c];
1494                                 }
1495                                 else {
1496                                         // GROUP_SUM
1497                                         // GROUP_MAX
1498                                         // GROUP_INCREMENTAL_SUM
1499                                         cn[c] = group_values[c];
1500                                 }
1501
1502                                 if(cn[c] < r->min) r->min = cn[c];
1503                                 if(cn[c] > r->max) r->max = cn[c];
1504
1505                                 // reset them for the next loop
1506                                 group_values[c] = 0;
1507                                 group_counts[c] = 0;
1508                                 group_options[c] = 0;
1509                         }
1510
1511                         added++;
1512                         group_count = 0;
1513                         add_this = 0;
1514                 }
1515         }
1516
1517         rrdr_done(r);
1518         //info("RRD2RRDR(): %s: END %ld loops made, %ld points generated", st->id, counter, rrdr_rows(r));
1519         //error("SHIFT: %s: wanted %ld points, got %ld", st->id, points, rrdr_rows(r));
1520         return r;
1521 }
1522
1523 int rrd2value(RRDSET *st, BUFFER *wb, calculated_number *n, BUFFER *dimensions, long points, long long after, long long before, int group_method, uint32_t options, time_t *latest_timestamp, int *value_is_null)
1524 {
1525         RRDR *r = rrd2rrdr(st, points, after, before, group_method, !(options & RRDR_OPTION_NOT_ALIGNED));
1526         if(!r) {
1527                 if(value_is_null) *value_is_null = 1;
1528                 return 500;
1529         }
1530
1531         if(rrdr_rows(r) == 0) {
1532                 rrdr_free(r);
1533                 if(value_is_null) *value_is_null = 1;
1534                 return 400;
1535         }
1536
1537         if(r->result_options & RRDR_RESULT_OPTION_RELATIVE)
1538                 wb->options |= WB_CONTENT_NO_CACHEABLE;
1539         else if(r->result_options & RRDR_RESULT_OPTION_ABSOLUTE)
1540                 wb->options |= WB_CONTENT_CACHEABLE;
1541
1542         options = rrdr_check_options(r, options, (dimensions)?buffer_tostring(dimensions):NULL);
1543
1544         if(dimensions)
1545                 rrdr_disable_not_selected_dimensions(r, buffer_tostring(dimensions));
1546
1547         if(latest_timestamp)
1548                 *latest_timestamp = r->before;
1549
1550         long i = (options & RRDR_OPTION_REVERSED)?rrdr_rows(r) - 1:0;
1551         *n = rrdr2value(r, i, options, value_is_null);
1552
1553         rrdr_free(r);
1554         return 200;
1555 }
1556
1557 int rrd2format(RRDSET *st, BUFFER *wb, BUFFER *dimensions, uint32_t format, long points, long long after, long long before, int group_method, uint32_t options, time_t *latest_timestamp)
1558 {
1559         RRDR *r = rrd2rrdr(st, points, after, before, group_method, !(options & RRDR_OPTION_NOT_ALIGNED));
1560         if(!r) {
1561                 buffer_strcat(wb, "Cannot generate output with these parameters on this chart.");
1562                 return 500;
1563         }
1564
1565         if(r->result_options & RRDR_RESULT_OPTION_RELATIVE)
1566                 wb->options |= WB_CONTENT_NO_CACHEABLE;
1567         else if(r->result_options & RRDR_RESULT_OPTION_ABSOLUTE)
1568                 wb->options |= WB_CONTENT_CACHEABLE;
1569
1570         options = rrdr_check_options(r, options, (dimensions)?buffer_tostring(dimensions):NULL);
1571
1572         if(dimensions)
1573                 rrdr_disable_not_selected_dimensions(r, buffer_tostring(dimensions));
1574
1575         if(latest_timestamp && rrdr_rows(r) > 0)
1576                 *latest_timestamp = r->before;
1577
1578         switch(format) {
1579         case DATASOURCE_SSV:
1580                 if(options & RRDR_OPTION_JSON_WRAP) {
1581                         wb->contenttype = CT_APPLICATION_JSON;
1582                         rrdr_json_wrapper_begin(r, wb, format, options, 1);
1583                         rrdr2ssv(r, wb, options, "", " ", "");
1584                         rrdr_json_wrapper_end(r, wb, format, options, 1);
1585                 }
1586                 else {
1587                         wb->contenttype = CT_TEXT_PLAIN;
1588                         rrdr2ssv(r, wb, options, "", " ", "");
1589                 }
1590                 break;
1591
1592         case DATASOURCE_SSV_COMMA:
1593                 if(options & RRDR_OPTION_JSON_WRAP) {
1594                         wb->contenttype = CT_APPLICATION_JSON;
1595                         rrdr_json_wrapper_begin(r, wb, format, options, 1);
1596                         rrdr2ssv(r, wb, options, "", ",", "");
1597                         rrdr_json_wrapper_end(r, wb, format, options, 1);
1598                 }
1599                 else {
1600                         wb->contenttype = CT_TEXT_PLAIN;
1601                         rrdr2ssv(r, wb, options, "", ",", "");
1602                 }
1603                 break;
1604
1605         case DATASOURCE_JS_ARRAY:
1606                 if(options & RRDR_OPTION_JSON_WRAP) {
1607                         wb->contenttype = CT_APPLICATION_JSON;
1608                         rrdr_json_wrapper_begin(r, wb, format, options, 0);
1609                         rrdr2ssv(r, wb, options, "[", ",", "]");
1610                         rrdr_json_wrapper_end(r, wb, format, options, 0);
1611                 }
1612                 else {
1613                         wb->contenttype = CT_APPLICATION_JSON;
1614                         rrdr2ssv(r, wb, options, "[", ",", "]");
1615                 }
1616                 break;
1617
1618         case DATASOURCE_CSV:
1619                 if(options & RRDR_OPTION_JSON_WRAP) {
1620                         wb->contenttype = CT_APPLICATION_JSON;
1621                         rrdr_json_wrapper_begin(r, wb, format, options, 1);
1622                         rrdr2csv(r, wb, options, "", ",", "\\n", "");
1623                         rrdr_json_wrapper_end(r, wb, format, options, 1);
1624                 }
1625                 else {
1626                         wb->contenttype = CT_TEXT_PLAIN;
1627                         rrdr2csv(r, wb, options, "", ",", "\r\n", "");
1628                 }
1629                 break;
1630
1631         case DATASOURCE_CSV_JSON_ARRAY:
1632                 wb->contenttype = CT_APPLICATION_JSON;
1633                 if(options & RRDR_OPTION_JSON_WRAP) {
1634                         rrdr_json_wrapper_begin(r, wb, format, options, 0);
1635                         buffer_strcat(wb, "[\n");
1636                         rrdr2csv(r, wb, options + RRDR_OPTION_LABEL_QUOTES, "[", ",", "]", ",\n");
1637                         buffer_strcat(wb, "\n]");
1638                         rrdr_json_wrapper_end(r, wb, format, options, 0);
1639                 }
1640                 else {
1641                         wb->contenttype = CT_TEXT_PLAIN;
1642                         buffer_strcat(wb, "[\n");
1643                         rrdr2csv(r, wb, options + RRDR_OPTION_LABEL_QUOTES, "[", ",", "]", ",\n");
1644                         buffer_strcat(wb, "\n]");
1645                 }
1646                 break;
1647
1648         case DATASOURCE_TSV:
1649                 if(options & RRDR_OPTION_JSON_WRAP) {
1650                         wb->contenttype = CT_APPLICATION_JSON;
1651                         rrdr_json_wrapper_begin(r, wb, format, options, 1);
1652                         rrdr2csv(r, wb, options, "", "\t", "\\n", "");
1653                         rrdr_json_wrapper_end(r, wb, format, options, 1);
1654                 }
1655                 else {
1656                         wb->contenttype = CT_TEXT_PLAIN;
1657                         rrdr2csv(r, wb, options, "", "\t", "\r\n", "");
1658                 }
1659                 break;
1660
1661         case DATASOURCE_HTML:
1662                 if(options & RRDR_OPTION_JSON_WRAP) {
1663                         wb->contenttype = CT_APPLICATION_JSON;
1664                         rrdr_json_wrapper_begin(r, wb, format, options, 1);
1665                         buffer_strcat(wb, "<html>\\n<center>\\n<table border=\\\"0\\\" cellpadding=\\\"5\\\" cellspacing=\\\"5\\\">\\n");
1666                         rrdr2csv(r, wb, options, "<tr><td>", "</td><td>", "</td></tr>\\n", "");
1667                         buffer_strcat(wb, "</table>\\n</center>\\n</html>\\n");
1668                         rrdr_json_wrapper_end(r, wb, format, options, 1);
1669                 }
1670                 else {
1671                         wb->contenttype = CT_TEXT_HTML;
1672                         buffer_strcat(wb, "<html>\n<center>\n<table border=\"0\" cellpadding=\"5\" cellspacing=\"5\">\n");
1673                         rrdr2csv(r, wb, options, "<tr><td>", "</td><td>", "</td></tr>\n", "");
1674                         buffer_strcat(wb, "</table>\n</center>\n</html>\n");
1675                 }
1676                 break;
1677
1678         case DATASOURCE_DATATABLE_JSONP:
1679                 wb->contenttype = CT_APPLICATION_X_JAVASCRIPT;
1680
1681                 if(options & RRDR_OPTION_JSON_WRAP)
1682                         rrdr_json_wrapper_begin(r, wb, format, options, 0);
1683
1684                 rrdr2json(r, wb, options, 1);
1685
1686                 if(options & RRDR_OPTION_JSON_WRAP)
1687                         rrdr_json_wrapper_end(r, wb, format, options, 0);
1688                 break;
1689
1690         case DATASOURCE_DATATABLE_JSON:
1691                 wb->contenttype = CT_APPLICATION_JSON;
1692
1693                 if(options & RRDR_OPTION_JSON_WRAP)
1694                         rrdr_json_wrapper_begin(r, wb, format, options, 0);
1695
1696                 rrdr2json(r, wb, options, 1);
1697
1698                 if(options & RRDR_OPTION_JSON_WRAP)
1699                         rrdr_json_wrapper_end(r, wb, format, options, 0);
1700                 break;
1701
1702         case DATASOURCE_JSONP:
1703                 wb->contenttype = CT_APPLICATION_X_JAVASCRIPT;
1704                 if(options & RRDR_OPTION_JSON_WRAP)
1705                         rrdr_json_wrapper_begin(r, wb, format, options, 0);
1706
1707                 rrdr2json(r, wb, options, 0);
1708
1709                 if(options & RRDR_OPTION_JSON_WRAP)
1710                         rrdr_json_wrapper_end(r, wb, format, options, 0);
1711                 break;
1712
1713         case DATASOURCE_JSON:
1714         default:
1715                 wb->contenttype = CT_APPLICATION_JSON;
1716
1717                 if(options & RRDR_OPTION_JSON_WRAP)
1718                         rrdr_json_wrapper_begin(r, wb, format, options, 0);
1719
1720                 rrdr2json(r, wb, options, 0);
1721
1722                 if(options & RRDR_OPTION_JSON_WRAP)
1723                         rrdr_json_wrapper_end(r, wb, format, options, 0);
1724                 break;
1725         }
1726
1727         rrdr_free(r);
1728         return 200;
1729 }
1730
1731 time_t rrd_stats_json(int type, RRDSET *st, BUFFER *wb, long points, long group, int group_method, time_t after, time_t before, int only_non_zero)
1732 {
1733         int c;
1734         pthread_rwlock_rdlock(&st->rwlock);
1735
1736
1737         // -------------------------------------------------------------------------
1738         // switch from JSON to google JSON
1739
1740         char kq[2] = "\"";
1741         char sq[2] = "\"";
1742         switch(type) {
1743                 case DATASOURCE_DATATABLE_JSON:
1744                 case DATASOURCE_DATATABLE_JSONP:
1745                         kq[0] = '\0';
1746                         sq[0] = '\'';
1747                         break;
1748
1749                 case DATASOURCE_JSON:
1750                 default:
1751                         break;
1752         }
1753
1754
1755         // -------------------------------------------------------------------------
1756         // validate the parameters
1757
1758         if(points < 1) points = 1;
1759         if(group < 1) group = 1;
1760
1761         if(before == 0 || before > rrdset_last_entry_t(st)) before = rrdset_last_entry_t(st);
1762         if(after  == 0 || after < rrdset_first_entry_t(st)) after = rrdset_first_entry_t(st);
1763
1764         // ---
1765
1766         // our return value (the last timestamp printed)
1767         // this is required to detect re-transmit in google JSONP
1768         time_t last_timestamp = 0;
1769
1770
1771         // -------------------------------------------------------------------------
1772         // find how many dimensions we have
1773
1774         int dimensions = 0;
1775         RRDDIM *rd;
1776         for( rd = st->dimensions ; rd ; rd = rd->next) dimensions++;
1777         if(!dimensions) {
1778                 pthread_rwlock_unlock(&st->rwlock);
1779                 buffer_strcat(wb, "No dimensions yet.");
1780                 return 0;
1781         }
1782
1783
1784         // -------------------------------------------------------------------------
1785         // prepare various strings, to speed up the loop
1786
1787         char overflow_annotation[201]; snprintfz(overflow_annotation, 200, ",{%sv%s:%sRESET OR OVERFLOW%s},{%sv%s:%sThe counters have been wrapped.%s}", kq, kq, sq, sq, kq, kq, sq, sq);
1788         char normal_annotation[201];   snprintfz(normal_annotation,   200, ",{%sv%s:null},{%sv%s:null}", kq, kq, kq, kq);
1789         char pre_date[51];             snprintfz(pre_date,             50, "            {%sc%s:[{%sv%s:%s", kq, kq, kq, kq, sq);
1790         char post_date[21];            snprintfz(post_date,            20, "%s}", sq);
1791         char pre_value[21];            snprintfz(pre_value,            20, ",{%sv%s:", kq, kq);
1792         char post_value[21];           strcpy(post_value,                  "}");
1793
1794
1795         // -------------------------------------------------------------------------
1796         // checks for debugging
1797
1798         if(st->debug) {
1799                 debug(D_RRD_STATS, "%s first_entry_t = %ld, last_entry_t = %ld, duration = %ld, after = %ld, before = %ld, duration = %ld, entries_to_show = %ld, group = %ld"
1800                         , st->id
1801                         , rrdset_first_entry_t(st)
1802                         , rrdset_last_entry_t(st)
1803                         , rrdset_last_entry_t(st) - rrdset_first_entry_t(st)
1804                         , after
1805                         , before
1806                         , before - after
1807                         , points
1808                         , group
1809                         );
1810
1811                 if(before < after)
1812                         debug(D_RRD_STATS, "WARNING: %s The newest value in the database (%ld) is earlier than the oldest (%ld)", st->name, before, after);
1813
1814                 if((before - after) > st->entries * st->update_every)
1815                         debug(D_RRD_STATS, "WARNING: %s The time difference between the oldest and the newest entries (%ld) is higher than the capacity of the database (%ld)", st->name, before - after, st->entries * st->update_every);
1816         }
1817
1818
1819         // -------------------------------------------------------------------------
1820         // temp arrays for keeping values per dimension
1821
1822         calculated_number group_values[dimensions]; // keep sums when grouping
1823         int               print_hidden[dimensions]; // keep hidden flags
1824         int               found_non_zero[dimensions];
1825         int               found_non_existing[dimensions];
1826
1827         // initialize them
1828         for( rd = st->dimensions, c = 0 ; rd && c < dimensions ; rd = rd->next, c++) {
1829                 group_values[c] = 0;
1830                 print_hidden[c] = (rd->flags & RRDDIM_FLAG_HIDDEN)?1:0;
1831                 found_non_zero[c] = 0;
1832                 found_non_existing[c] = 0;
1833         }
1834
1835
1836         // error("OLD: points=%d after=%d before=%d group=%d, duration=%d", entries_to_show, before - (st->update_every * group * entries_to_show), before, group, before - after + 1);
1837         // rrd2array(st, entries_to_show, before - (st->update_every * group * entries_to_show), before, group_method, only_non_zero);
1838         // rrd2rrdr(st, entries_to_show, before - (st->update_every * group * entries_to_show), before, group_method);
1839
1840         // -------------------------------------------------------------------------
1841         // remove dimensions that contain only zeros
1842
1843         int max_loop = 1;
1844         if(only_non_zero) max_loop = 2;
1845
1846         for(; max_loop ; max_loop--) {
1847
1848                 // -------------------------------------------------------------------------
1849                 // print the JSON header
1850
1851                 buffer_sprintf(wb, "{\n %scols%s:\n     [\n", kq, kq);
1852                 buffer_sprintf(wb, "            {%sid%s:%s%s,%slabel%s:%stime%s,%spattern%s:%s%s,%stype%s:%sdatetime%s},\n", kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq);
1853                 buffer_sprintf(wb, "            {%sid%s:%s%s,%slabel%s:%s%s,%spattern%s:%s%s,%stype%s:%sstring%s,%sp%s:{%srole%s:%sannotation%s}},\n", kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, kq, kq, sq, sq);
1854                 buffer_sprintf(wb, "            {%sid%s:%s%s,%slabel%s:%s%s,%spattern%s:%s%s,%stype%s:%sstring%s,%sp%s:{%srole%s:%sannotationText%s}}", kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, kq, kq, sq, sq);
1855
1856                 // print the header for each dimension
1857                 // and update the print_hidden array for the dimensions that should be hidden
1858                 int pc = 0;
1859                 for( rd = st->dimensions, c = 0 ; rd && c < dimensions ; rd = rd->next, c++) {
1860                         if(!print_hidden[c]) {
1861                                 pc++;
1862                                 buffer_sprintf(wb, ",\n         {%sid%s:%s%s,%slabel%s:%s%s%s,%spattern%s:%s%s,%stype%s:%snumber%s}", kq, kq, sq, sq, kq, kq, sq, rd->name, sq, kq, kq, sq, sq, kq, kq, sq, sq);
1863                         }
1864                 }
1865                 if(!pc) {
1866                         buffer_sprintf(wb, ",\n         {%sid%s:%s%s,%slabel%s:%s%s%s,%spattern%s:%s%s,%stype%s:%snumber%s}", kq, kq, sq, sq, kq, kq, sq, "no data", sq, kq, kq, sq, sq, kq, kq, sq, sq);
1867                 }
1868
1869                 // print the begin of row data
1870                 buffer_sprintf(wb, "\n  ],\n    %srows%s:\n     [\n", kq, kq);
1871
1872
1873                 // -------------------------------------------------------------------------
1874                 // the main loop
1875
1876                 int annotate_reset = 0;
1877                 int annotation_count = 0;
1878
1879                 long    t = rrdset_time2slot(st, before),
1880                                 stop_at_t = rrdset_time2slot(st, after),
1881                                 stop_now = 0;
1882
1883                 t -= t % group;
1884
1885                 time_t  now = rrdset_slot2time(st, t),
1886                                 dt = st->update_every;
1887
1888                 long count = 0, printed = 0, group_count = 0;
1889                 last_timestamp = 0;
1890
1891                 if(st->debug) debug(D_RRD_STATS, "%s: REQUEST after:%u before:%u, points:%ld, group:%ld, CHART cur:%ld first: %u last:%u, CALC start_t:%ld, stop_t:%ld"
1892                                         , st->id
1893                                         , (uint32_t)after
1894                                         , (uint32_t)before
1895                                         , points
1896                                         , group
1897                                         , st->current_entry
1898                                         , (uint32_t)rrdset_first_entry_t(st)
1899                                         , (uint32_t)rrdset_last_entry_t(st)
1900                                         , t
1901                                         , stop_at_t
1902                                         );
1903
1904                 long counter = 0;
1905                 for(; !stop_now ; now -= dt, t--, counter++) {
1906                         if(t < 0) t = st->entries - 1;
1907                         if(t == stop_at_t) stop_now = counter;
1908
1909                         int print_this = 0;
1910
1911                         if(st->debug) debug(D_RRD_STATS, "%s t = %ld, count = %ld, group_count = %ld, printed = %ld, now = %ld, %s %s"
1912                                         , st->id
1913                                         , t
1914                                         , count + 1
1915                                         , group_count + 1
1916                                         , printed
1917                                         , now
1918                                         , (group_count + 1 == group)?"PRINT":"  -  "
1919                                         , (now >= after && now <= before)?"RANGE":"  -  "
1920                                         );
1921
1922
1923                         // make sure we return data in the proper time range
1924                         if(now > before) continue;
1925                         if(now < after) break;
1926
1927                         //if(rrdset_slot2time(st, t) != now)
1928                         //      error("%s: slot=%ld, now=%ld, slot2time=%ld, diff=%ld, last_entry_t=%ld, rrdset_last_slot=%ld", st->id, t, now, rrdset_slot2time(st,t), now - rrdset_slot2time(st,t), rrdset_last_entry_t(st), rrdset_last_slot(st));
1929
1930                         count++;
1931                         group_count++;
1932
1933                         // check if we have to print this now
1934                         if(group_count == group) {
1935                                 if(printed >= points) {
1936                                         // debug(D_RRD_STATS, "Already printed all rows. Stopping.");
1937                                         break;
1938                                 }
1939
1940                                 // generate the local date time
1941                                 struct tm tmbuf, *tm = localtime_r(&now, &tmbuf);
1942                                 if(!tm) { error("localtime() failed."); continue; }
1943                                 if(now > last_timestamp) last_timestamp = now;
1944
1945                                 if(printed) buffer_strcat(wb, "]},\n");
1946                                 buffer_strcat(wb, pre_date);
1947                                 buffer_jsdate(wb, tm->tm_year + 1900, tm->tm_mon, tm->tm_mday, tm->tm_hour, tm->tm_min, tm->tm_sec);
1948                                 buffer_strcat(wb, post_date);
1949
1950                                 print_this = 1;
1951                         }
1952
1953                         // do the calculations
1954                         for(rd = st->dimensions, c = 0 ; rd && c < dimensions ; rd = rd->next, c++) {
1955                                 storage_number n = rd->values[t];
1956                                 calculated_number value = unpack_storage_number(n);
1957
1958                                 if(!does_storage_number_exist(n)) {
1959                                         value = 0.0;
1960                                         found_non_existing[c]++;
1961                                 }
1962                                 if(did_storage_number_reset(n)) annotate_reset = 1;
1963
1964                                 switch(group_method) {
1965                                         case GROUP_MAX:
1966                                                 if(abs(value) > abs(group_values[c])) group_values[c] = value;
1967                                                 break;
1968
1969                                         case GROUP_SUM:
1970                                                 group_values[c] += value;
1971                                                 break;
1972
1973                                         default:
1974                                         case GROUP_AVERAGE:
1975                                                 group_values[c] += value;
1976                                                 if(print_this) group_values[c] /= ( group_count - found_non_existing[c] );
1977                                                 break;
1978                                 }
1979                         }
1980
1981                         if(print_this) {
1982                                 if(annotate_reset) {
1983                                         annotation_count++;
1984                                         buffer_strcat(wb, overflow_annotation);
1985                                         annotate_reset = 0;
1986                                 }
1987                                 else
1988                                         buffer_strcat(wb, normal_annotation);
1989
1990                                 pc = 0;
1991                                 for(c = 0 ; c < dimensions ; c++) {
1992                                         if(found_non_existing[c] == group_count) {
1993                                                 // all entries are non-existing
1994                                                 pc++;
1995                                                 buffer_strcat(wb, pre_value);
1996                                                 buffer_strcat(wb, "null");
1997                                                 buffer_strcat(wb, post_value);
1998                                         }
1999                                         else if(!print_hidden[c]) {
2000                                                 pc++;
2001                                                 buffer_strcat(wb, pre_value);
2002                                                 buffer_rrd_value(wb, group_values[c]);
2003                                                 buffer_strcat(wb, post_value);
2004
2005                                                 if(group_values[c]) found_non_zero[c]++;
2006                                         }
2007
2008                                         // reset them for the next loop
2009                                         group_values[c] = 0;
2010                                         found_non_existing[c] = 0;
2011                                 }
2012
2013                                 // if all dimensions are hidden, print a null
2014                                 if(!pc) {
2015                                         buffer_strcat(wb, pre_value);
2016                                         buffer_strcat(wb, "null");
2017                                         buffer_strcat(wb, post_value);
2018                                 }
2019
2020                                 printed++;
2021                                 group_count = 0;
2022                         }
2023                 }
2024
2025                 if(printed) buffer_strcat(wb, "]}");
2026                 buffer_strcat(wb, "\n   ]\n}\n");
2027
2028                 if(only_non_zero && max_loop > 1) {
2029                         int changed = 0;
2030                         for(rd = st->dimensions, c = 0 ; rd && c < dimensions ; rd = rd->next, c++) {
2031                                 group_values[c] = 0;
2032                                 found_non_existing[c] = 0;
2033
2034                                 if(!print_hidden[c] && !found_non_zero[c]) {
2035                                         changed = 1;
2036                                         print_hidden[c] = 1;
2037                                 }
2038                         }
2039
2040                         if(changed) buffer_flush(wb);
2041                         else break;
2042                 }
2043                 else break;
2044
2045         } // max_loop
2046
2047         debug(D_RRD_STATS, "RRD_STATS_JSON: %s total %lu bytes", st->name, wb->len);
2048
2049         pthread_rwlock_unlock(&st->rwlock);
2050         return last_timestamp;
2051 }