src/rrd2json.c

   1 #include "common.h"
   2
   3 void rrd_stats_api_v1_chart_with_data(RRDSET *st, BUFFER *wb, size_t *dimensions_count, size_t *memory_used)
   4 {
   5     rrdset_rdlock(st);
   6
   7     buffer_sprintf(wb,
   8         "\t\t{\n"
   9         "\t\t\t\"id\": \"%s\",\n"
  10         "\t\t\t\"name\": \"%s\",\n"
  11         "\t\t\t\"type\": \"%s\",\n"
  12         "\t\t\t\"family\": \"%s\",\n"
  13         "\t\t\t\"context\": \"%s\",\n"
  14         "\t\t\t\"title\": \"%s\",\n"
  15         "\t\t\t\"priority\": %ld,\n"
  16         "\t\t\t\"enabled\": %s,\n"
  17         "\t\t\t\"units\": \"%s\",\n"
  18         "\t\t\t\"data_url\": \"/api/v1/data?chart=%s\",\n"
  19         "\t\t\t\"chart_type\": \"%s\",\n"
  20         "\t\t\t\"duration\": %ld,\n"
  21         "\t\t\t\"first_entry\": %ld,\n"
  22         "\t\t\t\"last_entry\": %ld,\n"
  23         "\t\t\t\"update_every\": %d,\n"
  24         "\t\t\t\"dimensions\": {\n"
  25         , st->id
  26         , st->name
  27         , st->type
  28         , st->family
  29         , st->context
  30         , st->title
  31         , st->priority
  32         , rrdset_flag_check(st, RRDSET_FLAG_ENABLED)?"true":"false"
  33         , st->units
  34         , st->name
  35         , rrdset_type_name(st->chart_type)
  36         , st->entries * st->update_every
  37         , rrdset_first_entry_t(st)
  38         , rrdset_last_entry_t(st)
  39         , st->update_every
  40         );
  41
  42     unsigned long memory = st->memsize;
  43
  44     size_t dimensions = 0;
  45     RRDDIM *rd;
  46     rrddim_foreach_read(rd, st) {
  47         if(rrddim_flag_check(rd, RRDDIM_FLAG_HIDDEN)) continue;
  48
  49         memory += rd->memsize;
  50
  51         buffer_sprintf(wb,
  52             "%s"
  53             "\t\t\t\t\"%s\": { \"name\": \"%s\" }"
  54             , dimensions?",\n":""
  55             , rd->id
  56             , rd->name
  57             );
  58
  59         dimensions++;
  60     }
  61
  62     if(dimensions_count) *dimensions_count += dimensions;
  63     if(memory_used) *memory_used += memory;
  64
  65     buffer_strcat(wb, "\n\t\t\t},\n\t\t\t\"green\": ");
  66     buffer_rrd_value(wb, st->green);
  67     buffer_strcat(wb, ",\n\t\t\t\"red\": ");
  68     buffer_rrd_value(wb, st->red);
  69
  70     buffer_sprintf(wb,
  71         "\n\t\t}"
  72         );
  73
  74     rrdset_unlock(st);
  75 }
  76
  77 void rrd_stats_api_v1_chart(RRDSET *st, BUFFER *wb) {
  78     rrd_stats_api_v1_chart_with_data(st, wb, NULL, NULL);
  79 }
  80
  81 void rrd_stats_api_v1_charts(RRDHOST *host, BUFFER *wb)
  82 {
  83     size_t c, dimensions = 0, memory = 0, alarms = 0;
  84     RRDSET *st;
  85
  86     buffer_sprintf(wb, "{\n"
  87            "\t\"hostname\": \"%s\""
  88         ",\n\t\"version\": \"%s\""
  89         ",\n\t\"os\": \"%s\""
  90         ",\n\t\"update_every\": %d"
  91         ",\n\t\"history\": %d"
  92         ",\n\t\"charts\": {"
  93         , host->hostname
  94         , program_version
  95         , host->os
  96         , host->rrd_update_every
  97         , host->rrd_history_entries
  98         );
  99
 100     c = 0;
 101     rrdhost_rdlock(host);
 102     rrdset_foreach_read(st, host) {
 103         if(rrdset_flag_check(st, RRDSET_FLAG_ENABLED) && st->dimensions) {
 104             if(c) buffer_strcat(wb, ",");
 105             buffer_strcat(wb, "\n\t\t\"");
 106             buffer_strcat(wb, st->id);
 107             buffer_strcat(wb, "\": ");
 108             rrd_stats_api_v1_chart_with_data(st, wb, &dimensions, &memory);
 109             c++;
 110         }
 111     }
 112
 113     RRDCALC *rc;
 114     for(rc = host->alarms; rc ; rc = rc->next) {
 115         if(rc->rrdset)
 116             alarms++;
 117     }
 118     rrdhost_unlock(host);
 119
 120     buffer_sprintf(wb, "\n\t}"
 121                     ",\n\t\"charts_count\": %zu"
 122                     ",\n\t\"dimensions_count\": %zu"
 123                     ",\n\t\"alarms_count\": %zu"
 124                     ",\n\t\"rrd_memory_bytes\": %zu"
 125                     ",\n\t\"hosts_count\": %zu"
 126                     ",\n\t\"hosts\": ["
 127                    , c
 128                    , dimensions
 129                    , alarms
 130                    , memory
 131                    , rrd_hosts_available
 132     );
 133
 134     if(unlikely(rrd_hosts_available > 1)) {
 135         rrd_rdlock();
 136         RRDHOST *h;
 137         rrdhost_foreach_read(h)
 138             buffer_sprintf(wb,
 139                     "%s\n\t\t{"
 140                     "\n\t\t\t\"hostname\": \"%s\""
 141                     "\n\t\t}"
 142                     , (h != localhost)?",":""
 143                     , h->hostname
 144             );
 145         rrd_unlock();
 146     }
 147     else {
 148         buffer_sprintf(wb,
 149                 "\n\t\t{"
 150                 "\n\t\t\t\"hostname\": \"%s\""
 151                 "\n\t\t}"
 152                 , host->hostname
 153         );
 154     }
 155
 156     buffer_sprintf(wb, "\n\t]\n}\n");
 157 }
 158
 159 // ----------------------------------------------------------------------------
 160 // PROMETHEUS
 161 // /api/v1/allmetrics?format=prometheus
 162
 163 static inline size_t prometheus_name_copy(char *d, const char *s, size_t usable) {
 164     size_t n;
 165
 166     for(n = 0; *s && n < usable ; d++, s++, n++) {
 167         register char c = *s;
 168
 169         if(unlikely(!isalnum(c))) *d = '_';
 170         else *d = c;
 171     }
 172     *d = '\0';
 173
 174     return n;
 175 }
 176
 177 #define PROMETHEUS_ELEMENT_MAX 256
 178
 179 void rrd_stats_api_v1_charts_allmetrics_prometheus(RRDHOST *host, BUFFER *wb) {
 180     rrdhost_rdlock(host);
 181
 182     char hostname[PROMETHEUS_ELEMENT_MAX + 1];
 183     prometheus_name_copy(hostname, host->hostname, PROMETHEUS_ELEMENT_MAX);
 184
 185     // for each chart
 186     RRDSET *st;
 187     rrdset_foreach_read(st, host) {
 188         char chart[PROMETHEUS_ELEMENT_MAX + 1];
 189         prometheus_name_copy(chart, st->id, PROMETHEUS_ELEMENT_MAX);
 190
 191         buffer_strcat(wb, "\n");
 192         if(rrdset_flag_check(st, RRDSET_FLAG_ENABLED) && st->dimensions) {
 193             rrdset_rdlock(st);
 194
 195             // for each dimension
 196             RRDDIM *rd;
 197             rrddim_foreach_read(rd, st) {
 198                 if(rd->collections_counter) {
 199                     char dimension[PROMETHEUS_ELEMENT_MAX + 1];
 200                     prometheus_name_copy(dimension, rd->id, PROMETHEUS_ELEMENT_MAX);
 201
 202                     // buffer_sprintf(wb, "# HELP %s.%s %s\n", st->id, rd->id, st->units);
 203
 204                     switch(rd->algorithm) {
 205                         case RRD_ALGORITHM_INCREMENTAL:
 206                         case RRD_ALGORITHM_PCENT_OVER_DIFF_TOTAL:
 207                             buffer_sprintf(wb, "# TYPE %s_%s counter\n", chart, dimension);
 208                             break;
 209
 210                         default:
 211                             buffer_sprintf(wb, "# TYPE %s_%s gauge\n", chart, dimension);
 212                             break;
 213                     }
 214
 215                     // calculated_number n = (calculated_number)rd->last_collected_value * (calculated_number)(abs(rd->multiplier)) / (calculated_number)(abs(rd->divisor));
 216                     // buffer_sprintf(wb, "%s.%s " CALCULATED_NUMBER_FORMAT " %llu\n", st->id, rd->id, n,
 217                     //        (unsigned long long)((rd->last_collected_time.tv_sec * 1000) + (rd->last_collected_time.tv_usec / 1000)));
 218
 219                     buffer_sprintf(wb, "%s_%s{instance=\"%s\"} " COLLECTED_NUMBER_FORMAT " %llu\n",
 220                             chart, dimension, hostname, rd->last_collected_value,
 221                             (unsigned long long)((rd->last_collected_time.tv_sec * 1000) + (rd->last_collected_time.tv_usec / 1000)));
 222
 223                 }
 224             }
 225
 226             rrdset_unlock(st);
 227         }
 228     }
 229
 230     rrdhost_unlock(host);
 231 }
 232
 233 // ----------------------------------------------------------------------------
 234 // BASH
 235 // /api/v1/allmetrics?format=bash
 236
 237 static inline size_t shell_name_copy(char *d, const char *s, size_t usable) {
 238     size_t n;
 239
 240     for(n = 0; *s && n < usable ; d++, s++, n++) {
 241         register char c = *s;
 242
 243         if(unlikely(!isalnum(c))) *d = '_';
 244         else *d = (char)toupper(c);
 245     }
 246     *d = '\0';
 247
 248     return n;
 249 }
 250
 251 #define SHELL_ELEMENT_MAX 100
 252
 253 void rrd_stats_api_v1_charts_allmetrics_shell(RRDHOST *host, BUFFER *wb) {
 254     rrdhost_rdlock(host);
 255
 256     // for each chart
 257     RRDSET *st;
 258     rrdset_foreach_read(st, host) {
 259         calculated_number total = 0.0;
 260         char chart[SHELL_ELEMENT_MAX + 1];
 261         shell_name_copy(chart, st->id, SHELL_ELEMENT_MAX);
 262
 263         buffer_sprintf(wb, "\n# chart: %s (name: %s)\n", st->id, st->name);
 264         if(rrdset_flag_check(st, RRDSET_FLAG_ENABLED) && st->dimensions) {
 265             rrdset_rdlock(st);
 266
 267             // for each dimension
 268             RRDDIM *rd;
 269             rrddim_foreach_read(rd, st) {
 270                 if(rd->collections_counter) {
 271                     char dimension[SHELL_ELEMENT_MAX + 1];
 272                     shell_name_copy(dimension, rd->id, SHELL_ELEMENT_MAX);
 273
 274                     calculated_number n = rd->last_stored_value;
 275
 276                     if(isnan(n) || isinf(n))
 277                         buffer_sprintf(wb, "NETDATA_%s_%s=\"\"      # %s\n", chart, dimension, st->units);
 278                     else {
 279                         if(rd->multiplier < 0 || rd->divisor < 0) n = -n;
 280                         n = roundl(n);
 281                         if(!rrddim_flag_check(rd, RRDDIM_FLAG_HIDDEN)) total += n;
 282                         buffer_sprintf(wb, "NETDATA_%s_%s=\"%0.0Lf\"      # %s\n", chart, dimension, n, st->units);
 283                     }
 284                 }
 285             }
 286
 287             total = roundl(total);
 288             buffer_sprintf(wb, "NETDATA_%s_VISIBLETOTAL=\"%0.0Lf\"      # %s\n", chart, total, st->units);
 289             rrdset_unlock(st);
 290         }
 291     }
 292
 293     buffer_strcat(wb, "\n# NETDATA ALARMS RUNNING\n");
 294
 295     RRDCALC *rc;
 296     for(rc = host->alarms; rc ;rc = rc->next) {
 297         if(!rc->rrdset) continue;
 298
 299         char chart[SHELL_ELEMENT_MAX + 1];
 300         shell_name_copy(chart, rc->rrdset->id, SHELL_ELEMENT_MAX);
 301
 302         char alarm[SHELL_ELEMENT_MAX + 1];
 303         shell_name_copy(alarm, rc->name, SHELL_ELEMENT_MAX);
 304
 305         calculated_number n = rc->value;
 306
 307         if(isnan(n) || isinf(n))
 308             buffer_sprintf(wb, "NETDATA_ALARM_%s_%s_VALUE=\"\"      # %s\n", chart, alarm, rc->units);
 309         else {
 310             n = roundl(n);
 311             buffer_sprintf(wb, "NETDATA_ALARM_%s_%s_VALUE=\"%0.0Lf\"      # %s\n", chart, alarm, n, rc->units);
 312         }
 313
 314         buffer_sprintf(wb, "NETDATA_ALARM_%s_%s_STATUS=\"%s\"\n", chart, alarm, rrdcalc_status2string(rc->status));
 315     }
 316
 317     rrdhost_unlock(host);
 318 }
 319
 320 // ----------------------------------------------------------------------------
 321
 322 unsigned long rrd_stats_one_json(RRDSET *st, char *options, BUFFER *wb)
 323 {
 324     time_t now = now_realtime_sec();
 325
 326     rrdset_rdlock(st);
 327
 328     buffer_sprintf(wb,
 329         "\t\t{\n"
 330         "\t\t\t\"id\": \"%s\",\n"
 331         "\t\t\t\"name\": \"%s\",\n"
 332         "\t\t\t\"type\": \"%s\",\n"
 333         "\t\t\t\"family\": \"%s\",\n"
 334         "\t\t\t\"context\": \"%s\",\n"
 335         "\t\t\t\"title\": \"%s\",\n"
 336         "\t\t\t\"priority\": %ld,\n"
 337         "\t\t\t\"enabled\": %d,\n"
 338         "\t\t\t\"units\": \"%s\",\n"
 339         "\t\t\t\"url\": \"/data/%s/%s\",\n"
 340         "\t\t\t\"chart_type\": \"%s\",\n"
 341         "\t\t\t\"counter\": %lu,\n"
 342         "\t\t\t\"entries\": %ld,\n"
 343         "\t\t\t\"first_entry_t\": %ld,\n"
 344         "\t\t\t\"last_entry\": %lu,\n"
 345         "\t\t\t\"last_entry_t\": %ld,\n"
 346         "\t\t\t\"last_entry_secs_ago\": %ld,\n"
 347         "\t\t\t\"update_every\": %d,\n"
 348         "\t\t\t\"isdetail\": %d,\n"
 349         "\t\t\t\"usec_since_last_update\": %llu,\n"
 350         "\t\t\t\"collected_total\": " TOTAL_NUMBER_FORMAT ",\n"
 351         "\t\t\t\"last_collected_total\": " TOTAL_NUMBER_FORMAT ",\n"
 352         "\t\t\t\"dimensions\": [\n"
 353         , st->id
 354         , st->name
 355         , st->type
 356         , st->family
 357         , st->context
 358         , st->title
 359         , st->priority
 360         , rrdset_flag_check(st, RRDSET_FLAG_ENABLED)?1:0
 361         , st->units
 362         , st->name, options?options:""
 363         , rrdset_type_name(st->chart_type)
 364         , st->counter
 365         , st->entries
 366         , rrdset_first_entry_t(st)
 367         , rrdset_last_slot(st)
 368         , rrdset_last_entry_t(st)
 369         , (now < rrdset_last_entry_t(st)) ? (time_t)0 : now - rrdset_last_entry_t(st)
 370         , st->update_every
 371         , rrdset_flag_check(st, RRDSET_FLAG_DETAIL)?1:0
 372         , st->usec_since_last_update
 373         , st->collected_total
 374         , st->last_collected_total
 375         );
 376
 377     unsigned long memory = st->memsize;
 378
 379     RRDDIM *rd;
 380     rrddim_foreach_read(rd, st) {
 381
 382         memory += rd->memsize;
 383
 384         buffer_sprintf(wb,
 385             "\t\t\t\t{\n"
 386             "\t\t\t\t\t\"id\": \"%s\",\n"
 387             "\t\t\t\t\t\"name\": \"%s\",\n"
 388             "\t\t\t\t\t\"entries\": %ld,\n"
 389             "\t\t\t\t\t\"isHidden\": %d,\n"
 390             "\t\t\t\t\t\"algorithm\": \"%s\",\n"
 391             "\t\t\t\t\t\"multiplier\": " COLLECTED_NUMBER_FORMAT ",\n"
 392             "\t\t\t\t\t\"divisor\": " COLLECTED_NUMBER_FORMAT ",\n"
 393             "\t\t\t\t\t\"last_entry_t\": %ld,\n"
 394             "\t\t\t\t\t\"collected_value\": " COLLECTED_NUMBER_FORMAT ",\n"
 395             "\t\t\t\t\t\"calculated_value\": " CALCULATED_NUMBER_FORMAT ",\n"
 396             "\t\t\t\t\t\"last_collected_value\": " COLLECTED_NUMBER_FORMAT ",\n"
 397             "\t\t\t\t\t\"last_calculated_value\": " CALCULATED_NUMBER_FORMAT ",\n"
 398             "\t\t\t\t\t\"memory\": %lu\n"
 399             "\t\t\t\t}%s\n"
 400             , rd->id
 401             , rd->name
 402             , rd->entries
 403             , rrddim_flag_check(rd, RRDDIM_FLAG_HIDDEN)?1:0
 404             , rrd_algorithm_name(rd->algorithm)
 405             , rd->multiplier
 406             , rd->divisor
 407             , rd->last_collected_time.tv_sec
 408             , rd->collected_value
 409             , rd->calculated_value
 410             , rd->last_collected_value
 411             , rd->last_calculated_value
 412             , rd->memsize
 413             , rd->next?",":""
 414             );
 415     }
 416
 417     buffer_sprintf(wb,
 418         "\t\t\t],\n"
 419         "\t\t\t\"memory\" : %lu\n"
 420         "\t\t}"
 421         , memory
 422         );
 423
 424     rrdset_unlock(st);
 425     return memory;
 426 }
 427
 428 #define RRD_GRAPH_JSON_HEADER "{\n\t\"charts\": [\n"
 429 #define RRD_GRAPH_JSON_FOOTER "\n\t]\n}\n"
 430
 431 void rrd_stats_graph_json(RRDSET *st, char *options, BUFFER *wb)
 432 {
 433     buffer_strcat(wb, RRD_GRAPH_JSON_HEADER);
 434     rrd_stats_one_json(st, options, wb);
 435     buffer_strcat(wb, RRD_GRAPH_JSON_FOOTER);
 436 }
 437
 438 void rrd_stats_all_json(RRDHOST *host, BUFFER *wb)
 439 {
 440     unsigned long memory = 0;
 441     long c = 0;
 442     RRDSET *st;
 443
 444     buffer_strcat(wb, RRD_GRAPH_JSON_HEADER);
 445
 446     rrdhost_rdlock(host);
 447     rrdset_foreach_read(st, host) {
 448         if(rrdset_flag_check(st, RRDSET_FLAG_ENABLED) && st->dimensions) {
 449             if(c) buffer_strcat(wb, ",\n");
 450             memory += rrd_stats_one_json(st, NULL, wb);
 451             c++;
 452         }
 453     }
 454     rrdhost_unlock(host);
 455
 456     buffer_sprintf(wb, "\n\t],\n"
 457         "\t\"hostname\": \"%s\",\n"
 458         "\t\"update_every\": %d,\n"
 459         "\t\"history\": %d,\n"
 460         "\t\"memory\": %lu\n"
 461         "}\n"
 462         , host->hostname
 463         , host->rrd_update_every
 464         , host->rrd_history_entries
 465         , memory
 466         );
 467 }
 468
 469
 470
 471 // ----------------------------------------------------------------------------
 472
 473 // RRDR dimension options
 474 #define RRDR_EMPTY      0x01 // the dimension contains / the value is empty (null)
 475 #define RRDR_RESET      0x02 // the dimension contains / the value is reset
 476 #define RRDR_HIDDEN     0x04 // the dimension contains / the value is hidden
 477 #define RRDR_NONZERO    0x08 // the dimension contains / the value is non-zero
 478 #define RRDR_SELECTED   0x10 // the dimension is selected
 479
 480 // RRDR result options
 481 #define RRDR_RESULT_OPTION_ABSOLUTE 0x00000001
 482 #define RRDR_RESULT_OPTION_RELATIVE 0x00000002
 483
 484 typedef struct rrdresult {
 485     RRDSET *st;         // the chart this result refers to
 486
 487     uint32_t result_options;    // RRDR_RESULT_OPTION_*
 488
 489     int d;                  // the number of dimensions
 490     long n;                 // the number of values in the arrays
 491     long rows;              // the number of rows used
 492
 493     uint8_t *od;            // the options for the dimensions
 494
 495     time_t *t;              // array of n timestamps
 496     calculated_number *v;   // array n x d values
 497     uint8_t *o;             // array n x d options
 498
 499     long c;                 // current line ( -1 ~ n ), ( -1 = none, use rrdr_rows() to get number of rows )
 500
 501     long group;             // how many collected values were grouped for each row
 502     int update_every;       // what is the suggested update frequency in seconds
 503
 504     calculated_number min;
 505     calculated_number max;
 506
 507     time_t before;
 508     time_t after;
 509
 510     int has_st_lock;        // if st is read locked by us
 511 } RRDR;
 512
 513 #define rrdr_rows(r) ((r)->rows)
 514
 515 /*
 516 static void rrdr_dump(RRDR *r)
 517 {
 518     long c, i;
 519     RRDDIM *d;
 520
 521     fprintf(stderr, "\nCHART %s (%s)\n", r->st->id, r->st->name);
 522
 523     for(c = 0, d = r->st->dimensions; d ;c++, d = d->next) {
 524         fprintf(stderr, "DIMENSION %s (%s), %s%s%s%s\n"
 525                 , d->id
 526                 , d->name
 527                 , (r->od[c] & RRDR_EMPTY)?"EMPTY ":""
 528                 , (r->od[c] & RRDR_RESET)?"RESET ":""
 529                 , (r->od[c] & RRDR_HIDDEN)?"HIDDEN ":""
 530                 , (r->od[c] & RRDR_NONZERO)?"NONZERO ":""
 531                 );
 532     }
 533
 534     if(r->rows <= 0) {
 535         fprintf(stderr, "RRDR does not have any values in it.\n");
 536         return;
 537     }
 538
 539     fprintf(stderr, "RRDR includes %d values in it:\n", r->rows);
 540
 541     // for each line in the array
 542     for(i = 0; i < r->rows ;i++) {
 543         calculated_number *cn = &r->v[ i * r->d ];
 544         uint8_t *co = &r->o[ i * r->d ];
 545
 546         // print the id and the timestamp of the line
 547         fprintf(stderr, "%ld %ld ", i + 1, r->t[i]);
 548
 549         // for each dimension
 550         for(c = 0, d = r->st->dimensions; d ;c++, d = d->next) {
 551             if(unlikely(r->od[c] & RRDR_HIDDEN)) continue;
 552             if(unlikely(!(r->od[c] & RRDR_NONZERO))) continue;
 553
 554             if(co[c] & RRDR_EMPTY)
 555                 fprintf(stderr, "null ");
 556             else
 557                 fprintf(stderr, CALCULATED_NUMBER_FORMAT " %s%s%s%s "
 558                     , cn[c]
 559                     , (co[c] & RRDR_EMPTY)?"E":" "
 560                     , (co[c] & RRDR_RESET)?"R":" "
 561                     , (co[c] & RRDR_HIDDEN)?"H":" "
 562                     , (co[c] & RRDR_NONZERO)?"N":" "
 563                     );
 564         }
 565
 566         fprintf(stderr, "\n");
 567     }
 568 }
 569 */
 570
 571 void rrdr_disable_not_selected_dimensions(RRDR *r, uint32_t options, const char *dims) {
 572     rrdset_check_rdlock(r->st);
 573
 574     if(unlikely(!dims || !*dims)) return;
 575
 576     char b[strlen(dims) + 1];
 577     char *o = b, *tok;
 578     strcpy(o, dims);
 579
 580     long c, dims_selected = 0, dims_not_hidden_not_zero = 0;
 581     RRDDIM *d;
 582
 583     // disable all of them
 584     for(c = 0, d = r->st->dimensions; d ;c++, d = d->next)
 585         r->od[c] |= RRDR_HIDDEN;
 586
 587     while(o && *o && (tok = mystrsep(&o, ",|"))) {
 588         if(!*tok) continue;
 589
 590         uint32_t hash = simple_hash(tok);
 591
 592         // find it and enable it
 593         for(c = 0, d = r->st->dimensions; d ;c++, d = d->next) {
 594             if(unlikely((hash == d->hash && !strcmp(d->id, tok)) || (hash == d->hash_name && !strcmp(d->name, tok)))) {
 595
 596                 if(likely(r->od[c] & RRDR_HIDDEN)) {
 597                     r->od[c] |= RRDR_SELECTED;
 598                     r->od[c] &= ~RRDR_HIDDEN;
 599                     dims_selected++;
 600                 }
 601
 602                 // since the user needs this dimension
 603                 // make it appear as NONZERO, to return it
 604                 // even if the dimension has only zeros
 605                 // unless option non_zero is set
 606                 if(likely(!(options & RRDR_OPTION_NONZERO)))
 607                     r->od[c] |= RRDR_NONZERO;
 608
 609                 // count the visible dimensions
 610                 if(likely(r->od[c] & RRDR_NONZERO))
 611                     dims_not_hidden_not_zero++;
 612             }
 613         }
 614     }
 615
 616     // check if all dimensions are hidden
 617     if(unlikely(!dims_not_hidden_not_zero && dims_selected)) {
 618         // there are a few selected dimensions
 619         // but they are all zero
 620         // enable the selected ones
 621         // to avoid returning an empty chart
 622         for(c = 0, d = r->st->dimensions; d ;c++, d = d->next)
 623             if(unlikely(r->od[c] & RRDR_SELECTED))
 624                 r->od[c] |= RRDR_NONZERO;
 625     }
 626 }
 627
 628 void rrdr_buffer_print_format(BUFFER *wb, uint32_t format)
 629 {
 630     switch(format) {
 631     case DATASOURCE_JSON:
 632         buffer_strcat(wb, DATASOURCE_FORMAT_JSON);
 633         break;
 634
 635     case DATASOURCE_DATATABLE_JSON:
 636         buffer_strcat(wb, DATASOURCE_FORMAT_DATATABLE_JSON);
 637         break;
 638
 639     case DATASOURCE_DATATABLE_JSONP:
 640         buffer_strcat(wb, DATASOURCE_FORMAT_DATATABLE_JSONP);
 641         break;
 642
 643     case DATASOURCE_JSONP:
 644         buffer_strcat(wb, DATASOURCE_FORMAT_JSONP);
 645         break;
 646
 647     case DATASOURCE_SSV:
 648         buffer_strcat(wb, DATASOURCE_FORMAT_SSV);
 649         break;
 650
 651     case DATASOURCE_CSV:
 652         buffer_strcat(wb, DATASOURCE_FORMAT_CSV);
 653         break;
 654
 655     case DATASOURCE_TSV:
 656         buffer_strcat(wb, DATASOURCE_FORMAT_TSV);
 657         break;
 658
 659     case DATASOURCE_HTML:
 660         buffer_strcat(wb, DATASOURCE_FORMAT_HTML);
 661         break;
 662
 663     case DATASOURCE_JS_ARRAY:
 664         buffer_strcat(wb, DATASOURCE_FORMAT_JS_ARRAY);
 665         break;
 666
 667     case DATASOURCE_SSV_COMMA:
 668         buffer_strcat(wb, DATASOURCE_FORMAT_SSV_COMMA);
 669         break;
 670
 671     default:
 672         buffer_strcat(wb, "unknown");
 673         break;
 674     }
 675 }
 676
 677 uint32_t rrdr_check_options(RRDR *r, uint32_t options, const char *dims)
 678 {
 679     rrdset_check_rdlock(r->st);
 680
 681     (void)dims;
 682
 683     if(options & RRDR_OPTION_NONZERO) {
 684         long i;
 685
 686         // commented due to #1514
 687
 688         //if(dims && *dims) {
 689             // the caller wants specific dimensions
 690             // disable NONZERO option
 691             // to make sure we don't accidentally prevent
 692             // the specific dimensions from being returned
 693             // i = 0;
 694         //}
 695         //else {
 696             // find how many dimensions are not zero
 697             long c;
 698             RRDDIM *rd;
 699             for(c = 0, i = 0, rd = r->st->dimensions; rd && c < r->d ; c++, rd = rd->next) {
 700                 if(unlikely(r->od[c] & RRDR_HIDDEN)) continue;
 701                 if(unlikely(!(r->od[c] & RRDR_NONZERO))) continue;
 702                 i++;
 703             }
 704         //}
 705
 706         // if with nonzero we get i = 0 (no dimensions will be returned)
 707         // disable nonzero to show all dimensions
 708         if(!i) options &= ~RRDR_OPTION_NONZERO;
 709     }
 710
 711     return options;
 712 }
 713
 714 void rrdr_json_wrapper_begin(RRDR *r, BUFFER *wb, uint32_t format, uint32_t options, int string_value)
 715 {
 716     rrdset_check_rdlock(r->st);
 717
 718     long rows = rrdr_rows(r);
 719     long c, i;
 720     RRDDIM *rd;
 721
 722     //info("JSONWRAPPER(): %s: BEGIN", r->st->id);
 723     char kq[2] = "",                    // key quote
 724         sq[2] = "";                     // string quote
 725
 726     if( options & RRDR_OPTION_GOOGLE_JSON ) {
 727         kq[0] = '\0';
 728         sq[0] = '\'';
 729     }
 730     else {
 731         kq[0] = '"';
 732         sq[0] = '"';
 733     }
 734
 735     buffer_sprintf(wb, "{\n"
 736             "   %sapi%s: 1,\n"
 737             "   %sid%s: %s%s%s,\n"
 738             "   %sname%s: %s%s%s,\n"
 739             "   %sview_update_every%s: %d,\n"
 740             "   %supdate_every%s: %d,\n"
 741             "   %sfirst_entry%s: %u,\n"
 742             "   %slast_entry%s: %u,\n"
 743             "   %sbefore%s: %u,\n"
 744             "   %safter%s: %u,\n"
 745             "   %sdimension_names%s: ["
 746             , kq, kq
 747             , kq, kq, sq, r->st->id, sq
 748             , kq, kq, sq, r->st->name, sq
 749             , kq, kq, r->update_every
 750             , kq, kq, r->st->update_every
 751             , kq, kq, (uint32_t)rrdset_first_entry_t(r->st)
 752             , kq, kq, (uint32_t)rrdset_last_entry_t(r->st)
 753             , kq, kq, (uint32_t)r->before
 754             , kq, kq, (uint32_t)r->after
 755             , kq, kq);
 756
 757     for(c = 0, i = 0, rd = r->st->dimensions; rd && c < r->d ;c++, rd = rd->next) {
 758         if(unlikely(r->od[c] & RRDR_HIDDEN)) continue;
 759         if(unlikely((options & RRDR_OPTION_NONZERO) && !(r->od[c] & RRDR_NONZERO))) continue;
 760
 761         if(i) buffer_strcat(wb, ", ");
 762         buffer_strcat(wb, sq);
 763         buffer_strcat(wb, rd->name);
 764         buffer_strcat(wb, sq);
 765         i++;
 766     }
 767     if(!i) {
 768 #ifdef NETDATA_INTERNAL_CHECKS
 769         error("RRDR is empty for %s (RRDR has %d dimensions, options is 0x%08x)", r->st->id, r->d, options);
 770 #endif
 771         rows = 0;
 772         buffer_strcat(wb, sq);
 773         buffer_strcat(wb, "no data");
 774         buffer_strcat(wb, sq);
 775     }
 776
 777     buffer_sprintf(wb, "],\n"
 778             "   %sdimension_ids%s: ["
 779             , kq, kq);
 780
 781     for(c = 0, i = 0, rd = r->st->dimensions; rd && c < r->d ;c++, rd = rd->next) {
 782         if(unlikely(r->od[c] & RRDR_HIDDEN)) continue;
 783         if(unlikely((options & RRDR_OPTION_NONZERO) && !(r->od[c] & RRDR_NONZERO))) continue;
 784
 785         if(i) buffer_strcat(wb, ", ");
 786         buffer_strcat(wb, sq);
 787         buffer_strcat(wb, rd->id);
 788         buffer_strcat(wb, sq);
 789         i++;
 790     }
 791     if(!i) {
 792         rows = 0;
 793         buffer_strcat(wb, sq);
 794         buffer_strcat(wb, "no data");
 795         buffer_strcat(wb, sq);
 796     }
 797
 798     buffer_sprintf(wb, "],\n"
 799             "   %slatest_values%s: ["
 800             , kq, kq);
 801
 802     for(c = 0, i = 0, rd = r->st->dimensions; rd && c < r->d ;c++, rd = rd->next) {
 803         if(unlikely(r->od[c] & RRDR_HIDDEN)) continue;
 804         if(unlikely((options & RRDR_OPTION_NONZERO) && !(r->od[c] & RRDR_NONZERO))) continue;
 805
 806         if(i) buffer_strcat(wb, ", ");
 807         i++;
 808
 809         storage_number n = rd->values[rrdset_last_slot(r->st)];
 810
 811         if(!does_storage_number_exist(n))
 812             buffer_strcat(wb, "null");
 813         else
 814             buffer_rrd_value(wb, unpack_storage_number(n));
 815     }
 816     if(!i) {
 817         rows = 0;
 818         buffer_strcat(wb, "null");
 819     }
 820
 821     buffer_sprintf(wb, "],\n"
 822             "   %sview_latest_values%s: ["
 823             , kq, kq);
 824
 825     i = 0;
 826     if(rows) {
 827         for(c = 0, i = 0, rd = r->st->dimensions; rd && c < r->d ;c++, rd = rd->next) {
 828             if(unlikely(r->od[c] & RRDR_HIDDEN)) continue;
 829             if(unlikely((options & RRDR_OPTION_NONZERO) && !(r->od[c] & RRDR_NONZERO))) continue;
 830
 831             if(i) buffer_strcat(wb, ", ");
 832             i++;
 833
 834             calculated_number *cn = &r->v[ (0) * r->d ];
 835             uint8_t *co = &r->o[ (0) * r->d ];
 836
 837             if(co[c] & RRDR_EMPTY)
 838                 buffer_strcat(wb, "null");
 839             else
 840                 buffer_rrd_value(wb, cn[c]);
 841         }
 842     }
 843     if(!i) {
 844         rows = 0;
 845         buffer_strcat(wb, "null");
 846     }
 847
 848     buffer_sprintf(wb, "],\n"
 849             "   %sdimensions%s: %ld,\n"
 850             "   %spoints%s: %ld,\n"
 851             "   %sformat%s: %s"
 852             , kq, kq, i
 853             , kq, kq, rows
 854             , kq, kq, sq
 855             );
 856
 857     rrdr_buffer_print_format(wb, format);
 858
 859     buffer_sprintf(wb, "%s,\n"
 860             "   %sresult%s: "
 861             , sq
 862             , kq, kq
 863             );
 864
 865     if(string_value) buffer_strcat(wb, sq);
 866     //info("JSONWRAPPER(): %s: END", r->st->id);
 867 }
 868
 869 void rrdr_json_wrapper_end(RRDR *r, BUFFER *wb, uint32_t format, uint32_t options, int string_value)
 870 {
 871     (void)format;
 872
 873     char kq[2] = "",                    // key quote
 874         sq[2] = "";                     // string quote
 875
 876     if( options & RRDR_OPTION_GOOGLE_JSON ) {
 877         kq[0] = '\0';
 878         sq[0] = '\'';
 879     }
 880     else {
 881         kq[0] = '"';
 882         sq[0] = '"';
 883     }
 884
 885     if(string_value) buffer_strcat(wb, sq);
 886
 887     buffer_sprintf(wb, ",\n %smin%s: ", kq, kq);
 888     buffer_rrd_value(wb, r->min);
 889     buffer_sprintf(wb, ",\n %smax%s: ", kq, kq);
 890     buffer_rrd_value(wb, r->max);
 891     buffer_strcat(wb, "\n}\n");
 892 }
 893
 894 #define JSON_DATES_JS 1
 895 #define JSON_DATES_TIMESTAMP 2
 896
 897 static void rrdr2json(RRDR *r, BUFFER *wb, uint32_t options, int datatable)
 898 {
 899     rrdset_check_rdlock(r->st);
 900
 901     //info("RRD2JSON(): %s: BEGIN", r->st->id);
 902     int row_annotations = 0, dates, dates_with_new = 0;
 903     char kq[2] = "",                    // key quote
 904         sq[2] = "",                     // string quote
 905         pre_label[101] = "",            // before each label
 906         post_label[101] = "",           // after each label
 907         pre_date[101] = "",             // the beginning of line, to the date
 908         post_date[101] = "",            // closing the date
 909         pre_value[101] = "",            // before each value
 910         post_value[101] = "",           // after each value
 911         post_line[101] = "",            // at the end of each row
 912         normal_annotation[201] = "",    // default row annotation
 913         overflow_annotation[201] = "",  // overflow row annotation
 914         data_begin[101] = "",           // between labels and values
 915         finish[101] = "";               // at the end of everything
 916
 917     if(datatable) {
 918         dates = JSON_DATES_JS;
 919         if( options & RRDR_OPTION_GOOGLE_JSON ) {
 920             kq[0] = '\0';
 921             sq[0] = '\'';
 922         }
 923         else {
 924             kq[0] = '"';
 925             sq[0] = '"';
 926         }
 927         row_annotations = 1;
 928         snprintfz(pre_date,   100, "        {%sc%s:[{%sv%s:%s", kq, kq, kq, kq, sq);
 929         snprintfz(post_date,  100, "%s}", sq);
 930         snprintfz(pre_label,  100, ",\n     {%sid%s:%s%s,%slabel%s:%s", kq, kq, sq, sq, kq, kq, sq);
 931         snprintfz(post_label, 100, "%s,%spattern%s:%s%s,%stype%s:%snumber%s}", sq, kq, kq, sq, sq, kq, kq, sq, sq);
 932         snprintfz(pre_value,  100, ",{%sv%s:", kq, kq);
 933         strcpy(post_value,         "}");
 934         strcpy(post_line,          "]}");
 935         snprintfz(data_begin, 100, "\n  ],\n    %srows%s:\n [\n", kq, kq);
 936         strcpy(finish,             "\n  ]\n}");
 937
 938         snprintfz(overflow_annotation, 200, ",{%sv%s:%sRESET OR OVERFLOW%s},{%sv%s:%sThe counters have been wrapped.%s}", kq, kq, sq, sq, kq, kq, sq, sq);
 939         snprintfz(normal_annotation,   200, ",{%sv%s:null},{%sv%s:null}", kq, kq, kq, kq);
 940
 941         buffer_sprintf(wb, "{\n %scols%s:\n [\n", kq, kq);
 942         buffer_sprintf(wb, "        {%sid%s:%s%s,%slabel%s:%stime%s,%spattern%s:%s%s,%stype%s:%sdatetime%s},\n", kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq);
 943         buffer_sprintf(wb, "        {%sid%s:%s%s,%slabel%s:%s%s,%spattern%s:%s%s,%stype%s:%sstring%s,%sp%s:{%srole%s:%sannotation%s}},\n", kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, kq, kq, sq, sq);
 944         buffer_sprintf(wb, "        {%sid%s:%s%s,%slabel%s:%s%s,%spattern%s:%s%s,%stype%s:%sstring%s,%sp%s:{%srole%s:%sannotationText%s}}", kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, kq, kq, sq, sq);
 945
 946         // remove the valueobjects flag
 947         // google wants its own keys
 948         if(options & RRDR_OPTION_OBJECTSROWS)
 949             options &= ~RRDR_OPTION_OBJECTSROWS;
 950     }
 951     else {
 952         kq[0] = '"';
 953         sq[0] = '"';
 954         if(options & RRDR_OPTION_GOOGLE_JSON) {
 955             dates = JSON_DATES_JS;
 956             dates_with_new = 1;
 957         }
 958         else {
 959             dates = JSON_DATES_TIMESTAMP;
 960             dates_with_new = 0;
 961         }
 962         if( options & RRDR_OPTION_OBJECTSROWS )
 963             strcpy(pre_date, "      { ");
 964         else
 965             strcpy(pre_date, "      [ ");
 966         strcpy(pre_label,  ", \"");
 967         strcpy(post_label, "\"");
 968         strcpy(pre_value,  ", ");
 969         if( options & RRDR_OPTION_OBJECTSROWS )
 970             strcpy(post_line, "}");
 971         else
 972             strcpy(post_line, "]");
 973         snprintfz(data_begin, 100, "],\n    %sdata%s:\n [\n", kq, kq);
 974         strcpy(finish,             "\n  ]\n}");
 975
 976         buffer_sprintf(wb, "{\n %slabels%s: [", kq, kq);
 977         buffer_sprintf(wb, "%stime%s", sq, sq);
 978     }
 979
 980     // -------------------------------------------------------------------------
 981     // print the JSON header
 982
 983     long c, i;
 984     RRDDIM *rd;
 985
 986     // print the header lines
 987     for(c = 0, i = 0, rd = r->st->dimensions; rd && c < r->d ;c++, rd = rd->next) {
 988         if(unlikely(r->od[c] & RRDR_HIDDEN)) continue;
 989         if(unlikely((options & RRDR_OPTION_NONZERO) && !(r->od[c] & RRDR_NONZERO))) continue;
 990
 991         buffer_strcat(wb, pre_label);
 992         buffer_strcat(wb, rd->name);
 993         buffer_strcat(wb, post_label);
 994         i++;
 995     }
 996     if(!i) {
 997         buffer_strcat(wb, pre_label);
 998         buffer_strcat(wb, "no data");
 999         buffer_strcat(wb, post_label);
1000     }
1001
1002     // print the begin of row data
1003     buffer_strcat(wb, data_begin);
1004
1005     // if all dimensions are hidden, print a null
1006     if(!i) {
1007         buffer_strcat(wb, finish);
1008         return;
1009     }
1010
1011     long start = 0, end = rrdr_rows(r), step = 1;
1012     if((options & RRDR_OPTION_REVERSED)) {
1013         start = rrdr_rows(r) - 1;
1014         end = -1;
1015         step = -1;
1016     }
1017
1018     // for each line in the array
1019     calculated_number total = 1;
1020     for(i = start; i != end ;i += step) {
1021         calculated_number *cn = &r->v[ i * r->d ];
1022         uint8_t *co = &r->o[ i * r->d ];
1023
1024         time_t now = r->t[i];
1025
1026         if(dates == JSON_DATES_JS) {
1027             // generate the local date time
1028             struct tm tmbuf, *tm = localtime_r(&now, &tmbuf);
1029             if(!tm) { error("localtime_r() failed."); continue; }
1030
1031             if(likely(i != start)) buffer_strcat(wb, ",\n");
1032             buffer_strcat(wb, pre_date);
1033
1034             if( options & RRDR_OPTION_OBJECTSROWS )
1035                 buffer_sprintf(wb, "%stime%s: ", kq, kq);
1036
1037             if(dates_with_new)
1038                 buffer_strcat(wb, "new ");
1039
1040             buffer_jsdate(wb, tm->tm_year + 1900, tm->tm_mon, tm->tm_mday, tm->tm_hour, tm->tm_min, tm->tm_sec);
1041
1042             buffer_strcat(wb, post_date);
1043
1044             if(row_annotations) {
1045                 // google supports one annotation per row
1046                 int annotation_found = 0;
1047                 for(c = 0, rd = r->st->dimensions; rd ;c++, rd = rd->next) {
1048                     if(co[c] & RRDR_RESET) {
1049                         buffer_strcat(wb, overflow_annotation);
1050                         annotation_found = 1;
1051                         break;
1052                     }
1053                 }
1054                 if(!annotation_found)
1055                     buffer_strcat(wb, normal_annotation);
1056             }
1057         }
1058         else {
1059             // print the timestamp of the line
1060             if(likely(i != start)) buffer_strcat(wb, ",\n");
1061             buffer_strcat(wb, pre_date);
1062
1063             if( options & RRDR_OPTION_OBJECTSROWS )
1064                 buffer_sprintf(wb, "%stime%s: ", kq, kq);
1065
1066             buffer_rrd_value(wb, (calculated_number)r->t[i]);
1067             // in ms
1068             if(options & RRDR_OPTION_MILLISECONDS) buffer_strcat(wb, "000");
1069
1070             buffer_strcat(wb, post_date);
1071         }
1072
1073         if(unlikely(options & RRDR_OPTION_PERCENTAGE)) {
1074             total = 0;
1075             for(c = 0, rd = r->st->dimensions; rd && c < r->d ;c++, rd = rd->next) {
1076                 calculated_number n = cn[c];
1077
1078                 if(likely((options & RRDR_OPTION_ABSOLUTE) && n < 0))
1079                     n = -n;
1080
1081                 total += n;
1082             }
1083             // prevent a division by zero
1084             if(total == 0) total = 1;
1085         }
1086
1087         // for each dimension
1088         for(c = 0, rd = r->st->dimensions; rd && c < r->d ;c++, rd = rd->next) {
1089             if(unlikely(r->od[c] & RRDR_HIDDEN)) continue;
1090             if(unlikely((options & RRDR_OPTION_NONZERO) && !(r->od[c] & RRDR_NONZERO))) continue;
1091
1092             calculated_number n = cn[c];
1093
1094             buffer_strcat(wb, pre_value);
1095
1096             if( options & RRDR_OPTION_OBJECTSROWS )
1097                 buffer_sprintf(wb, "%s%s%s: ", kq, rd->name, kq);
1098
1099             if(co[c] & RRDR_EMPTY) {
1100                 if(options & RRDR_OPTION_NULL2ZERO)
1101                     buffer_strcat(wb, "0");
1102                 else
1103                     buffer_strcat(wb, "null");
1104             }
1105             else {
1106                 if(unlikely((options & RRDR_OPTION_ABSOLUTE) && n < 0))
1107                     n = -n;
1108
1109                 if(unlikely(options & RRDR_OPTION_PERCENTAGE))
1110                     n = n * 100 / total;
1111
1112                 buffer_rrd_value(wb, n);
1113             }
1114
1115             buffer_strcat(wb, post_value);
1116         }
1117
1118         buffer_strcat(wb, post_line);
1119     }
1120
1121     buffer_strcat(wb, finish);
1122     //info("RRD2JSON(): %s: END", r->st->id);
1123 }
1124
1125 static void rrdr2csv(RRDR *r, BUFFER *wb, uint32_t options, const char *startline, const char *separator, const char *endline, const char *betweenlines)
1126 {
1127     rrdset_check_rdlock(r->st);
1128
1129     //info("RRD2CSV(): %s: BEGIN", r->st->id);
1130     long c, i;
1131     RRDDIM *d;
1132
1133     // print the csv header
1134     for(c = 0, i = 0, d = r->st->dimensions; d && c < r->d ;c++, d = d->next) {
1135         if(unlikely(r->od[c] & RRDR_HIDDEN)) continue;
1136         if(unlikely((options & RRDR_OPTION_NONZERO) && !(r->od[c] & RRDR_NONZERO))) continue;
1137
1138         if(!i) {
1139             buffer_strcat(wb, startline);
1140             if(options & RRDR_OPTION_LABEL_QUOTES) buffer_strcat(wb, "\"");
1141             buffer_strcat(wb, "time");
1142             if(options & RRDR_OPTION_LABEL_QUOTES) buffer_strcat(wb, "\"");
1143         }
1144         buffer_strcat(wb, separator);
1145         if(options & RRDR_OPTION_LABEL_QUOTES) buffer_strcat(wb, "\"");
1146         buffer_strcat(wb, d->name);
1147         if(options & RRDR_OPTION_LABEL_QUOTES) buffer_strcat(wb, "\"");
1148         i++;
1149     }
1150     buffer_strcat(wb, endline);
1151
1152     if(!i) {
1153         // no dimensions present
1154         return;
1155     }
1156
1157     long start = 0, end = rrdr_rows(r), step = 1;
1158     if((options & RRDR_OPTION_REVERSED)) {
1159         start = rrdr_rows(r) - 1;
1160         end = -1;
1161         step = -1;
1162     }
1163
1164     // for each line in the array
1165     calculated_number total = 1;
1166     for(i = start; i != end ;i += step) {
1167         calculated_number *cn = &r->v[ i * r->d ];
1168         uint8_t *co = &r->o[ i * r->d ];
1169
1170         buffer_strcat(wb, betweenlines);
1171         buffer_strcat(wb, startline);
1172
1173         time_t now = r->t[i];
1174
1175         if((options & RRDR_OPTION_SECONDS) || (options & RRDR_OPTION_MILLISECONDS)) {
1176             // print the timestamp of the line
1177             buffer_rrd_value(wb, (calculated_number)now);
1178             // in ms
1179             if(options & RRDR_OPTION_MILLISECONDS) buffer_strcat(wb, "000");
1180         }
1181         else {
1182             // generate the local date time
1183             struct tm tmbuf, *tm = localtime_r(&now, &tmbuf);
1184             if(!tm) { error("localtime() failed."); continue; }
1185             buffer_date(wb, tm->tm_year + 1900, tm->tm_mon + 1, tm->tm_mday, tm->tm_hour, tm->tm_min, tm->tm_sec);
1186         }
1187
1188         if(unlikely(options & RRDR_OPTION_PERCENTAGE)) {
1189             total = 0;
1190             for(c = 0, d = r->st->dimensions; d && c < r->d ;c++, d = d->next) {
1191                 calculated_number n = cn[c];
1192
1193                 if(likely((options & RRDR_OPTION_ABSOLUTE) && n < 0))
1194                     n = -n;
1195
1196                 total += n;
1197             }
1198             // prevent a division by zero
1199             if(total == 0) total = 1;
1200         }
1201
1202         // for each dimension
1203         for(c = 0, d = r->st->dimensions; d && c < r->d ;c++, d = d->next) {
1204             if(unlikely(r->od[c] & RRDR_HIDDEN)) continue;
1205             if(unlikely((options & RRDR_OPTION_NONZERO) && !(r->od[c] & RRDR_NONZERO))) continue;
1206
1207             buffer_strcat(wb, separator);
1208
1209             calculated_number n = cn[c];
1210
1211             if(co[c] & RRDR_EMPTY) {
1212                 if(options & RRDR_OPTION_NULL2ZERO)
1213                     buffer_strcat(wb, "0");
1214                 else
1215                     buffer_strcat(wb, "null");
1216             }
1217             else {
1218                 if(unlikely((options & RRDR_OPTION_ABSOLUTE) && n < 0))
1219                     n = -n;
1220
1221                 if(unlikely(options & RRDR_OPTION_PERCENTAGE))
1222                     n = n * 100 / total;
1223
1224                 buffer_rrd_value(wb, n);
1225             }
1226         }
1227
1228         buffer_strcat(wb, endline);
1229     }
1230     //info("RRD2CSV(): %s: END", r->st->id);
1231 }
1232
1233 inline static calculated_number rrdr2value(RRDR *r, long i, uint32_t options, int *all_values_are_null) {
1234     rrdset_check_rdlock(r->st);
1235
1236     long c;
1237     RRDDIM *d;
1238
1239     calculated_number *cn = &r->v[ i * r->d ];
1240     uint8_t *co = &r->o[ i * r->d ];
1241
1242     calculated_number sum = 0, min = 0, max = 0, v;
1243     int all_null = 1, init = 1;
1244
1245     calculated_number total = 1;
1246     if(unlikely(options & RRDR_OPTION_PERCENTAGE)) {
1247         total = 0;
1248         for(c = 0, d = r->st->dimensions; d && c < r->d ;c++, d = d->next) {
1249             calculated_number n = cn[c];
1250
1251             if(likely((options & RRDR_OPTION_ABSOLUTE) && n < 0))
1252                 n = -n;
1253
1254             total += n;
1255         }
1256         // prevent a division by zero
1257         if(total == 0) total = 1;
1258     }
1259
1260     // for each dimension
1261     for(c = 0, d = r->st->dimensions; d && c < r->d ;c++, d = d->next) {
1262         if(unlikely(r->od[c] & RRDR_HIDDEN)) continue;
1263         if(unlikely((options & RRDR_OPTION_NONZERO) && !(r->od[c] & RRDR_NONZERO))) continue;
1264
1265         calculated_number n = cn[c];
1266
1267         if(likely((options & RRDR_OPTION_ABSOLUTE) && n < 0))
1268             n = -n;
1269
1270         if(unlikely(options & RRDR_OPTION_PERCENTAGE))
1271             n = n * 100 / total;
1272
1273         if(unlikely(init)) {
1274             if(n > 0) {
1275                 min = 0;
1276                 max = n;
1277             }
1278             else {
1279                 min = n;
1280                 max = 0;
1281             }
1282             init = 0;
1283         }
1284
1285         if(likely(!(co[c] & RRDR_EMPTY))) {
1286             all_null = 0;
1287             sum += n;
1288         }
1289
1290         if(n < min) min = n;
1291         if(n > max) max = n;
1292     }
1293
1294     if(unlikely(all_null)) {
1295         if(likely(all_values_are_null))
1296             *all_values_are_null = 1;
1297         return 0;
1298     }
1299     else {
1300         if(likely(all_values_are_null))
1301             *all_values_are_null = 0;
1302     }
1303
1304     if(options & RRDR_OPTION_MIN2MAX)
1305         v = max - min;
1306     else
1307         v = sum;
1308
1309     return v;
1310 }
1311
1312 static void rrdr2ssv(RRDR *r, BUFFER *wb, uint32_t options, const char *prefix, const char *separator, const char *suffix)
1313 {
1314     //info("RRD2SSV(): %s: BEGIN", r->st->id);
1315     long i;
1316
1317     buffer_strcat(wb, prefix);
1318     long start = 0, end = rrdr_rows(r), step = 1;
1319     if((options & RRDR_OPTION_REVERSED)) {
1320         start = rrdr_rows(r) - 1;
1321         end = -1;
1322         step = -1;
1323     }
1324
1325     // for each line in the array
1326     for(i = start; i != end ;i += step) {
1327         int all_values_are_null = 0;
1328         calculated_number v = rrdr2value(r, i, options, &all_values_are_null);
1329
1330         if(likely(i != start)) {
1331             if(r->min > v) r->min = v;
1332             if(r->max < v) r->max = v;
1333         }
1334         else {
1335             r->min = v;
1336             r->max = v;
1337         }
1338
1339         if(likely(i != start))
1340             buffer_strcat(wb, separator);
1341
1342         if(all_values_are_null) {
1343             if(options & RRDR_OPTION_NULL2ZERO)
1344                 buffer_strcat(wb, "0");
1345             else
1346                 buffer_strcat(wb, "null");
1347         }
1348         else
1349             buffer_rrd_value(wb, v);
1350     }
1351     buffer_strcat(wb, suffix);
1352     //info("RRD2SSV(): %s: END", r->st->id);
1353 }
1354
1355 inline static calculated_number *rrdr_line_values(RRDR *r)
1356 {
1357     return &r->v[ r->c * r->d ];
1358 }
1359
1360 inline static uint8_t *rrdr_line_options(RRDR *r)
1361 {
1362     return &r->o[ r->c * r->d ];
1363 }
1364
1365 inline static int rrdr_line_init(RRDR *r, time_t t)
1366 {
1367     r->c++;
1368
1369     if(unlikely(r->c >= r->n)) {
1370         error("requested to step above RRDR size for chart %s", r->st->name);
1371         r->c = r->n - 1;
1372     }
1373
1374     // save the time
1375     r->t[r->c] = t;
1376
1377     return 1;
1378 }
1379
1380 inline static void rrdr_lock_rrdset(RRDR *r) {
1381     if(unlikely(!r)) {
1382         error("NULL value given!");
1383         return;
1384     }
1385
1386     rrdset_rdlock(r->st);
1387     r->has_st_lock = 1;
1388 }
1389
1390 inline static void rrdr_unlock_rrdset(RRDR *r) {
1391     if(unlikely(!r)) {
1392         error("NULL value given!");
1393         return;
1394     }
1395
1396     if(likely(r->has_st_lock)) {
1397         rrdset_unlock(r->st);
1398         r->has_st_lock = 0;
1399     }
1400 }
1401
1402 inline static void rrdr_free(RRDR *r)
1403 {
1404     if(unlikely(!r)) {
1405         error("NULL value given!");
1406         return;
1407     }
1408
1409     rrdr_unlock_rrdset(r);
1410     freez(r->t);
1411     freez(r->v);
1412     freez(r->o);
1413     freez(r->od);
1414     freez(r);
1415 }
1416
1417 static inline void rrdr_done(RRDR *r)
1418 {
1419     r->rows = r->c + 1;
1420     r->c = 0;
1421 }
1422
1423 static RRDR *rrdr_create(RRDSET *st, long n)
1424 {
1425     if(unlikely(!st)) {
1426         error("NULL value given!");
1427         return NULL;
1428     }
1429
1430     RRDR *r = callocz(1, sizeof(RRDR));
1431     r->st = st;
1432
1433     rrdr_lock_rrdset(r);
1434
1435     RRDDIM *rd;
1436     rrddim_foreach_read(rd, st) r->d++;
1437
1438     r->n = n;
1439
1440     r->t = mallocz(n * sizeof(time_t));
1441     r->v = mallocz(n * r->d * sizeof(calculated_number));
1442     r->o = mallocz(n * r->d * sizeof(uint8_t));
1443     r->od = mallocz(r->d * sizeof(uint8_t));
1444
1445     // set the hidden flag on hidden dimensions
1446     int c;
1447     for(c = 0, rd = st->dimensions ; rd ; c++, rd = rd->next) {
1448         if(unlikely(rrddim_flag_check(rd, RRDDIM_FLAG_HIDDEN)))
1449             r->od[c] = RRDR_HIDDEN;
1450         else
1451             r->od[c] = 0;
1452     }
1453
1454     r->c = -1;
1455     r->group = 1;
1456     r->update_every = 1;
1457
1458     return r;
1459 }
1460
1461 RRDR *rrd2rrdr(RRDSET *st, long points, long long after, long long before, int group_method, int aligned)
1462 {
1463     int debug = rrdset_flag_check(st, RRDSET_FLAG_DEBUG)?1:0;
1464     int absolute_period_requested = -1;
1465
1466     time_t first_entry_t = rrdset_first_entry_t(st);
1467     time_t last_entry_t  = rrdset_last_entry_t(st);
1468
1469     if(before == 0 && after == 0) {
1470         // dump the all the data
1471         before = last_entry_t;
1472         after = first_entry_t;
1473         absolute_period_requested = 0;
1474     }
1475
1476     // allow relative for before (smaller than API_RELATIVE_TIME_MAX)
1477     if(((before < 0)?-before:before) <= API_RELATIVE_TIME_MAX) {
1478         if(abs(before) % st->update_every) {
1479             // make sure it is multiple of st->update_every
1480             if(before < 0) before = before - st->update_every - before % st->update_every;
1481             else           before = before + st->update_every - before % st->update_every;
1482         }
1483         if(before > 0) before = first_entry_t + before;
1484         else           before = last_entry_t  + before;
1485         absolute_period_requested = 0;
1486     }
1487
1488     // allow relative for after (smaller than API_RELATIVE_TIME_MAX)
1489     if(((after < 0)?-after:after) <= API_RELATIVE_TIME_MAX) {
1490         if(after == 0) after = -st->update_every;
1491         if(abs(after) % st->update_every) {
1492             // make sure it is multiple of st->update_every
1493             if(after < 0) after = after - st->update_every - after % st->update_every;
1494             else          after = after + st->update_every - after % st->update_every;
1495         }
1496         after = before + after;
1497         absolute_period_requested = 0;
1498     }
1499
1500     if(absolute_period_requested == -1)
1501         absolute_period_requested = 1;
1502
1503     // make sure they are within our timeframe
1504     if(before > last_entry_t)  before = last_entry_t;
1505     if(before < first_entry_t) before = first_entry_t;
1506
1507     if(after > last_entry_t)  after = last_entry_t;
1508     if(after < first_entry_t) after = first_entry_t;
1509
1510     // check if they are upside down
1511     if(after > before) {
1512         time_t tmp = before;
1513         before = after;
1514         after = tmp;
1515     }
1516
1517     // the duration of the chart
1518     time_t duration = before - after;
1519     long available_points = duration / st->update_every;
1520
1521     if(duration <= 0 || available_points <= 0)
1522         return rrdr_create(st, 1);
1523
1524     // check the wanted points
1525     if(points < 0) points = -points;
1526     if(points > available_points) points = available_points;
1527     if(points == 0) points = available_points;
1528
1529     // calculate proper grouping of source data
1530     long group = available_points / points;
1531     if(group <= 0) group = 1;
1532
1533     // round group to the closest integer
1534     if(available_points % points > points / 2) group++;
1535
1536     time_t after_new  = (aligned) ? (after  - (after  % (group * st->update_every))) : after;
1537     time_t before_new = (aligned) ? (before - (before % (group * st->update_every))) : before;
1538     long points_new   = (before_new - after_new) / st->update_every / group;
1539
1540     // find the starting and ending slots in our round robin db
1541     long    start_at_slot = rrdset_time2slot(st, before_new),
1542             stop_at_slot  = rrdset_time2slot(st, after_new);
1543
1544 #ifdef NETDATA_INTERNAL_CHECKS
1545     if(after_new < first_entry_t) {
1546         error("after_new %u is too small, minimum %u", (uint32_t)after_new, (uint32_t)first_entry_t);
1547     }
1548     if(after_new > last_entry_t) {
1549         error("after_new %u is too big, maximum %u", (uint32_t)after_new, (uint32_t)last_entry_t);
1550     }
1551     if(before_new < first_entry_t) {
1552         error("before_new %u is too small, minimum %u", (uint32_t)before_new, (uint32_t)first_entry_t);
1553     }
1554     if(before_new > last_entry_t) {
1555         error("before_new %u is too big, maximum %u", (uint32_t)before_new, (uint32_t)last_entry_t);
1556     }
1557     if(start_at_slot < 0 || start_at_slot >= st->entries) {
1558         error("start_at_slot is invalid %ld, expected 0 to %ld", start_at_slot, st->entries - 1);
1559     }
1560     if(stop_at_slot < 0 || stop_at_slot >= st->entries) {
1561         error("stop_at_slot is invalid %ld, expected 0 to %ld", stop_at_slot, st->entries - 1);
1562     }
1563     if(points_new > (before_new - after_new) / group / st->update_every + 1) {
1564         error("points_new %ld is more than points %ld", points_new, (before_new - after_new) / group / st->update_every + 1);
1565     }
1566 #endif
1567
1568     //info("RRD2RRDR(): %s: wanted %ld points, got %ld - group=%ld, wanted duration=%u, got %u - wanted %ld - %ld, got %ld - %ld", st->id, points, points_new, group, before - after, before_new - after_new, after, before, after_new, before_new);
1569
1570     after = after_new;
1571     before = before_new;
1572     duration = before - after;
1573     points = points_new;
1574
1575     // Now we have:
1576     // before = the end time of the calculation
1577     // after = the start time of the calculation
1578     // duration = the duration of the calculation
1579     // group = the number of source points to aggregate / group together
1580     // method = the method of grouping source points
1581     // points = the number of points to generate
1582
1583
1584     // -------------------------------------------------------------------------
1585     // initialize our result set
1586
1587     RRDR *r = rrdr_create(st, points);
1588     if(!r) {
1589 #ifdef NETDATA_INTERNAL_CHECKS
1590         error("Cannot create RRDR for %s, after=%u, before=%u, duration=%u, points=%ld", st->id, (uint32_t)after, (uint32_t)before, (uint32_t)duration, points);
1591 #endif
1592         return NULL;
1593     }
1594     if(!r->d) {
1595 #ifdef NETDATA_INTERNAL_CHECKS
1596         error("Returning empty RRDR (no dimensions in RRDSET) for %s, after=%u, before=%u, duration=%u, points=%ld", st->id, (uint32_t)after, (uint32_t)before, (uint32_t)duration, points);
1597 #endif
1598         return r;
1599     }
1600
1601     if(absolute_period_requested == 1)
1602         r->result_options |= RRDR_RESULT_OPTION_ABSOLUTE;
1603     else
1604         r->result_options |= RRDR_RESULT_OPTION_RELATIVE;
1605
1606     // find how many dimensions we have
1607     long dimensions = r->d;
1608
1609
1610     // -------------------------------------------------------------------------
1611     // checks for debugging
1612
1613     if(debug) debug(D_RRD_STATS, "INFO %s first_t: %u, last_t: %u, all_duration: %u, after: %u, before: %u, duration: %u, points: %ld, group: %ld"
1614             , st->id
1615             , (uint32_t)first_entry_t
1616             , (uint32_t)last_entry_t
1617             , (uint32_t)(last_entry_t - first_entry_t)
1618             , (uint32_t)after
1619             , (uint32_t)before
1620             , (uint32_t)duration
1621             , points
1622             , group
1623             );
1624
1625
1626     // -------------------------------------------------------------------------
1627     // temp arrays for keeping values per dimension
1628
1629     calculated_number   last_values[dimensions]; // keep the last value of each dimension
1630     calculated_number   group_values[dimensions]; // keep sums when grouping
1631     long                group_counts[dimensions]; // keep the number of values added to group_values
1632     uint8_t             group_options[dimensions];
1633     uint8_t             found_non_zero[dimensions];
1634
1635
1636     // initialize them
1637     RRDDIM *rd;
1638     long c;
1639     rrdset_check_rdlock(st);
1640     for( rd = st->dimensions, c = 0 ; rd && c < dimensions ; rd = rd->next, c++) {
1641         last_values[c] = 0;
1642         group_values[c] = (group_method == GROUP_MAX || group_method == GROUP_MIN)?NAN:0;
1643         group_counts[c] = 0;
1644         group_options[c] = 0;
1645         found_non_zero[c] = 0;
1646     }
1647
1648
1649     // -------------------------------------------------------------------------
1650     // the main loop
1651
1652     time_t  now = rrdset_slot2time(st, start_at_slot),
1653             dt = st->update_every,
1654             group_start_t = 0;
1655
1656     if(unlikely(debug)) debug(D_RRD_STATS, "BEGIN %s after_t: %u (stop_at_t: %ld), before_t: %u (start_at_t: %ld), start_t(now): %u, current_entry: %ld, entries: %ld"
1657             , st->id
1658             , (uint32_t)after
1659             , stop_at_slot
1660             , (uint32_t)before
1661             , start_at_slot
1662             , (uint32_t)now
1663             , st->current_entry
1664             , st->entries
1665             );
1666
1667     r->group = group;
1668     r->update_every = group * st->update_every;
1669     r->before = now;
1670     r->after = now;
1671
1672     //info("RRD2RRDR(): %s: STARTING", st->id);
1673
1674     long slot = start_at_slot, counter = 0, stop_now = 0, added = 0, group_count = 0, add_this = 0;
1675     for(; !stop_now ; now -= dt, slot--, counter++) {
1676         if(unlikely(slot < 0)) slot = st->entries - 1;
1677         if(unlikely(slot == stop_at_slot)) stop_now = counter;
1678
1679         if(unlikely(debug)) debug(D_RRD_STATS, "ROW %s slot: %ld, entries_counter: %ld, group_count: %ld, added: %ld, now: %ld, %s %s"
1680                 , st->id
1681                 , slot
1682                 , counter
1683                 , group_count + 1
1684                 , added
1685                 , now
1686                 , (group_count + 1 == group)?"PRINT":"  -  "
1687                 , (now >= after && now <= before)?"RANGE":"  -  "
1688                 );
1689
1690         // make sure we return data in the proper time range
1691         if(unlikely(now > before)) continue;
1692         if(unlikely(now < after)) break;
1693
1694         if(unlikely(group_count == 0)) {
1695             group_start_t = now;
1696         }
1697         group_count++;
1698
1699         if(unlikely(group_count == group)) {
1700             if(unlikely(added >= points)) break;
1701             add_this = 1;
1702         }
1703
1704         // do the calculations
1705         for(rd = st->dimensions, c = 0 ; rd && c < dimensions ; rd = rd->next, c++) {
1706             storage_number n = rd->values[slot];
1707             if(unlikely(!does_storage_number_exist(n))) continue;
1708
1709             group_counts[c]++;
1710
1711             calculated_number value = unpack_storage_number(n);
1712             if(likely(value != 0.0)) {
1713                 group_options[c] |= RRDR_NONZERO;
1714                 found_non_zero[c] = 1;
1715             }
1716
1717             if(unlikely(did_storage_number_reset(n)))
1718                 group_options[c] |= RRDR_RESET;
1719
1720             switch(group_method) {
1721                 case GROUP_MIN:
1722                     if(unlikely(isnan(group_values[c])) ||
1723                             fabsl(value) < fabsl(group_values[c]))
1724                         group_values[c] = value;
1725                     break;
1726
1727                 case GROUP_MAX:
1728                     if(unlikely(isnan(group_values[c])) ||
1729                             fabsl(value) > fabsl(group_values[c]))
1730                         group_values[c] = value;
1731                     break;
1732
1733                 default:
1734                 case GROUP_SUM:
1735                 case GROUP_AVERAGE:
1736                 case GROUP_UNDEFINED:
1737                     group_values[c] += value;
1738                     break;
1739
1740                 case GROUP_INCREMENTAL_SUM:
1741                     if(unlikely(slot == start_at_slot))
1742                         last_values[c] = value;
1743
1744                     group_values[c] += last_values[c] - value;
1745                     last_values[c] = value;
1746                     break;
1747             }
1748         }
1749
1750         // added it
1751         if(unlikely(add_this)) {
1752             if(unlikely(!rrdr_line_init(r, group_start_t))) break;
1753
1754             r->after = now;
1755
1756             calculated_number *cn = rrdr_line_values(r);
1757             uint8_t *co = rrdr_line_options(r);
1758
1759             for(rd = st->dimensions, c = 0 ; rd && c < dimensions ; rd = rd->next, c++) {
1760
1761                 // update the dimension options
1762                 if(likely(found_non_zero[c])) r->od[c] |= RRDR_NONZERO;
1763
1764                 // store the specific point options
1765                 co[c] = group_options[c];
1766
1767                 // store the value
1768                 if(unlikely(group_counts[c] == 0)) {
1769                     cn[c] = 0.0;
1770                     co[c] |= RRDR_EMPTY;
1771                     group_values[c] = (group_method == GROUP_MAX || group_method == GROUP_MIN)?NAN:0;
1772                 }
1773                 else {
1774                     switch(group_method) {
1775                         case GROUP_MIN:
1776                         case GROUP_MAX:
1777                             if(unlikely(isnan(group_values[c])))
1778                                 cn[c] = 0;
1779                             else {
1780                                 cn[c] = group_values[c];
1781                                 group_values[c] = NAN;
1782                             }
1783                             break;
1784
1785                         case GROUP_SUM:
1786                         case GROUP_INCREMENTAL_SUM:
1787                             cn[c] = group_values[c];
1788                             group_values[c] = 0;
1789                             break;
1790
1791                         default:
1792                         case GROUP_AVERAGE:
1793                         case GROUP_UNDEFINED:
1794                             cn[c] = group_values[c] / group_counts[c];
1795                             group_values[c] = 0;
1796                             break;
1797                     }
1798
1799                     if(cn[c] < r->min) r->min = cn[c];
1800                     if(cn[c] > r->max) r->max = cn[c];
1801                 }
1802
1803                 // reset for the next loop
1804                 group_counts[c] = 0;
1805                 group_options[c] = 0;
1806             }
1807
1808             added++;
1809             group_count = 0;
1810             add_this = 0;
1811         }
1812     }
1813
1814     rrdr_done(r);
1815     //info("RRD2RRDR(): %s: END %ld loops made, %ld points generated", st->id, counter, rrdr_rows(r));
1816     //error("SHIFT: %s: wanted %ld points, got %ld", st->id, points, rrdr_rows(r));
1817     return r;
1818 }
1819
1820 int rrd2value(RRDSET *st, BUFFER *wb, calculated_number *n, const char *dimensions, long points, long long after, long long before, int group_method, uint32_t options, time_t *db_after, time_t *db_before, int *value_is_null)
1821 {
1822     RRDR *r = rrd2rrdr(st, points, after, before, group_method, !(options & RRDR_OPTION_NOT_ALIGNED));
1823     if(!r) {
1824         if(value_is_null) *value_is_null = 1;
1825         return 500;
1826     }
1827
1828     if(rrdr_rows(r) == 0) {
1829         rrdr_free(r);
1830
1831         if(db_after)  *db_after  = 0;
1832         if(db_before) *db_before = 0;
1833         if(value_is_null) *value_is_null = 1;
1834
1835         return 400;
1836     }
1837
1838     if(r->result_options & RRDR_RESULT_OPTION_RELATIVE)
1839         buffer_no_cacheable(wb);
1840     else if(r->result_options & RRDR_RESULT_OPTION_ABSOLUTE)
1841         buffer_cacheable(wb);
1842
1843     options = rrdr_check_options(r, options, dimensions);
1844
1845     if(dimensions)
1846         rrdr_disable_not_selected_dimensions(r, options, dimensions);
1847
1848     if(db_after)  *db_after  = r->after;
1849     if(db_before) *db_before = r->before;
1850
1851     long i = (options & RRDR_OPTION_REVERSED)?rrdr_rows(r) - 1:0;
1852     *n = rrdr2value(r, i, options, value_is_null);
1853
1854     rrdr_free(r);
1855     return 200;
1856 }
1857
1858 int rrd2format(RRDSET *st, BUFFER *wb, BUFFER *dimensions, uint32_t format, long points, long long after, long long before, int group_method, uint32_t options, time_t *latest_timestamp)
1859 {
1860     RRDR *r = rrd2rrdr(st, points, after, before, group_method, !(options & RRDR_OPTION_NOT_ALIGNED));
1861     if(!r) {
1862         buffer_strcat(wb, "Cannot generate output with these parameters on this chart.");
1863         return 500;
1864     }
1865
1866     if(r->result_options & RRDR_RESULT_OPTION_RELATIVE)
1867         buffer_no_cacheable(wb);
1868     else if(r->result_options & RRDR_RESULT_OPTION_ABSOLUTE)
1869         buffer_cacheable(wb);
1870
1871     options = rrdr_check_options(r, options, (dimensions)?buffer_tostring(dimensions):NULL);
1872
1873     if(dimensions)
1874         rrdr_disable_not_selected_dimensions(r, options, buffer_tostring(dimensions));
1875
1876     if(latest_timestamp && rrdr_rows(r) > 0)
1877         *latest_timestamp = r->before;
1878
1879     switch(format) {
1880     case DATASOURCE_SSV:
1881         if(options & RRDR_OPTION_JSON_WRAP) {
1882             wb->contenttype = CT_APPLICATION_JSON;
1883             rrdr_json_wrapper_begin(r, wb, format, options, 1);
1884             rrdr2ssv(r, wb, options, "", " ", "");
1885             rrdr_json_wrapper_end(r, wb, format, options, 1);
1886         }
1887         else {
1888             wb->contenttype = CT_TEXT_PLAIN;
1889             rrdr2ssv(r, wb, options, "", " ", "");
1890         }
1891         break;
1892
1893     case DATASOURCE_SSV_COMMA:
1894         if(options & RRDR_OPTION_JSON_WRAP) {
1895             wb->contenttype = CT_APPLICATION_JSON;
1896             rrdr_json_wrapper_begin(r, wb, format, options, 1);
1897             rrdr2ssv(r, wb, options, "", ",", "");
1898             rrdr_json_wrapper_end(r, wb, format, options, 1);
1899         }
1900         else {
1901             wb->contenttype = CT_TEXT_PLAIN;
1902             rrdr2ssv(r, wb, options, "", ",", "");
1903         }
1904         break;
1905
1906     case DATASOURCE_JS_ARRAY:
1907         if(options & RRDR_OPTION_JSON_WRAP) {
1908             wb->contenttype = CT_APPLICATION_JSON;
1909             rrdr_json_wrapper_begin(r, wb, format, options, 0);
1910             rrdr2ssv(r, wb, options, "[", ",", "]");
1911             rrdr_json_wrapper_end(r, wb, format, options, 0);
1912         }
1913         else {
1914             wb->contenttype = CT_APPLICATION_JSON;
1915             rrdr2ssv(r, wb, options, "[", ",", "]");
1916         }
1917         break;
1918
1919     case DATASOURCE_CSV:
1920         if(options & RRDR_OPTION_JSON_WRAP) {
1921             wb->contenttype = CT_APPLICATION_JSON;
1922             rrdr_json_wrapper_begin(r, wb, format, options, 1);
1923             rrdr2csv(r, wb, options, "", ",", "\\n", "");
1924             rrdr_json_wrapper_end(r, wb, format, options, 1);
1925         }
1926         else {
1927             wb->contenttype = CT_TEXT_PLAIN;
1928             rrdr2csv(r, wb, options, "", ",", "\r\n", "");
1929         }
1930         break;
1931
1932     case DATASOURCE_CSV_JSON_ARRAY:
1933         wb->contenttype = CT_APPLICATION_JSON;
1934         if(options & RRDR_OPTION_JSON_WRAP) {
1935             rrdr_json_wrapper_begin(r, wb, format, options, 0);
1936             buffer_strcat(wb, "[\n");
1937             rrdr2csv(r, wb, options + RRDR_OPTION_LABEL_QUOTES, "[", ",", "]", ",\n");
1938             buffer_strcat(wb, "\n]");
1939             rrdr_json_wrapper_end(r, wb, format, options, 0);
1940         }
1941         else {
1942             wb->contenttype = CT_TEXT_PLAIN;
1943             buffer_strcat(wb, "[\n");
1944             rrdr2csv(r, wb, options + RRDR_OPTION_LABEL_QUOTES, "[", ",", "]", ",\n");
1945             buffer_strcat(wb, "\n]");
1946         }
1947         break;
1948
1949     case DATASOURCE_TSV:
1950         if(options & RRDR_OPTION_JSON_WRAP) {
1951             wb->contenttype = CT_APPLICATION_JSON;
1952             rrdr_json_wrapper_begin(r, wb, format, options, 1);
1953             rrdr2csv(r, wb, options, "", "\t", "\\n", "");
1954             rrdr_json_wrapper_end(r, wb, format, options, 1);
1955         }
1956         else {
1957             wb->contenttype = CT_TEXT_PLAIN;
1958             rrdr2csv(r, wb, options, "", "\t", "\r\n", "");
1959         }
1960         break;
1961
1962     case DATASOURCE_HTML:
1963         if(options & RRDR_OPTION_JSON_WRAP) {
1964             wb->contenttype = CT_APPLICATION_JSON;
1965             rrdr_json_wrapper_begin(r, wb, format, options, 1);
1966             buffer_strcat(wb, "<html>\\n<center>\\n<table border=\\\"0\\\" cellpadding=\\\"5\\\" cellspacing=\\\"5\\\">\\n");
1967             rrdr2csv(r, wb, options, "<tr><td>", "</td><td>", "</td></tr>\\n", "");
1968             buffer_strcat(wb, "</table>\\n</center>\\n</html>\\n");
1969             rrdr_json_wrapper_end(r, wb, format, options, 1);
1970         }
1971         else {
1972             wb->contenttype = CT_TEXT_HTML;
1973             buffer_strcat(wb, "<html>\n<center>\n<table border=\"0\" cellpadding=\"5\" cellspacing=\"5\">\n");
1974             rrdr2csv(r, wb, options, "<tr><td>", "</td><td>", "</td></tr>\n", "");
1975             buffer_strcat(wb, "</table>\n</center>\n</html>\n");
1976         }
1977         break;
1978
1979     case DATASOURCE_DATATABLE_JSONP:
1980         wb->contenttype = CT_APPLICATION_X_JAVASCRIPT;
1981
1982         if(options & RRDR_OPTION_JSON_WRAP)
1983             rrdr_json_wrapper_begin(r, wb, format, options, 0);
1984
1985         rrdr2json(r, wb, options, 1);
1986
1987         if(options & RRDR_OPTION_JSON_WRAP)
1988             rrdr_json_wrapper_end(r, wb, format, options, 0);
1989         break;
1990
1991     case DATASOURCE_DATATABLE_JSON:
1992         wb->contenttype = CT_APPLICATION_JSON;
1993
1994         if(options & RRDR_OPTION_JSON_WRAP)
1995             rrdr_json_wrapper_begin(r, wb, format, options, 0);
1996
1997         rrdr2json(r, wb, options, 1);
1998
1999         if(options & RRDR_OPTION_JSON_WRAP)
2000             rrdr_json_wrapper_end(r, wb, format, options, 0);
2001         break;
2002
2003     case DATASOURCE_JSONP:
2004         wb->contenttype = CT_APPLICATION_X_JAVASCRIPT;
2005         if(options & RRDR_OPTION_JSON_WRAP)
2006             rrdr_json_wrapper_begin(r, wb, format, options, 0);
2007
2008         rrdr2json(r, wb, options, 0);
2009
2010         if(options & RRDR_OPTION_JSON_WRAP)
2011             rrdr_json_wrapper_end(r, wb, format, options, 0);
2012         break;
2013
2014     case DATASOURCE_JSON:
2015     default:
2016         wb->contenttype = CT_APPLICATION_JSON;
2017
2018         if(options & RRDR_OPTION_JSON_WRAP)
2019             rrdr_json_wrapper_begin(r, wb, format, options, 0);
2020
2021         rrdr2json(r, wb, options, 0);
2022
2023         if(options & RRDR_OPTION_JSON_WRAP)
2024             rrdr_json_wrapper_end(r, wb, format, options, 0);
2025         break;
2026     }
2027
2028     rrdr_free(r);
2029     return 200;
2030 }
2031
2032 time_t rrd_stats_json(int type, RRDSET *st, BUFFER *wb, long points, long group, int group_method, time_t after, time_t before, int only_non_zero)
2033 {
2034     int c;
2035     rrdset_rdlock(st);
2036
2037
2038     // -------------------------------------------------------------------------
2039     // switch from JSON to google JSON
2040
2041     char kq[2] = "\"";
2042     char sq[2] = "\"";
2043     switch(type) {
2044         case DATASOURCE_DATATABLE_JSON:
2045         case DATASOURCE_DATATABLE_JSONP:
2046             kq[0] = '\0';
2047             sq[0] = '\'';
2048             break;
2049
2050         case DATASOURCE_JSON:
2051         default:
2052             break;
2053     }
2054
2055
2056     // -------------------------------------------------------------------------
2057     // validate the parameters
2058
2059     if(points < 1) points = 1;
2060     if(group < 1) group = 1;
2061
2062     if(before == 0 || before > rrdset_last_entry_t(st)) before = rrdset_last_entry_t(st);
2063     if(after  == 0 || after < rrdset_first_entry_t(st)) after = rrdset_first_entry_t(st);
2064
2065     // ---
2066
2067     // our return value (the last timestamp printed)
2068     // this is required to detect re-transmit in google JSONP
2069     time_t last_timestamp = 0;
2070
2071
2072     // -------------------------------------------------------------------------
2073     // find how many dimensions we have
2074
2075     int dimensions = 0;
2076     RRDDIM *rd;
2077     rrddim_foreach_read(rd, st) dimensions++;
2078     if(!dimensions) {
2079         rrdset_unlock(st);
2080         buffer_strcat(wb, "No dimensions yet.");
2081         return 0;
2082     }
2083
2084
2085     // -------------------------------------------------------------------------
2086     // prepare various strings, to speed up the loop
2087
2088     char overflow_annotation[201]; snprintfz(overflow_annotation, 200, ",{%sv%s:%sRESET OR OVERFLOW%s},{%sv%s:%sThe counters have been wrapped.%s}", kq, kq, sq, sq, kq, kq, sq, sq);
2089     char normal_annotation[201];   snprintfz(normal_annotation,   200, ",{%sv%s:null},{%sv%s:null}", kq, kq, kq, kq);
2090     char pre_date[51];             snprintfz(pre_date,             50, "        {%sc%s:[{%sv%s:%s", kq, kq, kq, kq, sq);
2091     char post_date[21];            snprintfz(post_date,            20, "%s}", sq);
2092     char pre_value[21];            snprintfz(pre_value,            20, ",{%sv%s:", kq, kq);
2093     char post_value[21];           strcpy(post_value,                  "}");
2094
2095
2096     // -------------------------------------------------------------------------
2097     // checks for debugging
2098
2099     if(rrdset_flag_check(st, RRDSET_FLAG_DEBUG)) {
2100         debug(D_RRD_STATS, "%s first_entry_t = %ld, last_entry_t = %ld, duration = %ld, after = %ld, before = %ld, duration = %ld, entries_to_show = %ld, group = %ld"
2101             , st->id
2102             , rrdset_first_entry_t(st)
2103             , rrdset_last_entry_t(st)
2104             , rrdset_last_entry_t(st) - rrdset_first_entry_t(st)
2105             , after
2106             , before
2107             , before - after
2108             , points
2109             , group
2110             );
2111
2112         if(before < after)
2113             debug(D_RRD_STATS, "WARNING: %s The newest value in the database (%ld) is earlier than the oldest (%ld)", st->name, before, after);
2114
2115         if((before - after) > st->entries * st->update_every)
2116             debug(D_RRD_STATS, "WARNING: %s The time difference between the oldest and the newest entries (%ld) is higher than the capacity of the database (%ld)", st->name, before - after, st->entries * st->update_every);
2117     }
2118
2119
2120     // -------------------------------------------------------------------------
2121     // temp arrays for keeping values per dimension
2122
2123     calculated_number group_values[dimensions]; // keep sums when grouping
2124     int               print_hidden[dimensions]; // keep hidden flags
2125     int               found_non_zero[dimensions];
2126     int               found_non_existing[dimensions];
2127
2128     // initialize them
2129     for( rd = st->dimensions, c = 0 ; rd && c < dimensions ; rd = rd->next, c++) {
2130         group_values[c] = 0;
2131         print_hidden[c] = rrddim_flag_check(rd, RRDDIM_FLAG_HIDDEN)?1:0;
2132         found_non_zero[c] = 0;
2133         found_non_existing[c] = 0;
2134     }
2135
2136
2137     // error("OLD: points=%d after=%d before=%d group=%d, duration=%d", entries_to_show, before - (st->update_every * group * entries_to_show), before, group, before - after + 1);
2138     // rrd2array(st, entries_to_show, before - (st->update_every * group * entries_to_show), before, group_method, only_non_zero);
2139     // rrd2rrdr(st, entries_to_show, before - (st->update_every * group * entries_to_show), before, group_method);
2140
2141     // -------------------------------------------------------------------------
2142     // remove dimensions that contain only zeros
2143
2144     int max_loop = 1;
2145     if(only_non_zero) max_loop = 2;
2146
2147     for(; max_loop ; max_loop--) {
2148
2149         // -------------------------------------------------------------------------
2150         // print the JSON header
2151
2152         buffer_sprintf(wb, "{\n %scols%s:\n [\n", kq, kq);
2153         buffer_sprintf(wb, "        {%sid%s:%s%s,%slabel%s:%stime%s,%spattern%s:%s%s,%stype%s:%sdatetime%s},\n", kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq);
2154         buffer_sprintf(wb, "        {%sid%s:%s%s,%slabel%s:%s%s,%spattern%s:%s%s,%stype%s:%sstring%s,%sp%s:{%srole%s:%sannotation%s}},\n", kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, kq, kq, sq, sq);
2155         buffer_sprintf(wb, "        {%sid%s:%s%s,%slabel%s:%s%s,%spattern%s:%s%s,%stype%s:%sstring%s,%sp%s:{%srole%s:%sannotationText%s}}", kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, kq, kq, sq, sq);
2156
2157         // print the header for each dimension
2158         // and update the print_hidden array for the dimensions that should be hidden
2159         int pc = 0;
2160         for( rd = st->dimensions, c = 0 ; rd && c < dimensions ; rd = rd->next, c++) {
2161             if(!print_hidden[c]) {
2162                 pc++;
2163                 buffer_sprintf(wb, ",\n     {%sid%s:%s%s,%slabel%s:%s%s%s,%spattern%s:%s%s,%stype%s:%snumber%s}", kq, kq, sq, sq, kq, kq, sq, rd->name, sq, kq, kq, sq, sq, kq, kq, sq, sq);
2164             }
2165         }
2166         if(!pc) {
2167             buffer_sprintf(wb, ",\n     {%sid%s:%s%s,%slabel%s:%s%s%s,%spattern%s:%s%s,%stype%s:%snumber%s}", kq, kq, sq, sq, kq, kq, sq, "no data", sq, kq, kq, sq, sq, kq, kq, sq, sq);
2168         }
2169
2170         // print the begin of row data
2171         buffer_sprintf(wb, "\n  ],\n    %srows%s:\n [\n", kq, kq);
2172
2173
2174         // -------------------------------------------------------------------------
2175         // the main loop
2176
2177         int annotate_reset = 0;
2178         int annotation_count = 0;
2179
2180         long    t = rrdset_time2slot(st, before),
2181                 stop_at_t = rrdset_time2slot(st, after),
2182                 stop_now = 0;
2183
2184         t -= t % group;
2185
2186         time_t  now = rrdset_slot2time(st, t),
2187                 dt = st->update_every;
2188
2189         long count = 0, printed = 0, group_count = 0;
2190         last_timestamp = 0;
2191
2192         if(unlikely(rrdset_flag_check(st, RRDSET_FLAG_DEBUG)))
2193             debug(D_RRD_STATS, "%s: REQUEST after:%u before:%u, points:%ld, group:%ld, CHART cur:%ld first: %u last:%u, CALC start_t:%ld, stop_t:%ld"
2194                     , st->id
2195                     , (uint32_t)after
2196                     , (uint32_t)before
2197                     , points
2198                     , group
2199                     , st->current_entry
2200                     , (uint32_t)rrdset_first_entry_t(st)
2201                     , (uint32_t)rrdset_last_entry_t(st)
2202                     , t
2203                     , stop_at_t
2204                     );
2205
2206         long counter = 0;
2207         for(; !stop_now ; now -= dt, t--, counter++) {
2208             if(t < 0) t = st->entries - 1;
2209             if(t == stop_at_t) stop_now = counter;
2210
2211             int print_this = 0;
2212
2213             if(unlikely(rrdset_flag_check(st, RRDSET_FLAG_DEBUG)))
2214                 debug(D_RRD_STATS, "%s t = %ld, count = %ld, group_count = %ld, printed = %ld, now = %ld, %s %s"
2215                     , st->id
2216                     , t
2217                     , count + 1
2218                     , group_count + 1
2219                     , printed
2220                     , now
2221                     , (group_count + 1 == group)?"PRINT":"  -  "
2222                     , (now >= after && now <= before)?"RANGE":"  -  "
2223                     );
2224
2225
2226             // make sure we return data in the proper time range
2227             if(now > before) continue;
2228             if(now < after) break;
2229
2230             //if(rrdset_slot2time(st, t) != now)
2231             //  error("%s: slot=%ld, now=%ld, slot2time=%ld, diff=%ld, last_entry_t=%ld, rrdset_last_slot=%ld", st->id, t, now, rrdset_slot2time(st,t), now - rrdset_slot2time(st,t), rrdset_last_entry_t(st), rrdset_last_slot(st));
2232
2233             count++;
2234             group_count++;
2235
2236             // check if we have to print this now
2237             if(group_count == group) {
2238                 if(printed >= points) {
2239                     // debug(D_RRD_STATS, "Already printed all rows. Stopping.");
2240                     break;
2241                 }
2242
2243                 // generate the local date time
2244                 struct tm tmbuf, *tm = localtime_r(&now, &tmbuf);
2245                 if(!tm) { error("localtime() failed."); continue; }
2246                 if(now > last_timestamp) last_timestamp = now;
2247
2248                 if(printed) buffer_strcat(wb, "]},\n");
2249                 buffer_strcat(wb, pre_date);
2250                 buffer_jsdate(wb, tm->tm_year + 1900, tm->tm_mon, tm->tm_mday, tm->tm_hour, tm->tm_min, tm->tm_sec);
2251                 buffer_strcat(wb, post_date);
2252
2253                 print_this = 1;
2254             }
2255
2256             // do the calculations
2257             for(rd = st->dimensions, c = 0 ; rd && c < dimensions ; rd = rd->next, c++) {
2258                 storage_number n = rd->values[t];
2259                 calculated_number value = unpack_storage_number(n);
2260
2261                 if(!does_storage_number_exist(n)) {
2262                     value = 0.0;
2263                     found_non_existing[c]++;
2264                 }
2265                 if(did_storage_number_reset(n)) annotate_reset = 1;
2266
2267                 switch(group_method) {
2268                     case GROUP_MAX:
2269                         if(abs(value) > abs(group_values[c])) group_values[c] = value;
2270                         break;
2271
2272                     case GROUP_SUM:
2273                         group_values[c] += value;
2274                         break;
2275
2276                     default:
2277                     case GROUP_AVERAGE:
2278                         group_values[c] += value;
2279                         if(print_this) group_values[c] /= ( group_count - found_non_existing[c] );
2280                         break;
2281                 }
2282             }
2283
2284             if(print_this) {
2285                 if(annotate_reset) {
2286                     annotation_count++;
2287                     buffer_strcat(wb, overflow_annotation);
2288                     annotate_reset = 0;
2289                 }
2290                 else
2291                     buffer_strcat(wb, normal_annotation);
2292
2293                 pc = 0;
2294                 for(c = 0 ; c < dimensions ; c++) {
2295                     if(found_non_existing[c] == group_count) {
2296                         // all entries are non-existing
2297                         pc++;
2298                         buffer_strcat(wb, pre_value);
2299                         buffer_strcat(wb, "null");
2300                         buffer_strcat(wb, post_value);
2301                     }
2302                     else if(!print_hidden[c]) {
2303                         pc++;
2304                         buffer_strcat(wb, pre_value);
2305                         buffer_rrd_value(wb, group_values[c]);
2306                         buffer_strcat(wb, post_value);
2307
2308                         if(group_values[c]) found_non_zero[c]++;
2309                     }
2310
2311                     // reset them for the next loop
2312                     group_values[c] = 0;
2313                     found_non_existing[c] = 0;
2314                 }
2315
2316                 // if all dimensions are hidden, print a null
2317                 if(!pc) {
2318                     buffer_strcat(wb, pre_value);
2319                     buffer_strcat(wb, "null");
2320                     buffer_strcat(wb, post_value);
2321                 }
2322
2323                 printed++;
2324                 group_count = 0;
2325             }
2326         }
2327
2328         if(printed) buffer_strcat(wb, "]}");
2329         buffer_strcat(wb, "\n   ]\n}\n");
2330
2331         if(only_non_zero && max_loop > 1) {
2332             int changed = 0;
2333             for(rd = st->dimensions, c = 0 ; rd && c < dimensions ; rd = rd->next, c++) {
2334                 group_values[c] = 0;
2335                 found_non_existing[c] = 0;
2336
2337                 if(!print_hidden[c] && !found_non_zero[c]) {
2338                     changed = 1;
2339                     print_hidden[c] = 1;
2340                 }
2341             }
2342
2343             if(changed) buffer_flush(wb);
2344             else break;
2345         }
2346         else break;
2347
2348     } // max_loop
2349
2350     debug(D_RRD_STATS, "RRD_STATS_JSON: %s total %zu bytes", st->name, wb->len);
2351
2352     rrdset_unlock(st);
2353     return last_timestamp;
2354 }