src/rrd2json.c

   1 #ifdef HAVE_CONFIG_H
   2 #include <config.h>
   3 #endif
   4 #include <pthread.h>
   5 #include <sys/time.h>
   6 #include <stdlib.h>
   7 #include <string.h>
   8
   9 #include "log.h"
  10 #include "common.h"
  11 #include "rrd2json.h"
  12
  13 #define HOSTNAME_MAX 1024
  14 char *hostname = "unknown";
  15
  16 void rrd_stats_api_v1_chart(RRDSET *st, BUFFER *wb)
  17 {
  18         pthread_rwlock_rdlock(&st->rwlock);
  19
  20         buffer_sprintf(wb,
  21                 "\t\t{\n"
  22                 "\t\t\t\"id\": \"%s\",\n"
  23                 "\t\t\t\"name\": \"%s\",\n"
  24                 "\t\t\t\"type\": \"%s\",\n"
  25                 "\t\t\t\"family\": \"%s\",\n"
  26                 "\t\t\t\"title\": \"%s\",\n"
  27                 "\t\t\t\"priority\": %ld,\n"
  28                 "\t\t\t\"enabled\": %s,\n"
  29                 "\t\t\t\"units\": \"%s\",\n"
  30                 "\t\t\t\"data_url\": \"/api/v1/data?chart=%s\",\n"
  31                 "\t\t\t\"chart_type\": \"%s\",\n"
  32                 "\t\t\t\"duration\": %ld,\n"
  33                 "\t\t\t\"first_entry\": %lu,\n"
  34                 "\t\t\t\"last_entry\": %lu,\n"
  35                 "\t\t\t\"update_every\": %d,\n"
  36                 "\t\t\t\"dimensions\": {\n"
  37                 , st->id
  38                 , st->name
  39                 , st->type
  40                 , st->family
  41                 , st->title
  42                 , st->priority
  43                 , st->enabled?"true":"false"
  44                 , st->units
  45                 , st->name
  46                 , rrdset_type_name(st->chart_type)
  47                 , st->entries * st->update_every
  48                 , rrdset_first_entry_t(st)
  49                 , rrdset_last_entry_t(st)
  50                 , st->update_every
  51                 );
  52
  53         unsigned long memory = st->memsize;
  54
  55         int c = 0;
  56         RRDDIM *rd;
  57         for(rd = st->dimensions; rd ; rd = rd->next) {
  58                 if(rd->flags & RRDDIM_FLAG_HIDDEN) continue;
  59
  60                 memory += rd->memsize;
  61
  62                 buffer_sprintf(wb,
  63                         "%s"
  64                         "\t\t\t\t\"%s\": { \"name\": \"%s\" }"
  65                         , c?",\n":""
  66                         , rd->id
  67                         , rd->name
  68                         );
  69
  70                 c++;
  71         }
  72
  73         buffer_sprintf(wb,
  74                 "\n\t\t\t}\n"
  75                 "\t\t}"
  76                 );
  77
  78         pthread_rwlock_unlock(&st->rwlock);
  79 }
  80
  81 void rrd_stats_api_v1_charts(BUFFER *wb)
  82 {
  83         long c;
  84         RRDSET *st;
  85
  86         buffer_sprintf(wb, "{\n"
  87                    "\t\"hostname\": \"%s\""
  88                 ",\n\t\"update_every\": %d"
  89                 ",\n\t\"history\": %d"
  90                 ",\n\t\"charts\": {"
  91                 , hostname
  92                 , rrd_update_every
  93                 , rrd_default_history_entries
  94                 );
  95
  96         pthread_rwlock_rdlock(&rrdset_root_rwlock);
  97         for(st = rrdset_root, c = 0; st ; st = st->next) {
  98                 if(st->enabled) {
  99                         if(c) buffer_strcat(wb, ",");
 100                         buffer_strcat(wb, "\n\t\t\"");
 101                         buffer_strcat(wb, st->id);
 102                         buffer_strcat(wb, "\": ");
 103                         rrd_stats_api_v1_chart(st, wb);
 104                         c++;
 105                 }
 106         }
 107         pthread_rwlock_unlock(&rrdset_root_rwlock);
 108
 109         buffer_strcat(wb, "\n\t}\n}\n");
 110 }
 111
 112
 113 unsigned long rrd_stats_one_json(RRDSET *st, char *options, BUFFER *wb)
 114 {
 115         time_t now = time(NULL);
 116
 117         pthread_rwlock_rdlock(&st->rwlock);
 118
 119         buffer_sprintf(wb,
 120                 "\t\t{\n"
 121                 "\t\t\t\"id\": \"%s\",\n"
 122                 "\t\t\t\"name\": \"%s\",\n"
 123                 "\t\t\t\"type\": \"%s\",\n"
 124                 "\t\t\t\"family\": \"%s\",\n"
 125                 "\t\t\t\"title\": \"%s\",\n"
 126                 "\t\t\t\"priority\": %ld,\n"
 127                 "\t\t\t\"enabled\": %d,\n"
 128                 "\t\t\t\"units\": \"%s\",\n"
 129                 "\t\t\t\"url\": \"/data/%s/%s\",\n"
 130                 "\t\t\t\"chart_type\": \"%s\",\n"
 131                 "\t\t\t\"counter\": %ld,\n"
 132                 "\t\t\t\"entries\": %ld,\n"
 133                 "\t\t\t\"first_entry_t\": %lu,\n"
 134                 "\t\t\t\"last_entry\": %ld,\n"
 135                 "\t\t\t\"last_entry_t\": %lu,\n"
 136                 "\t\t\t\"last_entry_secs_ago\": %lu,\n"
 137                 "\t\t\t\"update_every\": %d,\n"
 138                 "\t\t\t\"isdetail\": %d,\n"
 139                 "\t\t\t\"usec_since_last_update\": %llu,\n"
 140                 "\t\t\t\"collected_total\": " TOTAL_NUMBER_FORMAT ",\n"
 141                 "\t\t\t\"last_collected_total\": " TOTAL_NUMBER_FORMAT ",\n"
 142                 "\t\t\t\"dimensions\": [\n"
 143                 , st->id
 144                 , st->name
 145                 , st->type
 146                 , st->family
 147                 , st->title
 148                 , st->priority
 149                 , st->enabled
 150                 , st->units
 151                 , st->name, options?options:""
 152                 , rrdset_type_name(st->chart_type)
 153                 , st->counter
 154                 , st->entries
 155                 , rrdset_first_entry_t(st)
 156                 , rrdset_last_slot(st)
 157                 , rrdset_last_entry_t(st)
 158                 , (now < rrdset_last_entry_t(st)) ? (time_t)0 : now - rrdset_last_entry_t(st)
 159                 , st->update_every
 160                 , st->isdetail
 161                 , st->usec_since_last_update
 162                 , st->collected_total
 163                 , st->last_collected_total
 164                 );
 165
 166         unsigned long memory = st->memsize;
 167
 168         RRDDIM *rd;
 169         for(rd = st->dimensions; rd ; rd = rd->next) {
 170
 171                 memory += rd->memsize;
 172
 173                 buffer_sprintf(wb,
 174                         "\t\t\t\t{\n"
 175                         "\t\t\t\t\t\"id\": \"%s\",\n"
 176                         "\t\t\t\t\t\"name\": \"%s\",\n"
 177                         "\t\t\t\t\t\"entries\": %ld,\n"
 178                         "\t\t\t\t\t\"isHidden\": %d,\n"
 179                         "\t\t\t\t\t\"algorithm\": \"%s\",\n"
 180                         "\t\t\t\t\t\"multiplier\": %ld,\n"
 181                         "\t\t\t\t\t\"divisor\": %ld,\n"
 182                         "\t\t\t\t\t\"last_entry_t\": %lu,\n"
 183                         "\t\t\t\t\t\"collected_value\": " COLLECTED_NUMBER_FORMAT ",\n"
 184                         "\t\t\t\t\t\"calculated_value\": " CALCULATED_NUMBER_FORMAT ",\n"
 185                         "\t\t\t\t\t\"last_collected_value\": " COLLECTED_NUMBER_FORMAT ",\n"
 186                         "\t\t\t\t\t\"last_calculated_value\": " CALCULATED_NUMBER_FORMAT ",\n"
 187                         "\t\t\t\t\t\"memory\": %lu\n"
 188                         "\t\t\t\t}%s\n"
 189                         , rd->id
 190                         , rd->name
 191                         , rd->entries
 192                         , (rd->flags & RRDDIM_FLAG_HIDDEN)?1:0
 193                         , rrddim_algorithm_name(rd->algorithm)
 194                         , rd->multiplier
 195                         , rd->divisor
 196                         , rd->last_collected_time.tv_sec
 197                         , rd->collected_value
 198                         , rd->calculated_value
 199                         , rd->last_collected_value
 200                         , rd->last_calculated_value
 201                         , rd->memsize
 202                         , rd->next?",":""
 203                         );
 204         }
 205
 206         buffer_sprintf(wb,
 207                 "\t\t\t],\n"
 208                 "\t\t\t\"memory\" : %lu\n"
 209                 "\t\t}"
 210                 , memory
 211                 );
 212
 213         pthread_rwlock_unlock(&st->rwlock);
 214         return memory;
 215 }
 216
 217 #define RRD_GRAPH_JSON_HEADER "{\n\t\"charts\": [\n"
 218 #define RRD_GRAPH_JSON_FOOTER "\n\t]\n}\n"
 219
 220 void rrd_stats_graph_json(RRDSET *st, char *options, BUFFER *wb)
 221 {
 222         buffer_strcat(wb, RRD_GRAPH_JSON_HEADER);
 223         rrd_stats_one_json(st, options, wb);
 224         buffer_strcat(wb, RRD_GRAPH_JSON_FOOTER);
 225 }
 226
 227 void rrd_stats_all_json(BUFFER *wb)
 228 {
 229         unsigned long memory = 0;
 230         long c;
 231         RRDSET *st;
 232
 233         buffer_strcat(wb, RRD_GRAPH_JSON_HEADER);
 234
 235         pthread_rwlock_rdlock(&rrdset_root_rwlock);
 236         for(st = rrdset_root, c = 0; st ; st = st->next) {
 237                 if(st->enabled) {
 238                         if(c) buffer_strcat(wb, ",\n");
 239                         memory += rrd_stats_one_json(st, NULL, wb);
 240                         c++;
 241                 }
 242         }
 243         pthread_rwlock_unlock(&rrdset_root_rwlock);
 244
 245         buffer_sprintf(wb, "\n\t],\n"
 246                 "\t\"hostname\": \"%s\",\n"
 247                 "\t\"update_every\": %d,\n"
 248                 "\t\"history\": %d,\n"
 249                 "\t\"memory\": %lu\n"
 250                 "}\n"
 251                 , hostname
 252                 , rrd_update_every
 253                 , rrd_default_history_entries
 254                 , memory
 255                 );
 256 }
 257
 258
 259
 260 // ----------------------------------------------------------------------------
 261
 262 // RRDR options
 263 #define RRDR_EMPTY      0x01 // the dimension contains / the value is empty (null)
 264 #define RRDR_RESET      0x02 // the dimension contains / the value is reset
 265 #define RRDR_HIDDEN     0x04 // the dimension contains / the value is hidden
 266 #define RRDR_NONZERO    0x08 // the dimension contains / the value is non-zero
 267
 268
 269 typedef struct rrdresult {
 270         RRDSET *st;                     // the chart this result refers to
 271
 272         int d;                                  // the number of dimensions
 273         int n;                                  // the number of values in the arrays
 274
 275         uint8_t *od;                    // the options for the dimensions
 276
 277         time_t *t;                              // array of n timestamps
 278         calculated_number *v;   // array n x d values
 279         uint8_t *o;                             // array n x d options
 280
 281         int c;                                  // current line ( -1 ~ n ), ( -1 = none, use rrdr_rows() to get number of rows )
 282
 283         int has_st_lock;                // if st is read locked by us
 284 } RRDR;
 285
 286 #define rrdr_rows(r) ((r)->c + 1)
 287
 288 /*
 289 static void rrdr_dump(RRDR *r)
 290 {
 291         long c, i;
 292         RRDDIM *d;
 293
 294         fprintf(stderr, "\nCHART %s (%s)\n", r->st->id, r->st->name);
 295
 296         for(c = 0, d = r->st->dimensions; d ;c++, d = d->next) {
 297                 fprintf(stderr, "DIMENSION %s (%s), %s%s%s%s\n"
 298                                 , d->id
 299                                 , d->name
 300                                 , (r->od[c] & RRDR_EMPTY)?"EMPTY ":""
 301                                 , (r->od[c] & RRDR_RESET)?"RESET ":""
 302                                 , (r->od[c] & RRDR_HIDDEN)?"HIDDEN ":""
 303                                 , (r->od[c] & RRDR_NONZERO)?"NONZERO ":""
 304                                 );
 305         }
 306
 307         if(r->c < 0) {
 308                 fprintf(stderr, "RRDR does not have any values in it.\n");
 309                 return;
 310         }
 311
 312         fprintf(stderr, "RRDR includes %d values in it:\n", r->c + 1);
 313
 314         // for each line in the array
 315         for(i = 0; i <= r->c ;i++) {
 316                 calculated_number *cn = &r->v[ i * r->d ];
 317                 uint8_t *co = &r->o[ i * r->d ];
 318
 319                 // print the id and the timestamp of the line
 320                 fprintf(stderr, "%ld %ld ", i + 1, r->t[i]);
 321
 322                 // for each dimension
 323                 for(c = 0, d = r->st->dimensions; d ;c++, d = d->next) {
 324                         if(unlikely(r->od[c] & RRDR_HIDDEN)) continue;
 325                         if(unlikely(!(r->od[c] & RRDR_NONZERO))) continue;
 326
 327                         if(co[c] & RRDR_EMPTY)
 328                                 fprintf(stderr, "null ");
 329                         else
 330                                 fprintf(stderr, CALCULATED_NUMBER_FORMAT " %s%s%s%s "
 331                                         , cn[c]
 332                                         , (co[c] & RRDR_EMPTY)?"E":" "
 333                                         , (co[c] & RRDR_RESET)?"R":" "
 334                                         , (co[c] & RRDR_HIDDEN)?"H":" "
 335                                         , (co[c] & RRDR_NONZERO)?"N":" "
 336                                         );
 337                 }
 338
 339                 fprintf(stderr, "\n");
 340         }
 341 }
 342 */
 343
 344 void rrdr_disable_not_selected_dimensions(RRDR *r, const char *dims)
 345 {
 346         char b[strlen(dims) + 1];
 347         char *o = b, *tok;
 348         strcpy(o, dims);
 349
 350         long c;
 351         RRDDIM *d;
 352
 353         // disable all of them
 354         for(c = 0, d = r->st->dimensions; d ;c++, d = d->next)
 355                 r->od[c] |= RRDR_HIDDEN;
 356
 357         while(o && *o && (tok = mystrsep(&o, ", |"))) {
 358                 if(!*tok) continue;
 359
 360                 // find it and enable it
 361                 for(c = 0, d = r->st->dimensions; d ;c++, d = d->next) {
 362                         if(!strcmp(d->name, tok)) {
 363                                 r->od[c] &= ~RRDR_HIDDEN;
 364                         }
 365                 }
 366         }
 367 }
 368
 369 #define JSON_DATES_JS 1
 370 #define JSON_DATES_TIMESTAMP 2
 371
 372 static void rrdr2json(RRDR *r, BUFFER *wb, uint32_t options, int google)
 373 {
 374         int annotations = 0, dates = JSON_DATES_JS;
 375         char kq[2] = "",                                        // key quote
 376                 sq[2] = "",                                             // string quote
 377                 pre_label[101] = "",                    // before each label
 378                 post_label[101] = "",                   // after each label
 379                 pre_date[101] = "",                             // the beginning of line, to the date
 380                 post_date[101] = "",                    // closing the date
 381                 pre_value[101] = "",                    // before each value
 382                 post_value[101] = "",                   // after each value
 383                 post_line[101] = "",                    // at the end of each row
 384                 normal_annotation[201] = "",    // default row annotation
 385                 overflow_annotation[201] = "",  // overflow row annotation
 386                 data_begin[101] = "",                   // between labels and values
 387                 finish[101] = "";                               // at the end of everything
 388
 389         if(google) {
 390                 dates = JSON_DATES_JS;
 391                 kq[0] = '\0';
 392                 sq[0] = '\'';
 393                 annotations = 1;
 394                 snprintf(pre_date,   100, "             {%sc%s:[{%sv%s:%s", kq, kq, kq, kq, sq);
 395                 snprintf(post_date,  100, "%s}", sq);
 396                 snprintf(pre_label,  100, ",\n          {%sid%s:%s%s,%slabel%s:%s", kq, kq, sq, sq, kq, kq, sq);
 397                 snprintf(post_label, 100, "%s,%spattern%s:%s%s,%stype%s:%snumber%s}", sq, kq, kq, sq, sq, kq, kq, sq, sq);
 398                 snprintf(pre_value,  100, ",{%sv%s:", kq, kq);
 399                 snprintf(post_value, 100, "}");
 400                 snprintf(post_line,  100, "]}");
 401                 snprintf(data_begin, 100, "\n   ],\n    %srows%s:\n     [\n", kq, kq);
 402                 snprintf(finish,     100, "\n   ]\n}\n");
 403
 404                 snprintf(overflow_annotation, 200, ",{%sv%s:%sRESET OR OVERFLOW%s},{%sv%s:%sThe counters have been wrapped.%s}", kq, kq, sq, sq, kq, kq, sq, sq);
 405                 snprintf(normal_annotation,   200, ",{%sv%s:null},{%sv%s:null}", kq, kq, kq, kq);
 406
 407                 buffer_sprintf(wb, "{\n %scols%s:\n     [\n", kq, kq);
 408                 buffer_sprintf(wb, "            {%sid%s:%s%s,%slabel%s:%stime%s,%spattern%s:%s%s,%stype%s:%sdatetime%s},\n", kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq);
 409                 buffer_sprintf(wb, "            {%sid%s:%s%s,%slabel%s:%s%s,%spattern%s:%s%s,%stype%s:%sstring%s,%sp%s:{%srole%s:%sannotation%s}},\n", kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, kq, kq, sq, sq);
 410                 buffer_sprintf(wb, "            {%sid%s:%s%s,%slabel%s:%s%s,%spattern%s:%s%s,%stype%s:%sstring%s,%sp%s:{%srole%s:%sannotationText%s}}", kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, kq, kq, sq, sq);
 411         }
 412         else {
 413                 if((options & RRDR_OPTION_SECONDS) || (options & RRDR_OPTION_MILLISECONDS)) {
 414                         dates = JSON_DATES_TIMESTAMP;
 415                         snprintf(pre_date,   100, "             [");
 416                 }
 417                 else {
 418                         dates = JSON_DATES_JS;
 419                         snprintf(pre_date,   100, "             [new ");
 420                 }
 421                 kq[0] = '"';
 422                 sq[0] = '"';
 423                 snprintf(pre_label,  100, ", \"");
 424                 snprintf(post_label, 100, "\"");
 425                 snprintf(pre_value,  100, ", ");
 426                 snprintf(post_line,  100, "]");
 427                 snprintf(data_begin, 100, "],\n %sdata%s:\n     [\n", kq, kq);
 428                 snprintf(finish,     100, "\n   ]\n}\n");
 429
 430                 buffer_sprintf(wb, "{\n %slabels%s: [", kq, kq);
 431                 buffer_sprintf(wb, "%stime%s", sq, sq);
 432         }
 433
 434         // -------------------------------------------------------------------------
 435         // print the JSON header
 436
 437         long c, i;
 438         RRDDIM *rd;
 439
 440         // print the csv header
 441         for(c = 0, i = 0, rd = r->st->dimensions; rd ;c++, rd = rd->next) {
 442                 if(unlikely(r->od[c] & RRDR_HIDDEN)) continue;
 443                 if(unlikely((options & RRDR_OPTION_NONZERO) && !(r->od[c] & RRDR_NONZERO))) continue;
 444
 445                 buffer_strcat(wb, pre_label);
 446                 buffer_strcat(wb, rd->name);
 447                 buffer_strcat(wb, post_label);
 448                 i++;
 449         }
 450         if(!i) {
 451                 buffer_strcat(wb, pre_label);
 452                 buffer_strcat(wb, "no data");
 453                 buffer_strcat(wb, post_label);
 454         }
 455
 456         // print the begin of row data
 457         buffer_strcat(wb, data_begin);
 458
 459         // if all dimensions are hidden, print a null
 460         if(!i) {
 461                 buffer_strcat(wb, pre_value);
 462                 if(options & RRDR_OPTION_NULL2ZERO)
 463                         buffer_strcat(wb, "0");
 464                 else
 465                         buffer_strcat(wb, "null");
 466                 buffer_strcat(wb, post_value);
 467         }
 468
 469         long start = 0, end = rrdr_rows(r), step = 1;
 470         if((options & RRDR_OPTION_REVERSED)) {
 471                 start = rrdr_rows(r) - 1;
 472                 end = -1;
 473                 step = -1;
 474         }
 475
 476         // for each line in the array
 477         for(i = start; i != end ;i += step) {
 478                 calculated_number *cn = &r->v[ i * r->d ];
 479                 uint8_t *co = &r->o[ i * r->d ];
 480
 481                 time_t now = r->t[i];
 482
 483                 if(dates == JSON_DATES_JS) {
 484                         // generate the local date time
 485                         struct tm *tm = localtime(&now);
 486                         if(!tm) { error("localtime() failed."); continue; }
 487
 488                         if(likely(i != start)) buffer_strcat(wb, ",\n");
 489                         buffer_strcat(wb, pre_date);
 490                         buffer_jsdate(wb, tm->tm_year + 1900, tm->tm_mon, tm->tm_mday, tm->tm_hour, tm->tm_min, tm->tm_sec);
 491                         buffer_strcat(wb, post_date);
 492
 493                         if(annotations) {
 494                                 if(co[c] & RRDR_RESET)
 495                                         buffer_strcat(wb, overflow_annotation);
 496                                 else
 497                                         buffer_strcat(wb, normal_annotation);
 498                         }
 499                 }
 500                 else {
 501                         // print the timestamp of the line
 502                         if(likely(i != start)) buffer_strcat(wb, ",\n");
 503                         buffer_strcat(wb, pre_date);
 504                         buffer_rrd_value(wb, (calculated_number)r->t[i]);
 505                         // in ms
 506                         if(options & RRDR_OPTION_MILLISECONDS) buffer_strcat(wb, "000");
 507                         buffer_strcat(wb, post_date);
 508                 }
 509
 510                 // for each dimension
 511                 for(c = 0, rd = r->st->dimensions; rd ;c++, rd = rd->next) {
 512                         if(unlikely(r->od[c] & RRDR_HIDDEN)) continue;
 513                         if(unlikely((options & RRDR_OPTION_NONZERO) && !(r->od[c] & RRDR_NONZERO))) continue;
 514
 515                         calculated_number n = cn[c];
 516
 517                         if(co[c] & RRDR_EMPTY) {
 518                                 buffer_strcat(wb, pre_value);
 519                                 if(options & RRDR_OPTION_NULL2ZERO)
 520                                         buffer_strcat(wb, "0");
 521                                 else
 522                                         buffer_strcat(wb, "null");
 523                                 buffer_strcat(wb, post_value);
 524                         }
 525                         else if((options & RRDR_OPTION_ABSOLUTE)) {
 526                                 buffer_strcat(wb, pre_value);
 527                                 buffer_rrd_value(wb, (n<0)?-n:n);
 528                                 buffer_strcat(wb, post_value);
 529                         }
 530                         else {
 531                                 buffer_strcat(wb, pre_value);
 532                                 buffer_rrd_value(wb, n);
 533                                 buffer_strcat(wb, post_value);
 534                         }
 535                 }
 536
 537                 buffer_strcat(wb, post_line);
 538         }
 539
 540         buffer_strcat(wb, finish);
 541 }
 542
 543 static void rrdr2csv(RRDR *r, BUFFER *wb, uint32_t options, const char *startline, const char *separator, const char *endline)
 544 {
 545         long c, i;
 546         RRDDIM *d;
 547
 548         // print the csv header
 549         for(c = 0, i = 0, d = r->st->dimensions; d ;c++, d = d->next) {
 550                 if(unlikely(r->od[c] & RRDR_HIDDEN)) continue;
 551                 if(unlikely((options & RRDR_OPTION_NONZERO) && !(r->od[c] & RRDR_NONZERO))) continue;
 552
 553                 if(!i) {
 554                         buffer_strcat(wb, startline);
 555                         buffer_strcat(wb, "time");
 556                 }
 557                 buffer_strcat(wb, separator);
 558                 buffer_strcat(wb, d->name);
 559                 i++;
 560         }
 561         buffer_strcat(wb, endline);
 562
 563         if(!i) {
 564                 // no dimensions present
 565                 return;
 566         }
 567
 568         long start = 0, end = rrdr_rows(r), step = 1;
 569         if((options & RRDR_OPTION_REVERSED)) {
 570                 start = rrdr_rows(r) - 1;
 571                 end = -1;
 572                 step = -1;
 573         }
 574
 575         // for each line in the array
 576         for(i = start; i != end ;i += step) {
 577                 calculated_number *cn = &r->v[ i * r->d ];
 578                 uint8_t *co = &r->o[ i * r->d ];
 579
 580                 buffer_strcat(wb, startline);
 581
 582                 time_t now = r->t[i];
 583
 584                 if((options & RRDR_OPTION_SECONDS) || (options & RRDR_OPTION_MILLISECONDS)) {
 585                         // print the timestamp of the line
 586                         buffer_rrd_value(wb, (calculated_number)now);
 587                         // in ms
 588                         if(options & RRDR_OPTION_MILLISECONDS) buffer_strcat(wb, "000");
 589                 }
 590                 else {
 591                         // generate the local date time
 592                         struct tm *tm = localtime(&now);
 593                         if(!tm) { error("localtime() failed."); continue; }
 594                         buffer_date(wb, tm->tm_year + 1900, tm->tm_mon, tm->tm_mday, tm->tm_hour, tm->tm_min, tm->tm_sec);
 595                 }
 596
 597                 // for each dimension
 598                 for(c = 0, d = r->st->dimensions; d ;c++, d = d->next) {
 599                         if(unlikely(r->od[c] & RRDR_HIDDEN)) continue;
 600                         if(unlikely((options & RRDR_OPTION_NONZERO) && !(r->od[c] & RRDR_NONZERO))) continue;
 601
 602                         buffer_strcat(wb, separator);
 603
 604                         calculated_number n = cn[c];
 605
 606                         if(co[c] & RRDR_EMPTY) {
 607                                 if(options & RRDR_OPTION_NULL2ZERO)
 608                                         buffer_strcat(wb, "0");
 609                                 else
 610                                         buffer_strcat(wb, "null");
 611                         }
 612                         else if((options & RRDR_OPTION_ABSOLUTE))
 613                                 buffer_rrd_value(wb, (n<0)?-n:n);
 614                         else
 615                                 buffer_rrd_value(wb, n);
 616                 }
 617
 618                 buffer_strcat(wb, endline);
 619         }
 620 }
 621
 622 static void rrdr2ssv(RRDR *r, BUFFER *out, uint32_t options, const char *prefix, const char *separator, const char *suffix)
 623 {
 624         long c, i;
 625         RRDDIM *d;
 626
 627         buffer_strcat(out, prefix);
 628         long start = 0, end = rrdr_rows(r), step = 1;
 629         if((options & RRDR_OPTION_REVERSED)) {
 630                 start = rrdr_rows(r) - 1;
 631                 end = -1;
 632                 step = -1;
 633         }
 634
 635         // for each line in the array
 636         for(i = start; i != end ;i += step) {
 637
 638                 calculated_number *cn = &r->v[ i * r->d ];
 639                 uint8_t *co = &r->o[ i * r->d ];
 640
 641                 calculated_number sum = 0, min = 0, max = 0;
 642                 int all_null = 1, init = 1;
 643
 644                 // for each dimension
 645                 for(c = 0, d = r->st->dimensions; d ;c++, d = d->next) {
 646                         if(unlikely(r->od[c] & RRDR_HIDDEN)) continue;
 647                         if(unlikely((options & RRDR_OPTION_NONZERO) && !(r->od[c] & RRDR_NONZERO))) continue;
 648
 649                         calculated_number n = cn[c];
 650
 651                         if(unlikely(init)) {
 652                                 if(n > 0) {
 653                                         min = 0;
 654                                         max = n;
 655                                 }
 656                                 else {
 657                                         min = n;
 658                                         max = 0;
 659                                 }
 660                                 init = 0;
 661                         }
 662
 663                         if(likely(!(co[c] & RRDR_EMPTY))) {
 664                                 all_null = 0;
 665                                 if((options & RRDR_OPTION_ABSOLUTE) && n < 0) n = -n;
 666                                 sum += n;
 667                         }
 668
 669                         if(n < min) min = n;
 670                         if(n > max) max = n;
 671                 }
 672
 673                 if(likely(i != start))
 674                         buffer_strcat(out, separator);
 675
 676                 if(all_null) {
 677                         if(options & RRDR_OPTION_NULL2ZERO)
 678                                 buffer_strcat(out, "0");
 679                         else
 680                                 buffer_strcat(out, "null");
 681                 }
 682                 else if(options & RRDR_OPTION_MIN2MAX)
 683                         buffer_rrd_value(out, max - min);
 684                 else
 685                         buffer_rrd_value(out, sum);
 686         }
 687         buffer_strcat(out, suffix);
 688 }
 689
 690 inline static calculated_number *rrdr_line_values(RRDR *r)
 691 {
 692         return &r->v[ r->c * r->d ];
 693 }
 694
 695 inline static uint8_t *rrdr_line_options(RRDR *r)
 696 {
 697         return &r->o[ r->c * r->d ];
 698 }
 699
 700 inline static int rrdr_line_init(RRDR *r, time_t t)
 701 {
 702         r->c++;
 703         if(unlikely(r->c >= r->n)) {
 704                 r->c = r->n - 1;
 705                 return 0;
 706         }
 707
 708         // save the time
 709         r->t[r->c] = t;
 710
 711         return 1;
 712 }
 713
 714 inline static void rrdr_lock_rrdset(RRDR *r) {
 715         if(unlikely(!r)) {
 716                 error("NULL value given!");
 717                 return;
 718         }
 719
 720         pthread_rwlock_rdlock(&r->st->rwlock);
 721         r->has_st_lock = 1;
 722 }
 723
 724 inline static void rrdr_unlock_rrdset(RRDR *r) {
 725         if(unlikely(!r)) {
 726                 error("NULL value given!");
 727                 return;
 728         }
 729
 730         if(likely(r->has_st_lock)) {
 731                 pthread_rwlock_unlock(&r->st->rwlock);
 732                 r->has_st_lock = 0;
 733         }
 734 }
 735
 736 inline static void rrdr_free(RRDR *r)
 737 {
 738         if(unlikely(!r)) {
 739                 error("NULL value given!");
 740                 return;
 741         }
 742
 743         rrdr_unlock_rrdset(r);
 744         if(likely(r->t)) free(r->t);
 745         if(likely(r->v)) free(r->v);
 746         if(likely(r->o)) free(r->o);
 747         if(likely(r->od)) free(r->od);
 748         free(r);
 749 }
 750
 751 static RRDR *rrdr_create(RRDSET *st, int n)
 752 {
 753         if(unlikely(!st)) {
 754                 error("NULL value given!");
 755                 return NULL;
 756         }
 757
 758         RRDR *r = calloc(1, sizeof(RRDR));
 759         if(unlikely(!r)) goto cleanup;
 760
 761         r->st = st;
 762
 763         rrdr_lock_rrdset(r);
 764
 765         RRDDIM *rd;
 766         for(rd = st->dimensions ; rd ; rd = rd->next) r->d++;
 767
 768         r->n = n;
 769         r->t = malloc(n * sizeof(time_t));
 770         if(unlikely(!r->t)) goto cleanup;
 771
 772         r->t = malloc(n * sizeof(time_t));
 773         if(unlikely(!r->t)) goto cleanup;
 774
 775         r->v = malloc(n * r->d * sizeof(calculated_number));
 776         if(unlikely(!r->v)) goto cleanup;
 777
 778         r->o = malloc(n * r->d * sizeof(uint8_t));
 779         if(unlikely(!r->o)) goto cleanup;
 780
 781         r->od = calloc(r->d, sizeof(uint8_t));
 782         if(unlikely(!r->od)) goto cleanup;
 783
 784         r->c = -1;
 785
 786         return r;
 787
 788 cleanup:
 789         error("Cannot allocate memory");
 790         if(likely(r)) rrdr_free(r);
 791         return NULL;
 792 }
 793
 794 RRDR *rrd2rrdr(RRDSET *st, long points, long long after, long long before, int group_method)
 795 {
 796         int debug = st->debug;
 797
 798         time_t first_entry_t = rrdset_first_entry_t(st);
 799         time_t last_entry_t = rrdset_last_entry_t(st);
 800
 801         if(before == 0 && after == 0) {
 802                 before = last_entry_t;
 803                 after = first_entry_t;
 804         }
 805
 806         // allow relative for before and after
 807         if(before <= st->update_every * st->entries) before = last_entry_t + before;
 808         if(after <= st->update_every * st->entries) after = last_entry_t + after;
 809
 810         // make sure they are within our timeframe
 811         if(before > last_entry_t) before = last_entry_t;
 812         if(before < first_entry_t) before = first_entry_t;
 813
 814         if(after > last_entry_t) after = last_entry_t;
 815         if(after < first_entry_t) after = first_entry_t;
 816
 817         // check if they are upside down
 818         if(after > before) {
 819                 time_t tmp = before;
 820                 before = after;
 821                 after = tmp;
 822         }
 823
 824         // the duration of the chart
 825         time_t duration = before - after;
 826         if(duration <= 0) return NULL;
 827
 828         // check the required points
 829         if(points > duration / st->update_every) points = 0;
 830         if(points <= 0) points = duration / st->update_every;
 831
 832         // calculate proper grouping of source data
 833         long group = duration / points;
 834         if(group <= 0) group = 1;
 835         if(duration / group > points) group++;
 836
 837         // error("NEW: points=%d after=%d before=%d group=%d, duration=%d", points, after, before, group, duration);
 838
 839         // Now we have:
 840         // before = the end time of the calculation
 841         // after = the start time of the calculation
 842         // duration = the duration of the calculation
 843         // group = the number of source points to aggregate / group together
 844         // method = the method of grouping source points
 845         // points = the number of points to generate
 846
 847
 848         // -------------------------------------------------------------------------
 849         // initialize our result set
 850
 851         RRDR *r = rrdr_create(st, points);
 852         if(!r) return NULL;
 853         if(!r->d) {
 854                 rrdr_free(r);
 855                 return NULL;
 856         }
 857
 858         // find how many dimensions we have
 859         long dimensions = r->d;
 860
 861
 862         // -------------------------------------------------------------------------
 863         // checks for debugging
 864
 865         if(debug) debug(D_RRD_STATS, "INFO %s first_t: %lu, last_t: %lu, all_duration: %lu, after: %lu, before: %lu, duration: %lu, points: %ld, group: %ld"
 866                         , st->id
 867                         , first_entry_t
 868                         , last_entry_t
 869                         , last_entry_t - first_entry_t
 870                         , after
 871                         , before
 872                         , duration
 873                         , points
 874                         , group
 875                         );
 876
 877
 878         // -------------------------------------------------------------------------
 879         // temp arrays for keeping values per dimension
 880
 881         calculated_number       group_values[dimensions]; // keep sums when grouping
 882         long                            group_counts[dimensions]; // keep the number of values added to group_values
 883         uint8_t                         group_options[dimensions];
 884         uint8_t                         found_non_zero[dimensions];
 885
 886
 887         // initialize them
 888         RRDDIM *rd;
 889         long c;
 890         for( rd = st->dimensions, c = 0 ; rd && c < dimensions ; rd = rd->next, c++) {
 891                 group_values[c] = 0;
 892                 group_counts[c] = 0;
 893                 group_options[c] = 0;
 894                 found_non_zero[c] = 0;
 895         }
 896
 897
 898         // -------------------------------------------------------------------------
 899         // the main loop
 900
 901         long    start_at_slot = rrdset_time2slot(st, before), // rrdset_last_slot(st),
 902                         stop_at_slot = rrdset_time2slot(st, after);
 903
 904         time_t  now = rrdset_slot2time(st, start_at_slot),
 905                         dt = st->update_every,
 906                         group_start_t = 0;
 907
 908         if(unlikely(debug)) debug(D_RRD_STATS, "INFO  %s after_t: %lu (stop_at_t: %ld), before_t: %lu (start_at_t: %ld), start_t(now): %lu, current_entry: %ld, entries: %ld"
 909                         , st->id
 910                         , after
 911                         , stop_at_slot
 912                         , before
 913                         , start_at_slot
 914                         , now
 915                         , st->current_entry
 916                         , st->entries
 917                         );
 918
 919         // align to group for proper panning of data
 920         start_at_slot -= start_at_slot % group;
 921         stop_at_slot -= stop_at_slot % group;
 922         now = rrdset_slot2time(st, start_at_slot);
 923
 924         if(unlikely(debug)) debug(D_RRD_STATS, "BEGIN %s after_t: %lu (stop_at_t: %ld), before_t: %lu (start_at_t: %ld), start_t(now): %lu, current_entry: %ld, entries: %ld"
 925                         , st->id
 926                         , after
 927                         , stop_at_slot
 928                         , before
 929                         , start_at_slot
 930                         , now
 931                         , st->current_entry
 932                         , st->entries
 933                         );
 934
 935         long slot = start_at_slot, counter = 0, stop_now = 0, added = 0, group_count = 0, add_this = 0;
 936         for(; !stop_now ; now -= dt, slot--, counter++) {
 937                 if(unlikely(slot < 0)) slot = st->entries - 1;
 938                 if(unlikely(slot == stop_at_slot)) stop_now = counter;
 939
 940                 if(unlikely(debug)) debug(D_RRD_STATS, "ROW %s slot: %ld, entries_counter: %ld, group_count: %ld, added: %ld, now: %lu, %s %s"
 941                                 , st->id
 942                                 , slot
 943                                 , counter
 944                                 , group_count + 1
 945                                 , added
 946                                 , now
 947                                 , (group_count + 1 == group)?"PRINT":"  -  "
 948                                 , (now >= after && now <= before)?"RANGE":"  -  "
 949                                 );
 950
 951                 // make sure we return data in the proper time range
 952                 if(unlikely(now > before)) continue;
 953                 if(unlikely(now < after)) break;
 954
 955                 if(unlikely(group_count == 0)) group_start_t = now;
 956                 group_count++;
 957
 958                 if(unlikely(group_count == group)) {
 959                         if(unlikely(added >= points)) break;
 960                         add_this = 1;
 961                 }
 962
 963                 // do the calculations
 964                 for(rd = st->dimensions, c = 0 ; likely(rd && c < dimensions) ; rd = rd->next, c++) {
 965                         storage_number n = rd->values[slot];
 966                         if(unlikely(!does_storage_number_exist(n))) continue;
 967
 968                         group_counts[c]++;
 969
 970                         calculated_number value = unpack_storage_number(n);
 971                         if(likely(value != 0.0)) {
 972                                 group_options[c] |= RRDR_NONZERO;
 973                                 found_non_zero[c] = 1;
 974                         }
 975
 976                         if(unlikely(did_storage_number_reset(n)))
 977                                 group_options[c] |= RRDR_RESET;
 978
 979                         switch(group_method) {
 980                                 case GROUP_MAX:
 981                                         if(unlikely(abs(value) > abs(group_values[c])))
 982                                                 group_values[c] = value;
 983                                         break;
 984
 985                                 default:
 986                                 case GROUP_SUM:
 987                                 case GROUP_AVERAGE:
 988                                         group_values[c] += value;
 989                                         break;
 990                         }
 991                 }
 992
 993                 // added it
 994                 if(unlikely(add_this)) {
 995                         if(unlikely(!rrdr_line_init(r, group_start_t))) break;
 996
 997                         calculated_number *cn = rrdr_line_values(r);
 998                         uint8_t *co = rrdr_line_options(r);
 999
1000                         for(rd = st->dimensions, c = 0 ; likely(rd && c < dimensions) ; rd = rd->next, c++) {
1001
1002                                 // update the dimension options
1003                                 if(likely(found_non_zero[c])) r->od[c] |= RRDR_NONZERO;
1004                                 if(unlikely(rd->flags & RRDDIM_FLAG_HIDDEN)) r->od[c] |= RRDR_HIDDEN;
1005
1006                                 // store the specific point options
1007                                 co[c] = group_options[c];
1008
1009                                 // store the value
1010                                 if(unlikely(group_counts[c] == 0)) {
1011                                         cn[c] = 0.0;
1012                                         co[c] |= RRDR_EMPTY;
1013                                 }
1014                                 else if(unlikely(group_method == GROUP_AVERAGE)) {
1015                                         cn[c] = group_values[c] / group_counts[c];
1016                                 }
1017                                 else {
1018                                         cn[c] = group_values[c];
1019                                 }
1020
1021                                 // reset them for the next loop
1022                                 group_values[c] = 0;
1023                                 group_counts[c] = 0;
1024                                 group_options[c] = 0;
1025                         }
1026
1027                         added++;
1028                         group_count = 0;
1029                         add_this = 0;
1030                 }
1031         }
1032
1033         return r;
1034 }
1035
1036 int rrd2format(RRDSET *st, BUFFER *out, BUFFER *dimensions, uint32_t format, long points, long long after, long long before, int group_method, uint32_t options, time_t *latest_timestamp)
1037 {
1038         RRDR *rrdr = rrd2rrdr(st, points, after, before, group_method);
1039         if(!rrdr) {
1040                 buffer_strcat(out, "Cannot generate output with these parameters on this chart.");
1041                 return 500;
1042         }
1043
1044         if(dimensions)
1045                 rrdr_disable_not_selected_dimensions(rrdr, buffer_tostring(dimensions));
1046
1047         if(latest_timestamp && rrdr_rows(rrdr) > 0)
1048                 *latest_timestamp = rrdr->t[rrdr_rows(rrdr) - 1];
1049
1050         switch(format) {
1051         case DATASOURCE_SSV:
1052                 out->contenttype = CT_TEXT_PLAIN;
1053                 rrdr2ssv(rrdr, out, options, "", " ", "");
1054                 break;
1055
1056         case DATASOURCE_SSV_COMMA:
1057                 out->contenttype = CT_TEXT_PLAIN;
1058                 rrdr2ssv(rrdr, out, options, "", ",", "");
1059                 break;
1060
1061         case DATASOURCE_JS_ARRAY:
1062                 out->contenttype = CT_APPLICATION_JSON;
1063                 rrdr2ssv(rrdr, out, options, "[", ",", "]");
1064                 break;
1065
1066         case DATASOURCE_CSV:
1067                 out->contenttype = CT_TEXT_PLAIN;
1068                 rrdr2csv(rrdr, out, options, "", ",", "\r\n");
1069                 break;
1070
1071         case DATASOURCE_TSV:
1072                 out->contenttype = CT_TEXT_PLAIN;
1073                 rrdr2csv(rrdr, out, options, "", "\t", "\r\n");
1074                 break;
1075
1076         case DATASOURCE_HTML:
1077                 out->contenttype = CT_TEXT_HTML;
1078                 buffer_strcat(out, "<html>\n<center><table border=\"0\" cellpadding=\"5\" cellspacing=\"5\">");
1079                 rrdr2csv(rrdr, out, options, "<tr><td>", "</td><td>", "</td></tr>\n");
1080                 buffer_strcat(out, "</table>\n</center>\n</html>\n");
1081                 break;
1082
1083         case DATASOURCE_GOOGLE_JSONP:
1084                 out->contenttype = CT_APPLICATION_X_JAVASCRIPT;
1085                 rrdr2json(rrdr, out, options, 1);
1086                 break;
1087
1088         case DATASOURCE_GOOGLE_JSON:
1089                 out->contenttype = CT_APPLICATION_JSON;
1090                 rrdr2json(rrdr, out, options, 1);
1091                 break;
1092
1093         case DATASOURCE_JSONP:
1094                 out->contenttype = CT_APPLICATION_X_JAVASCRIPT;
1095                 rrdr2json(rrdr, out, options, 0);
1096                 break;
1097
1098         case DATASOURCE_JSON:
1099         default:
1100                 out->contenttype = CT_APPLICATION_JSON;
1101                 rrdr2json(rrdr, out, options, 0);
1102                 break;
1103         }
1104
1105         rrdr_free(rrdr);
1106         return 200;
1107 }
1108
1109 unsigned long rrd_stats_json(int type, RRDSET *st, BUFFER *wb, int points, int group, int group_method, time_t after, time_t before, int only_non_zero)
1110 {
1111         int c;
1112         pthread_rwlock_rdlock(&st->rwlock);
1113
1114
1115         // -------------------------------------------------------------------------
1116         // switch from JSON to google JSON
1117
1118         char kq[2] = "\"";
1119         char sq[2] = "\"";
1120         switch(type) {
1121                 case DATASOURCE_GOOGLE_JSON:
1122                 case DATASOURCE_GOOGLE_JSONP:
1123                         kq[0] = '\0';
1124                         sq[0] = '\'';
1125                         break;
1126
1127                 case DATASOURCE_JSON:
1128                 default:
1129                         break;
1130         }
1131
1132
1133         // -------------------------------------------------------------------------
1134         // validate the parameters
1135
1136         if(points < 1) points = 1;
1137         if(group < 1) group = 1;
1138
1139         if(before == 0 || before > rrdset_last_entry_t(st)) before = rrdset_last_entry_t(st);
1140         if(after  == 0 || after < rrdset_first_entry_t(st)) after = rrdset_first_entry_t(st);
1141
1142         // ---
1143
1144         // our return value (the last timestamp printed)
1145         // this is required to detect re-transmit in google JSONP
1146         time_t last_timestamp = 0;
1147
1148
1149         // -------------------------------------------------------------------------
1150         // find how many dimensions we have
1151
1152         int dimensions = 0;
1153         RRDDIM *rd;
1154         for( rd = st->dimensions ; rd ; rd = rd->next) dimensions++;
1155         if(!dimensions) {
1156                 pthread_rwlock_unlock(&st->rwlock);
1157                 buffer_strcat(wb, "No dimensions yet.");
1158                 return 0;
1159         }
1160
1161
1162         // -------------------------------------------------------------------------
1163         // prepare various strings, to speed up the loop
1164
1165         char overflow_annotation[201]; snprintf(overflow_annotation, 200, ",{%sv%s:%sRESET OR OVERFLOW%s},{%sv%s:%sThe counters have been wrapped.%s}", kq, kq, sq, sq, kq, kq, sq, sq);
1166         char normal_annotation[201];   snprintf(normal_annotation,   200, ",{%sv%s:null},{%sv%s:null}", kq, kq, kq, kq);
1167         char pre_date[51];             snprintf(pre_date,             50, "             {%sc%s:[{%sv%s:%s", kq, kq, kq, kq, sq);
1168         char post_date[21];            snprintf(post_date,            20, "%s}", sq);
1169         char pre_value[21];            snprintf(pre_value,            20, ",{%sv%s:", kq, kq);
1170         char post_value[21];           snprintf(post_value,           20, "}");
1171
1172
1173         // -------------------------------------------------------------------------
1174         // checks for debugging
1175
1176         if(st->debug) {
1177                 debug(D_RRD_STATS, "%s first_entry_t = %lu, last_entry_t = %lu, duration = %lu, after = %lu, before = %lu, duration = %lu, entries_to_show = %lu, group = %lu"
1178                         , st->id
1179                         , rrdset_first_entry_t(st)
1180                         , rrdset_last_entry_t(st)
1181                         , rrdset_last_entry_t(st) - rrdset_first_entry_t(st)
1182                         , after
1183                         , before
1184                         , before - after
1185                         , points
1186                         , group
1187                         );
1188
1189                 if(before < after)
1190                         debug(D_RRD_STATS, "WARNING: %s The newest value in the database (%lu) is earlier than the oldest (%lu)", st->name, before, after);
1191
1192                 if((before - after) > st->entries * st->update_every)
1193                         debug(D_RRD_STATS, "WARNING: %s The time difference between the oldest and the newest entries (%lu) is higher than the capacity of the database (%lu)", st->name, before - after, st->entries * st->update_every);
1194         }
1195
1196
1197         // -------------------------------------------------------------------------
1198         // temp arrays for keeping values per dimension
1199
1200         calculated_number group_values[dimensions]; // keep sums when grouping
1201         int               print_hidden[dimensions]; // keep hidden flags
1202         int               found_non_zero[dimensions];
1203         int               found_non_existing[dimensions];
1204
1205         // initialize them
1206         for( rd = st->dimensions, c = 0 ; rd && c < dimensions ; rd = rd->next, c++) {
1207                 group_values[c] = 0;
1208                 print_hidden[c] = (rd->flags & RRDDIM_FLAG_HIDDEN)?1:0;
1209                 found_non_zero[c] = 0;
1210                 found_non_existing[c] = 0;
1211         }
1212
1213
1214         // error("OLD: points=%d after=%d before=%d group=%d, duration=%d", entries_to_show, before - (st->update_every * group * entries_to_show), before, group, before - after + 1);
1215         // rrd2array(st, entries_to_show, before - (st->update_every * group * entries_to_show), before, group_method, only_non_zero);
1216         // rrd2rrdr(st, entries_to_show, before - (st->update_every * group * entries_to_show), before, group_method);
1217
1218         // -------------------------------------------------------------------------
1219         // remove dimensions that contain only zeros
1220
1221         int max_loop = 1;
1222         if(only_non_zero) max_loop = 2;
1223
1224         for(; max_loop ; max_loop--) {
1225
1226                 // -------------------------------------------------------------------------
1227                 // print the JSON header
1228
1229                 buffer_sprintf(wb, "{\n %scols%s:\n     [\n", kq, kq);
1230                 buffer_sprintf(wb, "            {%sid%s:%s%s,%slabel%s:%stime%s,%spattern%s:%s%s,%stype%s:%sdatetime%s},\n", kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq);
1231                 buffer_sprintf(wb, "            {%sid%s:%s%s,%slabel%s:%s%s,%spattern%s:%s%s,%stype%s:%sstring%s,%sp%s:{%srole%s:%sannotation%s}},\n", kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, kq, kq, sq, sq);
1232                 buffer_sprintf(wb, "            {%sid%s:%s%s,%slabel%s:%s%s,%spattern%s:%s%s,%stype%s:%sstring%s,%sp%s:{%srole%s:%sannotationText%s}}", kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, kq, kq, sq, sq);
1233
1234                 // print the header for each dimension
1235                 // and update the print_hidden array for the dimensions that should be hidden
1236                 int pc = 0;
1237                 for( rd = st->dimensions, c = 0 ; rd && c < dimensions ; rd = rd->next, c++) {
1238                         if(!print_hidden[c]) {
1239                                 pc++;
1240                                 buffer_sprintf(wb, ",\n         {%sid%s:%s%s,%slabel%s:%s%s%s,%spattern%s:%s%s,%stype%s:%snumber%s}", kq, kq, sq, sq, kq, kq, sq, rd->name, sq, kq, kq, sq, sq, kq, kq, sq, sq);
1241                         }
1242                 }
1243                 if(!pc) {
1244                         buffer_sprintf(wb, ",\n         {%sid%s:%s%s,%slabel%s:%s%s%s,%spattern%s:%s%s,%stype%s:%snumber%s}", kq, kq, sq, sq, kq, kq, sq, "no data", sq, kq, kq, sq, sq, kq, kq, sq, sq);
1245                 }
1246
1247                 // print the begin of row data
1248                 buffer_sprintf(wb, "\n  ],\n    %srows%s:\n     [\n", kq, kq);
1249
1250
1251                 // -------------------------------------------------------------------------
1252                 // the main loop
1253
1254                 int annotate_reset = 0;
1255                 int annotation_count = 0;
1256
1257                 long    t = rrdset_time2slot(st, before),
1258                                 stop_at_t = rrdset_time2slot(st, after),
1259                                 stop_now = 0;
1260
1261                 t -= t % group;
1262
1263                 time_t  now = rrdset_slot2time(st, t),
1264                                 dt = st->update_every;
1265
1266                 long count = 0, printed = 0, group_count = 0;
1267                 last_timestamp = 0;
1268
1269                 if(st->debug) debug(D_RRD_STATS, "%s: REQUEST after:%lu before:%lu, points:%d, group:%d, CHART cur:%ld first: %lu last:%lu, CALC start_t:%ld, stop_t:%ld"
1270                                         , st->id
1271                                         , after
1272                                         , before
1273                                         , points
1274                                         , group
1275                                         , st->current_entry
1276                                         , rrdset_first_entry_t(st)
1277                                         , rrdset_last_entry_t(st)
1278                                         , t
1279                                         , stop_at_t
1280                                         );
1281
1282                 for(; !stop_now ; now -= dt, t--) {
1283                         if(t < 0) t = st->entries - 1;
1284                         if(t == stop_at_t) stop_now = 1;
1285
1286                         int print_this = 0;
1287
1288                         if(st->debug) debug(D_RRD_STATS, "%s t = %ld, count = %ld, group_count = %ld, printed = %ld, now = %lu, %s %s"
1289                                         , st->id
1290                                         , t
1291                                         , count + 1
1292                                         , group_count + 1
1293                                         , printed
1294                                         , now
1295                                         , (group_count + 1 == group)?"PRINT":"  -  "
1296                                         , (now >= after && now <= before)?"RANGE":"  -  "
1297                                         );
1298
1299
1300                         // make sure we return data in the proper time range
1301                         if(now > before) continue;
1302                         if(now < after) break;
1303
1304                         //if(rrdset_slot2time(st, t) != now)
1305                         //      error("%s: slot=%ld, now=%ld, slot2time=%ld, diff=%ld, last_entry_t=%ld, rrdset_last_slot=%ld", st->id, t, now, rrdset_slot2time(st,t), now - rrdset_slot2time(st,t), rrdset_last_entry_t(st), rrdset_last_slot(st));
1306
1307                         count++;
1308                         group_count++;
1309
1310                         // check if we have to print this now
1311                         if(group_count == group) {
1312                                 if(printed >= points) {
1313                                         // debug(D_RRD_STATS, "Already printed all rows. Stopping.");
1314                                         break;
1315                                 }
1316
1317                                 // generate the local date time
1318                                 struct tm *tm = localtime(&now);
1319                                 if(!tm) { error("localtime() failed."); continue; }
1320                                 if(now > last_timestamp) last_timestamp = now;
1321
1322                                 if(printed) buffer_strcat(wb, "]},\n");
1323                                 buffer_strcat(wb, pre_date);
1324                                 buffer_jsdate(wb, tm->tm_year + 1900, tm->tm_mon, tm->tm_mday, tm->tm_hour, tm->tm_min, tm->tm_sec);
1325                                 buffer_strcat(wb, post_date);
1326
1327                                 print_this = 1;
1328                         }
1329
1330                         // do the calculations
1331                         for(rd = st->dimensions, c = 0 ; rd && c < dimensions ; rd = rd->next, c++) {
1332                                 storage_number n = rd->values[t];
1333                                 calculated_number value = unpack_storage_number(n);
1334
1335                                 if(!does_storage_number_exist(n)) {
1336                                         value = 0.0;
1337                                         found_non_existing[c]++;
1338                                 }
1339                                 if(did_storage_number_reset(n)) annotate_reset = 1;
1340
1341                                 switch(group_method) {
1342                                         case GROUP_MAX:
1343                                                 if(abs(value) > abs(group_values[c])) group_values[c] = value;
1344                                                 break;
1345
1346                                         case GROUP_SUM:
1347                                                 group_values[c] += value;
1348                                                 break;
1349
1350                                         default:
1351                                         case GROUP_AVERAGE:
1352                                                 group_values[c] += value;
1353                                                 if(print_this) group_values[c] /= ( group_count - found_non_existing[c] );
1354                                                 break;
1355                                 }
1356                         }
1357
1358                         if(print_this) {
1359                                 if(annotate_reset) {
1360                                         annotation_count++;
1361                                         buffer_strcat(wb, overflow_annotation);
1362                                         annotate_reset = 0;
1363                                 }
1364                                 else
1365                                         buffer_strcat(wb, normal_annotation);
1366
1367                                 pc = 0;
1368                                 for(c = 0 ; c < dimensions ; c++) {
1369                                         if(found_non_existing[c] == group_count) {
1370                                                 // all entries are non-existing
1371                                                 pc++;
1372                                                 buffer_strcat(wb, pre_value);
1373                                                 buffer_strcat(wb, "null");
1374                                                 buffer_strcat(wb, post_value);
1375                                         }
1376                                         else if(!print_hidden[c]) {
1377                                                 pc++;
1378                                                 buffer_strcat(wb, pre_value);
1379                                                 buffer_rrd_value(wb, group_values[c]);
1380                                                 buffer_strcat(wb, post_value);
1381
1382                                                 if(group_values[c]) found_non_zero[c]++;
1383                                         }
1384
1385                                         // reset them for the next loop
1386                                         group_values[c] = 0;
1387                                         found_non_existing[c] = 0;
1388                                 }
1389
1390                                 // if all dimensions are hidden, print a null
1391                                 if(!pc) {
1392                                         buffer_strcat(wb, pre_value);
1393                                         buffer_strcat(wb, "null");
1394                                         buffer_strcat(wb, post_value);
1395                                 }
1396
1397                                 printed++;
1398                                 group_count = 0;
1399                         }
1400                 }
1401
1402                 if(printed) buffer_strcat(wb, "]}");
1403                 buffer_strcat(wb, "\n   ]\n}\n");
1404
1405                 if(only_non_zero && max_loop > 1) {
1406                         int changed = 0;
1407                         for(rd = st->dimensions, c = 0 ; rd && c < dimensions ; rd = rd->next, c++) {
1408                                 group_values[c] = 0;
1409                                 found_non_existing[c] = 0;
1410
1411                                 if(!print_hidden[c] && !found_non_zero[c]) {
1412                                         changed = 1;
1413                                         print_hidden[c] = 1;
1414                                 }
1415                         }
1416
1417                         if(changed) buffer_flush(wb);
1418                         else break;
1419                 }
1420                 else break;
1421
1422         } // max_loop
1423
1424         debug(D_RRD_STATS, "RRD_STATS_JSON: %s total %ld bytes", st->name, wb->len);
1425
1426         pthread_rwlock_unlock(&st->rwlock);
1427         return last_timestamp;
1428 }