src/rrd2json.c

   1 #ifdef HAVE_CONFIG_H
   2 #include <config.h>
   3 #endif
   4 #include <pthread.h>
   5 #include <sys/time.h>
   6 #include <stdlib.h>
   7 #include <string.h>
   8
   9 #include "log.h"
  10 #include "common.h"
  11 #include "rrd2json.h"
  12
  13 #define HOSTNAME_MAX 1024
  14 char *hostname = "unknown";
  15
  16 void rrd_stats_api_v1_chart(RRDSET *st, BUFFER *wb)
  17 {
  18         pthread_rwlock_rdlock(&st->rwlock);
  19
  20         buffer_sprintf(wb,
  21                 "\t\t{\n"
  22                 "\t\t\t\"id\": \"%s\",\n"
  23                 "\t\t\t\"name\": \"%s\",\n"
  24                 "\t\t\t\"type\": \"%s\",\n"
  25                 "\t\t\t\"family\": \"%s\",\n"
  26                 "\t\t\t\"title\": \"%s\",\n"
  27                 "\t\t\t\"priority\": %ld,\n"
  28                 "\t\t\t\"enabled\": %s,\n"
  29                 "\t\t\t\"units\": \"%s\",\n"
  30                 "\t\t\t\"data_url\": \"/api/v1/data?chart=%s\",\n"
  31                 "\t\t\t\"chart_type\": \"%s\",\n"
  32                 "\t\t\t\"duration\": %ld,\n"
  33                 "\t\t\t\"first_entry\": %lu,\n"
  34                 "\t\t\t\"last_entry\": %lu,\n"
  35                 "\t\t\t\"update_every\": %d,\n"
  36                 "\t\t\t\"dimensions\": {\n"
  37                 , st->id
  38                 , st->name
  39                 , st->type
  40                 , st->family
  41                 , st->title
  42                 , st->priority
  43                 , st->enabled?"true":"false"
  44                 , st->units
  45                 , st->name
  46                 , rrdset_type_name(st->chart_type)
  47                 , st->entries * st->update_every
  48                 , rrdset_first_entry_t(st)
  49                 , rrdset_last_entry_t(st)
  50                 , st->update_every
  51                 );
  52
  53         unsigned long memory = st->memsize;
  54
  55         int c = 0;
  56         RRDDIM *rd;
  57         for(rd = st->dimensions; rd ; rd = rd->next) {
  58                 if(rd->flags & RRDDIM_FLAG_HIDDEN) continue;
  59
  60                 memory += rd->memsize;
  61
  62                 buffer_sprintf(wb,
  63                         "%s"
  64                         "\t\t\t\t\"%s\": { \"name\": \"%s\" }"
  65                         , c?",\n":""
  66                         , rd->id
  67                         , rd->name
  68                         );
  69
  70                 c++;
  71         }
  72
  73         buffer_sprintf(wb,
  74                 "\n\t\t\t}\n"
  75                 "\t\t}"
  76                 );
  77
  78         pthread_rwlock_unlock(&st->rwlock);
  79 }
  80
  81 void rrd_stats_api_v1_charts(BUFFER *wb)
  82 {
  83         long c;
  84         RRDSET *st;
  85
  86         buffer_sprintf(wb, "{\n"
  87                    "\t\"hostname\": \"%s\""
  88                 ",\n\t\"update_every\": %d"
  89                 ",\n\t\"history\": %d"
  90                 ",\n\t\"charts\": {"
  91                 , hostname
  92                 , rrd_update_every
  93                 , rrd_default_history_entries
  94                 );
  95
  96         pthread_rwlock_rdlock(&rrdset_root_rwlock);
  97         for(st = rrdset_root, c = 0; st ; st = st->next) {
  98                 if(st->enabled) {
  99                         if(c) buffer_strcat(wb, ",");
 100                         buffer_strcat(wb, "\n\t\t\"");
 101                         buffer_strcat(wb, st->id);
 102                         buffer_strcat(wb, "\": ");
 103                         rrd_stats_api_v1_chart(st, wb);
 104                         c++;
 105                 }
 106         }
 107         pthread_rwlock_unlock(&rrdset_root_rwlock);
 108
 109         buffer_strcat(wb, "\n\t}\n}\n");
 110 }
 111
 112
 113 unsigned long rrd_stats_one_json(RRDSET *st, char *options, BUFFER *wb)
 114 {
 115         time_t now = time(NULL);
 116
 117         pthread_rwlock_rdlock(&st->rwlock);
 118
 119         buffer_sprintf(wb,
 120                 "\t\t{\n"
 121                 "\t\t\t\"id\": \"%s\",\n"
 122                 "\t\t\t\"name\": \"%s\",\n"
 123                 "\t\t\t\"type\": \"%s\",\n"
 124                 "\t\t\t\"family\": \"%s\",\n"
 125                 "\t\t\t\"title\": \"%s\",\n"
 126                 "\t\t\t\"priority\": %ld,\n"
 127                 "\t\t\t\"enabled\": %d,\n"
 128                 "\t\t\t\"units\": \"%s\",\n"
 129                 "\t\t\t\"url\": \"/data/%s/%s\",\n"
 130                 "\t\t\t\"chart_type\": \"%s\",\n"
 131                 "\t\t\t\"counter\": %ld,\n"
 132                 "\t\t\t\"entries\": %ld,\n"
 133                 "\t\t\t\"first_entry_t\": %lu,\n"
 134                 "\t\t\t\"last_entry\": %ld,\n"
 135                 "\t\t\t\"last_entry_t\": %lu,\n"
 136                 "\t\t\t\"last_entry_secs_ago\": %lu,\n"
 137                 "\t\t\t\"update_every\": %d,\n"
 138                 "\t\t\t\"isdetail\": %d,\n"
 139                 "\t\t\t\"usec_since_last_update\": %llu,\n"
 140                 "\t\t\t\"collected_total\": " TOTAL_NUMBER_FORMAT ",\n"
 141                 "\t\t\t\"last_collected_total\": " TOTAL_NUMBER_FORMAT ",\n"
 142                 "\t\t\t\"dimensions\": [\n"
 143                 , st->id
 144                 , st->name
 145                 , st->type
 146                 , st->family
 147                 , st->title
 148                 , st->priority
 149                 , st->enabled
 150                 , st->units
 151                 , st->name, options?options:""
 152                 , rrdset_type_name(st->chart_type)
 153                 , st->counter
 154                 , st->entries
 155                 , rrdset_first_entry_t(st)
 156                 , rrdset_last_slot(st)
 157                 , rrdset_last_entry_t(st)
 158                 , (now < rrdset_last_entry_t(st)) ? (time_t)0 : now - rrdset_last_entry_t(st)
 159                 , st->update_every
 160                 , st->isdetail
 161                 , st->usec_since_last_update
 162                 , st->collected_total
 163                 , st->last_collected_total
 164                 );
 165
 166         unsigned long memory = st->memsize;
 167
 168         RRDDIM *rd;
 169         for(rd = st->dimensions; rd ; rd = rd->next) {
 170
 171                 memory += rd->memsize;
 172
 173                 buffer_sprintf(wb,
 174                         "\t\t\t\t{\n"
 175                         "\t\t\t\t\t\"id\": \"%s\",\n"
 176                         "\t\t\t\t\t\"name\": \"%s\",\n"
 177                         "\t\t\t\t\t\"entries\": %ld,\n"
 178                         "\t\t\t\t\t\"isHidden\": %d,\n"
 179                         "\t\t\t\t\t\"algorithm\": \"%s\",\n"
 180                         "\t\t\t\t\t\"multiplier\": %ld,\n"
 181                         "\t\t\t\t\t\"divisor\": %ld,\n"
 182                         "\t\t\t\t\t\"last_entry_t\": %lu,\n"
 183                         "\t\t\t\t\t\"collected_value\": " COLLECTED_NUMBER_FORMAT ",\n"
 184                         "\t\t\t\t\t\"calculated_value\": " CALCULATED_NUMBER_FORMAT ",\n"
 185                         "\t\t\t\t\t\"last_collected_value\": " COLLECTED_NUMBER_FORMAT ",\n"
 186                         "\t\t\t\t\t\"last_calculated_value\": " CALCULATED_NUMBER_FORMAT ",\n"
 187                         "\t\t\t\t\t\"memory\": %lu\n"
 188                         "\t\t\t\t}%s\n"
 189                         , rd->id
 190                         , rd->name
 191                         , rd->entries
 192                         , (rd->flags & RRDDIM_FLAG_HIDDEN)?1:0
 193                         , rrddim_algorithm_name(rd->algorithm)
 194                         , rd->multiplier
 195                         , rd->divisor
 196                         , rd->last_collected_time.tv_sec
 197                         , rd->collected_value
 198                         , rd->calculated_value
 199                         , rd->last_collected_value
 200                         , rd->last_calculated_value
 201                         , rd->memsize
 202                         , rd->next?",":""
 203                         );
 204         }
 205
 206         buffer_sprintf(wb,
 207                 "\t\t\t],\n"
 208                 "\t\t\t\"memory\" : %lu\n"
 209                 "\t\t}"
 210                 , memory
 211                 );
 212
 213         pthread_rwlock_unlock(&st->rwlock);
 214         return memory;
 215 }
 216
 217 #define RRD_GRAPH_JSON_HEADER "{\n\t\"charts\": [\n"
 218 #define RRD_GRAPH_JSON_FOOTER "\n\t]\n}\n"
 219
 220 void rrd_stats_graph_json(RRDSET *st, char *options, BUFFER *wb)
 221 {
 222         buffer_strcat(wb, RRD_GRAPH_JSON_HEADER);
 223         rrd_stats_one_json(st, options, wb);
 224         buffer_strcat(wb, RRD_GRAPH_JSON_FOOTER);
 225 }
 226
 227 void rrd_stats_all_json(BUFFER *wb)
 228 {
 229         unsigned long memory = 0;
 230         long c;
 231         RRDSET *st;
 232
 233         buffer_strcat(wb, RRD_GRAPH_JSON_HEADER);
 234
 235         pthread_rwlock_rdlock(&rrdset_root_rwlock);
 236         for(st = rrdset_root, c = 0; st ; st = st->next) {
 237                 if(st->enabled) {
 238                         if(c) buffer_strcat(wb, ",\n");
 239                         memory += rrd_stats_one_json(st, NULL, wb);
 240                         c++;
 241                 }
 242         }
 243         pthread_rwlock_unlock(&rrdset_root_rwlock);
 244
 245         buffer_sprintf(wb, "\n\t],\n"
 246                 "\t\"hostname\": \"%s\",\n"
 247                 "\t\"update_every\": %d,\n"
 248                 "\t\"history\": %d,\n"
 249                 "\t\"memory\": %lu\n"
 250                 "}\n"
 251                 , hostname
 252                 , rrd_update_every
 253                 , rrd_default_history_entries
 254                 , memory
 255                 );
 256 }
 257
 258
 259
 260 // ----------------------------------------------------------------------------
 261
 262 // RRDR options
 263 #define RRDR_EMPTY      0x01 // the dimension contains / the value is empty (null)
 264 #define RRDR_RESET      0x02 // the dimension contains / the value is reset
 265 #define RRDR_HIDDEN     0x04 // the dimension contains / the value is hidden
 266 #define RRDR_NONZERO    0x08 // the dimension contains / the value is non-zero
 267
 268
 269 typedef struct rrdresult {
 270         RRDSET *st;                     // the chart this result refers to
 271
 272         int group;                              // how many collected values were grouped for each row
 273         int update_every;               // what is the suggested update frequency in seconds
 274
 275         int d;                                  // the number of dimensions
 276         int n;                                  // the number of values in the arrays
 277         int rows;                       // the number of rows used
 278
 279         uint8_t *od;                    // the options for the dimensions
 280
 281         time_t *t;                              // array of n timestamps
 282         calculated_number *v;   // array n x d values
 283         uint8_t *o;                             // array n x d options
 284
 285         int c;                                  // current line ( -1 ~ n ), ( -1 = none, use rrdr_rows() to get number of rows )
 286
 287         int has_st_lock;                // if st is read locked by us
 288 } RRDR;
 289
 290 #define rrdr_rows(r) ((r)->rows)
 291
 292 /*
 293 static void rrdr_dump(RRDR *r)
 294 {
 295         long c, i;
 296         RRDDIM *d;
 297
 298         fprintf(stderr, "\nCHART %s (%s)\n", r->st->id, r->st->name);
 299
 300         for(c = 0, d = r->st->dimensions; d ;c++, d = d->next) {
 301                 fprintf(stderr, "DIMENSION %s (%s), %s%s%s%s\n"
 302                                 , d->id
 303                                 , d->name
 304                                 , (r->od[c] & RRDR_EMPTY)?"EMPTY ":""
 305                                 , (r->od[c] & RRDR_RESET)?"RESET ":""
 306                                 , (r->od[c] & RRDR_HIDDEN)?"HIDDEN ":""
 307                                 , (r->od[c] & RRDR_NONZERO)?"NONZERO ":""
 308                                 );
 309         }
 310
 311         if(r->rows <= 0) {
 312                 fprintf(stderr, "RRDR does not have any values in it.\n");
 313                 return;
 314         }
 315
 316         fprintf(stderr, "RRDR includes %d values in it:\n", r->rows);
 317
 318         // for each line in the array
 319         for(i = 0; i < r->rows ;i++) {
 320                 calculated_number *cn = &r->v[ i * r->d ];
 321                 uint8_t *co = &r->o[ i * r->d ];
 322
 323                 // print the id and the timestamp of the line
 324                 fprintf(stderr, "%ld %ld ", i + 1, r->t[i]);
 325
 326                 // for each dimension
 327                 for(c = 0, d = r->st->dimensions; d ;c++, d = d->next) {
 328                         if(unlikely(r->od[c] & RRDR_HIDDEN)) continue;
 329                         if(unlikely(!(r->od[c] & RRDR_NONZERO))) continue;
 330
 331                         if(co[c] & RRDR_EMPTY)
 332                                 fprintf(stderr, "null ");
 333                         else
 334                                 fprintf(stderr, CALCULATED_NUMBER_FORMAT " %s%s%s%s "
 335                                         , cn[c]
 336                                         , (co[c] & RRDR_EMPTY)?"E":" "
 337                                         , (co[c] & RRDR_RESET)?"R":" "
 338                                         , (co[c] & RRDR_HIDDEN)?"H":" "
 339                                         , (co[c] & RRDR_NONZERO)?"N":" "
 340                                         );
 341                 }
 342
 343                 fprintf(stderr, "\n");
 344         }
 345 }
 346 */
 347
 348 void rrdr_disable_not_selected_dimensions(RRDR *r, const char *dims)
 349 {
 350         char b[strlen(dims) + 1];
 351         char *o = b, *tok;
 352         strcpy(o, dims);
 353
 354         long c;
 355         RRDDIM *d;
 356
 357         // disable all of them
 358         for(c = 0, d = r->st->dimensions; d ;c++, d = d->next)
 359                 r->od[c] |= RRDR_HIDDEN;
 360
 361         while(o && *o && (tok = mystrsep(&o, ", |"))) {
 362                 if(!*tok) continue;
 363
 364                 // find it and enable it
 365                 for(c = 0, d = r->st->dimensions; d ;c++, d = d->next) {
 366                         if(!strcmp(d->name, tok)) {
 367                                 r->od[c] &= ~RRDR_HIDDEN;
 368                         }
 369                 }
 370         }
 371 }
 372
 373 #define JSON_DATES_JS 1
 374 #define JSON_DATES_TIMESTAMP 2
 375
 376 static void rrdr2json(RRDR *r, BUFFER *wb, uint32_t options, int datatable)
 377 {
 378         int row_annotations = 0, dates = JSON_DATES_JS, dates_with_new = 0;
 379         char kq[2] = "",                                        // key quote
 380                 sq[2] = "",                                             // string quote
 381                 pre_label[101] = "",                    // before each label
 382                 post_label[101] = "",                   // after each label
 383                 pre_date[101] = "",                             // the beginning of line, to the date
 384                 post_date[101] = "",                    // closing the date
 385                 pre_value[101] = "",                    // before each value
 386                 post_value[101] = "",                   // after each value
 387                 post_line[101] = "",                    // at the end of each row
 388                 normal_annotation[201] = "",    // default row annotation
 389                 overflow_annotation[201] = "",  // overflow row annotation
 390                 data_begin[101] = "",                   // between labels and values
 391                 finish[101] = "";                               // at the end of everything
 392
 393         if(datatable) {
 394                 dates = JSON_DATES_JS;
 395                 if( options & RRDR_OPTION_GOOGLE_JSON ) {
 396                         kq[0] = '\0';
 397                         sq[0] = '\'';
 398                 }
 399                 else {
 400                         kq[0] = '"';
 401                         sq[0] = '"';
 402                 }
 403                 row_annotations = 1;
 404                 snprintf(pre_date,   100, "             {%sc%s:[{%sv%s:%s", kq, kq, kq, kq, sq);
 405                 snprintf(post_date,  100, "%s}", sq);
 406                 snprintf(pre_label,  100, ",\n          {%sid%s:%s%s,%slabel%s:%s", kq, kq, sq, sq, kq, kq, sq);
 407                 snprintf(post_label, 100, "%s,%spattern%s:%s%s,%stype%s:%snumber%s}", sq, kq, kq, sq, sq, kq, kq, sq, sq);
 408                 snprintf(pre_value,  100, ",{%sv%s:", kq, kq);
 409                 snprintf(post_value, 100, "}");
 410                 snprintf(post_line,  100, "]}");
 411                 snprintf(data_begin, 100, "\n   ],\n    %srows%s:\n     [\n", kq, kq);
 412                 snprintf(finish,     100, "\n   ]\n}\n");
 413
 414                 snprintf(overflow_annotation, 200, ",{%sv%s:%sRESET OR OVERFLOW%s},{%sv%s:%sThe counters have been wrapped.%s}", kq, kq, sq, sq, kq, kq, sq, sq);
 415                 snprintf(normal_annotation,   200, ",{%sv%s:null},{%sv%s:null}", kq, kq, kq, kq);
 416
 417                 buffer_sprintf(wb, "{\n %supdate_every%s: %d,\n %scols%s:\n     [\n", kq, kq, r->update_every, kq, kq, kq, kq);
 418                 buffer_sprintf(wb, "            {%sid%s:%s%s,%slabel%s:%stime%s,%spattern%s:%s%s,%stype%s:%sdatetime%s},\n", kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq);
 419                 buffer_sprintf(wb, "            {%sid%s:%s%s,%slabel%s:%s%s,%spattern%s:%s%s,%stype%s:%sstring%s,%sp%s:{%srole%s:%sannotation%s}},\n", kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, kq, kq, sq, sq);
 420                 buffer_sprintf(wb, "            {%sid%s:%s%s,%slabel%s:%s%s,%spattern%s:%s%s,%stype%s:%sstring%s,%sp%s:{%srole%s:%sannotationText%s}}", kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, kq, kq, sq, sq);
 421
 422                 // remove the valueobjects flag
 423                 // google wants its own keys
 424                 if(options & RRDR_OPTION_OBJECTSROWS)
 425                         options &= ~RRDR_OPTION_OBJECTSROWS;
 426         }
 427         else {
 428                 kq[0] = '"';
 429                 sq[0] = '"';
 430                 if((options & RRDR_OPTION_SECONDS) || (options & RRDR_OPTION_MILLISECONDS)) {
 431                         dates = JSON_DATES_TIMESTAMP;
 432                         dates_with_new = 0;
 433                 }
 434                 else {
 435                         dates = JSON_DATES_JS;
 436                         dates_with_new = 1;
 437                 }
 438                 if( options & RRDR_OPTION_OBJECTSROWS )
 439                         snprintf(pre_date,   100, "             { ");
 440                 else
 441                         snprintf(pre_date,   100, "             [ ");
 442                 snprintf(pre_label,  100, ", \"");
 443                 snprintf(post_label, 100, "\"");
 444                 snprintf(pre_value,  100, ", ");
 445                 if( options & RRDR_OPTION_OBJECTSROWS )
 446                         snprintf(post_line,  100, "}");
 447                 else
 448                         snprintf(post_line,  100, "]");
 449                 snprintf(data_begin, 100, "],\n %sdata%s:\n     [\n", kq, kq);
 450                 snprintf(finish,     100, "\n   ]\n}\n");
 451
 452                 buffer_sprintf(wb, "{\n %supdate_every%s: %d,\n %slabels%s: [", kq, kq, r->update_every, kq, kq);
 453                 buffer_sprintf(wb, "%stime%s", sq, sq);
 454         }
 455
 456         // -------------------------------------------------------------------------
 457         // print the JSON header
 458
 459         long c, i;
 460         RRDDIM *rd;
 461
 462         // print the header lines
 463         for(c = 0, i = 0, rd = r->st->dimensions; rd ;c++, rd = rd->next) {
 464                 if(unlikely(r->od[c] & RRDR_HIDDEN)) continue;
 465                 if(unlikely((options & RRDR_OPTION_NONZERO) && !(r->od[c] & RRDR_NONZERO))) continue;
 466
 467                 buffer_strcat(wb, pre_label);
 468                 buffer_strcat(wb, rd->name);
 469                 buffer_strcat(wb, post_label);
 470                 i++;
 471         }
 472         if(!i) {
 473                 buffer_strcat(wb, pre_label);
 474                 buffer_strcat(wb, "no data");
 475                 buffer_strcat(wb, post_label);
 476         }
 477
 478         // print the begin of row data
 479         buffer_strcat(wb, data_begin);
 480
 481         // if all dimensions are hidden, print a null
 482         if(!i) {
 483                 buffer_strcat(wb, pre_value);
 484                 if(options & RRDR_OPTION_NULL2ZERO)
 485                         buffer_strcat(wb, "0");
 486                 else
 487                         buffer_strcat(wb, "null");
 488                 buffer_strcat(wb, post_value);
 489         }
 490
 491         long start = 0, end = rrdr_rows(r), step = 1;
 492         if((options & RRDR_OPTION_REVERSED)) {
 493                 start = rrdr_rows(r) - 1;
 494                 end = -1;
 495                 step = -1;
 496         }
 497
 498         // for each line in the array
 499         for(i = start; i != end ;i += step) {
 500                 calculated_number *cn = &r->v[ i * r->d ];
 501                 uint8_t *co = &r->o[ i * r->d ];
 502
 503                 time_t now = r->t[i];
 504
 505                 if(dates == JSON_DATES_JS) {
 506                         // generate the local date time
 507                         struct tm *tm = localtime(&now);
 508                         if(!tm) { error("localtime() failed."); continue; }
 509
 510                         if(likely(i != start)) buffer_strcat(wb, ",\n");
 511                         buffer_strcat(wb, pre_date);
 512
 513                         if( options & RRDR_OPTION_OBJECTSROWS )
 514                                 buffer_sprintf(wb, "%stime%s: ", kq, kq);
 515
 516                         if(dates_with_new)
 517                                 buffer_strcat(wb, "new ");
 518
 519                         buffer_jsdate(wb, tm->tm_year + 1900, tm->tm_mon, tm->tm_mday, tm->tm_hour, tm->tm_min, tm->tm_sec);
 520
 521                         buffer_strcat(wb, post_date);
 522
 523                         if(row_annotations) {
 524                                 // google supports one annotation per row
 525                                 int annotation_found = 0;
 526                                 for(c = 0, rd = r->st->dimensions; rd ;c++, rd = rd->next) {
 527                                         if(co[c] & RRDR_RESET) {
 528                                                 buffer_strcat(wb, overflow_annotation);
 529                                                 annotation_found = 1;
 530                                                 break;
 531                                         }
 532                                 }
 533                                 if(!annotation_found)
 534                                         buffer_strcat(wb, normal_annotation);
 535                         }
 536                 }
 537                 else {
 538                         // print the timestamp of the line
 539                         if(likely(i != start)) buffer_strcat(wb, ",\n");
 540                         buffer_strcat(wb, pre_date);
 541
 542                         if( options & RRDR_OPTION_OBJECTSROWS )
 543                                 buffer_sprintf(wb, "%stime%s: ", kq, kq);
 544
 545                         buffer_rrd_value(wb, (calculated_number)r->t[i]);
 546                         // in ms
 547                         if(options & RRDR_OPTION_MILLISECONDS) buffer_strcat(wb, "000");
 548
 549                         buffer_strcat(wb, post_date);
 550                 }
 551
 552                 // for each dimension
 553                 for(c = 0, rd = r->st->dimensions; rd ;c++, rd = rd->next) {
 554                         if(unlikely(r->od[c] & RRDR_HIDDEN)) continue;
 555                         if(unlikely((options & RRDR_OPTION_NONZERO) && !(r->od[c] & RRDR_NONZERO))) continue;
 556
 557                         calculated_number n = cn[c];
 558
 559                         buffer_strcat(wb, pre_value);
 560
 561                         if( options & RRDR_OPTION_OBJECTSROWS )
 562                                 buffer_sprintf(wb, "%s%s%s: ", kq, rd->name, kq);
 563
 564                         if(co[c] & RRDR_EMPTY) {
 565                                 if(options & RRDR_OPTION_NULL2ZERO)
 566                                         buffer_strcat(wb, "0");
 567                                 else
 568                                         buffer_strcat(wb, "null");
 569                         }
 570                         else if((options & RRDR_OPTION_ABSOLUTE))
 571                                 buffer_rrd_value(wb, (n<0)?-n:n);
 572                         else
 573                                 buffer_rrd_value(wb, n);
 574
 575                         buffer_strcat(wb, post_value);
 576                 }
 577
 578                 buffer_strcat(wb, post_line);
 579         }
 580
 581         buffer_strcat(wb, finish);
 582 }
 583
 584 static void rrdr2csv(RRDR *r, BUFFER *wb, uint32_t options, const char *startline, const char *separator, const char *endline)
 585 {
 586         long c, i;
 587         RRDDIM *d;
 588
 589         // print the csv header
 590         for(c = 0, i = 0, d = r->st->dimensions; d ;c++, d = d->next) {
 591                 if(unlikely(r->od[c] & RRDR_HIDDEN)) continue;
 592                 if(unlikely((options & RRDR_OPTION_NONZERO) && !(r->od[c] & RRDR_NONZERO))) continue;
 593
 594                 if(!i) {
 595                         buffer_strcat(wb, startline);
 596                         buffer_strcat(wb, "time");
 597                 }
 598                 buffer_strcat(wb, separator);
 599                 buffer_strcat(wb, d->name);
 600                 i++;
 601         }
 602         buffer_strcat(wb, endline);
 603
 604         if(!i) {
 605                 // no dimensions present
 606                 return;
 607         }
 608
 609         long start = 0, end = rrdr_rows(r), step = 1;
 610         if((options & RRDR_OPTION_REVERSED)) {
 611                 start = rrdr_rows(r) - 1;
 612                 end = -1;
 613                 step = -1;
 614         }
 615
 616         // for each line in the array
 617         for(i = start; i != end ;i += step) {
 618                 calculated_number *cn = &r->v[ i * r->d ];
 619                 uint8_t *co = &r->o[ i * r->d ];
 620
 621                 buffer_strcat(wb, startline);
 622
 623                 time_t now = r->t[i];
 624
 625                 if((options & RRDR_OPTION_SECONDS) || (options & RRDR_OPTION_MILLISECONDS)) {
 626                         // print the timestamp of the line
 627                         buffer_rrd_value(wb, (calculated_number)now);
 628                         // in ms
 629                         if(options & RRDR_OPTION_MILLISECONDS) buffer_strcat(wb, "000");
 630                 }
 631                 else {
 632                         // generate the local date time
 633                         struct tm *tm = localtime(&now);
 634                         if(!tm) { error("localtime() failed."); continue; }
 635                         buffer_date(wb, tm->tm_year + 1900, tm->tm_mon, tm->tm_mday, tm->tm_hour, tm->tm_min, tm->tm_sec);
 636                 }
 637
 638                 // for each dimension
 639                 for(c = 0, d = r->st->dimensions; d ;c++, d = d->next) {
 640                         if(unlikely(r->od[c] & RRDR_HIDDEN)) continue;
 641                         if(unlikely((options & RRDR_OPTION_NONZERO) && !(r->od[c] & RRDR_NONZERO))) continue;
 642
 643                         buffer_strcat(wb, separator);
 644
 645                         calculated_number n = cn[c];
 646
 647                         if(co[c] & RRDR_EMPTY) {
 648                                 if(options & RRDR_OPTION_NULL2ZERO)
 649                                         buffer_strcat(wb, "0");
 650                                 else
 651                                         buffer_strcat(wb, "null");
 652                         }
 653                         else if((options & RRDR_OPTION_ABSOLUTE))
 654                                 buffer_rrd_value(wb, (n<0)?-n:n);
 655                         else
 656                                 buffer_rrd_value(wb, n);
 657                 }
 658
 659                 buffer_strcat(wb, endline);
 660         }
 661 }
 662
 663 static void rrdr2ssv(RRDR *r, BUFFER *out, uint32_t options, const char *prefix, const char *separator, const char *suffix)
 664 {
 665         long c, i;
 666         RRDDIM *d;
 667
 668         buffer_strcat(out, prefix);
 669         long start = 0, end = rrdr_rows(r), step = 1;
 670         if((options & RRDR_OPTION_REVERSED)) {
 671                 start = rrdr_rows(r) - 1;
 672                 end = -1;
 673                 step = -1;
 674         }
 675
 676         // for each line in the array
 677         for(i = start; i != end ;i += step) {
 678
 679                 calculated_number *cn = &r->v[ i * r->d ];
 680                 uint8_t *co = &r->o[ i * r->d ];
 681
 682                 calculated_number sum = 0, min = 0, max = 0;
 683                 int all_null = 1, init = 1;
 684
 685                 // for each dimension
 686                 for(c = 0, d = r->st->dimensions; d ;c++, d = d->next) {
 687                         if(unlikely(r->od[c] & RRDR_HIDDEN)) continue;
 688                         if(unlikely((options & RRDR_OPTION_NONZERO) && !(r->od[c] & RRDR_NONZERO))) continue;
 689
 690                         calculated_number n = cn[c];
 691
 692                         if(unlikely(init)) {
 693                                 if(n > 0) {
 694                                         min = 0;
 695                                         max = n;
 696                                 }
 697                                 else {
 698                                         min = n;
 699                                         max = 0;
 700                                 }
 701                                 init = 0;
 702                         }
 703
 704                         if(likely(!(co[c] & RRDR_EMPTY))) {
 705                                 all_null = 0;
 706                                 if((options & RRDR_OPTION_ABSOLUTE) && n < 0) n = -n;
 707                                 sum += n;
 708                         }
 709
 710                         if(n < min) min = n;
 711                         if(n > max) max = n;
 712                 }
 713
 714                 if(likely(i != start))
 715                         buffer_strcat(out, separator);
 716
 717                 if(all_null) {
 718                         if(options & RRDR_OPTION_NULL2ZERO)
 719                                 buffer_strcat(out, "0");
 720                         else
 721                                 buffer_strcat(out, "null");
 722                 }
 723                 else if(options & RRDR_OPTION_MIN2MAX)
 724                         buffer_rrd_value(out, max - min);
 725                 else
 726                         buffer_rrd_value(out, sum);
 727         }
 728         buffer_strcat(out, suffix);
 729 }
 730
 731 inline static calculated_number *rrdr_line_values(RRDR *r)
 732 {
 733         return &r->v[ r->c * r->d ];
 734 }
 735
 736 inline static uint8_t *rrdr_line_options(RRDR *r)
 737 {
 738         return &r->o[ r->c * r->d ];
 739 }
 740
 741 inline static int rrdr_line_init(RRDR *r, time_t t)
 742 {
 743         r->c++;
 744
 745         if(unlikely(r->c >= r->n)) {
 746                 error("requested to step above RRDR size for chart %s", r->st->name);
 747                 r->c = r->n - 1;
 748         }
 749
 750         // save the time
 751         r->t[r->c] = t;
 752
 753         return 1;
 754 }
 755
 756 inline static void rrdr_lock_rrdset(RRDR *r) {
 757         if(unlikely(!r)) {
 758                 error("NULL value given!");
 759                 return;
 760         }
 761
 762         pthread_rwlock_rdlock(&r->st->rwlock);
 763         r->has_st_lock = 1;
 764 }
 765
 766 inline static void rrdr_unlock_rrdset(RRDR *r) {
 767         if(unlikely(!r)) {
 768                 error("NULL value given!");
 769                 return;
 770         }
 771
 772         if(likely(r->has_st_lock)) {
 773                 pthread_rwlock_unlock(&r->st->rwlock);
 774                 r->has_st_lock = 0;
 775         }
 776 }
 777
 778 inline static void rrdr_free(RRDR *r)
 779 {
 780         if(unlikely(!r)) {
 781                 error("NULL value given!");
 782                 return;
 783         }
 784
 785         rrdr_unlock_rrdset(r);
 786         if(likely(r->t)) free(r->t);
 787         if(likely(r->v)) free(r->v);
 788         if(likely(r->o)) free(r->o);
 789         if(likely(r->od)) free(r->od);
 790         free(r);
 791 }
 792
 793 inline void rrdr_done(RRDR *r)
 794 {
 795         r->rows = r->c + 1;
 796         r->c = 0;
 797 }
 798
 799 static RRDR *rrdr_create(RRDSET *st, int n)
 800 {
 801         if(unlikely(!st)) {
 802                 error("NULL value given!");
 803                 return NULL;
 804         }
 805
 806         RRDR *r = calloc(1, sizeof(RRDR));
 807         if(unlikely(!r)) goto cleanup;
 808
 809         r->st = st;
 810
 811         rrdr_lock_rrdset(r);
 812
 813         RRDDIM *rd;
 814         for(rd = st->dimensions ; rd ; rd = rd->next) r->d++;
 815
 816         r->n = n;
 817
 818         r->t = calloc(n, sizeof(time_t));
 819         if(unlikely(!r->t)) goto cleanup;
 820
 821         r->v = calloc(n * r->d, sizeof(calculated_number));
 822         if(unlikely(!r->v)) goto cleanup;
 823
 824         r->o = calloc(n * r->d, sizeof(uint8_t));
 825         if(unlikely(!r->o)) goto cleanup;
 826
 827         r->od = calloc(r->d, sizeof(uint8_t));
 828         if(unlikely(!r->od)) goto cleanup;
 829
 830         r->c = -1;
 831         r->rows = 0;
 832
 833         return r;
 834
 835 cleanup:
 836         error("Cannot allocate memory");
 837         if(likely(r)) rrdr_free(r);
 838         return NULL;
 839 }
 840
 841 RRDR *rrd2rrdr(RRDSET *st, long points, long long after, long long before, int group_method)
 842 {
 843         int debug = st->debug;
 844
 845         time_t first_entry_t = rrdset_first_entry_t(st);
 846         time_t last_entry_t = rrdset_last_entry_t(st);
 847
 848         if(before == 0 && after == 0) {
 849                 before = last_entry_t;
 850                 after = first_entry_t;
 851         }
 852
 853         // allow relative for before and after
 854         if(before <= st->update_every * st->entries)
 855                 before = last_entry_t + before;
 856
 857         if(after <= st->update_every * st->entries)
 858                 after = last_entry_t + after;
 859
 860         // make sure they are within our timeframe
 861         if(before > last_entry_t) before = last_entry_t;
 862         if(before < first_entry_t) before = first_entry_t;
 863
 864         if(after > last_entry_t) after = last_entry_t;
 865         if(after < first_entry_t) after = first_entry_t;
 866
 867         // check if they are upside down
 868         if(after > before) {
 869                 time_t tmp = before;
 870                 before = after;
 871                 after = tmp;
 872         }
 873
 874         // the duration of the chart
 875         time_t duration = before - after;
 876         if(duration <= 0) return NULL;
 877
 878         // how many points does the chart has?
 879         long available_points = duration / st->update_every;
 880
 881         // check the wanted points
 882         if(points < 0) points = -points;
 883         if(points > available_points) points = available_points;
 884         if(points == 0) points = available_points;
 885
 886         // calculate proper grouping of source data
 887         long group = available_points / points;
 888         if(group <= 0) group = 1;
 889
 890         // round group to the closest integer
 891         if(available_points % points > points / 2) group++;
 892
 893         // find the starting and ending slots in our round robin db
 894         long    start_at_slot = rrdset_time2slot(st, before),
 895                         stop_at_slot = rrdset_time2slot(st, after);
 896
 897         // align them, so that panning on the data
 898         // will always produce the same results
 899         start_at_slot -= start_at_slot % group;
 900         stop_at_slot -= stop_at_slot % group;
 901
 902         time_t after_new = rrdset_slot2time(st, stop_at_slot);
 903         time_t before_new = rrdset_slot2time(st, start_at_slot);
 904         long points_new = (before_new - after_new) / st->update_every / group;
 905
 906         if(after_new < first_entry_t) {
 907                 error("after_new %u is before min %u", after_new, first_entry_t);
 908                 after_new = first_entry_t;
 909                 points_new = (before_new - after_new) / st->update_every / group;
 910         }
 911         if(after_new > last_entry_t) {
 912                 error("after_new %u is after max %u", after_new, last_entry_t);
 913                 after_new = last_entry_t;
 914                 points_new = (before_new - after_new) / st->update_every / group;
 915         }
 916         if(before_new < first_entry_t) {
 917                 error("before_new %u is before min %u", before_new, first_entry_t);
 918                 before_new = first_entry_t;
 919                 points_new = (before_new - after_new) / st->update_every / group;
 920         }
 921         if(before_new > last_entry_t) {
 922                 error("before_new %u is after max %u", before_new, last_entry_t);
 923                 before_new = last_entry_t;
 924                 points_new = (before_new - after_new) / st->update_every / group;
 925         }
 926         if(start_at_slot < 0 || start_at_slot >= st->entries) {
 927                 error("start_at_slot is invalid %ld, expected %ld to %ld", start_at_slot, 0, st->entries - 1);
 928                 start_at_slot = st->current_entry;
 929                 points_new = (before_new - after_new) / st->update_every / group;
 930         }
 931         if(stop_at_slot < 0 || stop_at_slot >= st->entries) {
 932                 error("stop_at_slot is invalid %ld, expected %ld to %ld", stop_at_slot, 0, st->entries - 1);
 933                 stop_at_slot = 0;
 934                 points_new = (before_new - after_new) / st->update_every / group;
 935         }
 936         if(points_new > (before_new - after_new) / group / st->update_every + 1) {
 937                 error("points_new %ld is more than points %ld", points_new, (before_new - after_new) / group / st->update_every + 1);
 938         }
 939
 940         //error("SHIFT: %s: wanted %ld points, got %ld - group=%ld, wanted duration=%u, got %u - wanted %ld - %ld, got %ld - %ld", st->id, points, points_new, group, before - after, before_new - after_new, after, before, after_new, before_new);
 941
 942         after = after_new;
 943         before = before_new;
 944         duration = before - after;
 945         points = points_new;
 946
 947         // Now we have:
 948         // before = the end time of the calculation
 949         // after = the start time of the calculation
 950         // duration = the duration of the calculation
 951         // group = the number of source points to aggregate / group together
 952         // method = the method of grouping source points
 953         // points = the number of points to generate
 954
 955
 956         // -------------------------------------------------------------------------
 957         // initialize our result set
 958
 959         RRDR *r = rrdr_create(st, points);
 960         if(!r) return NULL;
 961         if(!r->d) {
 962                 rrdr_free(r);
 963                 return NULL;
 964         }
 965
 966         // find how many dimensions we have
 967         long dimensions = r->d;
 968
 969
 970         // -------------------------------------------------------------------------
 971         // checks for debugging
 972
 973         if(debug) debug(D_RRD_STATS, "INFO %s first_t: %lu, last_t: %lu, all_duration: %lu, after: %lu, before: %lu, duration: %lu, points: %ld, group: %ld"
 974                         , st->id
 975                         , first_entry_t
 976                         , last_entry_t
 977                         , last_entry_t - first_entry_t
 978                         , after
 979                         , before
 980                         , duration
 981                         , points
 982                         , group
 983                         );
 984
 985
 986         // -------------------------------------------------------------------------
 987         // temp arrays for keeping values per dimension
 988
 989         calculated_number       group_values[dimensions]; // keep sums when grouping
 990         long                            group_counts[dimensions]; // keep the number of values added to group_values
 991         uint8_t                         group_options[dimensions];
 992         uint8_t                         found_non_zero[dimensions];
 993
 994
 995         // initialize them
 996         RRDDIM *rd;
 997         long c;
 998         for( rd = st->dimensions, c = 0 ; rd && c < dimensions ; rd = rd->next, c++) {
 999                 group_values[c] = 0;
1000                 group_counts[c] = 0;
1001                 group_options[c] = 0;
1002                 found_non_zero[c] = 0;
1003         }
1004
1005
1006         // -------------------------------------------------------------------------
1007         // the main loop
1008
1009         time_t  now = rrdset_slot2time(st, start_at_slot),
1010                         dt = st->update_every,
1011                         group_start_t = 0;
1012
1013         if(unlikely(debug)) debug(D_RRD_STATS, "BEGIN %s after_t: %lu (stop_at_t: %ld), before_t: %lu (start_at_t: %ld), start_t(now): %lu, current_entry: %ld, entries: %ld"
1014                         , st->id
1015                         , after
1016                         , stop_at_slot
1017                         , before
1018                         , start_at_slot
1019                         , now
1020                         , st->current_entry
1021                         , st->entries
1022                         );
1023
1024         r->group = group;
1025         r->update_every = group * st->update_every;
1026
1027         long slot = start_at_slot, counter = 0, stop_now = 0, added = 0, group_count = 0, add_this = 0;
1028         for(; !stop_now ; now -= dt, slot--, counter++) {
1029                 if(unlikely(slot < 0)) slot = st->entries - 1;
1030                 if(unlikely(slot == stop_at_slot)) stop_now = counter;
1031
1032                 if(unlikely(debug)) debug(D_RRD_STATS, "ROW %s slot: %ld, entries_counter: %ld, group_count: %ld, added: %ld, now: %lu, %s %s"
1033                                 , st->id
1034                                 , slot
1035                                 , counter
1036                                 , group_count + 1
1037                                 , added
1038                                 , now
1039                                 , (group_count + 1 == group)?"PRINT":"  -  "
1040                                 , (now >= after && now <= before)?"RANGE":"  -  "
1041                                 );
1042
1043                 // make sure we return data in the proper time range
1044                 if(unlikely(now > before)) continue;
1045                 if(unlikely(now < after)) break;
1046
1047                 if(unlikely(group_count == 0)) group_start_t = now;
1048                 group_count++;
1049
1050                 if(unlikely(group_count == group)) {
1051                         if(unlikely(added >= points)) break;
1052                         add_this = 1;
1053                 }
1054
1055                 // do the calculations
1056                 for(rd = st->dimensions, c = 0 ; likely(rd && c < dimensions) ; rd = rd->next, c++) {
1057                         storage_number n = rd->values[slot];
1058                         if(unlikely(!does_storage_number_exist(n))) continue;
1059
1060                         group_counts[c]++;
1061
1062                         calculated_number value = unpack_storage_number(n);
1063                         if(likely(value != 0.0)) {
1064                                 group_options[c] |= RRDR_NONZERO;
1065                                 found_non_zero[c] = 1;
1066                         }
1067
1068                         if(unlikely(did_storage_number_reset(n)))
1069                                 group_options[c] |= RRDR_RESET;
1070
1071                         switch(group_method) {
1072                                 case GROUP_MAX:
1073                                         if(unlikely(abs(value) > abs(group_values[c])))
1074                                                 group_values[c] = value;
1075                                         break;
1076
1077                                 default:
1078                                 case GROUP_SUM:
1079                                 case GROUP_AVERAGE:
1080                                         group_values[c] += value;
1081                                         break;
1082                         }
1083                 }
1084
1085                 // added it
1086                 if(unlikely(add_this)) {
1087                         if(unlikely(!rrdr_line_init(r, group_start_t))) break;
1088
1089                         calculated_number *cn = rrdr_line_values(r);
1090                         uint8_t *co = rrdr_line_options(r);
1091
1092                         for(rd = st->dimensions, c = 0 ; likely(rd && c < dimensions) ; rd = rd->next, c++) {
1093
1094                                 // update the dimension options
1095                                 if(likely(found_non_zero[c])) r->od[c] |= RRDR_NONZERO;
1096                                 if(unlikely(rd->flags & RRDDIM_FLAG_HIDDEN)) r->od[c] |= RRDR_HIDDEN;
1097
1098                                 // store the specific point options
1099                                 co[c] = group_options[c];
1100
1101                                 // store the value
1102                                 if(unlikely(group_counts[c] == 0)) {
1103                                         cn[c] = 0.0;
1104                                         co[c] |= RRDR_EMPTY;
1105                                 }
1106                                 else if(unlikely(group_method == GROUP_AVERAGE)) {
1107                                         cn[c] = group_values[c] / group_counts[c];
1108                                 }
1109                                 else {
1110                                         cn[c] = group_values[c];
1111                                 }
1112
1113                                 // reset them for the next loop
1114                                 group_values[c] = 0;
1115                                 group_counts[c] = 0;
1116                                 group_options[c] = 0;
1117                         }
1118
1119                         added++;
1120                         group_count = 0;
1121                         add_this = 0;
1122                 }
1123         }
1124
1125         rrdr_done(r);
1126         //error("SHIFT: %s: wanted %ld points, got %ld", st->id, points, rrdr_rows(r));
1127         return r;
1128 }
1129
1130 int rrd2format(RRDSET *st, BUFFER *out, BUFFER *dimensions, uint32_t format, long points, long long after, long long before, int group_method, uint32_t options, time_t *latest_timestamp)
1131 {
1132         RRDR *rrdr = rrd2rrdr(st, points, after, before, group_method);
1133         if(!rrdr) {
1134                 buffer_strcat(out, "Cannot generate output with these parameters on this chart.");
1135                 return 500;
1136         }
1137
1138         if(dimensions)
1139                 rrdr_disable_not_selected_dimensions(rrdr, buffer_tostring(dimensions));
1140
1141         if(latest_timestamp && rrdr_rows(rrdr) > 0)
1142                 *latest_timestamp = rrdr->t[rrdr_rows(rrdr) - 1];
1143
1144         switch(format) {
1145         case DATASOURCE_SSV:
1146                 out->contenttype = CT_TEXT_PLAIN;
1147                 rrdr2ssv(rrdr, out, options, "", " ", "");
1148                 break;
1149
1150         case DATASOURCE_SSV_COMMA:
1151                 out->contenttype = CT_TEXT_PLAIN;
1152                 rrdr2ssv(rrdr, out, options, "", ",", "");
1153                 break;
1154
1155         case DATASOURCE_JS_ARRAY:
1156                 out->contenttype = CT_APPLICATION_JSON;
1157                 rrdr2ssv(rrdr, out, options, "[", ",", "]");
1158                 break;
1159
1160         case DATASOURCE_CSV:
1161                 out->contenttype = CT_TEXT_PLAIN;
1162                 rrdr2csv(rrdr, out, options, "", ",", "\r\n");
1163                 break;
1164
1165         case DATASOURCE_TSV:
1166                 out->contenttype = CT_TEXT_PLAIN;
1167                 rrdr2csv(rrdr, out, options, "", "\t", "\r\n");
1168                 break;
1169
1170         case DATASOURCE_HTML:
1171                 out->contenttype = CT_TEXT_HTML;
1172                 buffer_strcat(out, "<html>\n<center><table border=\"0\" cellpadding=\"5\" cellspacing=\"5\">");
1173                 rrdr2csv(rrdr, out, options, "<tr><td>", "</td><td>", "</td></tr>\n");
1174                 buffer_strcat(out, "</table>\n</center>\n</html>\n");
1175                 break;
1176
1177         case DATASOURCE_DATATABLE_JSONP:
1178                 out->contenttype = CT_APPLICATION_X_JAVASCRIPT;
1179                 rrdr2json(rrdr, out, options, 1);
1180                 break;
1181
1182         case DATASOURCE_DATATABLE_JSON:
1183                 out->contenttype = CT_APPLICATION_JSON;
1184                 rrdr2json(rrdr, out, options, 1);
1185                 break;
1186
1187         case DATASOURCE_JSONP:
1188                 out->contenttype = CT_APPLICATION_X_JAVASCRIPT;
1189                 rrdr2json(rrdr, out, options, 0);
1190                 break;
1191
1192         case DATASOURCE_JSON:
1193         default:
1194                 out->contenttype = CT_APPLICATION_JSON;
1195                 rrdr2json(rrdr, out, options, 0);
1196                 break;
1197         }
1198
1199         rrdr_free(rrdr);
1200         return 200;
1201 }
1202
1203 unsigned long rrd_stats_json(int type, RRDSET *st, BUFFER *wb, int points, int group, int group_method, time_t after, time_t before, int only_non_zero)
1204 {
1205         int c;
1206         pthread_rwlock_rdlock(&st->rwlock);
1207
1208
1209         // -------------------------------------------------------------------------
1210         // switch from JSON to google JSON
1211
1212         char kq[2] = "\"";
1213         char sq[2] = "\"";
1214         switch(type) {
1215                 case DATASOURCE_DATATABLE_JSON:
1216                 case DATASOURCE_DATATABLE_JSONP:
1217                         kq[0] = '\0';
1218                         sq[0] = '\'';
1219                         break;
1220
1221                 case DATASOURCE_JSON:
1222                 default:
1223                         break;
1224         }
1225
1226
1227         // -------------------------------------------------------------------------
1228         // validate the parameters
1229
1230         if(points < 1) points = 1;
1231         if(group < 1) group = 1;
1232
1233         if(before == 0 || before > rrdset_last_entry_t(st)) before = rrdset_last_entry_t(st);
1234         if(after  == 0 || after < rrdset_first_entry_t(st)) after = rrdset_first_entry_t(st);
1235
1236         // ---
1237
1238         // our return value (the last timestamp printed)
1239         // this is required to detect re-transmit in google JSONP
1240         time_t last_timestamp = 0;
1241
1242
1243         // -------------------------------------------------------------------------
1244         // find how many dimensions we have
1245
1246         int dimensions = 0;
1247         RRDDIM *rd;
1248         for( rd = st->dimensions ; rd ; rd = rd->next) dimensions++;
1249         if(!dimensions) {
1250                 pthread_rwlock_unlock(&st->rwlock);
1251                 buffer_strcat(wb, "No dimensions yet.");
1252                 return 0;
1253         }
1254
1255
1256         // -------------------------------------------------------------------------
1257         // prepare various strings, to speed up the loop
1258
1259         char overflow_annotation[201]; snprintf(overflow_annotation, 200, ",{%sv%s:%sRESET OR OVERFLOW%s},{%sv%s:%sThe counters have been wrapped.%s}", kq, kq, sq, sq, kq, kq, sq, sq);
1260         char normal_annotation[201];   snprintf(normal_annotation,   200, ",{%sv%s:null},{%sv%s:null}", kq, kq, kq, kq);
1261         char pre_date[51];             snprintf(pre_date,             50, "             {%sc%s:[{%sv%s:%s", kq, kq, kq, kq, sq);
1262         char post_date[21];            snprintf(post_date,            20, "%s}", sq);
1263         char pre_value[21];            snprintf(pre_value,            20, ",{%sv%s:", kq, kq);
1264         char post_value[21];           snprintf(post_value,           20, "}");
1265
1266
1267         // -------------------------------------------------------------------------
1268         // checks for debugging
1269
1270         if(st->debug) {
1271                 debug(D_RRD_STATS, "%s first_entry_t = %lu, last_entry_t = %lu, duration = %lu, after = %lu, before = %lu, duration = %lu, entries_to_show = %lu, group = %lu"
1272                         , st->id
1273                         , rrdset_first_entry_t(st)
1274                         , rrdset_last_entry_t(st)
1275                         , rrdset_last_entry_t(st) - rrdset_first_entry_t(st)
1276                         , after
1277                         , before
1278                         , before - after
1279                         , points
1280                         , group
1281                         );
1282
1283                 if(before < after)
1284                         debug(D_RRD_STATS, "WARNING: %s The newest value in the database (%lu) is earlier than the oldest (%lu)", st->name, before, after);
1285
1286                 if((before - after) > st->entries * st->update_every)
1287                         debug(D_RRD_STATS, "WARNING: %s The time difference between the oldest and the newest entries (%lu) is higher than the capacity of the database (%lu)", st->name, before - after, st->entries * st->update_every);
1288         }
1289
1290
1291         // -------------------------------------------------------------------------
1292         // temp arrays for keeping values per dimension
1293
1294         calculated_number group_values[dimensions]; // keep sums when grouping
1295         int               print_hidden[dimensions]; // keep hidden flags
1296         int               found_non_zero[dimensions];
1297         int               found_non_existing[dimensions];
1298
1299         // initialize them
1300         for( rd = st->dimensions, c = 0 ; rd && c < dimensions ; rd = rd->next, c++) {
1301                 group_values[c] = 0;
1302                 print_hidden[c] = (rd->flags & RRDDIM_FLAG_HIDDEN)?1:0;
1303                 found_non_zero[c] = 0;
1304                 found_non_existing[c] = 0;
1305         }
1306
1307
1308         // error("OLD: points=%d after=%d before=%d group=%d, duration=%d", entries_to_show, before - (st->update_every * group * entries_to_show), before, group, before - after + 1);
1309         // rrd2array(st, entries_to_show, before - (st->update_every * group * entries_to_show), before, group_method, only_non_zero);
1310         // rrd2rrdr(st, entries_to_show, before - (st->update_every * group * entries_to_show), before, group_method);
1311
1312         // -------------------------------------------------------------------------
1313         // remove dimensions that contain only zeros
1314
1315         int max_loop = 1;
1316         if(only_non_zero) max_loop = 2;
1317
1318         for(; max_loop ; max_loop--) {
1319
1320                 // -------------------------------------------------------------------------
1321                 // print the JSON header
1322
1323                 buffer_sprintf(wb, "{\n %scols%s:\n     [\n", kq, kq);
1324                 buffer_sprintf(wb, "            {%sid%s:%s%s,%slabel%s:%stime%s,%spattern%s:%s%s,%stype%s:%sdatetime%s},\n", kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq);
1325                 buffer_sprintf(wb, "            {%sid%s:%s%s,%slabel%s:%s%s,%spattern%s:%s%s,%stype%s:%sstring%s,%sp%s:{%srole%s:%sannotation%s}},\n", kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, kq, kq, sq, sq);
1326                 buffer_sprintf(wb, "            {%sid%s:%s%s,%slabel%s:%s%s,%spattern%s:%s%s,%stype%s:%sstring%s,%sp%s:{%srole%s:%sannotationText%s}}", kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, kq, kq, sq, sq);
1327
1328                 // print the header for each dimension
1329                 // and update the print_hidden array for the dimensions that should be hidden
1330                 int pc = 0;
1331                 for( rd = st->dimensions, c = 0 ; rd && c < dimensions ; rd = rd->next, c++) {
1332                         if(!print_hidden[c]) {
1333                                 pc++;
1334                                 buffer_sprintf(wb, ",\n         {%sid%s:%s%s,%slabel%s:%s%s%s,%spattern%s:%s%s,%stype%s:%snumber%s}", kq, kq, sq, sq, kq, kq, sq, rd->name, sq, kq, kq, sq, sq, kq, kq, sq, sq);
1335                         }
1336                 }
1337                 if(!pc) {
1338                         buffer_sprintf(wb, ",\n         {%sid%s:%s%s,%slabel%s:%s%s%s,%spattern%s:%s%s,%stype%s:%snumber%s}", kq, kq, sq, sq, kq, kq, sq, "no data", sq, kq, kq, sq, sq, kq, kq, sq, sq);
1339                 }
1340
1341                 // print the begin of row data
1342                 buffer_sprintf(wb, "\n  ],\n    %srows%s:\n     [\n", kq, kq);
1343
1344
1345                 // -------------------------------------------------------------------------
1346                 // the main loop
1347
1348                 int annotate_reset = 0;
1349                 int annotation_count = 0;
1350
1351                 long    t = rrdset_time2slot(st, before),
1352                                 stop_at_t = rrdset_time2slot(st, after),
1353                                 stop_now = 0;
1354
1355                 t -= t % group;
1356
1357                 time_t  now = rrdset_slot2time(st, t),
1358                                 dt = st->update_every;
1359
1360                 long count = 0, printed = 0, group_count = 0;
1361                 last_timestamp = 0;
1362
1363                 if(st->debug) debug(D_RRD_STATS, "%s: REQUEST after:%lu before:%lu, points:%d, group:%d, CHART cur:%ld first: %lu last:%lu, CALC start_t:%ld, stop_t:%ld"
1364                                         , st->id
1365                                         , after
1366                                         , before
1367                                         , points
1368                                         , group
1369                                         , st->current_entry
1370                                         , rrdset_first_entry_t(st)
1371                                         , rrdset_last_entry_t(st)
1372                                         , t
1373                                         , stop_at_t
1374                                         );
1375
1376                 for(; !stop_now ; now -= dt, t--) {
1377                         if(t < 0) t = st->entries - 1;
1378                         if(t == stop_at_t) stop_now = 1;
1379
1380                         int print_this = 0;
1381
1382                         if(st->debug) debug(D_RRD_STATS, "%s t = %ld, count = %ld, group_count = %ld, printed = %ld, now = %lu, %s %s"
1383                                         , st->id
1384                                         , t
1385                                         , count + 1
1386                                         , group_count + 1
1387                                         , printed
1388                                         , now
1389                                         , (group_count + 1 == group)?"PRINT":"  -  "
1390                                         , (now >= after && now <= before)?"RANGE":"  -  "
1391                                         );
1392
1393
1394                         // make sure we return data in the proper time range
1395                         if(now > before) continue;
1396                         if(now < after) break;
1397
1398                         //if(rrdset_slot2time(st, t) != now)
1399                         //      error("%s: slot=%ld, now=%ld, slot2time=%ld, diff=%ld, last_entry_t=%ld, rrdset_last_slot=%ld", st->id, t, now, rrdset_slot2time(st,t), now - rrdset_slot2time(st,t), rrdset_last_entry_t(st), rrdset_last_slot(st));
1400
1401                         count++;
1402                         group_count++;
1403
1404                         // check if we have to print this now
1405                         if(group_count == group) {
1406                                 if(printed >= points) {
1407                                         // debug(D_RRD_STATS, "Already printed all rows. Stopping.");
1408                                         break;
1409                                 }
1410
1411                                 // generate the local date time
1412                                 struct tm *tm = localtime(&now);
1413                                 if(!tm) { error("localtime() failed."); continue; }
1414                                 if(now > last_timestamp) last_timestamp = now;
1415
1416                                 if(printed) buffer_strcat(wb, "]},\n");
1417                                 buffer_strcat(wb, pre_date);
1418                                 buffer_jsdate(wb, tm->tm_year + 1900, tm->tm_mon, tm->tm_mday, tm->tm_hour, tm->tm_min, tm->tm_sec);
1419                                 buffer_strcat(wb, post_date);
1420
1421                                 print_this = 1;
1422                         }
1423
1424                         // do the calculations
1425                         for(rd = st->dimensions, c = 0 ; rd && c < dimensions ; rd = rd->next, c++) {
1426                                 storage_number n = rd->values[t];
1427                                 calculated_number value = unpack_storage_number(n);
1428
1429                                 if(!does_storage_number_exist(n)) {
1430                                         value = 0.0;
1431                                         found_non_existing[c]++;
1432                                 }
1433                                 if(did_storage_number_reset(n)) annotate_reset = 1;
1434
1435                                 switch(group_method) {
1436                                         case GROUP_MAX:
1437                                                 if(abs(value) > abs(group_values[c])) group_values[c] = value;
1438                                                 break;
1439
1440                                         case GROUP_SUM:
1441                                                 group_values[c] += value;
1442                                                 break;
1443
1444                                         default:
1445                                         case GROUP_AVERAGE:
1446                                                 group_values[c] += value;
1447                                                 if(print_this) group_values[c] /= ( group_count - found_non_existing[c] );
1448                                                 break;
1449                                 }
1450                         }
1451
1452                         if(print_this) {
1453                                 if(annotate_reset) {
1454                                         annotation_count++;
1455                                         buffer_strcat(wb, overflow_annotation);
1456                                         annotate_reset = 0;
1457                                 }
1458                                 else
1459                                         buffer_strcat(wb, normal_annotation);
1460
1461                                 pc = 0;
1462                                 for(c = 0 ; c < dimensions ; c++) {
1463                                         if(found_non_existing[c] == group_count) {
1464                                                 // all entries are non-existing
1465                                                 pc++;
1466                                                 buffer_strcat(wb, pre_value);
1467                                                 buffer_strcat(wb, "null");
1468                                                 buffer_strcat(wb, post_value);
1469                                         }
1470                                         else if(!print_hidden[c]) {
1471                                                 pc++;
1472                                                 buffer_strcat(wb, pre_value);
1473                                                 buffer_rrd_value(wb, group_values[c]);
1474                                                 buffer_strcat(wb, post_value);
1475
1476                                                 if(group_values[c]) found_non_zero[c]++;
1477                                         }
1478
1479                                         // reset them for the next loop
1480                                         group_values[c] = 0;
1481                                         found_non_existing[c] = 0;
1482                                 }
1483
1484                                 // if all dimensions are hidden, print a null
1485                                 if(!pc) {
1486                                         buffer_strcat(wb, pre_value);
1487                                         buffer_strcat(wb, "null");
1488                                         buffer_strcat(wb, post_value);
1489                                 }
1490
1491                                 printed++;
1492                                 group_count = 0;
1493                         }
1494                 }
1495
1496                 if(printed) buffer_strcat(wb, "]}");
1497                 buffer_strcat(wb, "\n   ]\n}\n");
1498
1499                 if(only_non_zero && max_loop > 1) {
1500                         int changed = 0;
1501                         for(rd = st->dimensions, c = 0 ; rd && c < dimensions ; rd = rd->next, c++) {
1502                                 group_values[c] = 0;
1503                                 found_non_existing[c] = 0;
1504
1505                                 if(!print_hidden[c] && !found_non_zero[c]) {
1506                                         changed = 1;
1507                                         print_hidden[c] = 1;
1508                                 }
1509                         }
1510
1511                         if(changed) buffer_flush(wb);
1512                         else break;
1513                 }
1514                 else break;
1515
1516         } // max_loop
1517
1518         debug(D_RRD_STATS, "RRD_STATS_JSON: %s total %ld bytes", st->name, wb->len);
1519
1520         pthread_rwlock_unlock(&st->rwlock);
1521         return last_timestamp;
1522 }