src/rrd2json.c

   1 #ifdef HAVE_CONFIG_H
   2 #include <config.h>
   3 #endif
   4 #include <pthread.h>
   5 #include <sys/time.h>
   6 #include <stdlib.h>
   7 #include <string.h>
   8
   9 #include "log.h"
  10 #include "common.h"
  11 #include "rrd2json.h"
  12
  13 #define HOSTNAME_MAX 1024
  14 char *hostname = "unknown";
  15
  16 void rrd_stats_api_v1_chart(RRDSET *st, BUFFER *wb)
  17 {
  18         pthread_rwlock_rdlock(&st->rwlock);
  19
  20         buffer_sprintf(wb,
  21                 "\t\t{\n"
  22                 "\t\t\t\"id\": \"%s\",\n"
  23                 "\t\t\t\"name\": \"%s\",\n"
  24                 "\t\t\t\"type\": \"%s\",\n"
  25                 "\t\t\t\"family\": \"%s\",\n"
  26                 "\t\t\t\"title\": \"%s\",\n"
  27                 "\t\t\t\"priority\": %ld,\n"
  28                 "\t\t\t\"enabled\": %s,\n"
  29                 "\t\t\t\"units\": \"%s\",\n"
  30                 "\t\t\t\"data_url\": \"/api/v1/data?chart=%s\",\n"
  31                 "\t\t\t\"chart_type\": \"%s\",\n"
  32                 "\t\t\t\"duration\": %ld,\n"
  33                 "\t\t\t\"first_entry\": %lu,\n"
  34                 "\t\t\t\"last_entry\": %lu,\n"
  35                 "\t\t\t\"update_every\": %d,\n"
  36                 "\t\t\t\"dimensions\": {\n"
  37                 , st->id
  38                 , st->name
  39                 , st->type
  40                 , st->family
  41                 , st->title
  42                 , st->priority
  43                 , st->enabled?"true":"false"
  44                 , st->units
  45                 , st->name
  46                 , rrdset_type_name(st->chart_type)
  47                 , st->entries * st->update_every
  48                 , rrdset_first_entry_t(st)
  49                 , rrdset_last_entry_t(st)
  50                 , st->update_every
  51                 );
  52
  53         unsigned long memory = st->memsize;
  54
  55         int c = 0;
  56         RRDDIM *rd;
  57         for(rd = st->dimensions; rd ; rd = rd->next) {
  58                 if(rd->flags & RRDDIM_FLAG_HIDDEN) continue;
  59
  60                 memory += rd->memsize;
  61
  62                 buffer_sprintf(wb,
  63                         "%s"
  64                         "\t\t\t\t\"%s\": { \"name\": \"%s\" }"
  65                         , c?",\n":""
  66                         , rd->id
  67                         , rd->name
  68                         );
  69
  70                 c++;
  71         }
  72
  73         buffer_sprintf(wb,
  74                 "\n\t\t\t}\n"
  75                 "\t\t}"
  76                 );
  77
  78         pthread_rwlock_unlock(&st->rwlock);
  79 }
  80
  81 void rrd_stats_api_v1_charts(BUFFER *wb)
  82 {
  83         long c;
  84         RRDSET *st;
  85
  86         buffer_sprintf(wb, "{\n"
  87                    "\t\"hostname\": \"%s\""
  88                 ",\n\t\"update_every\": %d"
  89                 ",\n\t\"history\": %d"
  90                 ",\n\t\"charts\": {"
  91                 , hostname
  92                 , rrd_update_every
  93                 , rrd_default_history_entries
  94                 );
  95
  96         pthread_rwlock_rdlock(&rrdset_root_rwlock);
  97         for(st = rrdset_root, c = 0; st ; st = st->next) {
  98                 if(st->enabled) {
  99                         if(c) buffer_strcat(wb, ",");
 100                         buffer_strcat(wb, "\n\t\t\"");
 101                         buffer_strcat(wb, st->id);
 102                         buffer_strcat(wb, "\": ");
 103                         rrd_stats_api_v1_chart(st, wb);
 104                         c++;
 105                 }
 106         }
 107         pthread_rwlock_unlock(&rrdset_root_rwlock);
 108
 109         buffer_strcat(wb, "\n\t}\n}\n");
 110 }
 111
 112
 113 unsigned long rrd_stats_one_json(RRDSET *st, char *options, BUFFER *wb)
 114 {
 115         time_t now = time(NULL);
 116
 117         pthread_rwlock_rdlock(&st->rwlock);
 118
 119         buffer_sprintf(wb,
 120                 "\t\t{\n"
 121                 "\t\t\t\"id\": \"%s\",\n"
 122                 "\t\t\t\"name\": \"%s\",\n"
 123                 "\t\t\t\"type\": \"%s\",\n"
 124                 "\t\t\t\"family\": \"%s\",\n"
 125                 "\t\t\t\"title\": \"%s\",\n"
 126                 "\t\t\t\"priority\": %ld,\n"
 127                 "\t\t\t\"enabled\": %d,\n"
 128                 "\t\t\t\"units\": \"%s\",\n"
 129                 "\t\t\t\"url\": \"/data/%s/%s\",\n"
 130                 "\t\t\t\"chart_type\": \"%s\",\n"
 131                 "\t\t\t\"counter\": %ld,\n"
 132                 "\t\t\t\"entries\": %ld,\n"
 133                 "\t\t\t\"first_entry_t\": %lu,\n"
 134                 "\t\t\t\"last_entry\": %ld,\n"
 135                 "\t\t\t\"last_entry_t\": %lu,\n"
 136                 "\t\t\t\"last_entry_secs_ago\": %lu,\n"
 137                 "\t\t\t\"update_every\": %d,\n"
 138                 "\t\t\t\"isdetail\": %d,\n"
 139                 "\t\t\t\"usec_since_last_update\": %llu,\n"
 140                 "\t\t\t\"collected_total\": " TOTAL_NUMBER_FORMAT ",\n"
 141                 "\t\t\t\"last_collected_total\": " TOTAL_NUMBER_FORMAT ",\n"
 142                 "\t\t\t\"dimensions\": [\n"
 143                 , st->id
 144                 , st->name
 145                 , st->type
 146                 , st->family
 147                 , st->title
 148                 , st->priority
 149                 , st->enabled
 150                 , st->units
 151                 , st->name, options?options:""
 152                 , rrdset_type_name(st->chart_type)
 153                 , st->counter
 154                 , st->entries
 155                 , rrdset_first_entry_t(st)
 156                 , rrdset_last_slot(st)
 157                 , rrdset_last_entry_t(st)
 158                 , (now < rrdset_last_entry_t(st)) ? (time_t)0 : now - rrdset_last_entry_t(st)
 159                 , st->update_every
 160                 , st->isdetail
 161                 , st->usec_since_last_update
 162                 , st->collected_total
 163                 , st->last_collected_total
 164                 );
 165
 166         unsigned long memory = st->memsize;
 167
 168         RRDDIM *rd;
 169         for(rd = st->dimensions; rd ; rd = rd->next) {
 170
 171                 memory += rd->memsize;
 172
 173                 buffer_sprintf(wb,
 174                         "\t\t\t\t{\n"
 175                         "\t\t\t\t\t\"id\": \"%s\",\n"
 176                         "\t\t\t\t\t\"name\": \"%s\",\n"
 177                         "\t\t\t\t\t\"entries\": %ld,\n"
 178                         "\t\t\t\t\t\"isHidden\": %d,\n"
 179                         "\t\t\t\t\t\"algorithm\": \"%s\",\n"
 180                         "\t\t\t\t\t\"multiplier\": %ld,\n"
 181                         "\t\t\t\t\t\"divisor\": %ld,\n"
 182                         "\t\t\t\t\t\"last_entry_t\": %lu,\n"
 183                         "\t\t\t\t\t\"collected_value\": " COLLECTED_NUMBER_FORMAT ",\n"
 184                         "\t\t\t\t\t\"calculated_value\": " CALCULATED_NUMBER_FORMAT ",\n"
 185                         "\t\t\t\t\t\"last_collected_value\": " COLLECTED_NUMBER_FORMAT ",\n"
 186                         "\t\t\t\t\t\"last_calculated_value\": " CALCULATED_NUMBER_FORMAT ",\n"
 187                         "\t\t\t\t\t\"memory\": %lu\n"
 188                         "\t\t\t\t}%s\n"
 189                         , rd->id
 190                         , rd->name
 191                         , rd->entries
 192                         , (rd->flags & RRDDIM_FLAG_HIDDEN)?1:0
 193                         , rrddim_algorithm_name(rd->algorithm)
 194                         , rd->multiplier
 195                         , rd->divisor
 196                         , rd->last_collected_time.tv_sec
 197                         , rd->collected_value
 198                         , rd->calculated_value
 199                         , rd->last_collected_value
 200                         , rd->last_calculated_value
 201                         , rd->memsize
 202                         , rd->next?",":""
 203                         );
 204         }
 205
 206         buffer_sprintf(wb,
 207                 "\t\t\t],\n"
 208                 "\t\t\t\"memory\" : %lu\n"
 209                 "\t\t}"
 210                 , memory
 211                 );
 212
 213         pthread_rwlock_unlock(&st->rwlock);
 214         return memory;
 215 }
 216
 217 #define RRD_GRAPH_JSON_HEADER "{\n\t\"charts\": [\n"
 218 #define RRD_GRAPH_JSON_FOOTER "\n\t]\n}\n"
 219
 220 void rrd_stats_graph_json(RRDSET *st, char *options, BUFFER *wb)
 221 {
 222         buffer_strcat(wb, RRD_GRAPH_JSON_HEADER);
 223         rrd_stats_one_json(st, options, wb);
 224         buffer_strcat(wb, RRD_GRAPH_JSON_FOOTER);
 225 }
 226
 227 void rrd_stats_all_json(BUFFER *wb)
 228 {
 229         unsigned long memory = 0;
 230         long c;
 231         RRDSET *st;
 232
 233         buffer_strcat(wb, RRD_GRAPH_JSON_HEADER);
 234
 235         pthread_rwlock_rdlock(&rrdset_root_rwlock);
 236         for(st = rrdset_root, c = 0; st ; st = st->next) {
 237                 if(st->enabled) {
 238                         if(c) buffer_strcat(wb, ",\n");
 239                         memory += rrd_stats_one_json(st, NULL, wb);
 240                         c++;
 241                 }
 242         }
 243         pthread_rwlock_unlock(&rrdset_root_rwlock);
 244
 245         buffer_sprintf(wb, "\n\t],\n"
 246                 "\t\"hostname\": \"%s\",\n"
 247                 "\t\"update_every\": %d,\n"
 248                 "\t\"history\": %d,\n"
 249                 "\t\"memory\": %lu\n"
 250                 "}\n"
 251                 , hostname
 252                 , rrd_update_every
 253                 , rrd_default_history_entries
 254                 , memory
 255                 );
 256 }
 257
 258
 259
 260 // ----------------------------------------------------------------------------
 261
 262 // RRDR options
 263 #define RRDR_EMPTY      0x01 // the dimension contains / the value is empty (null)
 264 #define RRDR_RESET      0x02 // the dimension contains / the value is reset
 265 #define RRDR_HIDDEN     0x04 // the dimension contains / the value is hidden
 266 #define RRDR_NONZERO    0x08 // the dimension contains / the value is non-zero
 267
 268
 269 typedef struct rrdresult {
 270         RRDSET *st;                     // the chart this result refers to
 271
 272         int d;                                  // the number of dimensions
 273         int n;                                  // the number of values in the arrays
 274         int rows;                       // the number of rows used
 275
 276         uint8_t *od;                    // the options for the dimensions
 277
 278         time_t *t;                              // array of n timestamps
 279         calculated_number *v;   // array n x d values
 280         uint8_t *o;                             // array n x d options
 281
 282         int c;                                  // current line ( -1 ~ n ), ( -1 = none, use rrdr_rows() to get number of rows )
 283
 284         int group;                              // how many collected values were grouped for each row
 285         int update_every;               // what is the suggested update frequency in seconds
 286
 287         calculated_number min;
 288         calculated_number max;
 289
 290         time_t before;
 291         time_t after;
 292
 293         int has_st_lock;                // if st is read locked by us
 294 } RRDR;
 295
 296 #define rrdr_rows(r) ((r)->rows)
 297
 298 /*
 299 static void rrdr_dump(RRDR *r)
 300 {
 301         long c, i;
 302         RRDDIM *d;
 303
 304         fprintf(stderr, "\nCHART %s (%s)\n", r->st->id, r->st->name);
 305
 306         for(c = 0, d = r->st->dimensions; d ;c++, d = d->next) {
 307                 fprintf(stderr, "DIMENSION %s (%s), %s%s%s%s\n"
 308                                 , d->id
 309                                 , d->name
 310                                 , (r->od[c] & RRDR_EMPTY)?"EMPTY ":""
 311                                 , (r->od[c] & RRDR_RESET)?"RESET ":""
 312                                 , (r->od[c] & RRDR_HIDDEN)?"HIDDEN ":""
 313                                 , (r->od[c] & RRDR_NONZERO)?"NONZERO ":""
 314                                 );
 315         }
 316
 317         if(r->rows <= 0) {
 318                 fprintf(stderr, "RRDR does not have any values in it.\n");
 319                 return;
 320         }
 321
 322         fprintf(stderr, "RRDR includes %d values in it:\n", r->rows);
 323
 324         // for each line in the array
 325         for(i = 0; i < r->rows ;i++) {
 326                 calculated_number *cn = &r->v[ i * r->d ];
 327                 uint8_t *co = &r->o[ i * r->d ];
 328
 329                 // print the id and the timestamp of the line
 330                 fprintf(stderr, "%ld %ld ", i + 1, r->t[i]);
 331
 332                 // for each dimension
 333                 for(c = 0, d = r->st->dimensions; d ;c++, d = d->next) {
 334                         if(unlikely(r->od[c] & RRDR_HIDDEN)) continue;
 335                         if(unlikely(!(r->od[c] & RRDR_NONZERO))) continue;
 336
 337                         if(co[c] & RRDR_EMPTY)
 338                                 fprintf(stderr, "null ");
 339                         else
 340                                 fprintf(stderr, CALCULATED_NUMBER_FORMAT " %s%s%s%s "
 341                                         , cn[c]
 342                                         , (co[c] & RRDR_EMPTY)?"E":" "
 343                                         , (co[c] & RRDR_RESET)?"R":" "
 344                                         , (co[c] & RRDR_HIDDEN)?"H":" "
 345                                         , (co[c] & RRDR_NONZERO)?"N":" "
 346                                         );
 347                 }
 348
 349                 fprintf(stderr, "\n");
 350         }
 351 }
 352 */
 353
 354 void rrdr_disable_not_selected_dimensions(RRDR *r, const char *dims)
 355 {
 356         char b[strlen(dims) + 1];
 357         char *o = b, *tok;
 358         strcpy(o, dims);
 359
 360         long c;
 361         RRDDIM *d;
 362
 363         // disable all of them
 364         for(c = 0, d = r->st->dimensions; d ;c++, d = d->next)
 365                 r->od[c] |= RRDR_HIDDEN;
 366
 367         while(o && *o && (tok = mystrsep(&o, ", |"))) {
 368                 if(!*tok) continue;
 369
 370                 // find it and enable it
 371                 for(c = 0, d = r->st->dimensions; d ;c++, d = d->next) {
 372                         if(!strcmp(d->name, tok)) {
 373                                 r->od[c] &= ~RRDR_HIDDEN;
 374                         }
 375                 }
 376         }
 377 }
 378
 379 void rrdr_buffer_print_format(BUFFER *wb, uint32_t format)
 380 {
 381         switch(format) {
 382         case DATASOURCE_JSON:
 383                 buffer_strcat(wb, DATASOURCE_FORMAT_JSON);
 384                 break;
 385
 386         case DATASOURCE_DATATABLE_JSON:
 387                 buffer_strcat(wb, DATASOURCE_FORMAT_DATATABLE_JSON);
 388                 break;
 389
 390         case DATASOURCE_DATATABLE_JSONP:
 391                 buffer_strcat(wb, DATASOURCE_FORMAT_DATATABLE_JSONP);
 392                 break;
 393
 394         case DATASOURCE_JSONP:
 395                 buffer_strcat(wb, DATASOURCE_FORMAT_JSONP);
 396                 break;
 397
 398         case DATASOURCE_SSV:
 399                 buffer_strcat(wb, DATASOURCE_FORMAT_SSV);
 400                 break;
 401
 402         case DATASOURCE_CSV:
 403                 buffer_strcat(wb, DATASOURCE_FORMAT_CSV);
 404                 break;
 405
 406         case DATASOURCE_TSV:
 407                 buffer_strcat(wb, DATASOURCE_FORMAT_TSV);
 408                 break;
 409
 410         case DATASOURCE_HTML:
 411                 buffer_strcat(wb, DATASOURCE_FORMAT_HTML);
 412                 break;
 413
 414         case DATASOURCE_JS_ARRAY:
 415                 buffer_strcat(wb, DATASOURCE_FORMAT_JS_ARRAY);
 416                 break;
 417
 418         case DATASOURCE_SSV_COMMA:
 419                 buffer_strcat(wb, DATASOURCE_FORMAT_SSV_COMMA);
 420                 break;
 421
 422         default:
 423                 buffer_strcat(wb, "unknown");
 424                 break;
 425         }
 426 }
 427
 428 void rrdr_json_wrapper_begin(RRDR *r, BUFFER *wb, uint32_t format, uint32_t options, int string_value)
 429 {
 430         char kq[2] = "",                                        // key quote
 431                 sq[2] = "";                                             // string quote
 432
 433         if( options & RRDR_OPTION_GOOGLE_JSON ) {
 434                 kq[0] = '\0';
 435                 sq[0] = '\'';
 436         }
 437         else {
 438                 kq[0] = '"';
 439                 sq[0] = '"';
 440         }
 441
 442         buffer_sprintf(wb, "{\n"
 443                         "       %sid%s: %s%s%s,\n"
 444                         "       %sname%s: %s%s%s,\n"
 445                         "       %supdate_every%s: %d,\n"
 446                         "       %smin%s: "
 447                         , kq, kq, sq, r->st->id, sq
 448                         , kq, kq, sq, r->st->name, sq
 449                         , kq, kq, r->update_every
 450                         , kq, kq);
 451
 452         buffer_rrd_value(wb, r->min);
 453         buffer_sprintf(wb, ",\n %smax%s: ", kq, kq);
 454         buffer_rrd_value(wb, r->max);
 455         buffer_sprintf(wb, ",\n"
 456                         "       %sbefore%s: %u,\n"
 457                         "       %safter%s: %u,\n"
 458                         "       %sdimension_names%s: ["
 459                         , kq, kq, r->before
 460                         , kq, kq, r->after
 461                         , kq, kq);
 462
 463
 464         long c, i;
 465         RRDDIM *rd;
 466         for(c = 0, i = 0, rd = r->st->dimensions; rd ;c++, rd = rd->next) {
 467                 if(unlikely(r->od[c] & RRDR_HIDDEN)) continue;
 468                 if(unlikely((options & RRDR_OPTION_NONZERO) && !(r->od[c] & RRDR_NONZERO))) continue;
 469
 470                 if(i) buffer_strcat(wb, ", ");
 471                 buffer_strcat(wb, sq);
 472                 buffer_strcat(wb, rd->name);
 473                 buffer_strcat(wb, sq);
 474                 i++;
 475         }
 476         buffer_sprintf(wb, "],\n"
 477                         "       %sdimension_ids%s: ["
 478                         , kq, kq);
 479
 480         for(c = 0, i = 0, rd = r->st->dimensions; rd ;c++, rd = rd->next) {
 481                 if(unlikely(r->od[c] & RRDR_HIDDEN)) continue;
 482                 if(unlikely((options & RRDR_OPTION_NONZERO) && !(r->od[c] & RRDR_NONZERO))) continue;
 483
 484                 if(i) buffer_strcat(wb, ", ");
 485                 buffer_strcat(wb, sq);
 486                 buffer_strcat(wb, rd->id);
 487                 buffer_strcat(wb, sq);
 488                 i++;
 489         }
 490         buffer_sprintf(wb, "],\n"
 491                         "       %sdimensions%s: %d,\n"
 492                         "       %spoints%s: %d,\n"
 493                         "       %sformat%s: %s"
 494                         , kq, kq, i
 495                         , kq, kq, rrdr_rows(r)
 496                         , kq, kq, sq
 497                         );
 498
 499         rrdr_buffer_print_format(wb, format);
 500
 501         buffer_sprintf(wb, "%s,\n"
 502                         "       %sresult%s: "
 503                         , sq
 504                         , kq, kq
 505                         );
 506
 507         if(string_value) buffer_strcat(wb, sq);
 508 }
 509
 510 void rrdr_json_wrapper_end(RRDR *r, BUFFER *wb, uint32_t format, uint32_t options, int string_value)
 511 {
 512         if(r) {;}
 513         if(format) {;}
 514
 515         char sq[2] = "";                                                // string quote
 516
 517         if( options & RRDR_OPTION_GOOGLE_JSON )
 518                 sq[0] = '\'';
 519         else
 520                 sq[0] = '"';
 521
 522         if(string_value) buffer_strcat(wb, sq);
 523         buffer_strcat(wb, "\n}\n");
 524 }
 525
 526 #define JSON_DATES_JS 1
 527 #define JSON_DATES_TIMESTAMP 2
 528
 529 static void rrdr2json(RRDR *r, BUFFER *wb, uint32_t options, int datatable)
 530 {
 531         int row_annotations = 0, dates = JSON_DATES_JS, dates_with_new = 0;
 532         char kq[2] = "",                                        // key quote
 533                 sq[2] = "",                                             // string quote
 534                 pre_label[101] = "",                    // before each label
 535                 post_label[101] = "",                   // after each label
 536                 pre_date[101] = "",                             // the beginning of line, to the date
 537                 post_date[101] = "",                    // closing the date
 538                 pre_value[101] = "",                    // before each value
 539                 post_value[101] = "",                   // after each value
 540                 post_line[101] = "",                    // at the end of each row
 541                 normal_annotation[201] = "",    // default row annotation
 542                 overflow_annotation[201] = "",  // overflow row annotation
 543                 data_begin[101] = "",                   // between labels and values
 544                 finish[101] = "";                               // at the end of everything
 545
 546         if(datatable) {
 547                 dates = JSON_DATES_JS;
 548                 if( options & RRDR_OPTION_GOOGLE_JSON ) {
 549                         kq[0] = '\0';
 550                         sq[0] = '\'';
 551                 }
 552                 else {
 553                         kq[0] = '"';
 554                         sq[0] = '"';
 555                 }
 556                 row_annotations = 1;
 557                 snprintf(pre_date,   100, "             {%sc%s:[{%sv%s:%s", kq, kq, kq, kq, sq);
 558                 snprintf(post_date,  100, "%s}", sq);
 559                 snprintf(pre_label,  100, ",\n          {%sid%s:%s%s,%slabel%s:%s", kq, kq, sq, sq, kq, kq, sq);
 560                 snprintf(post_label, 100, "%s,%spattern%s:%s%s,%stype%s:%snumber%s}", sq, kq, kq, sq, sq, kq, kq, sq, sq);
 561                 snprintf(pre_value,  100, ",{%sv%s:", kq, kq);
 562                 snprintf(post_value, 100, "}");
 563                 snprintf(post_line,  100, "]}");
 564                 snprintf(data_begin, 100, "\n   ],\n    %srows%s:\n     [\n", kq, kq);
 565                 snprintf(finish,     100, "\n   ]\n}\n");
 566
 567                 snprintf(overflow_annotation, 200, ",{%sv%s:%sRESET OR OVERFLOW%s},{%sv%s:%sThe counters have been wrapped.%s}", kq, kq, sq, sq, kq, kq, sq, sq);
 568                 snprintf(normal_annotation,   200, ",{%sv%s:null},{%sv%s:null}", kq, kq, kq, kq);
 569
 570                 buffer_sprintf(wb, "{\n %scols%s:\n     [\n", kq, kq, kq, kq);
 571                 buffer_sprintf(wb, "            {%sid%s:%s%s,%slabel%s:%stime%s,%spattern%s:%s%s,%stype%s:%sdatetime%s},\n", kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq);
 572                 buffer_sprintf(wb, "            {%sid%s:%s%s,%slabel%s:%s%s,%spattern%s:%s%s,%stype%s:%sstring%s,%sp%s:{%srole%s:%sannotation%s}},\n", kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, kq, kq, sq, sq);
 573                 buffer_sprintf(wb, "            {%sid%s:%s%s,%slabel%s:%s%s,%spattern%s:%s%s,%stype%s:%sstring%s,%sp%s:{%srole%s:%sannotationText%s}}", kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, kq, kq, sq, sq);
 574
 575                 // remove the valueobjects flag
 576                 // google wants its own keys
 577                 if(options & RRDR_OPTION_OBJECTSROWS)
 578                         options &= ~RRDR_OPTION_OBJECTSROWS;
 579         }
 580         else {
 581                 kq[0] = '"';
 582                 sq[0] = '"';
 583                 if((options & RRDR_OPTION_SECONDS) || (options & RRDR_OPTION_MILLISECONDS)) {
 584                         dates = JSON_DATES_TIMESTAMP;
 585                         dates_with_new = 0;
 586                 }
 587                 else {
 588                         dates = JSON_DATES_JS;
 589                         dates_with_new = 1;
 590                 }
 591                 if( options & RRDR_OPTION_OBJECTSROWS )
 592                         snprintf(pre_date,   100, "             { ");
 593                 else
 594                         snprintf(pre_date,   100, "             [ ");
 595                 snprintf(pre_label,  100, ", \"");
 596                 snprintf(post_label, 100, "\"");
 597                 snprintf(pre_value,  100, ", ");
 598                 if( options & RRDR_OPTION_OBJECTSROWS )
 599                         snprintf(post_line,  100, "}");
 600                 else
 601                         snprintf(post_line,  100, "]");
 602                 snprintf(data_begin, 100, "],\n %sdata%s:\n     [\n", kq, kq);
 603                 snprintf(finish,     100, "\n   ]\n}\n");
 604
 605                 buffer_sprintf(wb, "{\n %slabels%s: [", kq, kq);
 606                 buffer_sprintf(wb, "%stime%s", sq, sq);
 607         }
 608
 609         // -------------------------------------------------------------------------
 610         // print the JSON header
 611
 612         long c, i;
 613         RRDDIM *rd;
 614
 615         // print the header lines
 616         for(c = 0, i = 0, rd = r->st->dimensions; rd ;c++, rd = rd->next) {
 617                 if(unlikely(r->od[c] & RRDR_HIDDEN)) continue;
 618                 if(unlikely((options & RRDR_OPTION_NONZERO) && !(r->od[c] & RRDR_NONZERO))) continue;
 619
 620                 buffer_strcat(wb, pre_label);
 621                 buffer_strcat(wb, rd->name);
 622                 buffer_strcat(wb, post_label);
 623                 i++;
 624         }
 625         if(!i) {
 626                 buffer_strcat(wb, pre_label);
 627                 buffer_strcat(wb, "no data");
 628                 buffer_strcat(wb, post_label);
 629         }
 630
 631         // print the begin of row data
 632         buffer_strcat(wb, data_begin);
 633
 634         // if all dimensions are hidden, print a null
 635         if(!i) {
 636                 buffer_strcat(wb, pre_value);
 637                 if(options & RRDR_OPTION_NULL2ZERO)
 638                         buffer_strcat(wb, "0");
 639                 else
 640                         buffer_strcat(wb, "null");
 641                 buffer_strcat(wb, post_value);
 642         }
 643
 644         long start = 0, end = rrdr_rows(r), step = 1;
 645         if((options & RRDR_OPTION_REVERSED)) {
 646                 start = rrdr_rows(r) - 1;
 647                 end = -1;
 648                 step = -1;
 649         }
 650
 651         // for each line in the array
 652         for(i = start; i != end ;i += step) {
 653                 calculated_number *cn = &r->v[ i * r->d ];
 654                 uint8_t *co = &r->o[ i * r->d ];
 655
 656                 time_t now = r->t[i];
 657
 658                 if(dates == JSON_DATES_JS) {
 659                         // generate the local date time
 660                         struct tm *tm = localtime(&now);
 661                         if(!tm) { error("localtime() failed."); continue; }
 662
 663                         if(likely(i != start)) buffer_strcat(wb, ",\n");
 664                         buffer_strcat(wb, pre_date);
 665
 666                         if( options & RRDR_OPTION_OBJECTSROWS )
 667                                 buffer_sprintf(wb, "%stime%s: ", kq, kq);
 668
 669                         if(dates_with_new)
 670                                 buffer_strcat(wb, "new ");
 671
 672                         buffer_jsdate(wb, tm->tm_year + 1900, tm->tm_mon, tm->tm_mday, tm->tm_hour, tm->tm_min, tm->tm_sec);
 673
 674                         buffer_strcat(wb, post_date);
 675
 676                         if(row_annotations) {
 677                                 // google supports one annotation per row
 678                                 int annotation_found = 0;
 679                                 for(c = 0, rd = r->st->dimensions; rd ;c++, rd = rd->next) {
 680                                         if(co[c] & RRDR_RESET) {
 681                                                 buffer_strcat(wb, overflow_annotation);
 682                                                 annotation_found = 1;
 683                                                 break;
 684                                         }
 685                                 }
 686                                 if(!annotation_found)
 687                                         buffer_strcat(wb, normal_annotation);
 688                         }
 689                 }
 690                 else {
 691                         // print the timestamp of the line
 692                         if(likely(i != start)) buffer_strcat(wb, ",\n");
 693                         buffer_strcat(wb, pre_date);
 694
 695                         if( options & RRDR_OPTION_OBJECTSROWS )
 696                                 buffer_sprintf(wb, "%stime%s: ", kq, kq);
 697
 698                         buffer_rrd_value(wb, (calculated_number)r->t[i]);
 699                         // in ms
 700                         if(options & RRDR_OPTION_MILLISECONDS) buffer_strcat(wb, "000");
 701
 702                         buffer_strcat(wb, post_date);
 703                 }
 704
 705                 // for each dimension
 706                 for(c = 0, rd = r->st->dimensions; rd ;c++, rd = rd->next) {
 707                         if(unlikely(r->od[c] & RRDR_HIDDEN)) continue;
 708                         if(unlikely((options & RRDR_OPTION_NONZERO) && !(r->od[c] & RRDR_NONZERO))) continue;
 709
 710                         calculated_number n = cn[c];
 711
 712                         buffer_strcat(wb, pre_value);
 713
 714                         if( options & RRDR_OPTION_OBJECTSROWS )
 715                                 buffer_sprintf(wb, "%s%s%s: ", kq, rd->name, kq);
 716
 717                         if(co[c] & RRDR_EMPTY) {
 718                                 if(options & RRDR_OPTION_NULL2ZERO)
 719                                         buffer_strcat(wb, "0");
 720                                 else
 721                                         buffer_strcat(wb, "null");
 722                         }
 723                         else if((options & RRDR_OPTION_ABSOLUTE))
 724                                 buffer_rrd_value(wb, (n<0)?-n:n);
 725                         else
 726                                 buffer_rrd_value(wb, n);
 727
 728                         buffer_strcat(wb, post_value);
 729                 }
 730
 731                 buffer_strcat(wb, post_line);
 732         }
 733
 734         buffer_strcat(wb, finish);
 735 }
 736
 737 static void rrdr2csv(RRDR *r, BUFFER *wb, uint32_t options, const char *startline, const char *separator, const char *endline)
 738 {
 739         long c, i;
 740         RRDDIM *d;
 741
 742         // print the csv header
 743         for(c = 0, i = 0, d = r->st->dimensions; d ;c++, d = d->next) {
 744                 if(unlikely(r->od[c] & RRDR_HIDDEN)) continue;
 745                 if(unlikely((options & RRDR_OPTION_NONZERO) && !(r->od[c] & RRDR_NONZERO))) continue;
 746
 747                 if(!i) {
 748                         buffer_strcat(wb, startline);
 749                         buffer_strcat(wb, "time");
 750                 }
 751                 buffer_strcat(wb, separator);
 752                 buffer_strcat(wb, d->name);
 753                 i++;
 754         }
 755         buffer_strcat(wb, endline);
 756
 757         if(!i) {
 758                 // no dimensions present
 759                 return;
 760         }
 761
 762         long start = 0, end = rrdr_rows(r), step = 1;
 763         if((options & RRDR_OPTION_REVERSED)) {
 764                 start = rrdr_rows(r) - 1;
 765                 end = -1;
 766                 step = -1;
 767         }
 768
 769         // for each line in the array
 770         for(i = start; i != end ;i += step) {
 771                 calculated_number *cn = &r->v[ i * r->d ];
 772                 uint8_t *co = &r->o[ i * r->d ];
 773
 774                 buffer_strcat(wb, startline);
 775
 776                 time_t now = r->t[i];
 777
 778                 if((options & RRDR_OPTION_SECONDS) || (options & RRDR_OPTION_MILLISECONDS)) {
 779                         // print the timestamp of the line
 780                         buffer_rrd_value(wb, (calculated_number)now);
 781                         // in ms
 782                         if(options & RRDR_OPTION_MILLISECONDS) buffer_strcat(wb, "000");
 783                 }
 784                 else {
 785                         // generate the local date time
 786                         struct tm *tm = localtime(&now);
 787                         if(!tm) { error("localtime() failed."); continue; }
 788                         buffer_date(wb, tm->tm_year + 1900, tm->tm_mon, tm->tm_mday, tm->tm_hour, tm->tm_min, tm->tm_sec);
 789                 }
 790
 791                 // for each dimension
 792                 for(c = 0, d = r->st->dimensions; d ;c++, d = d->next) {
 793                         if(unlikely(r->od[c] & RRDR_HIDDEN)) continue;
 794                         if(unlikely((options & RRDR_OPTION_NONZERO) && !(r->od[c] & RRDR_NONZERO))) continue;
 795
 796                         buffer_strcat(wb, separator);
 797
 798                         calculated_number n = cn[c];
 799
 800                         if(co[c] & RRDR_EMPTY) {
 801                                 if(options & RRDR_OPTION_NULL2ZERO)
 802                                         buffer_strcat(wb, "0");
 803                                 else
 804                                         buffer_strcat(wb, "null");
 805                         }
 806                         else if((options & RRDR_OPTION_ABSOLUTE))
 807                                 buffer_rrd_value(wb, (n<0)?-n:n);
 808                         else
 809                                 buffer_rrd_value(wb, n);
 810                 }
 811
 812                 buffer_strcat(wb, endline);
 813         }
 814 }
 815
 816 static void rrdr2ssv(RRDR *r, BUFFER *wb, uint32_t options, const char *prefix, const char *separator, const char *suffix)
 817 {
 818         long c, i;
 819         RRDDIM *d;
 820
 821         buffer_strcat(wb, prefix);
 822         long start = 0, end = rrdr_rows(r), step = 1;
 823         if((options & RRDR_OPTION_REVERSED)) {
 824                 start = rrdr_rows(r) - 1;
 825                 end = -1;
 826                 step = -1;
 827         }
 828
 829         // for each line in the array
 830         for(i = start; i != end ;i += step) {
 831
 832                 calculated_number *cn = &r->v[ i * r->d ];
 833                 uint8_t *co = &r->o[ i * r->d ];
 834
 835                 calculated_number sum = 0, min = 0, max = 0;
 836                 int all_null = 1, init = 1;
 837
 838                 // for each dimension
 839                 for(c = 0, d = r->st->dimensions; d ;c++, d = d->next) {
 840                         if(unlikely(r->od[c] & RRDR_HIDDEN)) continue;
 841                         if(unlikely((options & RRDR_OPTION_NONZERO) && !(r->od[c] & RRDR_NONZERO))) continue;
 842
 843                         calculated_number n = cn[c];
 844
 845                         if(unlikely(init)) {
 846                                 if(n > 0) {
 847                                         min = 0;
 848                                         max = n;
 849                                 }
 850                                 else {
 851                                         min = n;
 852                                         max = 0;
 853                                 }
 854                                 init = 0;
 855                         }
 856
 857                         if(likely(!(co[c] & RRDR_EMPTY))) {
 858                                 all_null = 0;
 859                                 if((options & RRDR_OPTION_ABSOLUTE) && n < 0) n = -n;
 860                                 sum += n;
 861                         }
 862
 863                         if(n < min) min = n;
 864                         if(n > max) max = n;
 865                 }
 866
 867                 if(likely(i != start))
 868                         buffer_strcat(wb, separator);
 869
 870                 if(all_null) {
 871                         if(options & RRDR_OPTION_NULL2ZERO)
 872                                 buffer_strcat(wb, "0");
 873                         else
 874                                 buffer_strcat(wb, "null");
 875                 }
 876                 else if(options & RRDR_OPTION_MIN2MAX)
 877                         buffer_rrd_value(wb, max - min);
 878                 else
 879                         buffer_rrd_value(wb, sum);
 880         }
 881         buffer_strcat(wb, suffix);
 882 }
 883
 884 inline static calculated_number *rrdr_line_values(RRDR *r)
 885 {
 886         return &r->v[ r->c * r->d ];
 887 }
 888
 889 inline static uint8_t *rrdr_line_options(RRDR *r)
 890 {
 891         return &r->o[ r->c * r->d ];
 892 }
 893
 894 inline static int rrdr_line_init(RRDR *r, time_t t)
 895 {
 896         r->c++;
 897
 898         if(unlikely(r->c >= r->n)) {
 899                 error("requested to step above RRDR size for chart %s", r->st->name);
 900                 r->c = r->n - 1;
 901         }
 902
 903         // save the time
 904         r->t[r->c] = t;
 905
 906         return 1;
 907 }
 908
 909 inline static void rrdr_lock_rrdset(RRDR *r) {
 910         if(unlikely(!r)) {
 911                 error("NULL value given!");
 912                 return;
 913         }
 914
 915         pthread_rwlock_rdlock(&r->st->rwlock);
 916         r->has_st_lock = 1;
 917 }
 918
 919 inline static void rrdr_unlock_rrdset(RRDR *r) {
 920         if(unlikely(!r)) {
 921                 error("NULL value given!");
 922                 return;
 923         }
 924
 925         if(likely(r->has_st_lock)) {
 926                 pthread_rwlock_unlock(&r->st->rwlock);
 927                 r->has_st_lock = 0;
 928         }
 929 }
 930
 931 inline static void rrdr_free(RRDR *r)
 932 {
 933         if(unlikely(!r)) {
 934                 error("NULL value given!");
 935                 return;
 936         }
 937
 938         rrdr_unlock_rrdset(r);
 939         if(likely(r->t)) free(r->t);
 940         if(likely(r->v)) free(r->v);
 941         if(likely(r->o)) free(r->o);
 942         if(likely(r->od)) free(r->od);
 943         free(r);
 944 }
 945
 946 inline void rrdr_done(RRDR *r)
 947 {
 948         r->rows = r->c + 1;
 949         r->c = 0;
 950 }
 951
 952 static RRDR *rrdr_create(RRDSET *st, int n)
 953 {
 954         if(unlikely(!st)) {
 955                 error("NULL value given!");
 956                 return NULL;
 957         }
 958
 959         RRDR *r = malloc(sizeof(RRDR));
 960         if(unlikely(!r)) goto cleanup;
 961
 962         r->st = st;
 963         r->has_st_lock = 0;
 964
 965         rrdr_lock_rrdset(r);
 966
 967         RRDDIM *rd;
 968         for(rd = st->dimensions ; rd ; rd = rd->next) r->d++;
 969
 970         r->n = n;
 971
 972         r->t = malloc(n * sizeof(time_t));
 973         if(unlikely(!r->t)) goto cleanup;
 974
 975         r->v = malloc(n * r->d * sizeof(calculated_number));
 976         if(unlikely(!r->v)) goto cleanup;
 977
 978         r->o = malloc(n * r->d * sizeof(uint8_t));
 979         if(unlikely(!r->o)) goto cleanup;
 980
 981         r->od = malloc(r->d * sizeof(uint8_t));
 982         if(unlikely(!r->od)) goto cleanup;
 983
 984         r->c = -1;
 985         r->rows = 0;
 986
 987         r->group = 1;
 988         r->update_every = 1;
 989
 990         r->min = 0.0;
 991         r->max = 0.0;
 992
 993         r->before = 0;
 994         r->after = 0;
 995
 996         return r;
 997
 998 cleanup:
 999         error("Cannot allocate memory");
1000         if(likely(r)) rrdr_free(r);
1001         return NULL;
1002 }
1003
1004 RRDR *rrd2rrdr(RRDSET *st, long points, long long after, long long before, int group_method)
1005 {
1006         int debug = st->debug;
1007
1008         time_t first_entry_t = rrdset_first_entry_t(st);
1009         time_t last_entry_t = rrdset_last_entry_t(st);
1010
1011         if(before == 0 && after == 0) {
1012                 before = last_entry_t;
1013                 after = first_entry_t;
1014         }
1015
1016         // allow relative for before and after
1017         if(before <= st->update_every * st->entries)
1018                 before = last_entry_t + before;
1019
1020         if(after <= st->update_every * st->entries)
1021                 after = last_entry_t + after;
1022
1023         // make sure they are within our timeframe
1024         if(before > last_entry_t) before = last_entry_t;
1025         if(before < first_entry_t) before = first_entry_t;
1026
1027         if(after > last_entry_t) after = last_entry_t;
1028         if(after < first_entry_t) after = first_entry_t;
1029
1030         // check if they are upside down
1031         if(after > before) {
1032                 time_t tmp = before;
1033                 before = after;
1034                 after = tmp;
1035         }
1036
1037         // the duration of the chart
1038         time_t duration = before - after;
1039         if(duration <= 0) return NULL;
1040
1041         // how many points does the chart has?
1042         long available_points = duration / st->update_every;
1043
1044         // check the wanted points
1045         if(points < 0) points = -points;
1046         if(points > available_points) points = available_points;
1047         if(points == 0) points = available_points;
1048
1049         // calculate proper grouping of source data
1050         long group = available_points / points;
1051         if(group <= 0) group = 1;
1052
1053         // round group to the closest integer
1054         if(available_points % points > points / 2) group++;
1055
1056         // find the starting and ending slots in our round robin db
1057         long    start_at_slot = rrdset_time2slot(st, before),
1058                         stop_at_slot = rrdset_time2slot(st, after);
1059
1060         // align them, so that panning on the data
1061         // will always produce the same results
1062         start_at_slot -= start_at_slot % group;
1063         stop_at_slot -= stop_at_slot % group;
1064
1065         time_t after_new = rrdset_slot2time(st, stop_at_slot);
1066         time_t before_new = rrdset_slot2time(st, start_at_slot);
1067         long points_new = (before_new - after_new) / st->update_every / group;
1068
1069         if(after_new < first_entry_t) {
1070                 error("after_new %u is before min %u", after_new, first_entry_t);
1071                 after_new = first_entry_t;
1072                 points_new = (before_new - after_new) / st->update_every / group;
1073         }
1074         if(after_new > last_entry_t) {
1075                 error("after_new %u is after max %u", after_new, last_entry_t);
1076                 after_new = last_entry_t;
1077                 points_new = (before_new - after_new) / st->update_every / group;
1078         }
1079         if(before_new < first_entry_t) {
1080                 error("before_new %u is before min %u", before_new, first_entry_t);
1081                 before_new = first_entry_t;
1082                 points_new = (before_new - after_new) / st->update_every / group;
1083         }
1084         if(before_new > last_entry_t) {
1085                 error("before_new %u is after max %u", before_new, last_entry_t);
1086                 before_new = last_entry_t;
1087                 points_new = (before_new - after_new) / st->update_every / group;
1088         }
1089         if(start_at_slot < 0 || start_at_slot >= st->entries) {
1090                 error("start_at_slot is invalid %ld, expected %ld to %ld", start_at_slot, 0, st->entries - 1);
1091                 start_at_slot = st->current_entry;
1092                 points_new = (before_new - after_new) / st->update_every / group;
1093         }
1094         if(stop_at_slot < 0 || stop_at_slot >= st->entries) {
1095                 error("stop_at_slot is invalid %ld, expected %ld to %ld", stop_at_slot, 0, st->entries - 1);
1096                 stop_at_slot = 0;
1097                 points_new = (before_new - after_new) / st->update_every / group;
1098         }
1099         if(points_new > (before_new - after_new) / group / st->update_every + 1) {
1100                 error("points_new %ld is more than points %ld", points_new, (before_new - after_new) / group / st->update_every + 1);
1101         }
1102
1103         //error("SHIFT: %s: wanted %ld points, got %ld - group=%ld, wanted duration=%u, got %u - wanted %ld - %ld, got %ld - %ld", st->id, points, points_new, group, before - after, before_new - after_new, after, before, after_new, before_new);
1104
1105         after = after_new;
1106         before = before_new;
1107         duration = before - after;
1108         points = points_new;
1109
1110         // Now we have:
1111         // before = the end time of the calculation
1112         // after = the start time of the calculation
1113         // duration = the duration of the calculation
1114         // group = the number of source points to aggregate / group together
1115         // method = the method of grouping source points
1116         // points = the number of points to generate
1117
1118
1119         // -------------------------------------------------------------------------
1120         // initialize our result set
1121
1122         RRDR *r = rrdr_create(st, points);
1123         if(!r) return NULL;
1124         if(!r->d) {
1125                 rrdr_free(r);
1126                 return NULL;
1127         }
1128
1129         // find how many dimensions we have
1130         long dimensions = r->d;
1131
1132
1133         // -------------------------------------------------------------------------
1134         // checks for debugging
1135
1136         if(debug) debug(D_RRD_STATS, "INFO %s first_t: %lu, last_t: %lu, all_duration: %lu, after: %lu, before: %lu, duration: %lu, points: %ld, group: %ld"
1137                         , st->id
1138                         , first_entry_t
1139                         , last_entry_t
1140                         , last_entry_t - first_entry_t
1141                         , after
1142                         , before
1143                         , duration
1144                         , points
1145                         , group
1146                         );
1147
1148
1149         // -------------------------------------------------------------------------
1150         // temp arrays for keeping values per dimension
1151
1152         calculated_number       group_values[dimensions]; // keep sums when grouping
1153         long                            group_counts[dimensions]; // keep the number of values added to group_values
1154         uint8_t                         group_options[dimensions];
1155         uint8_t                         found_non_zero[dimensions];
1156
1157
1158         // initialize them
1159         RRDDIM *rd;
1160         long c;
1161         for( rd = st->dimensions, c = 0 ; rd && c < dimensions ; rd = rd->next, c++) {
1162                 group_values[c] = 0;
1163                 group_counts[c] = 0;
1164                 group_options[c] = 0;
1165                 found_non_zero[c] = 0;
1166         }
1167
1168
1169         // -------------------------------------------------------------------------
1170         // the main loop
1171
1172         time_t  now = rrdset_slot2time(st, start_at_slot),
1173                         dt = st->update_every,
1174                         group_start_t = 0;
1175
1176         if(unlikely(debug)) debug(D_RRD_STATS, "BEGIN %s after_t: %lu (stop_at_t: %ld), before_t: %lu (start_at_t: %ld), start_t(now): %lu, current_entry: %ld, entries: %ld"
1177                         , st->id
1178                         , after
1179                         , stop_at_slot
1180                         , before
1181                         , start_at_slot
1182                         , now
1183                         , st->current_entry
1184                         , st->entries
1185                         );
1186
1187         r->group = group;
1188         r->update_every = group * st->update_every;
1189
1190         long slot = start_at_slot, counter = 0, stop_now = 0, added = 0, group_count = 0, add_this = 0;
1191         for(; !stop_now ; now -= dt, slot--, counter++) {
1192                 if(unlikely(slot < 0)) slot = st->entries - 1;
1193                 if(unlikely(slot == stop_at_slot)) stop_now = counter;
1194
1195                 if(unlikely(debug)) debug(D_RRD_STATS, "ROW %s slot: %ld, entries_counter: %ld, group_count: %ld, added: %ld, now: %lu, %s %s"
1196                                 , st->id
1197                                 , slot
1198                                 , counter
1199                                 , group_count + 1
1200                                 , added
1201                                 , now
1202                                 , (group_count + 1 == group)?"PRINT":"  -  "
1203                                 , (now >= after && now <= before)?"RANGE":"  -  "
1204                                 );
1205
1206                 // make sure we return data in the proper time range
1207                 if(unlikely(now > before)) continue;
1208                 if(unlikely(now < after)) break;
1209
1210                 if(unlikely(group_count == 0)) {
1211                         group_start_t = now;
1212                         if(unlikely(!r->before)) {
1213                                 r->before = group_start_t;
1214                                 r->after = group_start_t;
1215                         }
1216                 }
1217                 group_count++;
1218
1219                 if(unlikely(group_count == group)) {
1220                         if(unlikely(added >= points)) break;
1221                         add_this = 1;
1222                 }
1223
1224                 // do the calculations
1225                 for(rd = st->dimensions, c = 0 ; likely(rd && c < dimensions) ; rd = rd->next, c++) {
1226                         storage_number n = rd->values[slot];
1227                         if(unlikely(!does_storage_number_exist(n))) continue;
1228
1229                         group_counts[c]++;
1230
1231                         calculated_number value = unpack_storage_number(n);
1232                         if(likely(value != 0.0)) {
1233                                 group_options[c] |= RRDR_NONZERO;
1234                                 found_non_zero[c] = 1;
1235                         }
1236
1237                         if(unlikely(did_storage_number_reset(n)))
1238                                 group_options[c] |= RRDR_RESET;
1239
1240                         switch(group_method) {
1241                                 case GROUP_MAX:
1242                                         if(unlikely(abs(value) > abs(group_values[c])))
1243                                                 group_values[c] = value;
1244                                         break;
1245
1246                                 default:
1247                                 case GROUP_SUM:
1248                                 case GROUP_AVERAGE:
1249                                         group_values[c] += value;
1250                                         break;
1251                         }
1252                 }
1253
1254                 // added it
1255                 if(unlikely(add_this)) {
1256                         if(unlikely(!rrdr_line_init(r, group_start_t))) break;
1257
1258                         r->after = group_start_t;
1259
1260                         calculated_number *cn = rrdr_line_values(r);
1261                         uint8_t *co = rrdr_line_options(r);
1262
1263                         for(rd = st->dimensions, c = 0 ; likely(rd && c < dimensions) ; rd = rd->next, c++) {
1264
1265                                 // update the dimension options
1266                                 if(likely(found_non_zero[c])) r->od[c] |= RRDR_NONZERO;
1267                                 if(unlikely(rd->flags & RRDDIM_FLAG_HIDDEN)) r->od[c] |= RRDR_HIDDEN;
1268
1269                                 // store the specific point options
1270                                 co[c] = group_options[c];
1271
1272                                 // store the value
1273                                 if(unlikely(group_counts[c] == 0)) {
1274                                         cn[c] = 0.0;
1275                                         co[c] |= RRDR_EMPTY;
1276                                 }
1277                                 else if(unlikely(group_method == GROUP_AVERAGE)) {
1278                                         cn[c] = group_values[c] / group_counts[c];
1279                                 }
1280                                 else {
1281                                         cn[c] = group_values[c];
1282                                 }
1283
1284                                 if(cn[c] < r->min) r->min = cn[c];
1285                                 if(cn[c] > r->max) r->max = cn[c];
1286
1287                                 // reset them for the next loop
1288                                 group_values[c] = 0;
1289                                 group_counts[c] = 0;
1290                                 group_options[c] = 0;
1291                         }
1292
1293                         added++;
1294                         group_count = 0;
1295                         add_this = 0;
1296                 }
1297         }
1298         r->after += group * st->update_every;
1299
1300         rrdr_done(r);
1301         //error("SHIFT: %s: wanted %ld points, got %ld", st->id, points, rrdr_rows(r));
1302         return r;
1303 }
1304
1305 int rrd2format(RRDSET *st, BUFFER *wb, BUFFER *dimensions, uint32_t format, long points, long long after, long long before, int group_method, uint32_t options, time_t *latest_timestamp)
1306 {
1307         RRDR *r = rrd2rrdr(st, points, after, before, group_method);
1308         if(!r) {
1309                 buffer_strcat(wb, "Cannot generate output with these parameters on this chart.");
1310                 return 500;
1311         }
1312
1313         if(dimensions)
1314                 rrdr_disable_not_selected_dimensions(r, buffer_tostring(dimensions));
1315
1316         if(latest_timestamp && rrdr_rows(r) > 0)
1317                 *latest_timestamp = r->before;
1318
1319         switch(format) {
1320         case DATASOURCE_SSV:
1321                 if(options & RRDR_OPTION_JSON_WRAP) {
1322                         wb->contenttype = CT_APPLICATION_JSON;
1323                         rrdr_json_wrapper_begin(r, wb, format, options, 1);
1324                         rrdr2ssv(r, wb, options, "", " ", "");
1325                         rrdr_json_wrapper_end(r, wb, format, options, 1);
1326                 }
1327                 else {
1328                         wb->contenttype = CT_TEXT_PLAIN;
1329                         rrdr2ssv(r, wb, options, "", " ", "");
1330                 }
1331                 break;
1332
1333         case DATASOURCE_SSV_COMMA:
1334                 if(options & RRDR_OPTION_JSON_WRAP) {
1335                         wb->contenttype = CT_APPLICATION_JSON;
1336                         rrdr_json_wrapper_begin(r, wb, format, options, 1);
1337                         rrdr2ssv(r, wb, options, "", ",", "");
1338                         rrdr_json_wrapper_end(r, wb, format, options, 1);
1339                 }
1340                 else {
1341                         wb->contenttype = CT_TEXT_PLAIN;
1342                         rrdr2ssv(r, wb, options, "", ",", "");
1343                 }
1344                 break;
1345
1346         case DATASOURCE_JS_ARRAY:
1347                 if(options & RRDR_OPTION_JSON_WRAP) {
1348                         wb->contenttype = CT_APPLICATION_JSON;
1349                         rrdr_json_wrapper_begin(r, wb, format, options, 0);
1350                         rrdr2ssv(r, wb, options, "[", ",", "]");
1351                         rrdr_json_wrapper_end(r, wb, format, options, 0);
1352                 }
1353                 else {
1354                         wb->contenttype = CT_APPLICATION_JSON;
1355                         rrdr2ssv(r, wb, options, "[", ",", "]");
1356                 }
1357                 break;
1358
1359         case DATASOURCE_CSV:
1360                 if(options & RRDR_OPTION_JSON_WRAP) {
1361                         wb->contenttype = CT_APPLICATION_JSON;
1362                         rrdr_json_wrapper_begin(r, wb, format, options, 1);
1363                         rrdr2csv(r, wb, options, "", ",", "\\n");
1364                         rrdr_json_wrapper_end(r, wb, format, options, 1);
1365                 }
1366                 else {
1367                         wb->contenttype = CT_TEXT_PLAIN;
1368                         rrdr2csv(r, wb, options, "", ",", "\r\n");
1369                 }
1370                 break;
1371
1372         case DATASOURCE_TSV:
1373                 if(options & RRDR_OPTION_JSON_WRAP) {
1374                         wb->contenttype = CT_APPLICATION_JSON;
1375                         rrdr_json_wrapper_begin(r, wb, format, options, 1);
1376                         rrdr2csv(r, wb, options, "", "\t", "\\n");
1377                         rrdr_json_wrapper_end(r, wb, format, options, 1);
1378                 }
1379                 else {
1380                         wb->contenttype = CT_TEXT_PLAIN;
1381                         rrdr2csv(r, wb, options, "", "\t", "\r\n");
1382                 }
1383                 break;
1384
1385         case DATASOURCE_HTML:
1386                 if(options & RRDR_OPTION_JSON_WRAP) {
1387                         wb->contenttype = CT_APPLICATION_JSON;
1388                         rrdr_json_wrapper_begin(r, wb, format, options, 1);
1389                         buffer_strcat(wb, "<html>\\n<center>\\n<table border=\\\"0\\\" cellpadding=\\\"5\\\" cellspacing=\\\"5\\\">\\n");
1390                         rrdr2csv(r, wb, options, "<tr><td>", "</td><td>", "</td></tr>\\n");
1391                         buffer_strcat(wb, "</table>\\n</center>\\n</html>\\n");
1392                         rrdr_json_wrapper_end(r, wb, format, options, 1);
1393                 }
1394                 else {
1395                         wb->contenttype = CT_TEXT_HTML;
1396                         buffer_strcat(wb, "<html>\n<center>\n<table border=\"0\" cellpadding=\"5\" cellspacing=\"5\">\n");
1397                         rrdr2csv(r, wb, options, "<tr><td>", "</td><td>", "</td></tr>\n");
1398                         buffer_strcat(wb, "</table>\n</center>\n</html>\n");
1399                 }
1400                 break;
1401
1402         case DATASOURCE_DATATABLE_JSONP:
1403                 wb->contenttype = CT_APPLICATION_X_JAVASCRIPT;
1404
1405                 if(options & RRDR_OPTION_JSON_WRAP)
1406                         rrdr_json_wrapper_begin(r, wb, format, options, 0);
1407
1408                 rrdr2json(r, wb, options, 1);
1409
1410                 if(options & RRDR_OPTION_JSON_WRAP)
1411                         rrdr_json_wrapper_end(r, wb, format, options, 0);
1412                 break;
1413
1414         case DATASOURCE_DATATABLE_JSON:
1415                 wb->contenttype = CT_APPLICATION_JSON;
1416
1417                 if(options & RRDR_OPTION_JSON_WRAP)
1418                         rrdr_json_wrapper_begin(r, wb, format, options, 0);
1419
1420                 rrdr2json(r, wb, options, 1);
1421
1422                 if(options & RRDR_OPTION_JSON_WRAP)
1423                         rrdr_json_wrapper_end(r, wb, format, options, 0);
1424                 break;
1425
1426         case DATASOURCE_JSONP:
1427                 wb->contenttype = CT_APPLICATION_X_JAVASCRIPT;
1428                 if(options & RRDR_OPTION_JSON_WRAP)
1429                         rrdr_json_wrapper_begin(r, wb, format, options, 0);
1430
1431                 rrdr2json(r, wb, options, 0);
1432
1433                 if(options & RRDR_OPTION_JSON_WRAP)
1434                         rrdr_json_wrapper_end(r, wb, format, options, 0);
1435                 break;
1436
1437         case DATASOURCE_JSON:
1438         default:
1439                 wb->contenttype = CT_APPLICATION_JSON;
1440
1441                 if(options & RRDR_OPTION_JSON_WRAP)
1442                         rrdr_json_wrapper_begin(r, wb, format, options, 0);
1443
1444                 rrdr2json(r, wb, options, 0);
1445
1446                 if(options & RRDR_OPTION_JSON_WRAP)
1447                         rrdr_json_wrapper_end(r, wb, format, options, 0);
1448                 break;
1449         }
1450
1451         rrdr_free(r);
1452         return 200;
1453 }
1454
1455 unsigned long rrd_stats_json(int type, RRDSET *st, BUFFER *wb, int points, int group, int group_method, time_t after, time_t before, int only_non_zero)
1456 {
1457         int c;
1458         pthread_rwlock_rdlock(&st->rwlock);
1459
1460
1461         // -------------------------------------------------------------------------
1462         // switch from JSON to google JSON
1463
1464         char kq[2] = "\"";
1465         char sq[2] = "\"";
1466         switch(type) {
1467                 case DATASOURCE_DATATABLE_JSON:
1468                 case DATASOURCE_DATATABLE_JSONP:
1469                         kq[0] = '\0';
1470                         sq[0] = '\'';
1471                         break;
1472
1473                 case DATASOURCE_JSON:
1474                 default:
1475                         break;
1476         }
1477
1478
1479         // -------------------------------------------------------------------------
1480         // validate the parameters
1481
1482         if(points < 1) points = 1;
1483         if(group < 1) group = 1;
1484
1485         if(before == 0 || before > rrdset_last_entry_t(st)) before = rrdset_last_entry_t(st);
1486         if(after  == 0 || after < rrdset_first_entry_t(st)) after = rrdset_first_entry_t(st);
1487
1488         // ---
1489
1490         // our return value (the last timestamp printed)
1491         // this is required to detect re-transmit in google JSONP
1492         time_t last_timestamp = 0;
1493
1494
1495         // -------------------------------------------------------------------------
1496         // find how many dimensions we have
1497
1498         int dimensions = 0;
1499         RRDDIM *rd;
1500         for( rd = st->dimensions ; rd ; rd = rd->next) dimensions++;
1501         if(!dimensions) {
1502                 pthread_rwlock_unlock(&st->rwlock);
1503                 buffer_strcat(wb, "No dimensions yet.");
1504                 return 0;
1505         }
1506
1507
1508         // -------------------------------------------------------------------------
1509         // prepare various strings, to speed up the loop
1510
1511         char overflow_annotation[201]; snprintf(overflow_annotation, 200, ",{%sv%s:%sRESET OR OVERFLOW%s},{%sv%s:%sThe counters have been wrapped.%s}", kq, kq, sq, sq, kq, kq, sq, sq);
1512         char normal_annotation[201];   snprintf(normal_annotation,   200, ",{%sv%s:null},{%sv%s:null}", kq, kq, kq, kq);
1513         char pre_date[51];             snprintf(pre_date,             50, "             {%sc%s:[{%sv%s:%s", kq, kq, kq, kq, sq);
1514         char post_date[21];            snprintf(post_date,            20, "%s}", sq);
1515         char pre_value[21];            snprintf(pre_value,            20, ",{%sv%s:", kq, kq);
1516         char post_value[21];           snprintf(post_value,           20, "}");
1517
1518
1519         // -------------------------------------------------------------------------
1520         // checks for debugging
1521
1522         if(st->debug) {
1523                 debug(D_RRD_STATS, "%s first_entry_t = %lu, last_entry_t = %lu, duration = %lu, after = %lu, before = %lu, duration = %lu, entries_to_show = %lu, group = %lu"
1524                         , st->id
1525                         , rrdset_first_entry_t(st)
1526                         , rrdset_last_entry_t(st)
1527                         , rrdset_last_entry_t(st) - rrdset_first_entry_t(st)
1528                         , after
1529                         , before
1530                         , before - after
1531                         , points
1532                         , group
1533                         );
1534
1535                 if(before < after)
1536                         debug(D_RRD_STATS, "WARNING: %s The newest value in the database (%lu) is earlier than the oldest (%lu)", st->name, before, after);
1537
1538                 if((before - after) > st->entries * st->update_every)
1539                         debug(D_RRD_STATS, "WARNING: %s The time difference between the oldest and the newest entries (%lu) is higher than the capacity of the database (%lu)", st->name, before - after, st->entries * st->update_every);
1540         }
1541
1542
1543         // -------------------------------------------------------------------------
1544         // temp arrays for keeping values per dimension
1545
1546         calculated_number group_values[dimensions]; // keep sums when grouping
1547         int               print_hidden[dimensions]; // keep hidden flags
1548         int               found_non_zero[dimensions];
1549         int               found_non_existing[dimensions];
1550
1551         // initialize them
1552         for( rd = st->dimensions, c = 0 ; rd && c < dimensions ; rd = rd->next, c++) {
1553                 group_values[c] = 0;
1554                 print_hidden[c] = (rd->flags & RRDDIM_FLAG_HIDDEN)?1:0;
1555                 found_non_zero[c] = 0;
1556                 found_non_existing[c] = 0;
1557         }
1558
1559
1560         // error("OLD: points=%d after=%d before=%d group=%d, duration=%d", entries_to_show, before - (st->update_every * group * entries_to_show), before, group, before - after + 1);
1561         // rrd2array(st, entries_to_show, before - (st->update_every * group * entries_to_show), before, group_method, only_non_zero);
1562         // rrd2rrdr(st, entries_to_show, before - (st->update_every * group * entries_to_show), before, group_method);
1563
1564         // -------------------------------------------------------------------------
1565         // remove dimensions that contain only zeros
1566
1567         int max_loop = 1;
1568         if(only_non_zero) max_loop = 2;
1569
1570         for(; max_loop ; max_loop--) {
1571
1572                 // -------------------------------------------------------------------------
1573                 // print the JSON header
1574
1575                 buffer_sprintf(wb, "{\n %scols%s:\n     [\n", kq, kq);
1576                 buffer_sprintf(wb, "            {%sid%s:%s%s,%slabel%s:%stime%s,%spattern%s:%s%s,%stype%s:%sdatetime%s},\n", kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq);
1577                 buffer_sprintf(wb, "            {%sid%s:%s%s,%slabel%s:%s%s,%spattern%s:%s%s,%stype%s:%sstring%s,%sp%s:{%srole%s:%sannotation%s}},\n", kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, kq, kq, sq, sq);
1578                 buffer_sprintf(wb, "            {%sid%s:%s%s,%slabel%s:%s%s,%spattern%s:%s%s,%stype%s:%sstring%s,%sp%s:{%srole%s:%sannotationText%s}}", kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, kq, kq, sq, sq);
1579
1580                 // print the header for each dimension
1581                 // and update the print_hidden array for the dimensions that should be hidden
1582                 int pc = 0;
1583                 for( rd = st->dimensions, c = 0 ; rd && c < dimensions ; rd = rd->next, c++) {
1584                         if(!print_hidden[c]) {
1585                                 pc++;
1586                                 buffer_sprintf(wb, ",\n         {%sid%s:%s%s,%slabel%s:%s%s%s,%spattern%s:%s%s,%stype%s:%snumber%s}", kq, kq, sq, sq, kq, kq, sq, rd->name, sq, kq, kq, sq, sq, kq, kq, sq, sq);
1587                         }
1588                 }
1589                 if(!pc) {
1590                         buffer_sprintf(wb, ",\n         {%sid%s:%s%s,%slabel%s:%s%s%s,%spattern%s:%s%s,%stype%s:%snumber%s}", kq, kq, sq, sq, kq, kq, sq, "no data", sq, kq, kq, sq, sq, kq, kq, sq, sq);
1591                 }
1592
1593                 // print the begin of row data
1594                 buffer_sprintf(wb, "\n  ],\n    %srows%s:\n     [\n", kq, kq);
1595
1596
1597                 // -------------------------------------------------------------------------
1598                 // the main loop
1599
1600                 int annotate_reset = 0;
1601                 int annotation_count = 0;
1602
1603                 long    t = rrdset_time2slot(st, before),
1604                                 stop_at_t = rrdset_time2slot(st, after),
1605                                 stop_now = 0;
1606
1607                 t -= t % group;
1608
1609                 time_t  now = rrdset_slot2time(st, t),
1610                                 dt = st->update_every;
1611
1612                 long count = 0, printed = 0, group_count = 0;
1613                 last_timestamp = 0;
1614
1615                 if(st->debug) debug(D_RRD_STATS, "%s: REQUEST after:%lu before:%lu, points:%d, group:%d, CHART cur:%ld first: %lu last:%lu, CALC start_t:%ld, stop_t:%ld"
1616                                         , st->id
1617                                         , after
1618                                         , before
1619                                         , points
1620                                         , group
1621                                         , st->current_entry
1622                                         , rrdset_first_entry_t(st)
1623                                         , rrdset_last_entry_t(st)
1624                                         , t
1625                                         , stop_at_t
1626                                         );
1627
1628                 for(; !stop_now ; now -= dt, t--) {
1629                         if(t < 0) t = st->entries - 1;
1630                         if(t == stop_at_t) stop_now = 1;
1631
1632                         int print_this = 0;
1633
1634                         if(st->debug) debug(D_RRD_STATS, "%s t = %ld, count = %ld, group_count = %ld, printed = %ld, now = %lu, %s %s"
1635                                         , st->id
1636                                         , t
1637                                         , count + 1
1638                                         , group_count + 1
1639                                         , printed
1640                                         , now
1641                                         , (group_count + 1 == group)?"PRINT":"  -  "
1642                                         , (now >= after && now <= before)?"RANGE":"  -  "
1643                                         );
1644
1645
1646                         // make sure we return data in the proper time range
1647                         if(now > before) continue;
1648                         if(now < after) break;
1649
1650                         //if(rrdset_slot2time(st, t) != now)
1651                         //      error("%s: slot=%ld, now=%ld, slot2time=%ld, diff=%ld, last_entry_t=%ld, rrdset_last_slot=%ld", st->id, t, now, rrdset_slot2time(st,t), now - rrdset_slot2time(st,t), rrdset_last_entry_t(st), rrdset_last_slot(st));
1652
1653                         count++;
1654                         group_count++;
1655
1656                         // check if we have to print this now
1657                         if(group_count == group) {
1658                                 if(printed >= points) {
1659                                         // debug(D_RRD_STATS, "Already printed all rows. Stopping.");
1660                                         break;
1661                                 }
1662
1663                                 // generate the local date time
1664                                 struct tm *tm = localtime(&now);
1665                                 if(!tm) { error("localtime() failed."); continue; }
1666                                 if(now > last_timestamp) last_timestamp = now;
1667
1668                                 if(printed) buffer_strcat(wb, "]},\n");
1669                                 buffer_strcat(wb, pre_date);
1670                                 buffer_jsdate(wb, tm->tm_year + 1900, tm->tm_mon, tm->tm_mday, tm->tm_hour, tm->tm_min, tm->tm_sec);
1671                                 buffer_strcat(wb, post_date);
1672
1673                                 print_this = 1;
1674                         }
1675
1676                         // do the calculations
1677                         for(rd = st->dimensions, c = 0 ; rd && c < dimensions ; rd = rd->next, c++) {
1678                                 storage_number n = rd->values[t];
1679                                 calculated_number value = unpack_storage_number(n);
1680
1681                                 if(!does_storage_number_exist(n)) {
1682                                         value = 0.0;
1683                                         found_non_existing[c]++;
1684                                 }
1685                                 if(did_storage_number_reset(n)) annotate_reset = 1;
1686
1687                                 switch(group_method) {
1688                                         case GROUP_MAX:
1689                                                 if(abs(value) > abs(group_values[c])) group_values[c] = value;
1690                                                 break;
1691
1692                                         case GROUP_SUM:
1693                                                 group_values[c] += value;
1694                                                 break;
1695
1696                                         default:
1697                                         case GROUP_AVERAGE:
1698                                                 group_values[c] += value;
1699                                                 if(print_this) group_values[c] /= ( group_count - found_non_existing[c] );
1700                                                 break;
1701                                 }
1702                         }
1703
1704                         if(print_this) {
1705                                 if(annotate_reset) {
1706                                         annotation_count++;
1707                                         buffer_strcat(wb, overflow_annotation);
1708                                         annotate_reset = 0;
1709                                 }
1710                                 else
1711                                         buffer_strcat(wb, normal_annotation);
1712
1713                                 pc = 0;
1714                                 for(c = 0 ; c < dimensions ; c++) {
1715                                         if(found_non_existing[c] == group_count) {
1716                                                 // all entries are non-existing
1717                                                 pc++;
1718                                                 buffer_strcat(wb, pre_value);
1719                                                 buffer_strcat(wb, "null");
1720                                                 buffer_strcat(wb, post_value);
1721                                         }
1722                                         else if(!print_hidden[c]) {
1723                                                 pc++;
1724                                                 buffer_strcat(wb, pre_value);
1725                                                 buffer_rrd_value(wb, group_values[c]);
1726                                                 buffer_strcat(wb, post_value);
1727
1728                                                 if(group_values[c]) found_non_zero[c]++;
1729                                         }
1730
1731                                         // reset them for the next loop
1732                                         group_values[c] = 0;
1733                                         found_non_existing[c] = 0;
1734                                 }
1735
1736                                 // if all dimensions are hidden, print a null
1737                                 if(!pc) {
1738                                         buffer_strcat(wb, pre_value);
1739                                         buffer_strcat(wb, "null");
1740                                         buffer_strcat(wb, post_value);
1741                                 }
1742
1743                                 printed++;
1744                                 group_count = 0;
1745                         }
1746                 }
1747
1748                 if(printed) buffer_strcat(wb, "]}");
1749                 buffer_strcat(wb, "\n   ]\n}\n");
1750
1751                 if(only_non_zero && max_loop > 1) {
1752                         int changed = 0;
1753                         for(rd = st->dimensions, c = 0 ; rd && c < dimensions ; rd = rd->next, c++) {
1754                                 group_values[c] = 0;
1755                                 found_non_existing[c] = 0;
1756
1757                                 if(!print_hidden[c] && !found_non_zero[c]) {
1758                                         changed = 1;
1759                                         print_hidden[c] = 1;
1760                                 }
1761                         }
1762
1763                         if(changed) buffer_flush(wb);
1764                         else break;
1765                 }
1766                 else break;
1767
1768         } // max_loop
1769
1770         debug(D_RRD_STATS, "RRD_STATS_JSON: %s total %ld bytes", st->name, wb->len);
1771
1772         pthread_rwlock_unlock(&st->rwlock);
1773         return last_timestamp;
1774 }