src/rrd2json.c

   1 #include <pthread.h>
   2 #include <sys/time.h>
   3 #include <stdlib.h>
   4
   5 #include "log.h"
   6 #include "common.h"
   7 #include "rrd2json.h"
   8
   9 #define HOSTNAME_MAX 1024
  10 char *hostname = "unknown";
  11
  12 unsigned long rrd_stats_one_json(RRDSET *st, char *options, struct web_buffer *wb)
  13 {
  14         time_t now = time(NULL);
  15         web_buffer_increase(wb, 65536);
  16
  17         pthread_rwlock_rdlock(&st->rwlock);
  18
  19         web_buffer_printf(wb,
  20                 "\t\t{\n"
  21                 "\t\t\t\"id\": \"%s\",\n"
  22                 "\t\t\t\"name\": \"%s\",\n"
  23                 "\t\t\t\"type\": \"%s\",\n"
  24                 "\t\t\t\"family\": \"%s\",\n"
  25                 "\t\t\t\"title\": \"%s\",\n"
  26                 "\t\t\t\"priority\": %ld,\n"
  27                 "\t\t\t\"enabled\": %d,\n"
  28                 "\t\t\t\"units\": \"%s\",\n"
  29                 "\t\t\t\"url\": \"/data/%s/%s\",\n"
  30                 "\t\t\t\"chart_type\": \"%s\",\n"
  31                 "\t\t\t\"counter\": %ld,\n"
  32                 "\t\t\t\"entries\": %ld,\n"
  33                 "\t\t\t\"first_entry_t\": %lu,\n"
  34                 "\t\t\t\"last_entry\": %ld,\n"
  35                 "\t\t\t\"last_entry_t\": %lu,\n"
  36                 "\t\t\t\"last_entry_secs_ago\": %lu,\n"
  37                 "\t\t\t\"update_every\": %d,\n"
  38                 "\t\t\t\"isdetail\": %d,\n"
  39                 "\t\t\t\"usec_since_last_update\": %llu,\n"
  40                 "\t\t\t\"collected_total\": " TOTAL_NUMBER_FORMAT ",\n"
  41                 "\t\t\t\"last_collected_total\": " TOTAL_NUMBER_FORMAT ",\n"
  42                 "\t\t\t\"dimensions\": [\n"
  43                 , st->id
  44                 , st->name
  45                 , st->type
  46                 , st->family
  47                 , st->title
  48                 , st->priority
  49                 , st->enabled
  50                 , st->units
  51                 , st->name, options?options:""
  52                 , rrdset_type_name(st->chart_type)
  53                 , st->counter
  54                 , st->entries
  55                 , rrdset_first_entry_t(st)
  56                 , rrdset_last_slot(st)
  57                 , rrdset_last_entry_t(st)
  58                 , (now < rrdset_last_entry_t(st)) ? (time_t)0 : now - rrdset_last_entry_t(st)
  59                 , st->update_every
  60                 , st->isdetail
  61                 , st->usec_since_last_update
  62                 , st->collected_total
  63                 , st->last_collected_total
  64                 );
  65
  66         unsigned long memory = st->memsize;
  67
  68         RRDDIM *rd;
  69         for(rd = st->dimensions; rd ; rd = rd->next) {
  70                 web_buffer_increase(wb, 4096);
  71
  72                 memory += rd->memsize;
  73
  74                 web_buffer_printf(wb,
  75                         "\t\t\t\t{\n"
  76                         "\t\t\t\t\t\"id\": \"%s\",\n"
  77                         "\t\t\t\t\t\"name\": \"%s\",\n"
  78                         "\t\t\t\t\t\"entries\": %ld,\n"
  79                         "\t\t\t\t\t\"isHidden\": %d,\n"
  80                         "\t\t\t\t\t\"algorithm\": \"%s\",\n"
  81                         "\t\t\t\t\t\"multiplier\": %ld,\n"
  82                         "\t\t\t\t\t\"divisor\": %ld,\n"
  83                         "\t\t\t\t\t\"last_entry_t\": %lu,\n"
  84                         "\t\t\t\t\t\"collected_value\": " COLLECTED_NUMBER_FORMAT ",\n"
  85                         "\t\t\t\t\t\"calculated_value\": " CALCULATED_NUMBER_FORMAT ",\n"
  86                         "\t\t\t\t\t\"last_collected_value\": " COLLECTED_NUMBER_FORMAT ",\n"
  87                         "\t\t\t\t\t\"last_calculated_value\": " CALCULATED_NUMBER_FORMAT ",\n"
  88                         "\t\t\t\t\t\"memory\": %lu\n"
  89                         "\t\t\t\t}%s\n"
  90                         , rd->id
  91                         , rd->name
  92                         , rd->entries
  93                         , rd->hidden
  94                         , rrddim_algorithm_name(rd->algorithm)
  95                         , rd->multiplier
  96                         , rd->divisor
  97                         , rd->last_collected_time.tv_sec
  98                         , rd->collected_value
  99                         , rd->calculated_value
 100                         , rd->last_collected_value
 101                         , rd->last_calculated_value
 102                         , rd->memsize
 103                         , rd->next?",":""
 104                         );
 105         }
 106
 107         web_buffer_printf(wb,
 108                 "\t\t\t],\n"
 109                 "\t\t\t\"memory\" : %lu\n"
 110                 "\t\t}"
 111                 , memory
 112                 );
 113
 114         pthread_rwlock_unlock(&st->rwlock);
 115         return memory;
 116 }
 117
 118 #define RRD_GRAPH_JSON_HEADER "{\n\t\"charts\": [\n"
 119 #define RRD_GRAPH_JSON_FOOTER "\n\t]\n}\n"
 120
 121 void rrd_stats_graph_json(RRDSET *st, char *options, struct web_buffer *wb)
 122 {
 123         web_buffer_increase(wb, 2048);
 124
 125         web_buffer_printf(wb, RRD_GRAPH_JSON_HEADER);
 126         rrd_stats_one_json(st, options, wb);
 127         web_buffer_printf(wb, RRD_GRAPH_JSON_FOOTER);
 128 }
 129
 130 void rrd_stats_all_json(struct web_buffer *wb)
 131 {
 132         web_buffer_increase(wb, 2048);
 133
 134         unsigned long memory = 0;
 135         long c;
 136         RRDSET *st;
 137
 138         web_buffer_printf(wb, RRD_GRAPH_JSON_HEADER);
 139
 140         pthread_rwlock_rdlock(&rrdset_root_rwlock);
 141         for(st = rrdset_root, c = 0; st ; st = st->next) {
 142                 if(st->enabled) {
 143                         if(c) web_buffer_printf(wb, "%s", ",\n");
 144                         memory += rrd_stats_one_json(st, NULL, wb);
 145                         c++;
 146                 }
 147         }
 148         pthread_rwlock_unlock(&rrdset_root_rwlock);
 149
 150         web_buffer_printf(wb, "\n\t],\n"
 151                 "\t\"hostname\": \"%s\",\n"
 152                 "\t\"update_every\": %d,\n"
 153                 "\t\"history\": %d,\n"
 154                 "\t\"memory\": %lu\n"
 155                 "}\n"
 156                 , hostname
 157                 , rrd_update_every
 158                 , rrd_default_history_entries
 159                 , memory
 160                 );
 161 }
 162
 163 unsigned long rrd_stats_json(int type, RRDSET *st, struct web_buffer *wb, int entries_to_show, int group, int group_method, time_t after, time_t before, int only_non_zero)
 164 {
 165         int c;
 166         pthread_rwlock_rdlock(&st->rwlock);
 167
 168
 169         // -------------------------------------------------------------------------
 170         // switch from JSON to google JSON
 171
 172         char kq[2] = "\"";
 173         char sq[2] = "\"";
 174         switch(type) {
 175                 case DATASOURCE_GOOGLE_JSON:
 176                 case DATASOURCE_GOOGLE_JSONP:
 177                         kq[0] = '\0';
 178                         sq[0] = '\'';
 179                         break;
 180
 181                 case DATASOURCE_JSON:
 182                 default:
 183                         break;
 184         }
 185
 186
 187         // -------------------------------------------------------------------------
 188         // validate the parameters
 189
 190         if(entries_to_show < 1) entries_to_show = 1;
 191         if(group < 1) group = 1;
 192
 193         // find the oldest entry of the round-robin
 194         long max_entries_init = (st->counter < (unsigned long)st->entries) ? st->counter : (unsigned long)st->entries;
 195
 196         time_t time_init = rrdset_last_entry_t(st);
 197
 198         if(before == 0 || before > time_init) before = time_init;
 199         if(after  == 0) after = rrdset_first_entry_t(st);
 200
 201
 202         // ---
 203
 204         // our return value (the last timestamp printed)
 205         // this is required to detect re-transmit in google JSONP
 206         time_t last_timestamp = 0;
 207
 208
 209         // -------------------------------------------------------------------------
 210         // find how many dimensions we have
 211
 212         int dimensions = 0;
 213         RRDDIM *rd;
 214         for( rd = st->dimensions ; rd ; rd = rd->next) dimensions++;
 215         if(!dimensions) {
 216                 pthread_rwlock_unlock(&st->rwlock);
 217                 web_buffer_printf(wb, "No dimensions yet.");
 218                 return 0;
 219         }
 220
 221
 222         // -------------------------------------------------------------------------
 223         // prepare various strings, to speed up the loop
 224
 225         char overflow_annotation[201]; snprintf(overflow_annotation, 200, ",{%sv%s:%sRESET OR OVERFLOW%s},{%sv%s:%sThe counters have been wrapped.%s}", kq, kq, sq, sq, kq, kq, sq, sq);
 226         char normal_annotation[201];   snprintf(normal_annotation,   200, ",{%sv%s:null},{%sv%s:null}", kq, kq, kq, kq);
 227         char pre_date[51];             snprintf(pre_date,             50, "             {%sc%s:[{%sv%s:%s", kq, kq, kq, kq, sq);
 228         char post_date[21];            snprintf(post_date,            20, "%s}", sq);
 229         char pre_value[21];            snprintf(pre_value,            20, ",{%sv%s:", kq, kq);
 230         char post_value[21];           snprintf(post_value,           20, "}");
 231
 232
 233         // -------------------------------------------------------------------------
 234         // checks for debuging
 235
 236         if(st->debug) {
 237                 debug(D_RRD_STATS, "%s first_entry_t = %lu, last_entry_t = %lu, duration = %lu, after = %lu, before = %lu, duration = %lu, entries_to_show = %lu, group = %lu, max_entries = %ld"
 238                         , st->id
 239                         , rrdset_first_entry_t(st)
 240                         , rrdset_last_entry_t(st)
 241                         , rrdset_last_entry_t(st) - rrdset_first_entry_t(st)
 242                         , after
 243                         , before
 244                         , before - after
 245                         , entries_to_show
 246                         , group
 247                         , max_entries_init
 248                         );
 249
 250                 if(before < after)
 251                         debug(D_RRD_STATS, "WARNING: %s The newest value in the database (%lu) is earlier than the oldest (%lu)", st->name, before, after);
 252
 253                 if((before - after) > st->entries * st->update_every)
 254                         debug(D_RRD_STATS, "WARNING: %s The time difference between the oldest and the newest entries (%lu) is higher than the capacity of the database (%lu)", st->name, before - after, st->entries * st->update_every);
 255         }
 256
 257
 258         // -------------------------------------------------------------------------
 259         // temp arrays for keeping values per dimension
 260
 261         calculated_number group_values[dimensions]; // keep sums when grouping
 262         int               print_hidden[dimensions]; // keep hidden flags
 263         int               found_non_zero[dimensions];
 264         int               found_non_existing[dimensions];
 265
 266         // initialize them
 267         for( rd = st->dimensions, c = 0 ; rd && c < dimensions ; rd = rd->next, c++) {
 268                 group_values[c] = 0;
 269                 print_hidden[c] = rd->hidden;
 270                 found_non_zero[c] = 0;
 271                 found_non_existing[c] = 0;
 272         }
 273
 274
 275         // error("OLD: points=%d after=%d before=%d group=%d, duration=%d", entries_to_show, before - (st->update_every * group * entries_to_show), before, group, before - after + 1);
 276         // rrd2array(st, entries_to_show, before - (st->update_every * group * entries_to_show), before, group_method, only_non_zero);
 277
 278         // -------------------------------------------------------------------------
 279         // remove dimensions that contain only zeros
 280
 281         int max_loop = 1;
 282         if(only_non_zero) max_loop = 2;
 283
 284         for(; max_loop ; max_loop--) {
 285
 286                 // -------------------------------------------------------------------------
 287                 // print the JSON header
 288
 289                 web_buffer_printf(wb, "{\n      %scols%s:\n     [\n", kq, kq);
 290                 web_buffer_printf(wb, "         {%sid%s:%s%s,%slabel%s:%stime%s,%spattern%s:%s%s,%stype%s:%sdatetime%s},\n", kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq);
 291                 web_buffer_printf(wb, "         {%sid%s:%s%s,%slabel%s:%s%s,%spattern%s:%s%s,%stype%s:%sstring%s,%sp%s:{%srole%s:%sannotation%s}},\n", kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, kq, kq, sq, sq);
 292                 web_buffer_printf(wb, "         {%sid%s:%s%s,%slabel%s:%s%s,%spattern%s:%s%s,%stype%s:%sstring%s,%sp%s:{%srole%s:%sannotationText%s}}", kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, kq, kq, sq, sq);
 293
 294                 // print the header for each dimension
 295                 // and update the print_hidden array for the dimensions that should be hidden
 296                 int pc = 0;
 297                 for( rd = st->dimensions, c = 0 ; rd && c < dimensions ; rd = rd->next, c++) {
 298                         if(!print_hidden[c]) {
 299                                 pc++;
 300                                 web_buffer_printf(wb, ",\n              {%sid%s:%s%s,%slabel%s:%s%s%s,%spattern%s:%s%s,%stype%s:%snumber%s}", kq, kq, sq, sq, kq, kq, sq, rd->name, sq, kq, kq, sq, sq, kq, kq, sq, sq);
 301                         }
 302                 }
 303                 if(!pc) {
 304                         web_buffer_printf(wb, ",\n              {%sid%s:%s%s,%slabel%s:%s%s%s,%spattern%s:%s%s,%stype%s:%snumber%s}", kq, kq, sq, sq, kq, kq, sq, "no data", sq, kq, kq, sq, sq, kq, kq, sq, sq);
 305                 }
 306
 307                 // print the begin of row data
 308                 web_buffer_printf(wb, "\n       ],\n    %srows%s:\n     [\n", kq, kq);
 309
 310
 311                 // -------------------------------------------------------------------------
 312                 // the main loop
 313
 314                 int annotate_reset = 0;
 315                 int annotation_count = 0;
 316
 317                 // to allow grouping on the same values, we need a pad
 318                 long pad = before % group;
 319
 320                 // the minimum line length we expect
 321                 int line_size = 4096 + (dimensions * 200);
 322
 323                 time_t now = time_init;
 324                 long max_entries = max_entries_init;
 325
 326                 long t;
 327
 328                 long count = 0, printed = 0, group_count = 0;
 329                 last_timestamp = 0;
 330
 331                 long expected_to_start_at_slot = rrdset_time2slot(st, before);
 332
 333                 for(t = rrdset_last_slot(st); max_entries ; now -= st->update_every, t--, max_entries--) {
 334                         if(t < 0) t = st->entries - 1;
 335
 336                         int print_this = 0;
 337
 338                         if(st->debug) {
 339                                 debug(D_RRD_STATS, "%s t = %ld, count = %ld, group_count = %ld, printed = %ld, now = %lu, %s %s"
 340                                         , st->id
 341                                         , t
 342                                         , count + 1
 343                                         , group_count + 1
 344                                         , printed
 345                                         , now
 346                                         , (((count + 1 - pad) % group) == 0)?"PRINT":"  -  "
 347                                         , (now >= after && now <= before)?"RANGE":"  -  "
 348                                         );
 349                         }
 350
 351                         // make sure we return data in the proper time range
 352                         if(now < after || now > before) continue;
 353
 354                         if(expected_to_start_at_slot != -999999) {
 355                                 error("%s: Expected to start on slot %ld, started on %ld, %s", st->id, expected_to_start_at_slot, t, (t == expected_to_start_at_slot)?"OK":"ERROR");
 356                                 expected_to_start_at_slot = -999999;
 357                         }
 358
 359                         count++;
 360                         group_count++;
 361
 362                         // check if we have to print this now
 363                         if(((count - pad) % group) == 0) {
 364                                 if(printed >= entries_to_show) {
 365                                         // debug(D_RRD_STATS, "Already printed all rows. Stopping.");
 366                                         break;
 367                                 }
 368
 369                                 if(group_count != group) {
 370                                         // this is an incomplete group, skip it.
 371                                         for( rd = st->dimensions, c = 0 ; rd && c < dimensions ; rd = rd->next, c++) {
 372                                                 group_values[c] = 0;
 373                                                 found_non_existing[c] = 0;
 374                                         }
 375
 376                                         group_count = 0;
 377                                         continue;
 378                                 }
 379
 380                                 // check if we may exceed the buffer provided
 381                                 web_buffer_increase(wb, line_size);
 382
 383                                 // generate the local date time
 384                                 struct tm *tm = localtime(&now);
 385                                 if(!tm) { error("localtime() failed."); continue; }
 386                                 if(now > last_timestamp) last_timestamp = now;
 387
 388                                 if(printed) web_buffer_strcpy(wb, "]},\n");
 389                                 web_buffer_strcpy(wb, pre_date);
 390                                 web_buffer_jsdate(wb, tm->tm_year + 1900, tm->tm_mon, tm->tm_mday, tm->tm_hour, tm->tm_min, tm->tm_sec);
 391                                 web_buffer_strcpy(wb, post_date);
 392
 393                                 print_this = 1;
 394                         }
 395
 396                         // do the calculations
 397                         for(rd = st->dimensions, c = 0 ; rd && c < dimensions ; rd = rd->next, c++) {
 398                                 storage_number n = rd->values[t];
 399                                 calculated_number value = unpack_storage_number(n);
 400
 401                                 if(!does_storage_number_exist(n)) {
 402                                         value = 0.0;
 403                                         found_non_existing[c]++;
 404                                 }
 405                                 if(did_storage_number_reset(n)) annotate_reset = 1;
 406
 407                                 switch(group_method) {
 408                                         case GROUP_MAX:
 409                                                 if(abs(value) > abs(group_values[c])) group_values[c] = value;
 410                                                 break;
 411
 412                                         case GROUP_SUM:
 413                                                 group_values[c] += value;
 414                                                 break;
 415
 416                                         default:
 417                                         case GROUP_AVERAGE:
 418                                                 group_values[c] += value;
 419                                                 if(print_this) group_values[c] /= ( group_count - found_non_existing[c] );
 420                                                 break;
 421                                 }
 422                         }
 423
 424                         if(print_this) {
 425                                 if(annotate_reset) {
 426                                         annotation_count++;
 427                                         web_buffer_strcpy(wb, overflow_annotation);
 428                                         annotate_reset = 0;
 429                                 }
 430                                 else
 431                                         web_buffer_strcpy(wb, normal_annotation);
 432
 433                                 pc = 0;
 434                                 for(c = 0 ; c < dimensions ; c++) {
 435                                         if(found_non_existing[c] == group_count) {
 436                                                 // all entries are non-existing
 437                                                 pc++;
 438                                                 web_buffer_strcpy(wb, pre_value);
 439                                                 web_buffer_strcpy(wb, "null");
 440                                                 web_buffer_strcpy(wb, post_value);
 441                                         }
 442                                         else if(!print_hidden[c]) {
 443                                                 pc++;
 444                                                 web_buffer_strcpy(wb, pre_value);
 445                                                 web_buffer_rrd_value(wb, group_values[c]);
 446                                                 web_buffer_strcpy(wb, post_value);
 447
 448                                                 if(group_values[c]) found_non_zero[c]++;
 449                                         }
 450
 451                                         // reset them for the next loop
 452                                         group_values[c] = 0;
 453                                         found_non_existing[c] = 0;
 454                                 }
 455
 456                                 // if all dimensions are hidden, print a null
 457                                 if(!pc) {
 458                                         web_buffer_strcpy(wb, pre_value);
 459                                         web_buffer_strcpy(wb, "null");
 460                                         web_buffer_strcpy(wb, post_value);
 461                                 }
 462
 463                                 printed++;
 464                                 group_count = 0;
 465                         }
 466                 }
 467
 468                 if(printed) web_buffer_printf(wb, "]}");
 469                 web_buffer_printf(wb, "\n       ]\n}\n");
 470
 471                 if(only_non_zero && max_loop > 1) {
 472                         int changed = 0;
 473                         for(rd = st->dimensions, c = 0 ; rd && c < dimensions ; rd = rd->next, c++) {
 474                                 group_values[c] = 0;
 475                                 found_non_existing[c] = 0;
 476
 477                                 if(!print_hidden[c] && !found_non_zero[c]) {
 478                                         changed = 1;
 479                                         print_hidden[c] = 1;
 480                                 }
 481                         }
 482
 483                         if(changed) web_buffer_reset(wb);
 484                         else break;
 485                 }
 486                 else break;
 487
 488         } // max_loop
 489
 490         debug(D_RRD_STATS, "RRD_STATS_JSON: %s total %ld bytes", st->name, wb->bytes);
 491
 492         pthread_rwlock_unlock(&st->rwlock);
 493         return last_timestamp;
 494 }
 495
 496
 497 // ----------------------------------------------------------------------------
 498
 499 // RRDR options
 500 #define RRDR_EMPTY      0x01
 501 #define RRDR_RESET      0x02
 502 #define RRDR_HIDDEN     0x04
 503 #define RRDR_NONZERO    0x08
 504
 505
 506 typedef struct rrdresult {
 507         RRDSET *st;                     // the chart this result refers to
 508
 509         int d;                                  // the number of dimensions
 510         int n;                                  // the number of values in the arrays
 511
 512         time_t *t;                              // array of timestamps
 513         calculated_number *v;   // array n x d values
 514         uint8_t *o;                             // array n x d options
 515
 516         int c;                                  // current line (n)
 517
 518         int has_st_lock;                // if st is read locked by us
 519 } RRDR;
 520
 521 inline static calculated_number *rrdr_line_values(RRDR *r)
 522 {
 523         return &r->v[ r->c * r->d ];
 524 }
 525
 526 inline static uint8_t *rrdr_line_options(RRDR *r)
 527 {
 528         return &r->o[ r->c * r->d ];
 529 }
 530
 531 inline static int rrdr_line_next(RRDR *r, time_t t)
 532 {
 533         // save the time
 534         r->t[r->c] = t;
 535
 536         r->c++;
 537         if(unlikely(r->c >= r->n)) {
 538                 r->c--;
 539                 return 0;
 540         }
 541
 542         return 1;
 543 }
 544
 545 inline static void rrdr_lock_rrdset(RRDR *r) {
 546         if(unlikely(!r)) {
 547                 error("NULL value given!");
 548                 return;
 549         }
 550
 551         pthread_rwlock_rdlock(&r->st->rwlock);
 552         r->has_st_lock = 1;
 553 }
 554
 555 inline static void rrdr_unlock_rrdset(RRDR *r) {
 556         if(unlikely(!r)) {
 557                 error("NULL value given!");
 558                 return;
 559         }
 560
 561         if(likely(r->has_st_lock)) {
 562                 pthread_rwlock_unlock(&r->st->rwlock);
 563                 r->has_st_lock = 0;
 564         }
 565 }
 566
 567 inline static void rrdr_free(RRDR *r)
 568 {
 569         if(unlikely(!r)) {
 570                 error("NULL value given!");
 571                 return;
 572         }
 573
 574         rrdr_unlock_rrdset(r);
 575         if(likely(r->t)) free(r->t);
 576         if(likely(r->v)) free(r->v);
 577         if(likely(r->o)) free(r->o);
 578         free(r);
 579 }
 580
 581 static RRDR *rrdr_create(RRDSET *st, int n)
 582 {
 583         if(unlikely(!st)) {
 584                 error("NULL value given!");
 585                 return NULL;
 586         }
 587
 588         RRDR *r = calloc(1, sizeof(RRDR));
 589         if(unlikely(!r)) goto cleanup;
 590
 591         r->st = st;
 592
 593         rrdr_lock_rrdset(r);
 594
 595         RRDDIM *rd;
 596         for(rd = st->dimensions ; rd ; rd = rd->next) r->d++;
 597
 598         r->n = n;
 599         r->t = malloc(sizeof(time_t) * n);
 600         if(unlikely(!r->t)) goto cleanup;
 601
 602         r->t = malloc(sizeof(time_t) * n);
 603         if(unlikely(!r->t)) goto cleanup;
 604
 605         r->v = malloc(sizeof(calculated_number) * n * r->d);
 606         if(unlikely(!r->v)) goto cleanup;
 607
 608         r->o = malloc(sizeof(calculated_number) * n * r->d);
 609         if(unlikely(!r->o)) goto cleanup;
 610
 611         return r;
 612
 613 cleanup:
 614         error("Cannot allocate memory");
 615         if(likely(r)) rrdr_free(r);
 616         return NULL;
 617 }
 618
 619 RRDR *rrd2rrdr(RRDSET *st, long points, time_t after, time_t before, int group_method)
 620 {
 621         time_t first_entry_t = rrdset_first_entry_t(st);
 622         time_t last_entry_t = rrdset_last_entry_t(st);
 623
 624         // allow relative for before and after
 625         if(before <= st->update_every * st->entries) before = last_entry_t + before;
 626         if(after <= st->update_every * st->entries) after = last_entry_t + after;
 627
 628         // make sure they are within our timeframe
 629         if(before > last_entry_t) before = last_entry_t;
 630         if(before < first_entry_t) before = first_entry_t;
 631
 632         if(after > last_entry_t) after = last_entry_t;
 633         if(after < first_entry_t) after = first_entry_t;
 634
 635         // check if they are upside down
 636         if(after > before) {
 637                 time_t t = before;
 638                 before = after;
 639                 after = t;
 640         }
 641
 642         // the duration of the chart
 643         time_t duration = before - after;
 644         if(duration <= 0) return NULL;
 645
 646         // check the required points
 647         if(points <= 0) points = duration;
 648
 649         // calculate proper grouping of source data
 650         int group = duration / points;
 651         if(group <= 0) group = 1;
 652         if(duration / group > points) group++;
 653
 654         // align timestamps to group
 655         before -= before % group;
 656         after -= after % group;
 657         duration = before - after;
 658
 659         // error("NEW: points=%d after=%d before=%d group=%d, duration=%d", points, after, before, group, duration);
 660
 661         // Now we have:
 662         // before = the end time of the calculation
 663         // after = the start time of the calculation
 664         // duration = the duration of the calculation
 665         // group = the number of source points to aggregate / group together
 666         // method = the method of grouping source points
 667         // points = the number of points to generate
 668
 669
 670         // -------------------------------------------------------------------------
 671         // initialize our result set
 672
 673         RRDR *r = rrdr_create(st, points);
 674         if(!r) return NULL;
 675         if(!r->d) {
 676                 rrdr_free(r);
 677                 return NULL;
 678         }
 679
 680         // find how many dimensions we have
 681         long dimensions = r->d;
 682
 683
 684         // -------------------------------------------------------------------------
 685
 686         // how many entries can we use from the source data?
 687         long max_entries = (st->counter < (unsigned long)st->entries) ? st->counter : (unsigned long)st->entries;
 688
 689
 690         // -------------------------------------------------------------------------
 691         // checks for debugging
 692
 693         if(st->debug) {
 694                 debug(D_RRD_STATS, "%s first_entry_t = %lu, last_entry_t = %lu, duration = %lu, after = %lu, before = %lu, duration = %lu, entries_to_show = %lu, group = %lu, max_entries = %ld"
 695                         , st->id
 696                         , first_entry_t
 697                         , last_entry_t
 698                         , last_entry_t - first_entry_t
 699                         , after
 700                         , before
 701                         , duration
 702                         , points
 703                         , group
 704                         , max_entries
 705                         );
 706         }
 707
 708
 709         // -------------------------------------------------------------------------
 710         // temp arrays for keeping values per dimension
 711
 712         calculated_number       group_values[dimensions]; // keep sums when grouping
 713         long                            group_counts[dimensions]; // keep the number of values added to group_values
 714         uint8_t                         group_options[dimensions];
 715         uint8_t                         found_non_zero[dimensions];
 716
 717
 718         // initialize them
 719         RRDDIM *rd;
 720         long c;
 721         for( rd = st->dimensions, c = 0 ; rd && c < dimensions ; rd = rd->next, c++) {
 722                 group_values[c] = 0;
 723                 group_counts[c] = 0;
 724                 group_options[c] = 0;
 725                 found_non_zero[c] = 0;
 726         }
 727
 728
 729         // -------------------------------------------------------------------------
 730         // the main loop
 731
 732         int debug = st->debug;
 733
 734         time_t  now = last_entry_t,
 735                         dt = st->update_every;
 736
 737         long    t = rrdset_time2slot(st, before), // rrdset_last_slot(st),
 738                         count = 0,
 739                         added = 0,
 740                         group_count = group,
 741                         add_this = 0;
 742
 743         for( ; max_entries ; now -= dt, t--, max_entries--) {
 744                 if(unlikely(t < 0)) c = st->entries - 1;
 745
 746                 if(unlikely(debug)) debug(D_RRD_STATS, "%s c = %ld, count = %ld, group_count = %ld, added = %ld, now = %lu, %s %s"
 747                                 , st->id
 748                                 , t
 749                                 , count + 1
 750                                 , group_count + 1
 751                                 , added
 752                                 , now
 753                                 , (group_count == 0)?"PRINT":"  -  "
 754                                 , (now >= after && now <= before)?"RANGE":"  -  "
 755                                 );
 756
 757                 // make sure we return data in the proper time range
 758                 if(unlikely(now < after || now > before)) continue;
 759
 760                 count++;
 761                 group_count++;
 762
 763                 if(unlikely(group_count == group)) {
 764                         if(unlikely(added >= points)) break;
 765                         add_this = 1;
 766                 }
 767
 768                 // do the calculations
 769                 for(rd = st->dimensions, c = 0 ; likely(rd && c < dimensions) ; rd = rd->next, c++) {
 770                         storage_number n = rd->values[t];
 771                         if(unlikely(!does_storage_number_exist(n))) continue;
 772
 773                         group_counts[c]++;
 774
 775                         calculated_number value = unpack_storage_number(n);
 776                         if(value != 0.0) {
 777                                 group_options[c] |= RRDR_NONZERO;
 778                                 found_non_zero[c] = 1;
 779                         }
 780
 781                         if(unlikely(did_storage_number_reset(n)))
 782                                 group_options[c] |= RRDR_RESET;
 783
 784                         switch(group_method) {
 785                                 case GROUP_MAX:
 786                                         if(unlikely(abs(value) > abs(group_values[c])))
 787                                                 group_values[c] = value;
 788                                         break;
 789
 790                                 default:
 791                                 case GROUP_SUM:
 792                                 case GROUP_AVERAGE:
 793                                         group_values[c] += value;
 794                                         break;
 795
 796                                         group_values[c] += value;
 797                                         break;
 798                         }
 799                 }
 800
 801                 // added it
 802                 if(unlikely(add_this)) {
 803                         calculated_number *cn = rrdr_line_values(r);
 804                         uint8_t *co = rrdr_line_options(r);
 805
 806                         for(rd = st->dimensions, c = 0 ; likely(rd && c < dimensions) ; rd = rd->next, c++) {
 807                                 if(rd->hidden) group_options[c] |= RRDR_HIDDEN;
 808
 809                                 co[c] = group_options[c];
 810
 811                                 if(group_counts[c] == 0) {
 812                                         cn[c] = 0.0;
 813                                         co[c] |= RRDR_EMPTY;
 814                                 }
 815                                 else if(unlikely(group_method == GROUP_AVERAGE)) {
 816                                         cn[c] = group_values[c] / group_counts[c];
 817                                 }
 818                                 else {
 819                                         cn[c] = group_values[c];
 820                                 }
 821
 822                                 // reset them for the next loop
 823                                 group_values[c] = 0;
 824                                 group_counts[c] = 0;
 825                                 group_options[c] = 0;
 826                         }
 827
 828                         added++;
 829                         group_count = 0;
 830                         add_this = 0;
 831                 }
 832
 833                 rrdr_line_next(r, now);
 834         }
 835
 836         return r;
 837 }
 838
 839
 840 /*
 841 unsigned long rrdset2json(int type, RRDSET *st, struct web_buffer *wb, int entries_to_show, int group, int group_method, time_t after, time_t before, int only_non_zero) {
 842
 843         if(!st->dimensions) {
 844                 web_buffer_printf(wb, "No dimensions yet.");
 845                 return 0;
 846         }
 847
 848         int c;
 849         pthread_rwlock_rdlock(&st->rwlock);
 850
 851         // -------------------------------------------------------------------------
 852         // validate the parameters
 853
 854         if(entries_to_show < 1) entries_to_show = 1;
 855         if(group < 1) group = 1;
 856
 857         // make sure current_entry is within limits
 858         long current_entry = (long)st->current_entry - (long)1;
 859         if(current_entry < 0) current_entry = 0;
 860         else if(current_entry >= st->entries) current_entry = st->entries - 1;
 861
 862         // find the oldest entry of the round-robin
 863         long max_entries = (st->counter < (unsigned long)st->entries) ? st->counter : (unsigned long)st->entries;
 864         if(entries_to_show > max_entries) entries_to_show = max_entries;
 865
 866         if(before == 0 || before > st->last_updated.tv_sec) before = st->last_updated.tv_sec;
 867         if(after  == 0 || after  < rrdset_first_entry_t(st)) after = rrdset_first_entry_t(st);
 868
 869         if(entries_to_show > (before - after) / st->update_every / group) entries_to_show = (before - after) / st->update_every / group;
 870
 871         // -------------------------------------------------------------------------
 872         // checks for debuging
 873
 874         if(st->debug) {
 875                 debug(D_RRD_STATS, "%s first_entry_t = %lu, last_entry_t = %lu, duration = %lu, after = %lu, before = %lu, duration = %lu, entries_to_show = %lu, group = %lu, max_entries = %ld"
 876                         , st->id
 877                         , rrdset_first_entry_t(st)
 878                         , st->last_updated.tv_sec
 879                         , st->last_updated.tv_sec - rrdset_first_entry_t(st)
 880                         , after
 881                         , before
 882                         , before - after
 883                         , entries_to_show
 884                         , group
 885                         , max_entries
 886                         );
 887
 888                 if(before < after)
 889                         debug(D_RRD_STATS, "WARNING: %s The newest value in the database (%lu) is earlier than the oldest (%lu)", st->name, before, after);
 890
 891                 if((before - after) > st->entries * st->update_every)
 892                         debug(D_RRD_STATS, "WARNING: %s The time difference between the oldest and the newest entries (%lu) is higher than the capacity of the database (%lu)", st->name, before - after, st->entries * st->update_every);
 893         }
 894
 895         // -------------------------------------------------------------------------
 896
 897         int dimensions = 0, i, j;
 898         RRDDIM *rd;
 899         for( rd = st->dimensions ; rd ; rd = rd->next) dimensions++;
 900
 901         uint32_t *annotations[dimensions];
 902         calculated_number *values[dimensions];
 903         int enabled[dimensions];
 904
 905         for( rd = st->dimensions, i = 0 ; rd && i < dimensions ; rd = rd->next, i++) {
 906                 annotations[i] = malloc(sizeof(uint32_t) * entries_to_show);
 907                 if(!annotations[i]) fatal("Cannot allocate %z bytes of memory.", sizeof(uint32_t) * entries_to_show);
 908
 909                 values[i] = malloc(sizeof(calculated_number) * entries_to_show);
 910                 if(!values[i]) fatal("Cannot allocate %z bytes of memory.", sizeof(calculated_number) * entries_to_show);
 911
 912                 calculated_number sum = rrddim2values(st, rd, values, annotations, entries_to_show, group, group_method, after, before);
 913                 if(only_non_zero && sum == 0) enabled[i] = 0;
 914                 else enabled[i] = 1;
 915         }
 916
 917         switch(type) {
 918                 case DATASOURCE_GOOGLE_JSON:
 919                 case DATASOURCE_GOOGLE_JSONP:
 920                         //rrdvalues2googlejson(st, rd, values, annotations, enabled, entries_to_show, );
 921                         break;
 922
 923                 case DATASOURCE_JSON:
 924                 default:
 925                         //rrdvalues2json();
 926                         break;
 927         }
 928
 929         //return ???;
 930 }
 931 */