From 8dd00ca3d2de1ac0df4ae34dff437b13fbb0c0e5 Mon Sep 17 00:00:00 2001 From: "Costa Tsaousis (ktsaou)" Date: Sun, 25 Oct 2015 14:58:22 +0200 Subject: [PATCH] optimized json generation loop; some two stage json generation progress --- src/rrd.c | 4 +- src/rrd.h | 7 +- src/rrd2json.c | 984 ++++++++++++++++++++++++------------------------- 3 files changed, 486 insertions(+), 509 deletions(-) diff --git a/src/rrd.c b/src/rrd.c index 20b958f4..92fd2452 100755 --- a/src/rrd.c +++ b/src/rrd.c @@ -17,7 +17,7 @@ #include "rrd.h" -#define RRD_DEFAULT_GAP_INTERPOLATIONS 10 +#define RRD_DEFAULT_GAP_INTERPOLATIONS 1 // ---------------------------------------------------------------------------- // globals @@ -421,7 +421,7 @@ RRDSET *rrdset_create(const char *type, const char *id, const char *name, const st->last_collected_time.tv_usec = 0; st->counter_done = 0; - st->gap_when_lost_iterations_above = config_get_number(st->id, "gap when lost iterations above", RRD_DEFAULT_GAP_INTERPOLATIONS); + st->gap_when_lost_iterations_above = config_get_number(st->id, "gap when lost iterations above", RRD_DEFAULT_GAP_INTERPOLATIONS) + 2; st->dimensions_index.root = NULL; st->dimensions_index.compar = rrddim_compare; diff --git a/src/rrd.h b/src/rrd.h index c179aafe..3443e25b 100755 --- a/src/rrd.h +++ b/src/rrd.h @@ -295,7 +295,12 @@ extern unsigned long long rrdset_done(RRDSET *st); ))) // get the timestamp of a specific slot in the round robin database -#define rrdset_slot2time(st, slot) () +#define rrdset_slot2time(st, slot) ( rrdset_last_entry_t(st) - \ + ((st)->update_every * ( \ + ( (slot) > rrdset_last_slot(st)) ? \ + ( (rrdset_last_slot(st) - slot + (st)->entries) ) : \ + ( (rrdset_last_slot(st) - slot) )) \ + )) // ---------------------------------------------------------------------------- // RRD DIMENSION functions diff --git a/src/rrd2json.c b/src/rrd2json.c index 248375ee..c96538f3 100755 --- a/src/rrd2json.c +++ b/src/rrd2json.c @@ -160,474 +160,206 @@ void rrd_stats_all_json(struct web_buffer *wb) ); } -unsigned long rrd_stats_json(int type, RRDSET *st, struct web_buffer *wb, int entries_to_show, int group, int group_method, time_t after, time_t before, int only_non_zero) -{ - int c; - pthread_rwlock_rdlock(&st->rwlock); - // ------------------------------------------------------------------------- - // switch from JSON to google JSON - - char kq[2] = "\""; - char sq[2] = "\""; - switch(type) { - case DATASOURCE_GOOGLE_JSON: - case DATASOURCE_GOOGLE_JSONP: - kq[0] = '\0'; - sq[0] = '\''; - break; +// ---------------------------------------------------------------------------- - case DATASOURCE_JSON: - default: - break; - } +// RRDR options +#define RRDR_EMPTY 0x01 +#define RRDR_RESET 0x02 +#define RRDR_HIDDEN 0x04 +#define RRDR_NONZERO 0x08 - // ------------------------------------------------------------------------- - // validate the parameters - - if(entries_to_show < 1) entries_to_show = 1; - if(group < 1) group = 1; - - // find the oldest entry of the round-robin - long max_entries_init = (st->counter < (unsigned long)st->entries) ? st->counter : (unsigned long)st->entries; - - time_t time_init = rrdset_last_entry_t(st); - - if(before == 0 || before > time_init) before = time_init; - if(after == 0) after = rrdset_first_entry_t(st); +typedef struct rrdresult { + RRDSET *st; // the chart this result refers to + int d; // the number of dimensions + int n; // the number of values in the arrays - // --- + uint8_t *od; // the options for the dimensions - // our return value (the last timestamp printed) - // this is required to detect re-transmit in google JSONP - time_t last_timestamp = 0; + time_t *t; // array of n timestamps + calculated_number *v; // array n x d values + uint8_t *o; // array n x d options + int c; // current line ( 0 ~ n ) - // ------------------------------------------------------------------------- - // find how many dimensions we have - - int dimensions = 0; - RRDDIM *rd; - for( rd = st->dimensions ; rd ; rd = rd->next) dimensions++; - if(!dimensions) { - pthread_rwlock_unlock(&st->rwlock); - web_buffer_printf(wb, "No dimensions yet."); - return 0; + int has_st_lock; // if st is read locked by us +} RRDR; + +static void rrdr_dump(RRDR *r) +{ + long c, i; + RRDDIM *d; + + fprintf(stderr, "\nCHART %s (%s)\n", r->st->id, r->st->name); + + for(c = 0, d = r->st->dimensions; d ;c++, d = d->next) { + fprintf(stderr, "DIMENSION %s (%s), %s%s%s%s\n" + , d->id + , d->name + , (r->od[c] & RRDR_EMPTY)?"EMPTY ":"" + , (r->od[c] & RRDR_RESET)?"RESET ":"" + , (r->od[c] & RRDR_HIDDEN)?"HIDDEN ":"" + , (r->od[c] & RRDR_NONZERO)?"NONZERO ":"" + ); } - - // ------------------------------------------------------------------------- - // prepare various strings, to speed up the loop - - char overflow_annotation[201]; snprintf(overflow_annotation, 200, ",{%sv%s:%sRESET OR OVERFLOW%s},{%sv%s:%sThe counters have been wrapped.%s}", kq, kq, sq, sq, kq, kq, sq, sq); - char normal_annotation[201]; snprintf(normal_annotation, 200, ",{%sv%s:null},{%sv%s:null}", kq, kq, kq, kq); - char pre_date[51]; snprintf(pre_date, 50, " {%sc%s:[{%sv%s:%s", kq, kq, kq, kq, sq); - char post_date[21]; snprintf(post_date, 20, "%s}", sq); - char pre_value[21]; snprintf(pre_value, 20, ",{%sv%s:", kq, kq); - char post_value[21]; snprintf(post_value, 20, "}"); + if(r->c < 0) { + fprintf(stderr, "RRDR does not have any values in it.\n"); + return; + } + fprintf(stderr, "RRDR includes %d values in it:\n", r->c + 1); + + // for each line in the array + for(i = 0; i <= r->c ;i++) { + calculated_number *cn = &r->v[ i * r->d ]; + uint8_t *co = &r->o[ i * r->d ]; + + // print the id and the timestamp of the line + fprintf(stderr, "%ld %ld ", i + 1, r->t[i]); + + // for each dimension + for(c = 0, d = r->st->dimensions; d ;c++, d = d->next) { + if(unlikely(r->od[c] & RRDR_HIDDEN)) continue; + if(unlikely(!(r->od[c] & RRDR_NONZERO))) continue; + + if(co[c] & RRDR_EMPTY) + fprintf(stderr, "null "); + else + fprintf(stderr, CALCULATED_NUMBER_FORMAT " %s%s%s%s " + , cn[c] + , (co[c] & RRDR_EMPTY)?"E":" " + , (co[c] & RRDR_RESET)?"R":" " + , (co[c] & RRDR_HIDDEN)?"H":" " + , (co[c] & RRDR_NONZERO)?"N":" " + ); + } - // ------------------------------------------------------------------------- - // checks for debuging - - if(st->debug) { - debug(D_RRD_STATS, "%s first_entry_t = %lu, last_entry_t = %lu, duration = %lu, after = %lu, before = %lu, duration = %lu, entries_to_show = %lu, group = %lu, max_entries = %ld" - , st->id - , rrdset_first_entry_t(st) - , rrdset_last_entry_t(st) - , rrdset_last_entry_t(st) - rrdset_first_entry_t(st) - , after - , before - , before - after - , entries_to_show - , group - , max_entries_init - ); + fprintf(stderr, "\n"); + } +} - if(before < after) - debug(D_RRD_STATS, "WARNING: %s The newest value in the database (%lu) is earlier than the oldest (%lu)", st->name, before, after); +inline static calculated_number *rrdr_line_values(RRDR *r) +{ + return &r->v[ r->c * r->d ]; +} - if((before - after) > st->entries * st->update_every) - debug(D_RRD_STATS, "WARNING: %s The time difference between the oldest and the newest entries (%lu) is higher than the capacity of the database (%lu)", st->name, before - after, st->entries * st->update_every); +inline static uint8_t *rrdr_line_options(RRDR *r) +{ + return &r->o[ r->c * r->d ]; +} + +inline static int rrdr_line_init(RRDR *r, time_t t) +{ + r->c++; + if(unlikely(r->c >= r->n)) { + r->c = r->n - 1; + return 0; } + // save the time + r->t[r->c] = t; - // ------------------------------------------------------------------------- - // temp arrays for keeping values per dimension - - calculated_number group_values[dimensions]; // keep sums when grouping - int print_hidden[dimensions]; // keep hidden flags - int found_non_zero[dimensions]; - int found_non_existing[dimensions]; + return 1; +} - // initialize them - for( rd = st->dimensions, c = 0 ; rd && c < dimensions ; rd = rd->next, c++) { - group_values[c] = 0; - print_hidden[c] = rd->hidden; - found_non_zero[c] = 0; - found_non_existing[c] = 0; +inline static void rrdr_lock_rrdset(RRDR *r) { + if(unlikely(!r)) { + error("NULL value given!"); + return; } + pthread_rwlock_rdlock(&r->st->rwlock); + r->has_st_lock = 1; +} - // error("OLD: points=%d after=%d before=%d group=%d, duration=%d", entries_to_show, before - (st->update_every * group * entries_to_show), before, group, before - after + 1); - // rrd2array(st, entries_to_show, before - (st->update_every * group * entries_to_show), before, group_method, only_non_zero); +inline static void rrdr_unlock_rrdset(RRDR *r) { + if(unlikely(!r)) { + error("NULL value given!"); + return; + } - // ------------------------------------------------------------------------- - // remove dimensions that contain only zeros + if(likely(r->has_st_lock)) { + pthread_rwlock_unlock(&r->st->rwlock); + r->has_st_lock = 0; + } +} - int max_loop = 1; - if(only_non_zero) max_loop = 2; +inline static void rrdr_free(RRDR *r) +{ + if(unlikely(!r)) { + error("NULL value given!"); + return; + } - for(; max_loop ; max_loop--) { + rrdr_unlock_rrdset(r); + if(likely(r->t)) free(r->t); + if(likely(r->v)) free(r->v); + if(likely(r->o)) free(r->o); + if(likely(r->od)) free(r->od); + free(r); +} - // ------------------------------------------------------------------------- - // print the JSON header - - web_buffer_printf(wb, "{\n %scols%s:\n [\n", kq, kq); - web_buffer_printf(wb, " {%sid%s:%s%s,%slabel%s:%stime%s,%spattern%s:%s%s,%stype%s:%sdatetime%s},\n", kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq); - web_buffer_printf(wb, " {%sid%s:%s%s,%slabel%s:%s%s,%spattern%s:%s%s,%stype%s:%sstring%s,%sp%s:{%srole%s:%sannotation%s}},\n", kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, kq, kq, sq, sq); - web_buffer_printf(wb, " {%sid%s:%s%s,%slabel%s:%s%s,%spattern%s:%s%s,%stype%s:%sstring%s,%sp%s:{%srole%s:%sannotationText%s}}", kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, kq, kq, sq, sq); +static RRDR *rrdr_create(RRDSET *st, int n) +{ + if(unlikely(!st)) { + error("NULL value given!"); + return NULL; + } - // print the header for each dimension - // and update the print_hidden array for the dimensions that should be hidden - int pc = 0; - for( rd = st->dimensions, c = 0 ; rd && c < dimensions ; rd = rd->next, c++) { - if(!print_hidden[c]) { - pc++; - web_buffer_printf(wb, ",\n {%sid%s:%s%s,%slabel%s:%s%s%s,%spattern%s:%s%s,%stype%s:%snumber%s}", kq, kq, sq, sq, kq, kq, sq, rd->name, sq, kq, kq, sq, sq, kq, kq, sq, sq); - } - } - if(!pc) { - web_buffer_printf(wb, ",\n {%sid%s:%s%s,%slabel%s:%s%s%s,%spattern%s:%s%s,%stype%s:%snumber%s}", kq, kq, sq, sq, kq, kq, sq, "no data", sq, kq, kq, sq, sq, kq, kq, sq, sq); - } + RRDR *r = calloc(1, sizeof(RRDR)); + if(unlikely(!r)) goto cleanup; - // print the begin of row data - web_buffer_printf(wb, "\n ],\n %srows%s:\n [\n", kq, kq); + r->st = st; + rrdr_lock_rrdset(r); - // ------------------------------------------------------------------------- - // the main loop + RRDDIM *rd; + for(rd = st->dimensions ; rd ; rd = rd->next) r->d++; - int annotate_reset = 0; - int annotation_count = 0; - - // to allow grouping on the same values, we need a pad - long pad = before % group; + r->n = n; + r->t = malloc(n * sizeof(time_t)); + if(unlikely(!r->t)) goto cleanup; - // the minimum line length we expect - int line_size = 4096 + (dimensions * 200); + r->t = malloc(n * sizeof(time_t)); + if(unlikely(!r->t)) goto cleanup; - time_t now = time_init; - long max_entries = max_entries_init; + r->v = malloc(n * r->d * sizeof(calculated_number)); + if(unlikely(!r->v)) goto cleanup; - long t; + r->o = malloc(n * r->d * sizeof(uint8_t)); + if(unlikely(!r->o)) goto cleanup; - long count = 0, printed = 0, group_count = 0; - last_timestamp = 0; + r->od = calloc(r->d, sizeof(uint8_t)); + if(unlikely(!r->od)) goto cleanup; - long expected_to_start_at_slot = rrdset_time2slot(st, before); + r->c = -1; - for(t = rrdset_last_slot(st); max_entries ; now -= st->update_every, t--, max_entries--) { - if(t < 0) t = st->entries - 1; + return r; - int print_this = 0; +cleanup: + error("Cannot allocate memory"); + if(likely(r)) rrdr_free(r); + return NULL; +} - if(st->debug) { - debug(D_RRD_STATS, "%s t = %ld, count = %ld, group_count = %ld, printed = %ld, now = %lu, %s %s" - , st->id - , t - , count + 1 - , group_count + 1 - , printed - , now - , (((count + 1 - pad) % group) == 0)?"PRINT":" - " - , (now >= after && now <= before)?"RANGE":" - " - ); - } +RRDR *rrd2rrdr(RRDSET *st, long points, time_t after, time_t before, int group_method) +{ + int debug = st->debug; - // make sure we return data in the proper time range - if(now < after || now > before) continue; + time_t first_entry_t = rrdset_first_entry_t(st); + time_t last_entry_t = rrdset_last_entry_t(st); - if(expected_to_start_at_slot != -999999) { - error("%s: Expected to start on slot %ld, started on %ld, %s", st->id, expected_to_start_at_slot, t, (t == expected_to_start_at_slot)?"OK":"ERROR"); - expected_to_start_at_slot = -999999; - } + // allow relative for before and after + if(before <= st->update_every * st->entries) before = last_entry_t + before; + if(after <= st->update_every * st->entries) after = last_entry_t + after; - count++; - group_count++; - - // check if we have to print this now - if(((count - pad) % group) == 0) { - if(printed >= entries_to_show) { - // debug(D_RRD_STATS, "Already printed all rows. Stopping."); - break; - } - - if(group_count != group) { - // this is an incomplete group, skip it. - for( rd = st->dimensions, c = 0 ; rd && c < dimensions ; rd = rd->next, c++) { - group_values[c] = 0; - found_non_existing[c] = 0; - } - - group_count = 0; - continue; - } - - // check if we may exceed the buffer provided - web_buffer_increase(wb, line_size); - - // generate the local date time - struct tm *tm = localtime(&now); - if(!tm) { error("localtime() failed."); continue; } - if(now > last_timestamp) last_timestamp = now; - - if(printed) web_buffer_strcpy(wb, "]},\n"); - web_buffer_strcpy(wb, pre_date); - web_buffer_jsdate(wb, tm->tm_year + 1900, tm->tm_mon, tm->tm_mday, tm->tm_hour, tm->tm_min, tm->tm_sec); - web_buffer_strcpy(wb, post_date); - - print_this = 1; - } - - // do the calculations - for(rd = st->dimensions, c = 0 ; rd && c < dimensions ; rd = rd->next, c++) { - storage_number n = rd->values[t]; - calculated_number value = unpack_storage_number(n); - - if(!does_storage_number_exist(n)) { - value = 0.0; - found_non_existing[c]++; - } - if(did_storage_number_reset(n)) annotate_reset = 1; - - switch(group_method) { - case GROUP_MAX: - if(abs(value) > abs(group_values[c])) group_values[c] = value; - break; - - case GROUP_SUM: - group_values[c] += value; - break; - - default: - case GROUP_AVERAGE: - group_values[c] += value; - if(print_this) group_values[c] /= ( group_count - found_non_existing[c] ); - break; - } - } - - if(print_this) { - if(annotate_reset) { - annotation_count++; - web_buffer_strcpy(wb, overflow_annotation); - annotate_reset = 0; - } - else - web_buffer_strcpy(wb, normal_annotation); - - pc = 0; - for(c = 0 ; c < dimensions ; c++) { - if(found_non_existing[c] == group_count) { - // all entries are non-existing - pc++; - web_buffer_strcpy(wb, pre_value); - web_buffer_strcpy(wb, "null"); - web_buffer_strcpy(wb, post_value); - } - else if(!print_hidden[c]) { - pc++; - web_buffer_strcpy(wb, pre_value); - web_buffer_rrd_value(wb, group_values[c]); - web_buffer_strcpy(wb, post_value); - - if(group_values[c]) found_non_zero[c]++; - } - - // reset them for the next loop - group_values[c] = 0; - found_non_existing[c] = 0; - } - - // if all dimensions are hidden, print a null - if(!pc) { - web_buffer_strcpy(wb, pre_value); - web_buffer_strcpy(wb, "null"); - web_buffer_strcpy(wb, post_value); - } - - printed++; - group_count = 0; - } - } - - if(printed) web_buffer_printf(wb, "]}"); - web_buffer_printf(wb, "\n ]\n}\n"); - - if(only_non_zero && max_loop > 1) { - int changed = 0; - for(rd = st->dimensions, c = 0 ; rd && c < dimensions ; rd = rd->next, c++) { - group_values[c] = 0; - found_non_existing[c] = 0; - - if(!print_hidden[c] && !found_non_zero[c]) { - changed = 1; - print_hidden[c] = 1; - } - } - - if(changed) web_buffer_reset(wb); - else break; - } - else break; - - } // max_loop - - debug(D_RRD_STATS, "RRD_STATS_JSON: %s total %ld bytes", st->name, wb->bytes); - - pthread_rwlock_unlock(&st->rwlock); - return last_timestamp; -} - - -// ---------------------------------------------------------------------------- - -// RRDR options -#define RRDR_EMPTY 0x01 -#define RRDR_RESET 0x02 -#define RRDR_HIDDEN 0x04 -#define RRDR_NONZERO 0x08 - - -typedef struct rrdresult { - RRDSET *st; // the chart this result refers to - - int d; // the number of dimensions - int n; // the number of values in the arrays - - time_t *t; // array of timestamps - calculated_number *v; // array n x d values - uint8_t *o; // array n x d options - - int c; // current line (n) - - int has_st_lock; // if st is read locked by us -} RRDR; - -inline static calculated_number *rrdr_line_values(RRDR *r) -{ - return &r->v[ r->c * r->d ]; -} - -inline static uint8_t *rrdr_line_options(RRDR *r) -{ - return &r->o[ r->c * r->d ]; -} - -inline static int rrdr_line_next(RRDR *r, time_t t) -{ - // save the time - r->t[r->c] = t; - - r->c++; - if(unlikely(r->c >= r->n)) { - r->c--; - return 0; - } - - return 1; -} - -inline static void rrdr_lock_rrdset(RRDR *r) { - if(unlikely(!r)) { - error("NULL value given!"); - return; - } - - pthread_rwlock_rdlock(&r->st->rwlock); - r->has_st_lock = 1; -} - -inline static void rrdr_unlock_rrdset(RRDR *r) { - if(unlikely(!r)) { - error("NULL value given!"); - return; - } - - if(likely(r->has_st_lock)) { - pthread_rwlock_unlock(&r->st->rwlock); - r->has_st_lock = 0; - } -} - -inline static void rrdr_free(RRDR *r) -{ - if(unlikely(!r)) { - error("NULL value given!"); - return; - } - - rrdr_unlock_rrdset(r); - if(likely(r->t)) free(r->t); - if(likely(r->v)) free(r->v); - if(likely(r->o)) free(r->o); - free(r); -} - -static RRDR *rrdr_create(RRDSET *st, int n) -{ - if(unlikely(!st)) { - error("NULL value given!"); - return NULL; - } - - RRDR *r = calloc(1, sizeof(RRDR)); - if(unlikely(!r)) goto cleanup; - - r->st = st; - - rrdr_lock_rrdset(r); - - RRDDIM *rd; - for(rd = st->dimensions ; rd ; rd = rd->next) r->d++; - - r->n = n; - r->t = malloc(sizeof(time_t) * n); - if(unlikely(!r->t)) goto cleanup; - - r->t = malloc(sizeof(time_t) * n); - if(unlikely(!r->t)) goto cleanup; - - r->v = malloc(sizeof(calculated_number) * n * r->d); - if(unlikely(!r->v)) goto cleanup; - - r->o = malloc(sizeof(calculated_number) * n * r->d); - if(unlikely(!r->o)) goto cleanup; - - return r; - -cleanup: - error("Cannot allocate memory"); - if(likely(r)) rrdr_free(r); - return NULL; -} - -RRDR *rrd2rrdr(RRDSET *st, long points, time_t after, time_t before, int group_method) -{ - time_t first_entry_t = rrdset_first_entry_t(st); - time_t last_entry_t = rrdset_last_entry_t(st); - - // allow relative for before and after - if(before <= st->update_every * st->entries) before = last_entry_t + before; - if(after <= st->update_every * st->entries) after = last_entry_t + after; - - // make sure they are within our timeframe - if(before > last_entry_t) before = last_entry_t; - if(before < first_entry_t) before = first_entry_t; + // make sure they are within our timeframe + if(before > last_entry_t) before = last_entry_t; + if(before < first_entry_t) before = first_entry_t; if(after > last_entry_t) after = last_entry_t; if(after < first_entry_t) after = first_entry_t; @@ -647,7 +379,7 @@ RRDR *rrd2rrdr(RRDSET *st, long points, time_t after, time_t before, int group_m if(points <= 0) points = duration; // calculate proper grouping of source data - int group = duration / points; + long group = duration / points; if(group <= 0) group = 1; if(duration / group > points) group++; @@ -681,17 +413,10 @@ RRDR *rrd2rrdr(RRDSET *st, long points, time_t after, time_t before, int group_m long dimensions = r->d; - // ------------------------------------------------------------------------- - - // how many entries can we use from the source data? - long max_entries = (st->counter < (unsigned long)st->entries) ? st->counter : (unsigned long)st->entries; - - // ------------------------------------------------------------------------- // checks for debugging - if(st->debug) { - debug(D_RRD_STATS, "%s first_entry_t = %lu, last_entry_t = %lu, duration = %lu, after = %lu, before = %lu, duration = %lu, entries_to_show = %lu, group = %lu, max_entries = %ld" + if(debug) debug(D_RRD_STATS, "INFO %s first_t: %lu, last_t: %lu, all_duration: %lu, after: %lu, before: %lu, duration: %lu, points: %ld, group: %ld" , st->id , first_entry_t , last_entry_t @@ -701,9 +426,7 @@ RRDR *rrd2rrdr(RRDSET *st, long points, time_t after, time_t before, int group_m , duration , points , group - , max_entries ); - } // ------------------------------------------------------------------------- @@ -729,35 +452,45 @@ RRDR *rrd2rrdr(RRDSET *st, long points, time_t after, time_t before, int group_m // ------------------------------------------------------------------------- // the main loop - int debug = st->debug; - - time_t now = last_entry_t, - dt = st->update_every; - long t = rrdset_time2slot(st, before), // rrdset_last_slot(st), - count = 0, + stop_at_t = rrdset_time2slot(st, after), added = 0, - group_count = group, - add_this = 0; + group_count = 0, + add_this = 0, + stop_now = 0; - for( ; max_entries ; now -= dt, t--, max_entries--) { - if(unlikely(t < 0)) c = st->entries - 1; + time_t now = rrdset_slot2time(st, t), + dt = st->update_every, + group_start_t = 0; - if(unlikely(debug)) debug(D_RRD_STATS, "%s c = %ld, count = %ld, group_count = %ld, added = %ld, now = %lu, %s %s" + if(debug) debug(D_RRD_STATS, "BEGIN %s after_t: %lu (stop slot %ld), before_t: %lu (start slot %ld), start_t(now): %lu" + , st->id + , after + , stop_at_t + , before + , t + , now + ); + + for( ; !stop_now ; now -= dt, t--) { + if(unlikely(t < 0)) c = st->entries - 1; + if(t == stop_at_t) stop_now = 1; + + if(debug) debug(D_RRD_STATS, "ROW %s c: %ld, group_count: %ld, added: %ld, now: %lu, %s %s" , st->id , t - , count + 1 , group_count + 1 , added , now - , (group_count == 0)?"PRINT":" - " + , (group_count + 1 == group)?"PRINT":" - " , (now >= after && now <= before)?"RANGE":" - " ); // make sure we return data in the proper time range - if(unlikely(now < after || now > before)) continue; + if(unlikely(now > before)) continue; + if(unlikely(now < after)) break; - count++; + if(group_count == 0) group_start_t = now; group_count++; if(unlikely(group_count == group)) { @@ -792,22 +525,26 @@ RRDR *rrd2rrdr(RRDSET *st, long points, time_t after, time_t before, int group_m case GROUP_AVERAGE: group_values[c] += value; break; - - group_values[c] += value; - break; } } // added it if(unlikely(add_this)) { + if(!rrdr_line_init(r, group_start_t)) break; + calculated_number *cn = rrdr_line_values(r); uint8_t *co = rrdr_line_options(r); for(rd = st->dimensions, c = 0 ; likely(rd && c < dimensions) ; rd = rd->next, c++) { - if(rd->hidden) group_options[c] |= RRDR_HIDDEN; + // update the dimension options + if(found_non_zero[c]) r->od[c] |= RRDR_NONZERO; + if(rd->hidden) r->od[c] |= RRDR_HIDDEN; + + // store the specific point options co[c] = group_options[c]; + // store the value if(group_counts[c] == 0) { cn[c] = 0.0; co[c] |= RRDR_EMPTY; @@ -829,24 +566,36 @@ RRDR *rrd2rrdr(RRDSET *st, long points, time_t after, time_t before, int group_m group_count = 0; add_this = 0; } - - rrdr_line_next(r, now); } - return r; + rrdr_dump(r); + rrdr_free(r); + return NULL; } +unsigned long rrd_stats_json(int type, RRDSET *st, struct web_buffer *wb, int entries_to_show, int group, int group_method, time_t after, time_t before, int only_non_zero) +{ + int c; + pthread_rwlock_rdlock(&st->rwlock); + -/* -unsigned long rrdset2json(int type, RRDSET *st, struct web_buffer *wb, int entries_to_show, int group, int group_method, time_t after, time_t before, int only_non_zero) { + // ------------------------------------------------------------------------- + // switch from JSON to google JSON - if(!st->dimensions) { - web_buffer_printf(wb, "No dimensions yet."); - return 0; + char kq[2] = "\""; + char sq[2] = "\""; + switch(type) { + case DATASOURCE_GOOGLE_JSON: + case DATASOURCE_GOOGLE_JSONP: + kq[0] = '\0'; + sq[0] = '\''; + break; + + case DATASOURCE_JSON: + default: + break; } - int c; - pthread_rwlock_rdlock(&st->rwlock); // ------------------------------------------------------------------------- // validate the parameters @@ -854,35 +603,57 @@ unsigned long rrdset2json(int type, RRDSET *st, struct web_buffer *wb, int entri if(entries_to_show < 1) entries_to_show = 1; if(group < 1) group = 1; - // make sure current_entry is within limits - long current_entry = (long)st->current_entry - (long)1; - if(current_entry < 0) current_entry = 0; - else if(current_entry >= st->entries) current_entry = st->entries - 1; + time_t time_init = rrdset_last_entry_t(st); + + if(before == 0 || before > time_init) before = time_init; + if(after == 0) after = rrdset_first_entry_t(st); + + + // --- + + // our return value (the last timestamp printed) + // this is required to detect re-transmit in google JSONP + time_t last_timestamp = 0; + + + // ------------------------------------------------------------------------- + // find how many dimensions we have + + int dimensions = 0; + RRDDIM *rd; + for( rd = st->dimensions ; rd ; rd = rd->next) dimensions++; + if(!dimensions) { + pthread_rwlock_unlock(&st->rwlock); + web_buffer_printf(wb, "No dimensions yet."); + return 0; + } - // find the oldest entry of the round-robin - long max_entries = (st->counter < (unsigned long)st->entries) ? st->counter : (unsigned long)st->entries; - if(entries_to_show > max_entries) entries_to_show = max_entries; - if(before == 0 || before > st->last_updated.tv_sec) before = st->last_updated.tv_sec; - if(after == 0 || after < rrdset_first_entry_t(st)) after = rrdset_first_entry_t(st); + // ------------------------------------------------------------------------- + // prepare various strings, to speed up the loop + + char overflow_annotation[201]; snprintf(overflow_annotation, 200, ",{%sv%s:%sRESET OR OVERFLOW%s},{%sv%s:%sThe counters have been wrapped.%s}", kq, kq, sq, sq, kq, kq, sq, sq); + char normal_annotation[201]; snprintf(normal_annotation, 200, ",{%sv%s:null},{%sv%s:null}", kq, kq, kq, kq); + char pre_date[51]; snprintf(pre_date, 50, " {%sc%s:[{%sv%s:%s", kq, kq, kq, kq, sq); + char post_date[21]; snprintf(post_date, 20, "%s}", sq); + char pre_value[21]; snprintf(pre_value, 20, ",{%sv%s:", kq, kq); + char post_value[21]; snprintf(post_value, 20, "}"); - if(entries_to_show > (before - after) / st->update_every / group) entries_to_show = (before - after) / st->update_every / group; // ------------------------------------------------------------------------- // checks for debuging if(st->debug) { - debug(D_RRD_STATS, "%s first_entry_t = %lu, last_entry_t = %lu, duration = %lu, after = %lu, before = %lu, duration = %lu, entries_to_show = %lu, group = %lu, max_entries = %ld" + debug(D_RRD_STATS, "%s first_entry_t = %lu, last_entry_t = %lu, duration = %lu, after = %lu, before = %lu, duration = %lu, entries_to_show = %lu, group = %lu" , st->id , rrdset_first_entry_t(st) - , st->last_updated.tv_sec - , st->last_updated.tv_sec - rrdset_first_entry_t(st) + , rrdset_last_entry_t(st) + , rrdset_last_entry_t(st) - rrdset_first_entry_t(st) , after , before , before - after , entries_to_show , group - , max_entries ); if(before < after) @@ -892,40 +663,241 @@ unsigned long rrdset2json(int type, RRDSET *st, struct web_buffer *wb, int entri debug(D_RRD_STATS, "WARNING: %s The time difference between the oldest and the newest entries (%lu) is higher than the capacity of the database (%lu)", st->name, before - after, st->entries * st->update_every); } + // ------------------------------------------------------------------------- + // temp arrays for keeping values per dimension - int dimensions = 0, i, j; - RRDDIM *rd; - for( rd = st->dimensions ; rd ; rd = rd->next) dimensions++; + calculated_number group_values[dimensions]; // keep sums when grouping + int print_hidden[dimensions]; // keep hidden flags + int found_non_zero[dimensions]; + int found_non_existing[dimensions]; - uint32_t *annotations[dimensions]; - calculated_number *values[dimensions]; - int enabled[dimensions]; + // initialize them + for( rd = st->dimensions, c = 0 ; rd && c < dimensions ; rd = rd->next, c++) { + group_values[c] = 0; + print_hidden[c] = rd->hidden; + found_non_zero[c] = 0; + found_non_existing[c] = 0; + } - for( rd = st->dimensions, i = 0 ; rd && i < dimensions ; rd = rd->next, i++) { - annotations[i] = malloc(sizeof(uint32_t) * entries_to_show); - if(!annotations[i]) fatal("Cannot allocate %z bytes of memory.", sizeof(uint32_t) * entries_to_show); - values[i] = malloc(sizeof(calculated_number) * entries_to_show); - if(!values[i]) fatal("Cannot allocate %z bytes of memory.", sizeof(calculated_number) * entries_to_show); + // error("OLD: points=%d after=%d before=%d group=%d, duration=%d", entries_to_show, before - (st->update_every * group * entries_to_show), before, group, before - after + 1); + // rrd2array(st, entries_to_show, before - (st->update_every * group * entries_to_show), before, group_method, only_non_zero); + // rrd2rrdr(st, entries_to_show, before - (st->update_every * group * entries_to_show), before, group_method); - calculated_number sum = rrddim2values(st, rd, values, annotations, entries_to_show, group, group_method, after, before); - if(only_non_zero && sum == 0) enabled[i] = 0; - else enabled[i] = 1; - } + // ------------------------------------------------------------------------- + // remove dimensions that contain only zeros - switch(type) { - case DATASOURCE_GOOGLE_JSON: - case DATASOURCE_GOOGLE_JSONP: - //rrdvalues2googlejson(st, rd, values, annotations, enabled, entries_to_show, ); - break; + int max_loop = 1; + if(only_non_zero) max_loop = 2; - case DATASOURCE_JSON: - default: - //rrdvalues2json(); - break; - } + for(; max_loop ; max_loop--) { - //return ???; + // ------------------------------------------------------------------------- + // print the JSON header + + web_buffer_printf(wb, "{\n %scols%s:\n [\n", kq, kq); + web_buffer_printf(wb, " {%sid%s:%s%s,%slabel%s:%stime%s,%spattern%s:%s%s,%stype%s:%sdatetime%s},\n", kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq); + web_buffer_printf(wb, " {%sid%s:%s%s,%slabel%s:%s%s,%spattern%s:%s%s,%stype%s:%sstring%s,%sp%s:{%srole%s:%sannotation%s}},\n", kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, kq, kq, sq, sq); + web_buffer_printf(wb, " {%sid%s:%s%s,%slabel%s:%s%s,%spattern%s:%s%s,%stype%s:%sstring%s,%sp%s:{%srole%s:%sannotationText%s}}", kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, kq, kq, sq, sq); + + // print the header for each dimension + // and update the print_hidden array for the dimensions that should be hidden + int pc = 0; + for( rd = st->dimensions, c = 0 ; rd && c < dimensions ; rd = rd->next, c++) { + if(!print_hidden[c]) { + pc++; + web_buffer_printf(wb, ",\n {%sid%s:%s%s,%slabel%s:%s%s%s,%spattern%s:%s%s,%stype%s:%snumber%s}", kq, kq, sq, sq, kq, kq, sq, rd->name, sq, kq, kq, sq, sq, kq, kq, sq, sq); + } + } + if(!pc) { + web_buffer_printf(wb, ",\n {%sid%s:%s%s,%slabel%s:%s%s%s,%spattern%s:%s%s,%stype%s:%snumber%s}", kq, kq, sq, sq, kq, kq, sq, "no data", sq, kq, kq, sq, sq, kq, kq, sq, sq); + } + + // print the begin of row data + web_buffer_printf(wb, "\n ],\n %srows%s:\n [\n", kq, kq); + + + // ------------------------------------------------------------------------- + // the main loop + + int annotate_reset = 0; + int annotation_count = 0; + + // to allow grouping on the same values, we need a pad + long pad = before % group; + + // the minimum line length we expect + int line_size = 4096 + (dimensions * 200); + + time_t now = time_init; + + long t = rrdset_time2slot(st, before), + stop_at_t = rrdset_time2slot(st, after), + stop_now = 0; + + long count = 0, printed = 0, group_count = 0; + last_timestamp = 0; + + for(; !stop_now ; now -= st->update_every, t--) { + if(t < 0) t = st->entries - 1; + if(t == stop_at_t) stop_now = 1; + + int print_this = 0; + + if(st->debug) { + debug(D_RRD_STATS, "%s t = %ld, count = %ld, group_count = %ld, printed = %ld, now = %lu, %s %s" + , st->id + , t + , count + 1 + , group_count + 1 + , printed + , now + , (((count + 1 - pad) % group) == 0)?"PRINT":" - " + , (now >= after && now <= before)?"RANGE":" - " + ); + } + + // make sure we return data in the proper time range + if(now > before) continue; + if(now < after) break; + + //if(rrdset_slot2time(st, t) != now) + // error("%s: slot=%ld, now=%ld, slot2time=%ld, diff=%ld, last_entry_t=%ld, rrdset_last_slot=%ld", st->id, t, now, rrdset_slot2time(st,t), now - rrdset_slot2time(st,t), rrdset_last_entry_t(st), rrdset_last_slot(st)); + + count++; + group_count++; + + // check if we have to print this now + if(((count - pad) % group) == 0) { + if(printed >= entries_to_show) { + // debug(D_RRD_STATS, "Already printed all rows. Stopping."); + break; + } + + if(group_count != group) { + // this is an incomplete group, skip it. + for( rd = st->dimensions, c = 0 ; rd && c < dimensions ; rd = rd->next, c++) { + group_values[c] = 0; + found_non_existing[c] = 0; + } + + group_count = 0; + continue; + } + + // check if we may exceed the buffer provided + web_buffer_increase(wb, line_size); + + // generate the local date time + struct tm *tm = localtime(&now); + if(!tm) { error("localtime() failed."); continue; } + if(now > last_timestamp) last_timestamp = now; + + if(printed) web_buffer_strcpy(wb, "]},\n"); + web_buffer_strcpy(wb, pre_date); + web_buffer_jsdate(wb, tm->tm_year + 1900, tm->tm_mon, tm->tm_mday, tm->tm_hour, tm->tm_min, tm->tm_sec); + web_buffer_strcpy(wb, post_date); + + print_this = 1; + } + + // do the calculations + for(rd = st->dimensions, c = 0 ; rd && c < dimensions ; rd = rd->next, c++) { + storage_number n = rd->values[t]; + calculated_number value = unpack_storage_number(n); + + if(!does_storage_number_exist(n)) { + value = 0.0; + found_non_existing[c]++; + } + if(did_storage_number_reset(n)) annotate_reset = 1; + + switch(group_method) { + case GROUP_MAX: + if(abs(value) > abs(group_values[c])) group_values[c] = value; + break; + + case GROUP_SUM: + group_values[c] += value; + break; + + default: + case GROUP_AVERAGE: + group_values[c] += value; + if(print_this) group_values[c] /= ( group_count - found_non_existing[c] ); + break; + } + } + + if(print_this) { + if(annotate_reset) { + annotation_count++; + web_buffer_strcpy(wb, overflow_annotation); + annotate_reset = 0; + } + else + web_buffer_strcpy(wb, normal_annotation); + + pc = 0; + for(c = 0 ; c < dimensions ; c++) { + if(found_non_existing[c] == group_count) { + // all entries are non-existing + pc++; + web_buffer_strcpy(wb, pre_value); + web_buffer_strcpy(wb, "null"); + web_buffer_strcpy(wb, post_value); + } + else if(!print_hidden[c]) { + pc++; + web_buffer_strcpy(wb, pre_value); + web_buffer_rrd_value(wb, group_values[c]); + web_buffer_strcpy(wb, post_value); + + if(group_values[c]) found_non_zero[c]++; + } + + // reset them for the next loop + group_values[c] = 0; + found_non_existing[c] = 0; + } + + // if all dimensions are hidden, print a null + if(!pc) { + web_buffer_strcpy(wb, pre_value); + web_buffer_strcpy(wb, "null"); + web_buffer_strcpy(wb, post_value); + } + + printed++; + group_count = 0; + } + } + + if(printed) web_buffer_printf(wb, "]}"); + web_buffer_printf(wb, "\n ]\n}\n"); + + if(only_non_zero && max_loop > 1) { + int changed = 0; + for(rd = st->dimensions, c = 0 ; rd && c < dimensions ; rd = rd->next, c++) { + group_values[c] = 0; + found_non_existing[c] = 0; + + if(!print_hidden[c] && !found_non_zero[c]) { + changed = 1; + print_hidden[c] = 1; + } + } + + if(changed) web_buffer_reset(wb); + else break; + } + else break; + + } // max_loop + + debug(D_RRD_STATS, "RRD_STATS_JSON: %s total %ld bytes", st->name, wb->bytes); + + pthread_rwlock_unlock(&st->rwlock); + return last_timestamp; } -*/ -- 2.39.2