X-Git-Url: https://arthur.barton.de/gitweb/?a=blobdiff_plain;f=src%2Frrd2json.c;h=ad453cb687f957aa35a2a99f1928cc2af4bbc07b;hb=7c0ba6a675cbca3a6fa1a0b79a37259ad4b52aee;hp=888c8f492981824faf82db1a8d42e2157d04b90e;hpb=7b3be13a1a3ecdc1730c5e4ed2e460ec30c574a1;p=netdata.git diff --git a/src/rrd2json.c b/src/rrd2json.c index 888c8f49..ad453cb6 100755 --- a/src/rrd2json.c +++ b/src/rrd2json.c @@ -2,8 +2,9 @@ #include #endif #include -#include #include +#include +#include #include "log.h" #include "common.h" @@ -12,19 +13,118 @@ #define HOSTNAME_MAX 1024 char *hostname = "unknown"; -unsigned long rrd_stats_one_json(RRDSET *st, char *options, struct web_buffer *wb) +void rrd_stats_api_v1_chart(RRDSET *st, BUFFER *wb) +{ + pthread_rwlock_rdlock(&st->rwlock); + + buffer_sprintf(wb, + "\t\t{\n" + "\t\t\t\"id\": \"%s\",\n" + "\t\t\t\"name\": \"%s\",\n" + "\t\t\t\"type\": \"%s\",\n" + "\t\t\t\"family\": \"%s\",\n" + "\t\t\t\"context\": \"%s\",\n" + "\t\t\t\"title\": \"%s\",\n" + "\t\t\t\"priority\": %ld,\n" + "\t\t\t\"enabled\": %s,\n" + "\t\t\t\"units\": \"%s\",\n" + "\t\t\t\"data_url\": \"/api/v1/data?chart=%s\",\n" + "\t\t\t\"chart_type\": \"%s\",\n" + "\t\t\t\"duration\": %ld,\n" + "\t\t\t\"first_entry\": %lu,\n" + "\t\t\t\"last_entry\": %lu,\n" + "\t\t\t\"update_every\": %d,\n" + "\t\t\t\"dimensions\": {\n" + , st->id + , st->name + , st->type + , st->family + , st->context + , st->title + , st->priority + , st->enabled?"true":"false" + , st->units + , st->name + , rrdset_type_name(st->chart_type) + , st->entries * st->update_every + , rrdset_first_entry_t(st) + , rrdset_last_entry_t(st) + , st->update_every + ); + + unsigned long memory = st->memsize; + + int c = 0; + RRDDIM *rd; + for(rd = st->dimensions; rd ; rd = rd->next) { + if(rd->flags & RRDDIM_FLAG_HIDDEN) continue; + + memory += rd->memsize; + + buffer_sprintf(wb, + "%s" + "\t\t\t\t\"%s\": { \"name\": \"%s\" }" + , c?",\n":"" + , rd->id + , rd->name + ); + + c++; + } + + buffer_sprintf(wb, + "\n\t\t\t}\n" + "\t\t}" + ); + + pthread_rwlock_unlock(&st->rwlock); +} + +void rrd_stats_api_v1_charts(BUFFER *wb) +{ + long c; + RRDSET *st; + + buffer_sprintf(wb, "{\n" + "\t\"hostname\": \"%s\"" + ",\n\t\"update_every\": %d" + ",\n\t\"history\": %d" + ",\n\t\"charts\": {" + , hostname + , rrd_update_every + , rrd_default_history_entries + ); + + pthread_rwlock_rdlock(&rrdset_root_rwlock); + for(st = rrdset_root, c = 0; st ; st = st->next) { + if(st->enabled) { + if(c) buffer_strcat(wb, ","); + buffer_strcat(wb, "\n\t\t\""); + buffer_strcat(wb, st->id); + buffer_strcat(wb, "\": "); + rrd_stats_api_v1_chart(st, wb); + c++; + } + } + pthread_rwlock_unlock(&rrdset_root_rwlock); + + buffer_strcat(wb, "\n\t}\n}\n"); +} + + +unsigned long rrd_stats_one_json(RRDSET *st, char *options, BUFFER *wb) { time_t now = time(NULL); - web_buffer_increase(wb, 65536); pthread_rwlock_rdlock(&st->rwlock); - web_buffer_printf(wb, + buffer_sprintf(wb, "\t\t{\n" "\t\t\t\"id\": \"%s\",\n" "\t\t\t\"name\": \"%s\",\n" "\t\t\t\"type\": \"%s\",\n" "\t\t\t\"family\": \"%s\",\n" + "\t\t\t\"context\": \"%s\",\n" "\t\t\t\"title\": \"%s\",\n" "\t\t\t\"priority\": %ld,\n" "\t\t\t\"enabled\": %d,\n" @@ -47,6 +147,7 @@ unsigned long rrd_stats_one_json(RRDSET *st, char *options, struct web_buffer *w , st->name , st->type , st->family + , st->context , st->title , st->priority , st->enabled @@ -70,11 +171,10 @@ unsigned long rrd_stats_one_json(RRDSET *st, char *options, struct web_buffer *w RRDDIM *rd; for(rd = st->dimensions; rd ; rd = rd->next) { - web_buffer_increase(wb, 4096); memory += rd->memsize; - web_buffer_printf(wb, + buffer_sprintf(wb, "\t\t\t\t{\n" "\t\t\t\t\t\"id\": \"%s\",\n" "\t\t\t\t\t\"name\": \"%s\",\n" @@ -107,7 +207,7 @@ unsigned long rrd_stats_one_json(RRDSET *st, char *options, struct web_buffer *w ); } - web_buffer_printf(wb, + buffer_sprintf(wb, "\t\t\t],\n" "\t\t\t\"memory\" : %lu\n" "\t\t}" @@ -121,36 +221,32 @@ unsigned long rrd_stats_one_json(RRDSET *st, char *options, struct web_buffer *w #define RRD_GRAPH_JSON_HEADER "{\n\t\"charts\": [\n" #define RRD_GRAPH_JSON_FOOTER "\n\t]\n}\n" -void rrd_stats_graph_json(RRDSET *st, char *options, struct web_buffer *wb) +void rrd_stats_graph_json(RRDSET *st, char *options, BUFFER *wb) { - web_buffer_increase(wb, 2048); - - web_buffer_printf(wb, RRD_GRAPH_JSON_HEADER); + buffer_strcat(wb, RRD_GRAPH_JSON_HEADER); rrd_stats_one_json(st, options, wb); - web_buffer_printf(wb, RRD_GRAPH_JSON_FOOTER); + buffer_strcat(wb, RRD_GRAPH_JSON_FOOTER); } -void rrd_stats_all_json(struct web_buffer *wb) +void rrd_stats_all_json(BUFFER *wb) { - web_buffer_increase(wb, 2048); - unsigned long memory = 0; long c; RRDSET *st; - web_buffer_printf(wb, RRD_GRAPH_JSON_HEADER); + buffer_strcat(wb, RRD_GRAPH_JSON_HEADER); pthread_rwlock_rdlock(&rrdset_root_rwlock); for(st = rrdset_root, c = 0; st ; st = st->next) { if(st->enabled) { - if(c) web_buffer_printf(wb, "%s", ",\n"); + if(c) buffer_strcat(wb, ",\n"); memory += rrd_stats_one_json(st, NULL, wb); c++; } } pthread_rwlock_unlock(&rrdset_root_rwlock); - - web_buffer_printf(wb, "\n\t],\n" + + buffer_sprintf(wb, "\n\t],\n" "\t\"hostname\": \"%s\",\n" "\t\"update_every\": %d,\n" "\t\"history\": %d,\n" @@ -167,18 +263,24 @@ void rrd_stats_all_json(struct web_buffer *wb) // ---------------------------------------------------------------------------- -// RRDR options -#define RRDR_EMPTY 0x01 -#define RRDR_RESET 0x02 -#define RRDR_HIDDEN 0x04 -#define RRDR_NONZERO 0x08 +// RRDR dimension options +#define RRDR_EMPTY 0x01 // the dimension contains / the value is empty (null) +#define RRDR_RESET 0x02 // the dimension contains / the value is reset +#define RRDR_HIDDEN 0x04 // the dimension contains / the value is hidden +#define RRDR_NONZERO 0x08 // the dimension contains / the value is non-zero +// RRDR result options +#define RRDR_RESULT_OPTION_ABSOLUTE 0x00000001 +#define RRDR_RESULT_OPTION_RELATIVE 0x00000002 typedef struct rrdresult { RRDSET *st; // the chart this result refers to + uint32_t result_options; // RRDR_RESULT_OPTION_* + int d; // the number of dimensions - int n; // the number of values in the arrays + long n; // the number of values in the arrays + long rows; // the number of rows used uint8_t *od; // the options for the dimensions @@ -186,11 +288,23 @@ typedef struct rrdresult { calculated_number *v; // array n x d values uint8_t *o; // array n x d options - int c; // current line ( 0 ~ n ) + long c; // current line ( -1 ~ n ), ( -1 = none, use rrdr_rows() to get number of rows ) + + long group; // how many collected values were grouped for each row + long update_every; // what is the suggested update frequency in seconds + + calculated_number min; + calculated_number max; + + time_t before; + time_t after; int has_st_lock; // if st is read locked by us } RRDR; +#define rrdr_rows(r) ((r)->rows) + +/* static void rrdr_dump(RRDR *r) { long c, i; @@ -209,15 +323,15 @@ static void rrdr_dump(RRDR *r) ); } - if(r->c < 0) { + if(r->rows <= 0) { fprintf(stderr, "RRDR does not have any values in it.\n"); return; } - fprintf(stderr, "RRDR includes %d values in it:\n", r->c + 1); + fprintf(stderr, "RRDR includes %d values in it:\n", r->rows); // for each line in the array - for(i = 0; i <= r->c ;i++) { + for(i = 0; i < r->rows ;i++) { calculated_number *cn = &r->v[ i * r->d ]; uint8_t *co = &r->o[ i * r->d ]; @@ -244,6 +358,734 @@ static void rrdr_dump(RRDR *r) fprintf(stderr, "\n"); } } +*/ + +void rrdr_disable_not_selected_dimensions(RRDR *r, const char *dims) +{ + char b[strlen(dims) + 1]; + char *o = b, *tok; + strcpy(o, dims); + + long c; + RRDDIM *d; + + // disable all of them + for(c = 0, d = r->st->dimensions; d ;c++, d = d->next) + r->od[c] |= RRDR_HIDDEN; + + while(o && *o && (tok = mystrsep(&o, ", |"))) { + if(!*tok) continue; + + // find it and enable it + for(c = 0, d = r->st->dimensions; d ;c++, d = d->next) { + if(!strcmp(d->name, tok)) { + r->od[c] &= ~RRDR_HIDDEN; + + // since the user needs this dimension + // make it appear as NONZERO, to return it + // even if the dimension has only zeros + r->od[c] |= RRDR_NONZERO; + } + } + } +} + +void rrdr_buffer_print_format(BUFFER *wb, uint32_t format) +{ + switch(format) { + case DATASOURCE_JSON: + buffer_strcat(wb, DATASOURCE_FORMAT_JSON); + break; + + case DATASOURCE_DATATABLE_JSON: + buffer_strcat(wb, DATASOURCE_FORMAT_DATATABLE_JSON); + break; + + case DATASOURCE_DATATABLE_JSONP: + buffer_strcat(wb, DATASOURCE_FORMAT_DATATABLE_JSONP); + break; + + case DATASOURCE_JSONP: + buffer_strcat(wb, DATASOURCE_FORMAT_JSONP); + break; + + case DATASOURCE_SSV: + buffer_strcat(wb, DATASOURCE_FORMAT_SSV); + break; + + case DATASOURCE_CSV: + buffer_strcat(wb, DATASOURCE_FORMAT_CSV); + break; + + case DATASOURCE_TSV: + buffer_strcat(wb, DATASOURCE_FORMAT_TSV); + break; + + case DATASOURCE_HTML: + buffer_strcat(wb, DATASOURCE_FORMAT_HTML); + break; + + case DATASOURCE_JS_ARRAY: + buffer_strcat(wb, DATASOURCE_FORMAT_JS_ARRAY); + break; + + case DATASOURCE_SSV_COMMA: + buffer_strcat(wb, DATASOURCE_FORMAT_SSV_COMMA); + break; + + default: + buffer_strcat(wb, "unknown"); + break; + } +} + +uint32_t rrdr_check_options(RRDR *r, uint32_t options, const char *dims) +{ + if(options & RRDR_OPTION_NONZERO) { + long i; + + if(dims && *dims) { + // the caller wants specific dimensions + // disable NONZERO option + // to make sure we don't accidentally prevent + // the specific dimensions from being returned + i = 0; + } + else { + // find how many dimensions are not zero + long c; + RRDDIM *rd; + for(c = 0, i = 0, rd = r->st->dimensions; rd && c < r->d ; c++, rd = rd->next) { + if(unlikely(r->od[c] & RRDR_HIDDEN)) continue; + if(unlikely(!(r->od[c] & RRDR_NONZERO))) continue; + i++; + } + } + + // if with nonzero we get i = 0 (no dimensions will be returned) + // disable nonzero to show all dimensions + if(!i) options &= ~RRDR_OPTION_NONZERO; + } + + return options; +} + +void rrdr_json_wrapper_begin(RRDR *r, BUFFER *wb, uint32_t format, uint32_t options, int string_value) +{ + long rows = rrdr_rows(r); + long c, i; + RRDDIM *rd; + + //info("JSONWRAPPER(): %s: BEGIN", r->st->id); + char kq[2] = "", // key quote + sq[2] = ""; // string quote + + if( options & RRDR_OPTION_GOOGLE_JSON ) { + kq[0] = '\0'; + sq[0] = '\''; + } + else { + kq[0] = '"'; + sq[0] = '"'; + } + + buffer_sprintf(wb, "{\n" + " %sapi%s: 1,\n" + " %sid%s: %s%s%s,\n" + " %sname%s: %s%s%s,\n" + " %sview_update_every%s: %d,\n" + " %supdate_every%s: %d,\n" + " %sfirst_entry%s: %u,\n" + " %slast_entry%s: %u,\n" + " %sbefore%s: %u,\n" + " %safter%s: %u,\n" + " %sdimension_names%s: [" + , kq, kq + , kq, kq, sq, r->st->id, sq + , kq, kq, sq, r->st->name, sq + , kq, kq, r->update_every + , kq, kq, r->st->update_every + , kq, kq, rrdset_first_entry_t(r->st) + , kq, kq, rrdset_last_entry_t(r->st) + , kq, kq, r->before + , kq, kq, r->after + , kq, kq); + + for(c = 0, i = 0, rd = r->st->dimensions; rd && c < r->d ;c++, rd = rd->next) { + if(unlikely(r->od[c] & RRDR_HIDDEN)) continue; + if(unlikely((options & RRDR_OPTION_NONZERO) && !(r->od[c] & RRDR_NONZERO))) continue; + + if(i) buffer_strcat(wb, ", "); + buffer_strcat(wb, sq); + buffer_strcat(wb, rd->name); + buffer_strcat(wb, sq); + i++; + } + if(!i) { +#ifdef NETDATA_INTERNAL_CHECKS + error("RRDR is empty for %s (RRDR has %d dimensions, options is 0x%08x)", r->st->id, r->d, options); +#endif + rows = 0; + buffer_strcat(wb, sq); + buffer_strcat(wb, "no data"); + buffer_strcat(wb, sq); + } + + buffer_sprintf(wb, "],\n" + " %sdimension_ids%s: [" + , kq, kq); + + for(c = 0, i = 0, rd = r->st->dimensions; rd && c < r->d ;c++, rd = rd->next) { + if(unlikely(r->od[c] & RRDR_HIDDEN)) continue; + if(unlikely((options & RRDR_OPTION_NONZERO) && !(r->od[c] & RRDR_NONZERO))) continue; + + if(i) buffer_strcat(wb, ", "); + buffer_strcat(wb, sq); + buffer_strcat(wb, rd->id); + buffer_strcat(wb, sq); + i++; + } + if(!i) { + rows = 0; + buffer_strcat(wb, sq); + buffer_strcat(wb, "no data"); + buffer_strcat(wb, sq); + } + + buffer_sprintf(wb, "],\n" + " %slatest_values%s: [" + , kq, kq); + + for(c = 0, i = 0, rd = r->st->dimensions; rd && c < r->d ;c++, rd = rd->next) { + if(unlikely(r->od[c] & RRDR_HIDDEN)) continue; + if(unlikely((options & RRDR_OPTION_NONZERO) && !(r->od[c] & RRDR_NONZERO))) continue; + + if(i) buffer_strcat(wb, ", "); + i++; + + storage_number n = rd->values[rrdset_last_slot(r->st)]; + + if(!does_storage_number_exist(n)) + buffer_strcat(wb, "null"); + else + buffer_rrd_value(wb, unpack_storage_number(n)); + } + if(!i) { + rows = 0; + buffer_strcat(wb, "null"); + } + + buffer_sprintf(wb, "],\n" + " %sview_latest_values%s: [" + , kq, kq); + + i = 0; + if(rows) { + for(c = 0, i = 0, rd = r->st->dimensions; rd && c < r->d ;c++, rd = rd->next) { + if(unlikely(r->od[c] & RRDR_HIDDEN)) continue; + if(unlikely((options & RRDR_OPTION_NONZERO) && !(r->od[c] & RRDR_NONZERO))) continue; + + if(i) buffer_strcat(wb, ", "); + i++; + + calculated_number *cn = &r->v[ (0) * r->d ]; + uint8_t *co = &r->o[ (0) * r->d ]; + + if(co[c] & RRDR_EMPTY) + buffer_strcat(wb, "null"); + else + buffer_rrd_value(wb, cn[c]); + } + } + if(!i) { + rows = 0; + buffer_strcat(wb, "null"); + } + + buffer_sprintf(wb, "],\n" + " %sdimensions%s: %d,\n" + " %spoints%s: %d,\n" + " %sformat%s: %s" + , kq, kq, i + , kq, kq, rows + , kq, kq, sq + ); + + rrdr_buffer_print_format(wb, format); + + buffer_sprintf(wb, "%s,\n" + " %sresult%s: " + , sq + , kq, kq + ); + + if(string_value) buffer_strcat(wb, sq); + //info("JSONWRAPPER(): %s: END", r->st->id); +} + +void rrdr_json_wrapper_end(RRDR *r, BUFFER *wb, uint32_t format, uint32_t options, int string_value) +{ + if(r) {;} + if(format) {;} + + char kq[2] = "", // key quote + sq[2] = ""; // string quote + + if( options & RRDR_OPTION_GOOGLE_JSON ) { + kq[0] = '\0'; + sq[0] = '\''; + } + else { + kq[0] = '"'; + sq[0] = '"'; + } + + if(string_value) buffer_strcat(wb, sq); + + buffer_sprintf(wb, ",\n %smin%s: ", kq, kq); + buffer_rrd_value(wb, r->min); + buffer_sprintf(wb, ",\n %smax%s: ", kq, kq); + buffer_rrd_value(wb, r->max); + buffer_strcat(wb, "\n}\n"); +} + +#define JSON_DATES_JS 1 +#define JSON_DATES_TIMESTAMP 2 + +static void rrdr2json(RRDR *r, BUFFER *wb, uint32_t options, int datatable) +{ + //info("RRD2JSON(): %s: BEGIN", r->st->id); + int row_annotations = 0, dates, dates_with_new = 0; + char kq[2] = "", // key quote + sq[2] = "", // string quote + pre_label[101] = "", // before each label + post_label[101] = "", // after each label + pre_date[101] = "", // the beginning of line, to the date + post_date[101] = "", // closing the date + pre_value[101] = "", // before each value + post_value[101] = "", // after each value + post_line[101] = "", // at the end of each row + normal_annotation[201] = "", // default row annotation + overflow_annotation[201] = "", // overflow row annotation + data_begin[101] = "", // between labels and values + finish[101] = ""; // at the end of everything + + if(datatable) { + dates = JSON_DATES_JS; + if( options & RRDR_OPTION_GOOGLE_JSON ) { + kq[0] = '\0'; + sq[0] = '\''; + } + else { + kq[0] = '"'; + sq[0] = '"'; + } + row_annotations = 1; + snprintf(pre_date, 100, " {%sc%s:[{%sv%s:%s", kq, kq, kq, kq, sq); + snprintf(post_date, 100, "%s}", sq); + snprintf(pre_label, 100, ",\n {%sid%s:%s%s,%slabel%s:%s", kq, kq, sq, sq, kq, kq, sq); + snprintf(post_label, 100, "%s,%spattern%s:%s%s,%stype%s:%snumber%s}", sq, kq, kq, sq, sq, kq, kq, sq, sq); + snprintf(pre_value, 100, ",{%sv%s:", kq, kq); + snprintf(post_value, 100, "}"); + snprintf(post_line, 100, "]}"); + snprintf(data_begin, 100, "\n ],\n %srows%s:\n [\n", kq, kq); + snprintf(finish, 100, "\n ]\n}"); + + snprintf(overflow_annotation, 200, ",{%sv%s:%sRESET OR OVERFLOW%s},{%sv%s:%sThe counters have been wrapped.%s}", kq, kq, sq, sq, kq, kq, sq, sq); + snprintf(normal_annotation, 200, ",{%sv%s:null},{%sv%s:null}", kq, kq, kq, kq); + + buffer_sprintf(wb, "{\n %scols%s:\n [\n", kq, kq, kq, kq); + buffer_sprintf(wb, " {%sid%s:%s%s,%slabel%s:%stime%s,%spattern%s:%s%s,%stype%s:%sdatetime%s},\n", kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq); + buffer_sprintf(wb, " {%sid%s:%s%s,%slabel%s:%s%s,%spattern%s:%s%s,%stype%s:%sstring%s,%sp%s:{%srole%s:%sannotation%s}},\n", kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, kq, kq, sq, sq); + buffer_sprintf(wb, " {%sid%s:%s%s,%slabel%s:%s%s,%spattern%s:%s%s,%stype%s:%sstring%s,%sp%s:{%srole%s:%sannotationText%s}}", kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, kq, kq, sq, sq); + + // remove the valueobjects flag + // google wants its own keys + if(options & RRDR_OPTION_OBJECTSROWS) + options &= ~RRDR_OPTION_OBJECTSROWS; + } + else { + kq[0] = '"'; + sq[0] = '"'; + if((options & RRDR_OPTION_SECONDS) || (options & RRDR_OPTION_MILLISECONDS)) { + dates = JSON_DATES_TIMESTAMP; + dates_with_new = 0; + } + else { + dates = JSON_DATES_JS; + dates_with_new = 1; + } + if( options & RRDR_OPTION_OBJECTSROWS ) + snprintf(pre_date, 100, " { "); + else + snprintf(pre_date, 100, " [ "); + snprintf(pre_label, 100, ", \""); + snprintf(post_label, 100, "\""); + snprintf(pre_value, 100, ", "); + if( options & RRDR_OPTION_OBJECTSROWS ) + snprintf(post_line, 100, "}"); + else + snprintf(post_line, 100, "]"); + snprintf(data_begin, 100, "],\n %sdata%s:\n [\n", kq, kq); + snprintf(finish, 100, "\n ]\n}"); + + buffer_sprintf(wb, "{\n %slabels%s: [", kq, kq); + buffer_sprintf(wb, "%stime%s", sq, sq); + } + + // ------------------------------------------------------------------------- + // print the JSON header + + long c, i; + RRDDIM *rd; + + // print the header lines + for(c = 0, i = 0, rd = r->st->dimensions; rd && c < r->d ;c++, rd = rd->next) { + if(unlikely(r->od[c] & RRDR_HIDDEN)) continue; + if(unlikely((options & RRDR_OPTION_NONZERO) && !(r->od[c] & RRDR_NONZERO))) continue; + + buffer_strcat(wb, pre_label); + buffer_strcat(wb, rd->name); + buffer_strcat(wb, post_label); + i++; + } + if(!i) { + buffer_strcat(wb, pre_label); + buffer_strcat(wb, "no data"); + buffer_strcat(wb, post_label); + } + + // print the begin of row data + buffer_strcat(wb, data_begin); + + // if all dimensions are hidden, print a null + if(!i) { + buffer_strcat(wb, finish); + return; + } + + long start = 0, end = rrdr_rows(r), step = 1; + if((options & RRDR_OPTION_REVERSED)) { + start = rrdr_rows(r) - 1; + end = -1; + step = -1; + } + + // for each line in the array + calculated_number total = 1; + for(i = start; i != end ;i += step) { + calculated_number *cn = &r->v[ i * r->d ]; + uint8_t *co = &r->o[ i * r->d ]; + + time_t now = r->t[i]; + + if(dates == JSON_DATES_JS) { + // generate the local date time + struct tm tmbuf, *tm = localtime_r(&now, &tmbuf); + if(!tm) { error("localtime_r() failed."); continue; } + + if(likely(i != start)) buffer_strcat(wb, ",\n"); + buffer_strcat(wb, pre_date); + + if( options & RRDR_OPTION_OBJECTSROWS ) + buffer_sprintf(wb, "%stime%s: ", kq, kq); + + if(dates_with_new) + buffer_strcat(wb, "new "); + + buffer_jsdate(wb, tm->tm_year + 1900, tm->tm_mon, tm->tm_mday, tm->tm_hour, tm->tm_min, tm->tm_sec); + + buffer_strcat(wb, post_date); + + if(row_annotations) { + // google supports one annotation per row + int annotation_found = 0; + for(c = 0, rd = r->st->dimensions; rd ;c++, rd = rd->next) { + if(co[c] & RRDR_RESET) { + buffer_strcat(wb, overflow_annotation); + annotation_found = 1; + break; + } + } + if(!annotation_found) + buffer_strcat(wb, normal_annotation); + } + } + else { + // print the timestamp of the line + if(likely(i != start)) buffer_strcat(wb, ",\n"); + buffer_strcat(wb, pre_date); + + if( options & RRDR_OPTION_OBJECTSROWS ) + buffer_sprintf(wb, "%stime%s: ", kq, kq); + + buffer_rrd_value(wb, (calculated_number)r->t[i]); + // in ms + if(options & RRDR_OPTION_MILLISECONDS) buffer_strcat(wb, "000"); + + buffer_strcat(wb, post_date); + } + + if(unlikely(options & RRDR_OPTION_PERCENTAGE)) { + total = 0; + for(c = 0, rd = r->st->dimensions; rd && c < r->d ;c++, rd = rd->next) { + calculated_number n = cn[c]; + + if(likely((options & RRDR_OPTION_ABSOLUTE) && n < 0)) + n = -n; + + total += n; + } + // prevent a division by zero + if(total == 0) total = 1; + } + + // for each dimension + for(c = 0, rd = r->st->dimensions; rd && c < r->d ;c++, rd = rd->next) { + if(unlikely(r->od[c] & RRDR_HIDDEN)) continue; + if(unlikely((options & RRDR_OPTION_NONZERO) && !(r->od[c] & RRDR_NONZERO))) continue; + + calculated_number n = cn[c]; + + buffer_strcat(wb, pre_value); + + if( options & RRDR_OPTION_OBJECTSROWS ) + buffer_sprintf(wb, "%s%s%s: ", kq, rd->name, kq); + + if(co[c] & RRDR_EMPTY) { + if(options & RRDR_OPTION_NULL2ZERO) + buffer_strcat(wb, "0"); + else + buffer_strcat(wb, "null"); + } + else { + if(unlikely((options & RRDR_OPTION_ABSOLUTE) && n < 0)) + n = -n; + + if(unlikely(options & RRDR_OPTION_PERCENTAGE)) + n = n * 100 / total; + + buffer_rrd_value(wb, n); + } + + buffer_strcat(wb, post_value); + } + + buffer_strcat(wb, post_line); + } + + buffer_strcat(wb, finish); + //info("RRD2JSON(): %s: END", r->st->id); +} + +static void rrdr2csv(RRDR *r, BUFFER *wb, uint32_t options, const char *startline, const char *separator, const char *endline, const char *betweenlines) +{ + //info("RRD2CSV(): %s: BEGIN", r->st->id); + long c, i; + RRDDIM *d; + + // print the csv header + for(c = 0, i = 0, d = r->st->dimensions; d && c < r->d ;c++, d = d->next) { + if(unlikely(r->od[c] & RRDR_HIDDEN)) continue; + if(unlikely((options & RRDR_OPTION_NONZERO) && !(r->od[c] & RRDR_NONZERO))) continue; + + if(!i) { + buffer_strcat(wb, startline); + if(options & RRDR_OPTION_LABEL_QUOTES) buffer_strcat(wb, "\""); + buffer_strcat(wb, "time"); + if(options & RRDR_OPTION_LABEL_QUOTES) buffer_strcat(wb, "\""); + } + buffer_strcat(wb, separator); + if(options & RRDR_OPTION_LABEL_QUOTES) buffer_strcat(wb, "\""); + buffer_strcat(wb, d->name); + if(options & RRDR_OPTION_LABEL_QUOTES) buffer_strcat(wb, "\""); + i++; + } + buffer_strcat(wb, endline); + + if(!i) { + // no dimensions present + return; + } + + long start = 0, end = rrdr_rows(r), step = 1; + if((options & RRDR_OPTION_REVERSED)) { + start = rrdr_rows(r) - 1; + end = -1; + step = -1; + } + + // for each line in the array + calculated_number total = 1; + for(i = start; i != end ;i += step) { + calculated_number *cn = &r->v[ i * r->d ]; + uint8_t *co = &r->o[ i * r->d ]; + + buffer_strcat(wb, betweenlines); + buffer_strcat(wb, startline); + + time_t now = r->t[i]; + + if((options & RRDR_OPTION_SECONDS) || (options & RRDR_OPTION_MILLISECONDS)) { + // print the timestamp of the line + buffer_rrd_value(wb, (calculated_number)now); + // in ms + if(options & RRDR_OPTION_MILLISECONDS) buffer_strcat(wb, "000"); + } + else { + // generate the local date time + struct tm tmbuf, *tm = localtime_r(&now, &tmbuf); + if(!tm) { error("localtime() failed."); continue; } + buffer_date(wb, tm->tm_year + 1900, tm->tm_mon, tm->tm_mday, tm->tm_hour, tm->tm_min, tm->tm_sec); + } + + if(unlikely(options & RRDR_OPTION_PERCENTAGE)) { + total = 0; + for(c = 0, d = r->st->dimensions; d && c < r->d ;c++, d = d->next) { + calculated_number n = cn[c]; + + if(likely((options & RRDR_OPTION_ABSOLUTE) && n < 0)) + n = -n; + + total += n; + } + // prevent a division by zero + if(total == 0) total = 1; + } + + // for each dimension + for(c = 0, d = r->st->dimensions; d && c < r->d ;c++, d = d->next) { + if(unlikely(r->od[c] & RRDR_HIDDEN)) continue; + if(unlikely((options & RRDR_OPTION_NONZERO) && !(r->od[c] & RRDR_NONZERO))) continue; + + buffer_strcat(wb, separator); + + calculated_number n = cn[c]; + + if(co[c] & RRDR_EMPTY) { + if(options & RRDR_OPTION_NULL2ZERO) + buffer_strcat(wb, "0"); + else + buffer_strcat(wb, "null"); + } + else { + if(unlikely((options & RRDR_OPTION_ABSOLUTE) && n < 0)) + n = -n; + + if(unlikely(options & RRDR_OPTION_PERCENTAGE)) + n = n * 100 / total; + + buffer_rrd_value(wb, n); + } + } + + buffer_strcat(wb, endline); + } + //info("RRD2CSV(): %s: END", r->st->id); +} + +static void rrdr2ssv(RRDR *r, BUFFER *wb, uint32_t options, const char *prefix, const char *separator, const char *suffix) +{ + //info("RRD2SSV(): %s: BEGIN", r->st->id); + long c, i; + RRDDIM *d; + + buffer_strcat(wb, prefix); + long start = 0, end = rrdr_rows(r), step = 1; + if((options & RRDR_OPTION_REVERSED)) { + start = rrdr_rows(r) - 1; + end = -1; + step = -1; + } + + // for each line in the array + calculated_number total = 1; + for(i = start; i != end ;i += step) { + + calculated_number *cn = &r->v[ i * r->d ]; + uint8_t *co = &r->o[ i * r->d ]; + + calculated_number sum = 0, min = 0, max = 0, v; + int all_null = 1, init = 1; + + if(unlikely(options & RRDR_OPTION_PERCENTAGE)) { + total = 0; + for(c = 0, d = r->st->dimensions; d && c < r->d ;c++, d = d->next) { + calculated_number n = cn[c]; + + if(likely((options & RRDR_OPTION_ABSOLUTE) && n < 0)) + n = -n; + + total += n; + } + // prevent a division by zero + if(total == 0) total = 1; + } + + // for each dimension + for(c = 0, d = r->st->dimensions; d && c < r->d ;c++, d = d->next) { + if(unlikely(r->od[c] & RRDR_HIDDEN)) continue; + if(unlikely((options & RRDR_OPTION_NONZERO) && !(r->od[c] & RRDR_NONZERO))) continue; + + calculated_number n = cn[c]; + + if(likely((options & RRDR_OPTION_ABSOLUTE) && n < 0)) + n = -n; + + if(unlikely(options & RRDR_OPTION_PERCENTAGE)) + n = n * 100 / total; + + if(unlikely(init)) { + if(n > 0) { + min = 0; + max = n; + } + else { + min = n; + max = 0; + } + init = 0; + } + + if(likely(!(co[c] & RRDR_EMPTY))) { + all_null = 0; + sum += n; + } + + if(n < min) min = n; + if(n > max) max = n; + } + + if(likely(i != start)) + buffer_strcat(wb, separator); + + if(all_null) { + if(options & RRDR_OPTION_NULL2ZERO) + buffer_strcat(wb, "0"); + else + buffer_strcat(wb, "null"); + } + else { + if(options & RRDR_OPTION_MIN2MAX) + v = max - min; + else + v = sum; + + if(likely(i != start)) { + if(r->min > v) r->min = v; + if(r->max < v) r->max = v; + } + else { + r->min = v; + r->max = v; + } + + buffer_rrd_value(wb, v); + } + } + buffer_strcat(wb, suffix); + //info("RRD2SSV(): %s: END", r->st->id); +} inline static calculated_number *rrdr_line_values(RRDR *r) { @@ -258,9 +1100,10 @@ inline static uint8_t *rrdr_line_options(RRDR *r) inline static int rrdr_line_init(RRDR *r, time_t t) { r->c++; + if(unlikely(r->c >= r->n)) { + error("requested to step above RRDR size for chart %s", r->st->name); r->c = r->n - 1; - return 0; } // save the time @@ -306,7 +1149,13 @@ inline static void rrdr_free(RRDR *r) free(r); } -static RRDR *rrdr_create(RRDSET *st, int n) +inline void rrdr_done(RRDR *r) +{ + r->rows = r->c + 1; + r->c = 0; +} + +static RRDR *rrdr_create(RRDSET *st, long n) { if(unlikely(!st)) { error("NULL value given!"); @@ -324,8 +1173,6 @@ static RRDR *rrdr_create(RRDSET *st, int n) for(rd = st->dimensions ; rd ; rd = rd->next) r->d++; r->n = n; - r->t = malloc(n * sizeof(time_t)); - if(unlikely(!r->t)) goto cleanup; r->t = malloc(n * sizeof(time_t)); if(unlikely(!r->t)) goto cleanup; @@ -336,29 +1183,56 @@ static RRDR *rrdr_create(RRDSET *st, int n) r->o = malloc(n * r->d * sizeof(uint8_t)); if(unlikely(!r->o)) goto cleanup; - r->od = calloc(r->d, sizeof(uint8_t)); + r->od = malloc(r->d * sizeof(uint8_t)); if(unlikely(!r->od)) goto cleanup; + // set the hidden flag on hidden dimensions + int c; + for(c = 0, rd = st->dimensions ; rd ; c++, rd = rd->next) { + if(unlikely(rd->flags & RRDDIM_FLAG_HIDDEN)) r->od[c] = RRDR_HIDDEN; + else r->od[c] = 0; + } + r->c = -1; + r->group = 1; + r->update_every = 1; + return r; cleanup: - error("Cannot allocate memory"); + error("Cannot allocate RRDR memory for %d entries", n); if(likely(r)) rrdr_free(r); return NULL; } -RRDR *rrd2rrdr(RRDSET *st, long points, time_t after, time_t before, int group_method) +RRDR *rrd2rrdr(RRDSET *st, long points, long long after, long long before, int group_method) { int debug = st->debug; + int absolute_period_requested = -1; time_t first_entry_t = rrdset_first_entry_t(st); time_t last_entry_t = rrdset_last_entry_t(st); + if(before == 0 && after == 0) { + before = last_entry_t; + after = first_entry_t; + absolute_period_requested = 0; + } + // allow relative for before and after - if(before <= st->update_every * st->entries) before = last_entry_t + before; - if(after <= st->update_every * st->entries) after = last_entry_t + after; + if(before <= st->update_every * st->entries) { + before = last_entry_t + before; + absolute_period_requested = 0; + } + + if(after <= st->update_every * st->entries) { + after = last_entry_t + after; + absolute_period_requested = 0; + } + + if(absolute_period_requested == -1) + absolute_period_requested = 1; // make sure they are within our timeframe if(before > last_entry_t) before = last_entry_t; @@ -369,24 +1243,68 @@ RRDR *rrd2rrdr(RRDSET *st, long points, time_t after, time_t before, int group_m // check if they are upside down if(after > before) { - time_t t = before; + time_t tmp = before; before = after; - after = t; + after = tmp; } // the duration of the chart time_t duration = before - after; - if(duration <= 0) return NULL; + long available_points = duration / st->update_every; - // check the required points - if(points <= 0) points = duration; + if(duration <= 0 || available_points <= 0) + return rrdr_create(st, 1); + + // check the wanted points + if(points < 0) points = -points; + if(points > available_points) points = available_points; + if(points == 0) points = available_points; // calculate proper grouping of source data - long group = duration / points; + long group = available_points / points; if(group <= 0) group = 1; - if(duration / group > points) group++; - // error("NEW: points=%d after=%d before=%d group=%d, duration=%d", points, after, before, group, duration); + // round group to the closest integer + if(available_points % points > points / 2) group++; + + time_t after_new = after - (after % (group * st->update_every)); + time_t before_new = before - (before % (group * st->update_every)); + long points_new = (before_new - after_new) / st->update_every / group; + + // find the starting and ending slots in our round robin db + long start_at_slot = rrdset_time2slot(st, before_new), + stop_at_slot = rrdset_time2slot(st, after_new); + +#ifdef NETDATA_INTERNAL_CHECKS + if(after_new < first_entry_t) { + error("after_new %u is too small, minimum %u", after_new, first_entry_t); + } + if(after_new > last_entry_t) { + error("after_new %u is too big, maximum %u", after_new, last_entry_t); + } + if(before_new < first_entry_t) { + error("before_new %u is too small, minimum %u", before_new, first_entry_t); + } + if(before_new > last_entry_t) { + error("before_new %u is too big, maximum %u", before_new, last_entry_t); + } + if(start_at_slot < 0 || start_at_slot >= st->entries) { + error("start_at_slot is invalid %ld, expected %ld to %ld", start_at_slot, 0, st->entries - 1); + } + if(stop_at_slot < 0 || stop_at_slot >= st->entries) { + error("stop_at_slot is invalid %ld, expected %ld to %ld", stop_at_slot, 0, st->entries - 1); + } + if(points_new > (before_new - after_new) / group / st->update_every + 1) { + error("points_new %ld is more than points %ld", points_new, (before_new - after_new) / group / st->update_every + 1); + } +#endif + + //info("RRD2RRDR(): %s: wanted %ld points, got %ld - group=%ld, wanted duration=%u, got %u - wanted %ld - %ld, got %ld - %ld", st->id, points, points_new, group, before - after, before_new - after_new, after, before, after_new, before_new); + + after = after_new; + before = before_new; + duration = before - after; + points = points_new; // Now we have: // before = the end time of the calculation @@ -401,11 +1319,23 @@ RRDR *rrd2rrdr(RRDSET *st, long points, time_t after, time_t before, int group_m // initialize our result set RRDR *r = rrdr_create(st, points); - if(!r) return NULL; - if(!r->d) { - rrdr_free(r); + if(!r) { +#ifdef NETDATA_INTERNAL_CHECKS + error("Cannot create RRDR for %s, after=%u, before=%u, duration=%u, points=%d", st->id, after, before, duration, points); +#endif return NULL; } + if(!r->d) { +#ifdef NETDATA_INTERNAL_CHECKS + error("Returning empty RRDR (no dimensions in RRDSET) for %s, after=%u, before=%u, duration=%u, points=%d", st->id, after, before, duration, points); +#endif + return r; + } + + if(absolute_period_requested == 1) + r->result_options |= RRDR_RESULT_OPTION_ABSOLUTE; + else + r->result_options |= RRDR_RESULT_OPTION_RELATIVE; // find how many dimensions we have long dimensions = r->d; @@ -450,36 +1380,37 @@ RRDR *rrd2rrdr(RRDSET *st, long points, time_t after, time_t before, int group_m // ------------------------------------------------------------------------- // the main loop - long t = rrdset_time2slot(st, before), // rrdset_last_slot(st), - stop_at_t = rrdset_time2slot(st, after), - added = 0, - group_count = 0, - add_this = 0, - stop_now = 0; - - // align to group for proper panning of data - t -= t % group; - - time_t now = rrdset_slot2time(st, t), + time_t now = rrdset_slot2time(st, start_at_slot), dt = st->update_every, group_start_t = 0; - if(debug) debug(D_RRD_STATS, "BEGIN %s after_t: %lu (stop slot %ld), before_t: %lu (start slot %ld), start_t(now): %lu" + if(unlikely(debug)) debug(D_RRD_STATS, "BEGIN %s after_t: %lu (stop_at_t: %ld), before_t: %lu (start_at_t: %ld), start_t(now): %lu, current_entry: %ld, entries: %ld" , st->id , after - , stop_at_t + , stop_at_slot , before - , t + , start_at_slot , now + , st->current_entry + , st->entries ); - for( ; !stop_now ; now -= dt, t--) { - if(unlikely(t < 0)) c = st->entries - 1; - if(t == stop_at_t) stop_now = 1; + r->group = group; + r->update_every = group * st->update_every; + r->before = now; + r->after = now; + + //info("RRD2RRDR(): %s: STARTING", st->id); - if(debug) debug(D_RRD_STATS, "ROW %s c: %ld, group_count: %ld, added: %ld, now: %lu, %s %s" + long slot = start_at_slot, counter = 0, stop_now = 0, added = 0, group_count = 0, add_this = 0; + for(; !stop_now ; now -= dt, slot--, counter++) { + if(unlikely(slot < 0)) slot = st->entries - 1; + if(unlikely(slot == stop_at_slot)) stop_now = counter; + + if(unlikely(debug)) debug(D_RRD_STATS, "ROW %s slot: %ld, entries_counter: %ld, group_count: %ld, added: %ld, now: %lu, %s %s" , st->id - , t + , slot + , counter , group_count + 1 , added , now @@ -491,7 +1422,9 @@ RRDR *rrd2rrdr(RRDSET *st, long points, time_t after, time_t before, int group_m if(unlikely(now > before)) continue; if(unlikely(now < after)) break; - if(group_count == 0) group_start_t = now; + if(unlikely(group_count == 0)) { + group_start_t = now; + } group_count++; if(unlikely(group_count == group)) { @@ -500,14 +1433,14 @@ RRDR *rrd2rrdr(RRDSET *st, long points, time_t after, time_t before, int group_m } // do the calculations - for(rd = st->dimensions, c = 0 ; likely(rd && c < dimensions) ; rd = rd->next, c++) { - storage_number n = rd->values[t]; + for(rd = st->dimensions, c = 0 ; rd && c < dimensions ; rd = rd->next, c++) { + storage_number n = rd->values[slot]; if(unlikely(!does_storage_number_exist(n))) continue; group_counts[c]++; calculated_number value = unpack_storage_number(n); - if(value != 0.0) { + if(likely(value != 0.0)) { group_options[c] |= RRDR_NONZERO; found_non_zero[c] = 1; } @@ -531,22 +1464,23 @@ RRDR *rrd2rrdr(RRDSET *st, long points, time_t after, time_t before, int group_m // added it if(unlikely(add_this)) { - if(!rrdr_line_init(r, group_start_t)) break; + if(unlikely(!rrdr_line_init(r, group_start_t))) break; + + r->after = now; calculated_number *cn = rrdr_line_values(r); uint8_t *co = rrdr_line_options(r); - for(rd = st->dimensions, c = 0 ; likely(rd && c < dimensions) ; rd = rd->next, c++) { + for(rd = st->dimensions, c = 0 ; rd && c < dimensions ; rd = rd->next, c++) { // update the dimension options - if(found_non_zero[c]) r->od[c] |= RRDR_NONZERO; - if(rd->flags & RRDDIM_FLAG_HIDDEN) r->od[c] |= RRDR_HIDDEN; + if(likely(found_non_zero[c])) r->od[c] |= RRDR_NONZERO; // store the specific point options co[c] = group_options[c]; // store the value - if(group_counts[c] == 0) { + if(unlikely(group_counts[c] == 0)) { cn[c] = 0.0; co[c] |= RRDR_EMPTY; } @@ -557,6 +1491,9 @@ RRDR *rrd2rrdr(RRDSET *st, long points, time_t after, time_t before, int group_m cn[c] = group_values[c]; } + if(cn[c] < r->min) r->min = cn[c]; + if(cn[c] > r->max) r->max = cn[c]; + // reset them for the next loop group_values[c] = 0; group_counts[c] = 0; @@ -569,12 +1506,187 @@ RRDR *rrd2rrdr(RRDSET *st, long points, time_t after, time_t before, int group_m } } - rrdr_dump(r); + rrdr_done(r); + //info("RRD2RRDR(): %s: END %ld loops made, %ld points generated", st->id, counter, rrdr_rows(r)); + //error("SHIFT: %s: wanted %ld points, got %ld", st->id, points, rrdr_rows(r)); + return r; +} + +int rrd2format(RRDSET *st, BUFFER *wb, BUFFER *dimensions, uint32_t format, long points, long long after, long long before, int group_method, uint32_t options, time_t *latest_timestamp) +{ + RRDR *r = rrd2rrdr(st, points, after, before, group_method); + if(!r) { + buffer_strcat(wb, "Cannot generate output with these parameters on this chart."); + return 500; + } + + if(r->result_options & RRDR_RESULT_OPTION_RELATIVE) + wb->options |= WB_CONTENT_NO_CACHEABLE; + else if(r->result_options & RRDR_RESULT_OPTION_ABSOLUTE) + wb->options |= WB_CONTENT_CACHEABLE; + + options = rrdr_check_options(r, options, (dimensions)?buffer_tostring(dimensions):NULL); + + if(dimensions) + rrdr_disable_not_selected_dimensions(r, buffer_tostring(dimensions)); + + if(latest_timestamp && rrdr_rows(r) > 0) + *latest_timestamp = r->before; + + switch(format) { + case DATASOURCE_SSV: + if(options & RRDR_OPTION_JSON_WRAP) { + wb->contenttype = CT_APPLICATION_JSON; + rrdr_json_wrapper_begin(r, wb, format, options, 1); + rrdr2ssv(r, wb, options, "", " ", ""); + rrdr_json_wrapper_end(r, wb, format, options, 1); + } + else { + wb->contenttype = CT_TEXT_PLAIN; + rrdr2ssv(r, wb, options, "", " ", ""); + } + break; + + case DATASOURCE_SSV_COMMA: + if(options & RRDR_OPTION_JSON_WRAP) { + wb->contenttype = CT_APPLICATION_JSON; + rrdr_json_wrapper_begin(r, wb, format, options, 1); + rrdr2ssv(r, wb, options, "", ",", ""); + rrdr_json_wrapper_end(r, wb, format, options, 1); + } + else { + wb->contenttype = CT_TEXT_PLAIN; + rrdr2ssv(r, wb, options, "", ",", ""); + } + break; + + case DATASOURCE_JS_ARRAY: + if(options & RRDR_OPTION_JSON_WRAP) { + wb->contenttype = CT_APPLICATION_JSON; + rrdr_json_wrapper_begin(r, wb, format, options, 0); + rrdr2ssv(r, wb, options, "[", ",", "]"); + rrdr_json_wrapper_end(r, wb, format, options, 0); + } + else { + wb->contenttype = CT_APPLICATION_JSON; + rrdr2ssv(r, wb, options, "[", ",", "]"); + } + break; + + case DATASOURCE_CSV: + if(options & RRDR_OPTION_JSON_WRAP) { + wb->contenttype = CT_APPLICATION_JSON; + rrdr_json_wrapper_begin(r, wb, format, options, 1); + rrdr2csv(r, wb, options, "", ",", "\\n", ""); + rrdr_json_wrapper_end(r, wb, format, options, 1); + } + else { + wb->contenttype = CT_TEXT_PLAIN; + rrdr2csv(r, wb, options, "", ",", "\r\n", ""); + } + break; + + case DATASOURCE_CSV_JSON_ARRAY: + wb->contenttype = CT_APPLICATION_JSON; + if(options & RRDR_OPTION_JSON_WRAP) { + rrdr_json_wrapper_begin(r, wb, format, options, 0); + buffer_strcat(wb, "[\n"); + rrdr2csv(r, wb, options + RRDR_OPTION_LABEL_QUOTES, "[", ",", "]", ",\n"); + buffer_strcat(wb, "\n]"); + rrdr_json_wrapper_end(r, wb, format, options, 0); + } + else { + wb->contenttype = CT_TEXT_PLAIN; + buffer_strcat(wb, "[\n"); + rrdr2csv(r, wb, options + RRDR_OPTION_LABEL_QUOTES, "[", ",", "]", ",\n"); + buffer_strcat(wb, "\n]"); + } + break; + + case DATASOURCE_TSV: + if(options & RRDR_OPTION_JSON_WRAP) { + wb->contenttype = CT_APPLICATION_JSON; + rrdr_json_wrapper_begin(r, wb, format, options, 1); + rrdr2csv(r, wb, options, "", "\t", "\\n", ""); + rrdr_json_wrapper_end(r, wb, format, options, 1); + } + else { + wb->contenttype = CT_TEXT_PLAIN; + rrdr2csv(r, wb, options, "", "\t", "\r\n", ""); + } + break; + + case DATASOURCE_HTML: + if(options & RRDR_OPTION_JSON_WRAP) { + wb->contenttype = CT_APPLICATION_JSON; + rrdr_json_wrapper_begin(r, wb, format, options, 1); + buffer_strcat(wb, "\\n
\\n\\n"); + rrdr2csv(r, wb, options, "\\n", ""); + buffer_strcat(wb, "
", "", "
\\n
\\n\\n"); + rrdr_json_wrapper_end(r, wb, format, options, 1); + } + else { + wb->contenttype = CT_TEXT_HTML; + buffer_strcat(wb, "\n
\n\n"); + rrdr2csv(r, wb, options, "\n", ""); + buffer_strcat(wb, "
", "", "
\n
\n\n"); + } + break; + + case DATASOURCE_DATATABLE_JSONP: + wb->contenttype = CT_APPLICATION_X_JAVASCRIPT; + + if(options & RRDR_OPTION_JSON_WRAP) + rrdr_json_wrapper_begin(r, wb, format, options, 0); + + rrdr2json(r, wb, options, 1); + + if(options & RRDR_OPTION_JSON_WRAP) + rrdr_json_wrapper_end(r, wb, format, options, 0); + break; + + case DATASOURCE_DATATABLE_JSON: + wb->contenttype = CT_APPLICATION_JSON; + + if(options & RRDR_OPTION_JSON_WRAP) + rrdr_json_wrapper_begin(r, wb, format, options, 0); + + rrdr2json(r, wb, options, 1); + + if(options & RRDR_OPTION_JSON_WRAP) + rrdr_json_wrapper_end(r, wb, format, options, 0); + break; + + case DATASOURCE_JSONP: + wb->contenttype = CT_APPLICATION_X_JAVASCRIPT; + if(options & RRDR_OPTION_JSON_WRAP) + rrdr_json_wrapper_begin(r, wb, format, options, 0); + + rrdr2json(r, wb, options, 0); + + if(options & RRDR_OPTION_JSON_WRAP) + rrdr_json_wrapper_end(r, wb, format, options, 0); + break; + + case DATASOURCE_JSON: + default: + wb->contenttype = CT_APPLICATION_JSON; + + if(options & RRDR_OPTION_JSON_WRAP) + rrdr_json_wrapper_begin(r, wb, format, options, 0); + + rrdr2json(r, wb, options, 0); + + if(options & RRDR_OPTION_JSON_WRAP) + rrdr_json_wrapper_end(r, wb, format, options, 0); + break; + } + rrdr_free(r); - return NULL; + return 200; } -unsigned long rrd_stats_json(int type, RRDSET *st, struct web_buffer *wb, int points, int group, int group_method, time_t after, time_t before, int only_non_zero) +time_t rrd_stats_json(int type, RRDSET *st, BUFFER *wb, long points, long group, int group_method, time_t after, time_t before, int only_non_zero) { int c; pthread_rwlock_rdlock(&st->rwlock); @@ -586,8 +1698,8 @@ unsigned long rrd_stats_json(int type, RRDSET *st, struct web_buffer *wb, int po char kq[2] = "\""; char sq[2] = "\""; switch(type) { - case DATASOURCE_GOOGLE_JSON: - case DATASOURCE_GOOGLE_JSONP: + case DATASOURCE_DATATABLE_JSON: + case DATASOURCE_DATATABLE_JSONP: kq[0] = '\0'; sq[0] = '\''; break; @@ -622,7 +1734,7 @@ unsigned long rrd_stats_json(int type, RRDSET *st, struct web_buffer *wb, int po for( rd = st->dimensions ; rd ; rd = rd->next) dimensions++; if(!dimensions) { pthread_rwlock_unlock(&st->rwlock); - web_buffer_printf(wb, "No dimensions yet."); + buffer_strcat(wb, "No dimensions yet."); return 0; } @@ -694,10 +1806,10 @@ unsigned long rrd_stats_json(int type, RRDSET *st, struct web_buffer *wb, int po // ------------------------------------------------------------------------- // print the JSON header - web_buffer_printf(wb, "{\n %scols%s:\n [\n", kq, kq); - web_buffer_printf(wb, " {%sid%s:%s%s,%slabel%s:%stime%s,%spattern%s:%s%s,%stype%s:%sdatetime%s},\n", kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq); - web_buffer_printf(wb, " {%sid%s:%s%s,%slabel%s:%s%s,%spattern%s:%s%s,%stype%s:%sstring%s,%sp%s:{%srole%s:%sannotation%s}},\n", kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, kq, kq, sq, sq); - web_buffer_printf(wb, " {%sid%s:%s%s,%slabel%s:%s%s,%spattern%s:%s%s,%stype%s:%sstring%s,%sp%s:{%srole%s:%sannotationText%s}}", kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, kq, kq, sq, sq); + buffer_sprintf(wb, "{\n %scols%s:\n [\n", kq, kq); + buffer_sprintf(wb, " {%sid%s:%s%s,%slabel%s:%stime%s,%spattern%s:%s%s,%stype%s:%sdatetime%s},\n", kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq); + buffer_sprintf(wb, " {%sid%s:%s%s,%slabel%s:%s%s,%spattern%s:%s%s,%stype%s:%sstring%s,%sp%s:{%srole%s:%sannotation%s}},\n", kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, kq, kq, sq, sq); + buffer_sprintf(wb, " {%sid%s:%s%s,%slabel%s:%s%s,%spattern%s:%s%s,%stype%s:%sstring%s,%sp%s:{%srole%s:%sannotationText%s}}", kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, kq, kq, sq, sq); // print the header for each dimension // and update the print_hidden array for the dimensions that should be hidden @@ -705,15 +1817,15 @@ unsigned long rrd_stats_json(int type, RRDSET *st, struct web_buffer *wb, int po for( rd = st->dimensions, c = 0 ; rd && c < dimensions ; rd = rd->next, c++) { if(!print_hidden[c]) { pc++; - web_buffer_printf(wb, ",\n {%sid%s:%s%s,%slabel%s:%s%s%s,%spattern%s:%s%s,%stype%s:%snumber%s}", kq, kq, sq, sq, kq, kq, sq, rd->name, sq, kq, kq, sq, sq, kq, kq, sq, sq); + buffer_sprintf(wb, ",\n {%sid%s:%s%s,%slabel%s:%s%s%s,%spattern%s:%s%s,%stype%s:%snumber%s}", kq, kq, sq, sq, kq, kq, sq, rd->name, sq, kq, kq, sq, sq, kq, kq, sq, sq); } } if(!pc) { - web_buffer_printf(wb, ",\n {%sid%s:%s%s,%slabel%s:%s%s%s,%spattern%s:%s%s,%stype%s:%snumber%s}", kq, kq, sq, sq, kq, kq, sq, "no data", sq, kq, kq, sq, sq, kq, kq, sq, sq); + buffer_sprintf(wb, ",\n {%sid%s:%s%s,%slabel%s:%s%s%s,%spattern%s:%s%s,%stype%s:%snumber%s}", kq, kq, sq, sq, kq, kq, sq, "no data", sq, kq, kq, sq, sq, kq, kq, sq, sq); } // print the begin of row data - web_buffer_printf(wb, "\n ],\n %srows%s:\n [\n", kq, kq); + buffer_sprintf(wb, "\n ],\n %srows%s:\n [\n", kq, kq); // ------------------------------------------------------------------------- @@ -722,9 +1834,6 @@ unsigned long rrd_stats_json(int type, RRDSET *st, struct web_buffer *wb, int po int annotate_reset = 0; int annotation_count = 0; - // the minimum line length we expect - int line_size = 4096 + (dimensions * 200); - long t = rrdset_time2slot(st, before), stop_at_t = rrdset_time2slot(st, after), stop_now = 0; @@ -750,9 +1859,10 @@ unsigned long rrd_stats_json(int type, RRDSET *st, struct web_buffer *wb, int po , stop_at_t ); - for(; !stop_now ; now -= dt, t--) { + long counter = 0; + for(; !stop_now ; now -= dt, t--, counter++) { if(t < 0) t = st->entries - 1; - if(t == stop_at_t) stop_now = 1; + if(t == stop_at_t) stop_now = counter; int print_this = 0; @@ -785,18 +1895,15 @@ unsigned long rrd_stats_json(int type, RRDSET *st, struct web_buffer *wb, int po break; } - // check if we may exceed the buffer provided - web_buffer_increase(wb, line_size); - // generate the local date time - struct tm *tm = localtime(&now); + struct tm tmbuf, *tm = localtime_r(&now, &tmbuf); if(!tm) { error("localtime() failed."); continue; } if(now > last_timestamp) last_timestamp = now; - if(printed) web_buffer_strcpy(wb, "]},\n"); - web_buffer_strcpy(wb, pre_date); - web_buffer_jsdate(wb, tm->tm_year + 1900, tm->tm_mon, tm->tm_mday, tm->tm_hour, tm->tm_min, tm->tm_sec); - web_buffer_strcpy(wb, post_date); + if(printed) buffer_strcat(wb, "]},\n"); + buffer_strcat(wb, pre_date); + buffer_jsdate(wb, tm->tm_year + 1900, tm->tm_mon, tm->tm_mday, tm->tm_hour, tm->tm_min, tm->tm_sec); + buffer_strcat(wb, post_date); print_this = 1; } @@ -832,26 +1939,26 @@ unsigned long rrd_stats_json(int type, RRDSET *st, struct web_buffer *wb, int po if(print_this) { if(annotate_reset) { annotation_count++; - web_buffer_strcpy(wb, overflow_annotation); + buffer_strcat(wb, overflow_annotation); annotate_reset = 0; } else - web_buffer_strcpy(wb, normal_annotation); + buffer_strcat(wb, normal_annotation); pc = 0; for(c = 0 ; c < dimensions ; c++) { if(found_non_existing[c] == group_count) { // all entries are non-existing pc++; - web_buffer_strcpy(wb, pre_value); - web_buffer_strcpy(wb, "null"); - web_buffer_strcpy(wb, post_value); + buffer_strcat(wb, pre_value); + buffer_strcat(wb, "null"); + buffer_strcat(wb, post_value); } else if(!print_hidden[c]) { pc++; - web_buffer_strcpy(wb, pre_value); - web_buffer_rrd_value(wb, group_values[c]); - web_buffer_strcpy(wb, post_value); + buffer_strcat(wb, pre_value); + buffer_rrd_value(wb, group_values[c]); + buffer_strcat(wb, post_value); if(group_values[c]) found_non_zero[c]++; } @@ -863,9 +1970,9 @@ unsigned long rrd_stats_json(int type, RRDSET *st, struct web_buffer *wb, int po // if all dimensions are hidden, print a null if(!pc) { - web_buffer_strcpy(wb, pre_value); - web_buffer_strcpy(wb, "null"); - web_buffer_strcpy(wb, post_value); + buffer_strcat(wb, pre_value); + buffer_strcat(wb, "null"); + buffer_strcat(wb, post_value); } printed++; @@ -873,8 +1980,8 @@ unsigned long rrd_stats_json(int type, RRDSET *st, struct web_buffer *wb, int po } } - if(printed) web_buffer_printf(wb, "]}"); - web_buffer_printf(wb, "\n ]\n}\n"); + if(printed) buffer_strcat(wb, "]}"); + buffer_strcat(wb, "\n ]\n}\n"); if(only_non_zero && max_loop > 1) { int changed = 0; @@ -888,14 +1995,14 @@ unsigned long rrd_stats_json(int type, RRDSET *st, struct web_buffer *wb, int po } } - if(changed) web_buffer_reset(wb); + if(changed) buffer_flush(wb); else break; } else break; } // max_loop - debug(D_RRD_STATS, "RRD_STATS_JSON: %s total %ld bytes", st->name, wb->bytes); + debug(D_RRD_STATS, "RRD_STATS_JSON: %s total %ld bytes", st->name, wb->len); pthread_rwlock_unlock(&st->rwlock); return last_timestamp;