]> arthur.barton.de Git - netdata.git/blobdiff - src/rrd2json.c
allow netdata to cleanup orphan hosts that have not pushed any metrics for some time...
[netdata.git] / src / rrd2json.c
index ac04ca7213e27dd66271651997a93948d2dea825..84db40433245964aa1be15cfbf422d81a688780d 100644 (file)
@@ -1,8 +1,8 @@
 #include "common.h"
 
-void rrd_stats_api_v1_chart(RRDSET *st, BUFFER *wb)
+void rrd_stats_api_v1_chart_with_data(RRDSET *st, BUFFER *wb, size_t *dimensions_count, size_t *memory_used)
 {
-    pthread_rwlock_rdlock(&st->rwlock);
+    rrdset_rdlock(st);
 
     buffer_sprintf(wb,
         "\t\t{\n"
@@ -29,7 +29,7 @@ void rrd_stats_api_v1_chart(RRDSET *st, BUFFER *wb)
         , st->context
         , st->title
         , st->priority
-        , st->enabled?"true":"false"
+        , rrdset_flag_check(st, RRDSET_FLAG_ENABLED)?"true":"false"
         , st->units
         , st->name
         , rrdset_type_name(st->chart_type)
@@ -41,24 +41,27 @@ void rrd_stats_api_v1_chart(RRDSET *st, BUFFER *wb)
 
     unsigned long memory = st->memsize;
 
-    int c = 0;
+    size_t dimensions = 0;
     RRDDIM *rd;
-    for(rd = st->dimensions; rd ; rd = rd->next) {
-        if(rd->flags & RRDDIM_FLAG_HIDDEN) continue;
+    rrddim_foreach_read(rd, st) {
+        if(rrddim_flag_check(rd, RRDDIM_FLAG_HIDDEN)) continue;
 
         memory += rd->memsize;
 
         buffer_sprintf(wb,
             "%s"
             "\t\t\t\t\"%s\": { \"name\": \"%s\" }"
-            , c?",\n":""
+            , dimensions?",\n":""
             , rd->id
             , rd->name
             );
 
-        c++;
+        dimensions++;
     }
 
+    if(dimensions_count) *dimensions_count += dimensions;
+    if(memory_used) *memory_used += memory;
+
     buffer_strcat(wb, "\n\t\t\t},\n\t\t\t\"green\": ");
     buffer_rrd_value(wb, st->green);
     buffer_strcat(wb, ",\n\t\t\t\"red\": ");
@@ -68,46 +71,233 @@ void rrd_stats_api_v1_chart(RRDSET *st, BUFFER *wb)
         "\n\t\t}"
         );
 
-    pthread_rwlock_unlock(&st->rwlock);
+    rrdset_unlock(st);
+}
+
+void rrd_stats_api_v1_chart(RRDSET *st, BUFFER *wb) {
+    rrd_stats_api_v1_chart_with_data(st, wb, NULL, NULL);
 }
 
-void rrd_stats_api_v1_charts(BUFFER *wb)
+void rrd_stats_api_v1_charts(RRDHOST *host, BUFFER *wb)
 {
-    long c;
+    size_t c, dimensions = 0, memory = 0, alarms = 0;
     RRDSET *st;
 
     buffer_sprintf(wb, "{\n"
            "\t\"hostname\": \"%s\""
+        ",\n\t\"version\": \"%s\""
+        ",\n\t\"os\": \"%s\""
         ",\n\t\"update_every\": %d"
         ",\n\t\"history\": %d"
         ",\n\t\"charts\": {"
-        , localhost.hostname
-        , rrd_update_every
-        , rrd_default_history_entries
+        , host->hostname
+        , program_version
+        , host->os
+        , host->rrd_update_every
+        , host->rrd_history_entries
         );
 
-    pthread_rwlock_rdlock(&localhost.rrdset_root_rwlock);
-    for(st = localhost.rrdset_root, c = 0; st ; st = st->next) {
-        if(st->enabled && st->dimensions) {
+    c = 0;
+    rrdhost_rdlock(host);
+    rrdset_foreach_read(st, host) {
+        if(rrdset_flag_check(st, RRDSET_FLAG_ENABLED) && st->dimensions) {
             if(c) buffer_strcat(wb, ",");
             buffer_strcat(wb, "\n\t\t\"");
             buffer_strcat(wb, st->id);
             buffer_strcat(wb, "\": ");
-            rrd_stats_api_v1_chart(st, wb);
+            rrd_stats_api_v1_chart_with_data(st, wb, &dimensions, &memory);
             c++;
         }
     }
-    pthread_rwlock_unlock(&localhost.rrdset_root_rwlock);
 
-    buffer_strcat(wb, "\n\t}\n}\n");
+    RRDCALC *rc;
+    for(rc = host->alarms; rc ; rc = rc->next) {
+        if(rc->rrdset)
+            alarms++;
+    }
+    rrdhost_unlock(host);
+
+    buffer_sprintf(wb, "\n\t}"
+                    ",\n\t\"charts_count\": %zu"
+                    ",\n\t\"dimensions_count\": %zu"
+                    ",\n\t\"alarms_count\": %zu"
+                    ",\n\t\"rrd_memory_bytes\": %zu"
+                    "\n}\n"
+                   , c
+                   , dimensions
+                   , alarms
+                   , memory
+    );
+}
+
+// ----------------------------------------------------------------------------
+// PROMETHEUS
+// /api/v1/allmetrics?format=prometheus
+
+static inline size_t prometheus_name_copy(char *d, const char *s, size_t usable) {
+    size_t n;
+
+    for(n = 0; *s && n < usable ; d++, s++, n++) {
+        register char c = *s;
+
+        if(unlikely(!isalnum(c))) *d = '_';
+        else *d = c;
+    }
+    *d = '\0';
+
+    return n;
+}
+
+#define PROMETHEUS_ELEMENT_MAX 256
+
+void rrd_stats_api_v1_charts_allmetrics_prometheus(RRDHOST *host, BUFFER *wb) {
+    rrdhost_rdlock(host);
+
+    char hostname[PROMETHEUS_ELEMENT_MAX + 1];
+    prometheus_name_copy(hostname, host->hostname, PROMETHEUS_ELEMENT_MAX);
+
+    // for each chart
+    RRDSET *st;
+    rrdset_foreach_read(st, host) {
+        char chart[PROMETHEUS_ELEMENT_MAX + 1];
+        prometheus_name_copy(chart, st->id, PROMETHEUS_ELEMENT_MAX);
+
+        buffer_strcat(wb, "\n");
+        if(rrdset_flag_check(st, RRDSET_FLAG_ENABLED) && st->dimensions) {
+            rrdset_rdlock(st);
+
+            // for each dimension
+            RRDDIM *rd;
+            rrddim_foreach_read(rd, st) {
+                if(rd->collections_counter) {
+                    char dimension[PROMETHEUS_ELEMENT_MAX + 1];
+                    prometheus_name_copy(dimension, rd->id, PROMETHEUS_ELEMENT_MAX);
+
+                    // buffer_sprintf(wb, "# HELP %s.%s %s\n", st->id, rd->id, st->units);
+
+                    switch(rd->algorithm) {
+                        case RRD_ALGORITHM_INCREMENTAL:
+                        case RRD_ALGORITHM_PCENT_OVER_DIFF_TOTAL:
+                            buffer_sprintf(wb, "# TYPE %s_%s counter\n", chart, dimension);
+                            break;
+
+                        default:
+                            buffer_sprintf(wb, "# TYPE %s_%s gauge\n", chart, dimension);
+                            break;
+                    }
+
+                    // calculated_number n = (calculated_number)rd->last_collected_value * (calculated_number)(abs(rd->multiplier)) / (calculated_number)(abs(rd->divisor));
+                    // buffer_sprintf(wb, "%s.%s " CALCULATED_NUMBER_FORMAT " %llu\n", st->id, rd->id, n,
+                    //        (unsigned long long)((rd->last_collected_time.tv_sec * 1000) + (rd->last_collected_time.tv_usec / 1000)));
+
+                    buffer_sprintf(wb, "%s_%s{instance=\"%s\"} " COLLECTED_NUMBER_FORMAT " %llu\n",
+                            chart, dimension, hostname, rd->last_collected_value,
+                            (unsigned long long)((rd->last_collected_time.tv_sec * 1000) + (rd->last_collected_time.tv_usec / 1000)));
+
+                }
+            }
+
+            rrdset_unlock(st);
+        }
+    }
+
+    rrdhost_unlock(host);
+}
+
+// ----------------------------------------------------------------------------
+// BASH
+// /api/v1/allmetrics?format=bash
+
+static inline size_t shell_name_copy(char *d, const char *s, size_t usable) {
+    size_t n;
+
+    for(n = 0; *s && n < usable ; d++, s++, n++) {
+        register char c = *s;
+
+        if(unlikely(!isalnum(c))) *d = '_';
+        else *d = (char)toupper(c);
+    }
+    *d = '\0';
+
+    return n;
 }
 
+#define SHELL_ELEMENT_MAX 100
+
+void rrd_stats_api_v1_charts_allmetrics_shell(RRDHOST *host, BUFFER *wb) {
+    rrdhost_rdlock(host);
+
+    // for each chart
+    RRDSET *st;
+    rrdset_foreach_read(st, host) {
+        calculated_number total = 0.0;
+        char chart[SHELL_ELEMENT_MAX + 1];
+        shell_name_copy(chart, st->id, SHELL_ELEMENT_MAX);
+
+        buffer_sprintf(wb, "\n# chart: %s (name: %s)\n", st->id, st->name);
+        if(rrdset_flag_check(st, RRDSET_FLAG_ENABLED) && st->dimensions) {
+            rrdset_rdlock(st);
+
+            // for each dimension
+            RRDDIM *rd;
+            rrddim_foreach_read(rd, st) {
+                if(rd->collections_counter) {
+                    char dimension[SHELL_ELEMENT_MAX + 1];
+                    shell_name_copy(dimension, rd->id, SHELL_ELEMENT_MAX);
+
+                    calculated_number n = rd->last_stored_value;
+
+                    if(isnan(n) || isinf(n))
+                        buffer_sprintf(wb, "NETDATA_%s_%s=\"\"      # %s\n", chart, dimension, st->units);
+                    else {
+                        if(rd->multiplier < 0 || rd->divisor < 0) n = -n;
+                        n = roundl(n);
+                        if(!rrddim_flag_check(rd, RRDDIM_FLAG_HIDDEN)) total += n;
+                        buffer_sprintf(wb, "NETDATA_%s_%s=\"%0.0Lf\"      # %s\n", chart, dimension, n, st->units);
+                    }
+                }
+            }
+
+            total = roundl(total);
+            buffer_sprintf(wb, "NETDATA_%s_VISIBLETOTAL=\"%0.0Lf\"      # %s\n", chart, total, st->units);
+            rrdset_unlock(st);
+        }
+    }
+
+    buffer_strcat(wb, "\n# NETDATA ALARMS RUNNING\n");
+
+    RRDCALC *rc;
+    for(rc = host->alarms; rc ;rc = rc->next) {
+        if(!rc->rrdset) continue;
+
+        char chart[SHELL_ELEMENT_MAX + 1];
+        shell_name_copy(chart, rc->rrdset->id, SHELL_ELEMENT_MAX);
+
+        char alarm[SHELL_ELEMENT_MAX + 1];
+        shell_name_copy(alarm, rc->name, SHELL_ELEMENT_MAX);
+
+        calculated_number n = rc->value;
+
+        if(isnan(n) || isinf(n))
+            buffer_sprintf(wb, "NETDATA_ALARM_%s_%s_VALUE=\"\"      # %s\n", chart, alarm, rc->units);
+        else {
+            n = roundl(n);
+            buffer_sprintf(wb, "NETDATA_ALARM_%s_%s_VALUE=\"%0.0Lf\"      # %s\n", chart, alarm, n, rc->units);
+        }
+
+        buffer_sprintf(wb, "NETDATA_ALARM_%s_%s_STATUS=\"%s\"\n", chart, alarm, rrdcalc_status2string(rc->status));
+    }
+
+    rrdhost_unlock(host);
+}
+
+// ----------------------------------------------------------------------------
 
 unsigned long rrd_stats_one_json(RRDSET *st, char *options, BUFFER *wb)
 {
-    time_t now = time(NULL);
+    time_t now = now_realtime_sec();
 
-    pthread_rwlock_rdlock(&st->rwlock);
+    rrdset_rdlock(st);
 
     buffer_sprintf(wb,
         "\t\t{\n"
@@ -141,7 +331,7 @@ unsigned long rrd_stats_one_json(RRDSET *st, char *options, BUFFER *wb)
         , st->context
         , st->title
         , st->priority
-        , st->enabled
+        , rrdset_flag_check(st, RRDSET_FLAG_ENABLED)?1:0
         , st->units
         , st->name, options?options:""
         , rrdset_type_name(st->chart_type)
@@ -152,7 +342,7 @@ unsigned long rrd_stats_one_json(RRDSET *st, char *options, BUFFER *wb)
         , rrdset_last_entry_t(st)
         , (now < rrdset_last_entry_t(st)) ? (time_t)0 : now - rrdset_last_entry_t(st)
         , st->update_every
-        , st->isdetail
+        , rrdset_flag_check(st, RRDSET_FLAG_DETAIL)?1:0
         , st->usec_since_last_update
         , st->collected_total
         , st->last_collected_total
@@ -161,7 +351,7 @@ unsigned long rrd_stats_one_json(RRDSET *st, char *options, BUFFER *wb)
     unsigned long memory = st->memsize;
 
     RRDDIM *rd;
-    for(rd = st->dimensions; rd ; rd = rd->next) {
+    rrddim_foreach_read(rd, st) {
 
         memory += rd->memsize;
 
@@ -172,8 +362,8 @@ unsigned long rrd_stats_one_json(RRDSET *st, char *options, BUFFER *wb)
             "\t\t\t\t\t\"entries\": %ld,\n"
             "\t\t\t\t\t\"isHidden\": %d,\n"
             "\t\t\t\t\t\"algorithm\": \"%s\",\n"
-            "\t\t\t\t\t\"multiplier\": %ld,\n"
-            "\t\t\t\t\t\"divisor\": %ld,\n"
+            "\t\t\t\t\t\"multiplier\": " COLLECTED_NUMBER_FORMAT ",\n"
+            "\t\t\t\t\t\"divisor\": " COLLECTED_NUMBER_FORMAT ",\n"
             "\t\t\t\t\t\"last_entry_t\": %ld,\n"
             "\t\t\t\t\t\"collected_value\": " COLLECTED_NUMBER_FORMAT ",\n"
             "\t\t\t\t\t\"calculated_value\": " CALCULATED_NUMBER_FORMAT ",\n"
@@ -184,8 +374,8 @@ unsigned long rrd_stats_one_json(RRDSET *st, char *options, BUFFER *wb)
             , rd->id
             , rd->name
             , rd->entries
-            , (rd->flags & RRDDIM_FLAG_HIDDEN)?1:0
-            , rrddim_algorithm_name(rd->algorithm)
+            , rrddim_flag_check(rd, RRDDIM_FLAG_HIDDEN)?1:0
+            , rrd_algorithm_name(rd->algorithm)
             , rd->multiplier
             , rd->divisor
             , rd->last_collected_time.tv_sec
@@ -205,7 +395,7 @@ unsigned long rrd_stats_one_json(RRDSET *st, char *options, BUFFER *wb)
         , memory
         );
 
-    pthread_rwlock_unlock(&st->rwlock);
+    rrdset_unlock(st);
     return memory;
 }
 
@@ -219,23 +409,23 @@ void rrd_stats_graph_json(RRDSET *st, char *options, BUFFER *wb)
     buffer_strcat(wb, RRD_GRAPH_JSON_FOOTER);
 }
 
-void rrd_stats_all_json(BUFFER *wb)
+void rrd_stats_all_json(RRDHOST *host, BUFFER *wb)
 {
     unsigned long memory = 0;
-    long c;
+    long c = 0;
     RRDSET *st;
 
     buffer_strcat(wb, RRD_GRAPH_JSON_HEADER);
 
-    pthread_rwlock_rdlock(&localhost.rrdset_root_rwlock);
-    for(st = localhost.rrdset_root, c = 0; st ; st = st->next) {
-        if(st->enabled && st->dimensions) {
+    rrdhost_rdlock(host);
+    rrdset_foreach_read(st, host) {
+        if(rrdset_flag_check(st, RRDSET_FLAG_ENABLED) && st->dimensions) {
             if(c) buffer_strcat(wb, ",\n");
             memory += rrd_stats_one_json(st, NULL, wb);
             c++;
         }
     }
-    pthread_rwlock_unlock(&localhost.rrdset_root_rwlock);
+    rrdhost_unlock(host);
 
     buffer_sprintf(wb, "\n\t],\n"
         "\t\"hostname\": \"%s\",\n"
@@ -243,9 +433,9 @@ void rrd_stats_all_json(BUFFER *wb)
         "\t\"history\": %d,\n"
         "\t\"memory\": %lu\n"
         "}\n"
-        , localhost.hostname
-        , rrd_update_every
-        , rrd_default_history_entries
+        , host->hostname
+        , host->rrd_update_every
+        , host->rrd_history_entries
         , memory
         );
 }
@@ -259,6 +449,7 @@ void rrd_stats_all_json(BUFFER *wb)
 #define RRDR_RESET      0x02 // the dimension contains / the value is reset
 #define RRDR_HIDDEN     0x04 // the dimension contains / the value is hidden
 #define RRDR_NONZERO    0x08 // the dimension contains / the value is non-zero
+#define RRDR_SELECTED   0x10 // the dimension is selected
 
 // RRDR result options
 #define RRDR_RESULT_OPTION_ABSOLUTE 0x00000001
@@ -351,13 +542,16 @@ static void rrdr_dump(RRDR *r)
 }
 */
 
-void rrdr_disable_not_selected_dimensions(RRDR *r, const char *dims)
-{
+void rrdr_disable_not_selected_dimensions(RRDR *r, uint32_t options, const char *dims) {
+    rrdset_check_rdlock(r->st);
+
+    if(unlikely(!dims || !*dims)) return;
+
     char b[strlen(dims) + 1];
     char *o = b, *tok;
     strcpy(o, dims);
 
-    long c;
+    long c, dims_selected = 0, dims_not_hidden_not_zero = 0;
     RRDDIM *d;
 
     // disable all of them
@@ -371,16 +565,38 @@ void rrdr_disable_not_selected_dimensions(RRDR *r, const char *dims)
 
         // find it and enable it
         for(c = 0, d = r->st->dimensions; d ;c++, d = d->next) {
-            if(unlikely((hash == d->hash && !strcmp(d->id, tok)) || !strcmp(d->name, tok))) {
-                r->od[c] &= ~RRDR_HIDDEN;
+            if(unlikely((hash == d->hash && !strcmp(d->id, tok)) || (hash == d->hash_name && !strcmp(d->name, tok)))) {
+
+                if(likely(r->od[c] & RRDR_HIDDEN)) {
+                    r->od[c] |= RRDR_SELECTED;
+                    r->od[c] &= ~RRDR_HIDDEN;
+                    dims_selected++;
+                }
 
                 // since the user needs this dimension
                 // make it appear as NONZERO, to return it
                 // even if the dimension has only zeros
-                r->od[c] |= RRDR_NONZERO;
+                // unless option non_zero is set
+                if(likely(!(options & RRDR_OPTION_NONZERO)))
+                    r->od[c] |= RRDR_NONZERO;
+
+                // count the visible dimensions
+                if(likely(r->od[c] & RRDR_NONZERO))
+                    dims_not_hidden_not_zero++;
             }
         }
     }
+
+    // check if all dimensions are hidden
+    if(unlikely(!dims_not_hidden_not_zero && dims_selected)) {
+        // there are a few selected dimensions
+        // but they are all zero
+        // enable the selected ones
+        // to avoid returning an empty chart
+        for(c = 0, d = r->st->dimensions; d ;c++, d = d->next)
+            if(unlikely(r->od[c] & RRDR_SELECTED))
+                r->od[c] |= RRDR_NONZERO;
+    }
 }
 
 void rrdr_buffer_print_format(BUFFER *wb, uint32_t format)
@@ -434,17 +650,23 @@ void rrdr_buffer_print_format(BUFFER *wb, uint32_t format)
 
 uint32_t rrdr_check_options(RRDR *r, uint32_t options, const char *dims)
 {
+    rrdset_check_rdlock(r->st);
+
+    (void)dims;
+
     if(options & RRDR_OPTION_NONZERO) {
         long i;
 
-        if(dims && *dims) {
+        // commented due to #1514
+
+        //if(dims && *dims) {
             // the caller wants specific dimensions
             // disable NONZERO option
             // to make sure we don't accidentally prevent
             // the specific dimensions from being returned
-            i = 0;
-        }
-        else {
+            // i = 0;
+        //}
+        //else {
             // find how many dimensions are not zero
             long c;
             RRDDIM *rd;
@@ -453,7 +675,7 @@ uint32_t rrdr_check_options(RRDR *r, uint32_t options, const char *dims)
                 if(unlikely(!(r->od[c] & RRDR_NONZERO))) continue;
                 i++;
             }
-        }
+        //}
 
         // if with nonzero we get i = 0 (no dimensions will be returned)
         // disable nonzero to show all dimensions
@@ -465,6 +687,8 @@ uint32_t rrdr_check_options(RRDR *r, uint32_t options, const char *dims)
 
 void rrdr_json_wrapper_begin(RRDR *r, BUFFER *wb, uint32_t format, uint32_t options, int string_value)
 {
+    rrdset_check_rdlock(r->st);
+
     long rows = rrdr_rows(r);
     long c, i;
     RRDDIM *rd;
@@ -646,6 +870,8 @@ void rrdr_json_wrapper_end(RRDR *r, BUFFER *wb, uint32_t format, uint32_t option
 
 static void rrdr2json(RRDR *r, BUFFER *wb, uint32_t options, int datatable)
 {
+    rrdset_check_rdlock(r->st);
+
     //info("RRD2JSON(): %s: BEGIN", r->st->id);
     int row_annotations = 0, dates, dates_with_new = 0;
     char kq[2] = "",                    // key quote
@@ -699,14 +925,14 @@ static void rrdr2json(RRDR *r, BUFFER *wb, uint32_t options, int datatable)
     else {
         kq[0] = '"';
         sq[0] = '"';
-        if((options & RRDR_OPTION_SECONDS) || (options & RRDR_OPTION_MILLISECONDS)) {
-            dates = JSON_DATES_TIMESTAMP;
-            dates_with_new = 0;
-        }
-        else {
+        if(options & RRDR_OPTION_GOOGLE_JSON) {
             dates = JSON_DATES_JS;
             dates_with_new = 1;
         }
+        else {
+            dates = JSON_DATES_TIMESTAMP;
+            dates_with_new = 0;
+        }
         if( options & RRDR_OPTION_OBJECTSROWS )
             strcpy(pre_date, "      { ");
         else
@@ -872,6 +1098,8 @@ static void rrdr2json(RRDR *r, BUFFER *wb, uint32_t options, int datatable)
 
 static void rrdr2csv(RRDR *r, BUFFER *wb, uint32_t options, const char *startline, const char *separator, const char *endline, const char *betweenlines)
 {
+    rrdset_check_rdlock(r->st);
+
     //info("RRD2CSV(): %s: BEGIN", r->st->id);
     long c, i;
     RRDDIM *d;
@@ -977,6 +1205,8 @@ static void rrdr2csv(RRDR *r, BUFFER *wb, uint32_t options, const char *startlin
 }
 
 inline static calculated_number rrdr2value(RRDR *r, long i, uint32_t options, int *all_values_are_null) {
+    rrdset_check_rdlock(r->st);
+
     long c;
     RRDDIM *d;
 
@@ -1127,7 +1357,7 @@ inline static void rrdr_lock_rrdset(RRDR *r) {
         return;
     }
 
-    pthread_rwlock_rdlock(&r->st->rwlock);
+    rrdset_rdlock(r->st);
     r->has_st_lock = 1;
 }
 
@@ -1138,7 +1368,7 @@ inline static void rrdr_unlock_rrdset(RRDR *r) {
     }
 
     if(likely(r->has_st_lock)) {
-        pthread_rwlock_unlock(&r->st->rwlock);
+        rrdset_unlock(r->st);
         r->has_st_lock = 0;
     }
 }
@@ -1177,7 +1407,7 @@ static RRDR *rrdr_create(RRDSET *st, long n)
     rrdr_lock_rrdset(r);
 
     RRDDIM *rd;
-    for(rd = st->dimensions ; rd ; rd = rd->next) r->d++;
+    rrddim_foreach_read(rd, st) r->d++;
 
     r->n = n;
 
@@ -1189,8 +1419,10 @@ static RRDR *rrdr_create(RRDSET *st, long n)
     // set the hidden flag on hidden dimensions
     int c;
     for(c = 0, rd = st->dimensions ; rd ; c++, rd = rd->next) {
-        if(unlikely(rd->flags & RRDDIM_FLAG_HIDDEN)) r->od[c] = RRDR_HIDDEN;
-        else r->od[c] = 0;
+        if(unlikely(rrddim_flag_check(rd, RRDDIM_FLAG_HIDDEN)))
+            r->od[c] = RRDR_HIDDEN;
+        else
+            r->od[c] = 0;
     }
 
     r->c = -1;
@@ -1202,7 +1434,7 @@ static RRDR *rrdr_create(RRDSET *st, long n)
 
 RRDR *rrd2rrdr(RRDSET *st, long points, long long after, long long before, int group_method, int aligned)
 {
-    int debug = st->debug;
+    int debug = rrdset_flag_check(st, RRDSET_FLAG_DEBUG)?1:0;
     int absolute_period_requested = -1;
 
     time_t first_entry_t = rrdset_first_entry_t(st);
@@ -1378,6 +1610,7 @@ RRDR *rrd2rrdr(RRDSET *st, long points, long long after, long long before, int g
     // initialize them
     RRDDIM *rd;
     long c;
+    rrdset_check_rdlock(st);
     for( rd = st->dimensions, c = 0 ; rd && c < dimensions ; rd = rd->next, c++) {
         last_values[c] = 0;
         group_values[c] = (group_method == GROUP_MAX || group_method == GROUP_MIN)?NAN:0;
@@ -1584,7 +1817,7 @@ int rrd2value(RRDSET *st, BUFFER *wb, calculated_number *n, const char *dimensio
     options = rrdr_check_options(r, options, dimensions);
 
     if(dimensions)
-        rrdr_disable_not_selected_dimensions(r, dimensions);
+        rrdr_disable_not_selected_dimensions(r, options, dimensions);
 
     if(db_after)  *db_after  = r->after;
     if(db_before) *db_before = r->before;
@@ -1612,7 +1845,7 @@ int rrd2format(RRDSET *st, BUFFER *wb, BUFFER *dimensions, uint32_t format, long
     options = rrdr_check_options(r, options, (dimensions)?buffer_tostring(dimensions):NULL);
 
     if(dimensions)
-        rrdr_disable_not_selected_dimensions(r, buffer_tostring(dimensions));
+        rrdr_disable_not_selected_dimensions(r, options, buffer_tostring(dimensions));
 
     if(latest_timestamp && rrdr_rows(r) > 0)
         *latest_timestamp = r->before;
@@ -1773,7 +2006,7 @@ int rrd2format(RRDSET *st, BUFFER *wb, BUFFER *dimensions, uint32_t format, long
 time_t rrd_stats_json(int type, RRDSET *st, BUFFER *wb, long points, long group, int group_method, time_t after, time_t before, int only_non_zero)
 {
     int c;
-    pthread_rwlock_rdlock(&st->rwlock);
+    rrdset_rdlock(st);
 
 
     // -------------------------------------------------------------------------
@@ -1815,9 +2048,9 @@ time_t rrd_stats_json(int type, RRDSET *st, BUFFER *wb, long points, long group,
 
     int dimensions = 0;
     RRDDIM *rd;
-    for( rd = st->dimensions ; rd ; rd = rd->next) dimensions++;
+    rrddim_foreach_read(rd, st) dimensions++;
     if(!dimensions) {
-        pthread_rwlock_unlock(&st->rwlock);
+        rrdset_unlock(st);
         buffer_strcat(wb, "No dimensions yet.");
         return 0;
     }
@@ -1837,7 +2070,7 @@ time_t rrd_stats_json(int type, RRDSET *st, BUFFER *wb, long points, long group,
     // -------------------------------------------------------------------------
     // checks for debugging
 
-    if(st->debug) {
+    if(rrdset_flag_check(st, RRDSET_FLAG_DEBUG)) {
         debug(D_RRD_STATS, "%s first_entry_t = %ld, last_entry_t = %ld, duration = %ld, after = %ld, before = %ld, duration = %ld, entries_to_show = %ld, group = %ld"
             , st->id
             , rrdset_first_entry_t(st)
@@ -1869,7 +2102,7 @@ time_t rrd_stats_json(int type, RRDSET *st, BUFFER *wb, long points, long group,
     // initialize them
     for( rd = st->dimensions, c = 0 ; rd && c < dimensions ; rd = rd->next, c++) {
         group_values[c] = 0;
-        print_hidden[c] = (rd->flags & RRDDIM_FLAG_HIDDEN)?1:0;
+        print_hidden[c] = rrddim_flag_check(rd, RRDDIM_FLAG_HIDDEN)?1:0;
         found_non_zero[c] = 0;
         found_non_existing[c] = 0;
     }
@@ -1930,7 +2163,8 @@ time_t rrd_stats_json(int type, RRDSET *st, BUFFER *wb, long points, long group,
         long count = 0, printed = 0, group_count = 0;
         last_timestamp = 0;
 
-        if(st->debug) debug(D_RRD_STATS, "%s: REQUEST after:%u before:%u, points:%ld, group:%ld, CHART cur:%ld first: %u last:%u, CALC start_t:%ld, stop_t:%ld"
+        if(unlikely(rrdset_flag_check(st, RRDSET_FLAG_DEBUG)))
+            debug(D_RRD_STATS, "%s: REQUEST after:%u before:%u, points:%ld, group:%ld, CHART cur:%ld first: %u last:%u, CALC start_t:%ld, stop_t:%ld"
                     , st->id
                     , (uint32_t)after
                     , (uint32_t)before
@@ -1950,7 +2184,8 @@ time_t rrd_stats_json(int type, RRDSET *st, BUFFER *wb, long points, long group,
 
             int print_this = 0;
 
-            if(st->debug) debug(D_RRD_STATS, "%s t = %ld, count = %ld, group_count = %ld, printed = %ld, now = %ld, %s %s"
+            if(unlikely(rrdset_flag_check(st, RRDSET_FLAG_DEBUG)))
+                debug(D_RRD_STATS, "%s t = %ld, count = %ld, group_count = %ld, printed = %ld, now = %ld, %s %s"
                     , st->id
                     , t
                     , count + 1
@@ -2086,8 +2321,8 @@ time_t rrd_stats_json(int type, RRDSET *st, BUFFER *wb, long points, long group,
 
     } // max_loop
 
-    debug(D_RRD_STATS, "RRD_STATS_JSON: %s total %lu bytes", st->name, wb->len);
+    debug(D_RRD_STATS, "RRD_STATS_JSON: %s total %zu bytes", st->name, wb->len);
 
-    pthread_rwlock_unlock(&st->rwlock);
+    rrdset_unlock(st);
     return last_timestamp;
 }