From 73212e3a8731012fee476e373f4a5b647f5126e5 Mon Sep 17 00:00:00 2001 From: "Costa Tsaousis (ktsaou)" Date: Wed, 1 Mar 2017 00:47:50 +0200 Subject: [PATCH] self-cleaning obsolete cgroups and network interfaces from memory; fixes #1163; fixes #1358 --- CMakeLists.txt | 2 +- conf.d/health.d/net.conf | 15 - src/Makefile.am | 1 + src/common.h | 1 + src/health.c | 34 +- src/proc_net_dev.c | 217 +++++++--- src/rrd.h | 13 +- src/rrd2json.c | 532 ++--------------------- src/rrd2json.h | 16 +- src/rrd2json_api_old.c | 487 +++++++++++++++++++++ src/rrd2json_api_old.h | 14 + src/rrdhost.c | 49 ++- src/rrdset.c | 64 ++- src/sys_fs_cgroup.c | 893 ++++++++++++++++++++++++++------------- src/web_api_old.c | 14 +- src/web_api_v1.c | 20 +- 16 files changed, 1452 insertions(+), 920 deletions(-) create mode 100644 src/rrd2json_api_old.c create mode 100644 src/rrd2json_api_old.h diff --git a/CMakeLists.txt b/CMakeLists.txt index 979144e0..8931d8f8 100755 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -124,7 +124,7 @@ set(NETDATA_SOURCE_FILES src/web_client.h src/web_server.c src/web_server.h - src/rrdhost.c src/rrdfamily.c src/rrdset.c src/rrddim.c src/health_log.c src/health_config.c src/health_json.c src/rrdcalc.c src/rrdcalctemplate.c src/rrdvar.c src/rrddimvar.c src/rrdsetvar.c src/rrdpush.c src/rrdpush.h src/web_api_old.c src/web_api_old.h src/web_api_v1.c src/web_api_v1.h) + src/rrdhost.c src/rrdfamily.c src/rrdset.c src/rrddim.c src/health_log.c src/health_config.c src/health_json.c src/rrdcalc.c src/rrdcalctemplate.c src/rrdvar.c src/rrddimvar.c src/rrdsetvar.c src/rrdpush.c src/rrdpush.h src/web_api_old.c src/web_api_old.h src/web_api_v1.c src/web_api_v1.h src/rrd2json_api_old.c src/rrd2json_api_old.h) set(APPS_PLUGIN_SOURCE_FILES src/appconfig.c diff --git a/conf.d/health.d/net.conf b/conf.d/health.d/net.conf index cac0bbbf..0232395a 100644 --- a/conf.d/health.d/net.conf +++ b/conf.d/health.d/net.conf @@ -1,18 +1,3 @@ -# ----------------------------------------------------------------------------- -# make sure we collect values for each interface - -template: interface_last_collected_secs - on: net.net -families: * - calc: $now - $last_collected_t - units: seconds ago - every: 10s - warn: $this > (($status >= $WARNING) ? ($update_every) : ( 5 * $update_every)) - crit: $this > (($status == $CRITICAL) ? ($update_every) : (60 * $update_every)) - delay: down 5m multiplier 1.5 max 1h - info: number of seconds since the last successful data collection - to: sysadmin - # ----------------------------------------------------------------------------- # dropped packets diff --git a/src/Makefile.am b/src/Makefile.am index a87cd7a8..3e6a79c1 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -75,6 +75,7 @@ netdata_SOURCES = \ rrddimvar.c \ rrdsetvar.c \ rrd2json.c rrd2json.h \ + rrd2json_api_old.c rrd2json_api_old.h \ rrdpush.c rrdpush.h \ storage_number.c storage_number.h \ unit_test.c unit_test.h \ diff --git a/src/common.h b/src/common.h index 3d864f42..f53cc557 100644 --- a/src/common.h +++ b/src/common.h @@ -206,6 +206,7 @@ #include "plugin_tc.h" #include "plugins_d.h" #include "rrd2json.h" +#include "rrd2json_api_old.h" #include "web_client.h" #include "web_server.h" #include "registry.h" diff --git a/src/health.c b/src/health.c index 0c7983ec..57210c77 100644 --- a/src/health.c +++ b/src/health.c @@ -281,6 +281,16 @@ static inline int rrdcalc_isrunnable(RRDCALC *rc, time_t now, time_t *next_run) return 0; } + if(unlikely(rrdset_flag_check(rc->rrdset, RRDSET_FLAG_OBSOLETE))) { + debug(D_HEALTH, "Health not running alarm '%s.%s'. The chart has been marked as obsolete", rc->chart?rc->chart:"NOCHART", rc->name); + return 0; + } + + if(unlikely(!rrdset_flag_check(rc->rrdset, RRDSET_FLAG_ENABLED))) { + debug(D_HEALTH, "Health not running alarm '%s.%s'. The chart is not enabled", rc->chart?rc->chart:"NOCHART", rc->name); + return 0; + } + if(unlikely(!rc->rrdset->last_collected_time.tv_sec || rc->rrdset->counter_done < 2)) { debug(D_HEALTH, "Health not running alarm '%s.%s'. Chart is not fully collected yet.", rc->chart?rc->chart:"NOCHART", rc->name); return 0; @@ -398,18 +408,18 @@ void *health_main(void *ptr) { /* time_t old_db_timestamp = rc->db_before; */ int value_is_null = 0; - int ret = rrd2value(rc->rrdset - , wb - , &rc->value - , rc->dimensions - , 1 - , rc->after - , rc->before - , rc->group - , rc->options - , &rc->db_after - , &rc->db_before - , &value_is_null + int ret = rrdset2value_api_v1(rc->rrdset + , wb + , &rc->value + , rc->dimensions + , 1 + , rc->after + , rc->before + , rc->group + , rc->options + , &rc->db_after + , &rc->db_before + , &value_is_null ); if(unlikely(ret != 200)) { diff --git a/src/proc_net_dev.c b/src/proc_net_dev.c index 9b942326..e0de84c7 100644 --- a/src/proc_net_dev.c +++ b/src/proc_net_dev.c @@ -8,6 +8,7 @@ struct netdev { // flags int configured; int enabled; + int updated; int do_bandwidth; int do_packets; @@ -67,26 +68,70 @@ struct netdev { struct netdev *next; }; -static struct netdev *netdev_root = NULL; +static struct netdev *netdev_root = NULL, *netdev_last_used = NULL; + +static size_t netdev_added = 0, netdev_found = 0; + +static void netdev_free(struct netdev *d) { + if(d->st_bandwidth) rrdset_flag_set(d->st_bandwidth, RRDSET_FLAG_OBSOLETE); + if(d->st_packets) rrdset_flag_set(d->st_packets, RRDSET_FLAG_OBSOLETE); + if(d->st_errors) rrdset_flag_set(d->st_errors, RRDSET_FLAG_OBSOLETE); + if(d->st_drops) rrdset_flag_set(d->st_drops, RRDSET_FLAG_OBSOLETE); + if(d->st_fifo) rrdset_flag_set(d->st_fifo, RRDSET_FLAG_OBSOLETE); + if(d->st_compressed) rrdset_flag_set(d->st_compressed, RRDSET_FLAG_OBSOLETE); + if(d->st_events) rrdset_flag_set(d->st_events, RRDSET_FLAG_OBSOLETE); + + freez(d->name); + freez(d); +} + +static void netdev_cleanup() { + if(likely(netdev_found == netdev_added)) return; + + struct netdev *d = netdev_root, *last = NULL; + while(d) { + if(unlikely(!d->updated)) { + // info("Removing network device '%s', linked after '%s'", d->name, last?last->name:"ROOT"); + + if(netdev_last_used == d) + netdev_last_used = last; + + struct netdev *t = d; + + if(d == netdev_root || !last) + netdev_root = d = d->next; + + else + last->next = d = d->next; + + t->next = NULL; + netdev_free(t); + } + else { + last = d; + d->updated = 0; + d = d->next; + } + } +} static struct netdev *get_netdev(const char *name) { - static struct netdev *last = NULL; struct netdev *d; uint32_t hash = simple_hash(name); // search it, from the last position to the end - for(d = last ; d ; d = d->next) { + for(d = netdev_last_used ; d ; d = d->next) { if(unlikely(hash == d->hash && !strcmp(name, d->name))) { - last = d->next; + netdev_last_used = d->next; return d; } } // search it from the beginning to the last position we used - for(d = netdev_root ; d != last ; d = d->next) { + for(d = netdev_root ; d != netdev_last_used ; d = d->next) { if(unlikely(hash == d->hash && !strcmp(name, d->name))) { - last = d->next; + netdev_last_used = d->next; return d; } } @@ -96,6 +141,7 @@ static struct netdev *get_netdev(const char *name) { d->name = strdupz(name); d->hash = simple_hash(d->name); d->len = strlen(d->name); + netdev_added++; // link it to the end if(netdev_root) { @@ -142,12 +188,16 @@ int do_proc_net_dev(int update_every, usec_t dt) { ff = procfile_readall(ff); if(unlikely(!ff)) return 0; // we return 0, so that we will retry to open it next time + netdev_found = 0; + size_t lines = procfile_lines(ff), l; for(l = 2; l < lines ;l++) { // require 17 words on each line if(unlikely(procfile_linewords(ff, l) < 17)) continue; struct netdev *d = get_netdev(procfile_lineword(ff, l, 0)); + d->updated = 1; + netdev_found++; if(unlikely(!d->configured)) { // this is the first time we see this interface @@ -223,19 +273,27 @@ int do_proc_net_dev(int update_every, usec_t dt) { if(d->do_bandwidth == CONFIG_BOOLEAN_YES) { if(unlikely(!d->st_bandwidth)) { - d->st_bandwidth = rrdset_find_bytype_localhost("net", d->name); - if(!d->st_bandwidth) - d->st_bandwidth = rrdset_create_localhost("net", d->name, NULL, d->name, "net.net", "Bandwidth" - , "kilobits/s", 7000, update_every, RRDSET_TYPE_AREA); + d->st_bandwidth = rrdset_create_localhost( + "net" + , d->name + , NULL + , d->name + , "net.net" + , "Bandwidth" + , "kilobits/s" + , 7000 + , update_every + , RRDSET_TYPE_AREA + ); d->rd_rbytes = rrddim_add(d->st_bandwidth, "received", NULL, 8, 1024, RRD_ALGORITHM_INCREMENTAL); d->rd_tbytes = rrddim_add(d->st_bandwidth, "sent", NULL, -8, 1024, RRD_ALGORITHM_INCREMENTAL); } else rrdset_next(d->st_bandwidth); - rrddim_set_by_pointer(d->st_bandwidth, d->rd_rbytes, d->rbytes); - rrddim_set_by_pointer(d->st_bandwidth, d->rd_tbytes, d->tbytes); + rrddim_set_by_pointer(d->st_bandwidth, d->rd_rbytes, (collected_number)d->rbytes); + rrddim_set_by_pointer(d->st_bandwidth, d->rd_tbytes, (collected_number)d->tbytes); rrdset_done(d->st_bandwidth); } @@ -246,12 +304,20 @@ int do_proc_net_dev(int update_every, usec_t dt) { if(d->do_packets == CONFIG_BOOLEAN_YES) { if(unlikely(!d->st_packets)) { - d->st_packets = rrdset_find_bytype_localhost("net_packets", d->name); - if(!d->st_packets) - d->st_packets = rrdset_create_localhost("net_packets", d->name, NULL, d->name, "net.packets" - , "Packets", "packets/s", 7001, update_every - , RRDSET_TYPE_LINE); + d->st_packets = rrdset_create_localhost( + "net_packets" + , d->name + , NULL + , d->name + , "net.packets" + , "Packets" + , "packets/s" + , 7001 + , update_every + , RRDSET_TYPE_LINE + ); + rrdset_flag_set(d->st_packets, RRDSET_FLAG_DETAIL); d->rd_rpackets = rrddim_add(d->st_packets, "received", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); @@ -260,9 +326,9 @@ int do_proc_net_dev(int update_every, usec_t dt) { } else rrdset_next(d->st_packets); - rrddim_set_by_pointer(d->st_packets, d->rd_rpackets, d->rpackets); - rrddim_set_by_pointer(d->st_packets, d->rd_tpackets, d->tpackets); - rrddim_set_by_pointer(d->st_packets, d->rd_rmulticast, d->rmulticast); + rrddim_set_by_pointer(d->st_packets, d->rd_rpackets, (collected_number)d->rpackets); + rrddim_set_by_pointer(d->st_packets, d->rd_tpackets, (collected_number)d->tpackets); + rrddim_set_by_pointer(d->st_packets, d->rd_rmulticast, (collected_number)d->rmulticast); rrdset_done(d->st_packets); } @@ -273,12 +339,19 @@ int do_proc_net_dev(int update_every, usec_t dt) { if(d->do_errors == CONFIG_BOOLEAN_YES) { if(unlikely(!d->st_errors)) { - d->st_errors = rrdset_find_bytype_localhost("net_errors", d->name); - if(!d->st_errors) - d->st_errors = rrdset_create_localhost("net_errors", d->name, NULL, d->name, "net.errors" - , "Interface Errors", "errors/s", 7002, update_every - , RRDSET_TYPE_LINE); + d->st_errors = rrdset_create_localhost( + "net_errors" + , d->name + , NULL + , d->name + , "net.errors" + , "Interface Errors" + , "errors/s" + , 7002 + , update_every + , RRDSET_TYPE_LINE + ); rrdset_flag_set(d->st_errors, RRDSET_FLAG_DETAIL); @@ -287,8 +360,8 @@ int do_proc_net_dev(int update_every, usec_t dt) { } else rrdset_next(d->st_errors); - rrddim_set_by_pointer(d->st_errors, d->rd_rerrors, d->rerrors); - rrddim_set_by_pointer(d->st_errors, d->rd_terrors, d->terrors); + rrddim_set_by_pointer(d->st_errors, d->rd_rerrors, (collected_number)d->rerrors); + rrddim_set_by_pointer(d->st_errors, d->rd_terrors, (collected_number)d->terrors); rrdset_done(d->st_errors); } @@ -299,12 +372,19 @@ int do_proc_net_dev(int update_every, usec_t dt) { if(d->do_drops == CONFIG_BOOLEAN_YES) { if(unlikely(!d->st_drops)) { - d->st_drops = rrdset_find_bytype_localhost("net_drops", d->name); - if(!d->st_drops) - d->st_drops = rrdset_create_localhost("net_drops", d->name, NULL, d->name, "net.drops" - , "Interface Drops", "drops/s", 7003, update_every - , RRDSET_TYPE_LINE); + d->st_drops = rrdset_create_localhost( + "net_drops" + , d->name + , NULL + , d->name + , "net.drops" + , "Interface Drops" + , "drops/s" + , 7003 + , update_every + , RRDSET_TYPE_LINE + ); rrdset_flag_set(d->st_drops, RRDSET_FLAG_DETAIL); @@ -313,8 +393,8 @@ int do_proc_net_dev(int update_every, usec_t dt) { } else rrdset_next(d->st_drops); - rrddim_set_by_pointer(d->st_drops, d->rd_rdrops, d->rdrops); - rrddim_set_by_pointer(d->st_drops, d->rd_tdrops, d->tdrops); + rrddim_set_by_pointer(d->st_drops, d->rd_rdrops, (collected_number)d->rdrops); + rrddim_set_by_pointer(d->st_drops, d->rd_tdrops, (collected_number)d->tdrops); rrdset_done(d->st_drops); } @@ -325,12 +405,19 @@ int do_proc_net_dev(int update_every, usec_t dt) { if(d->do_fifo == CONFIG_BOOLEAN_YES) { if(unlikely(!d->st_fifo)) { - d->st_fifo = rrdset_find_bytype_localhost("net_fifo", d->name); - if(!d->st_fifo) - d->st_fifo = rrdset_create_localhost("net_fifo", d->name, NULL, d->name, "net.fifo" - , "Interface FIFO Buffer Errors", "errors", 7004, update_every - , RRDSET_TYPE_LINE); + d->st_fifo = rrdset_create_localhost( + "net_fifo" + , d->name + , NULL + , d->name + , "net.fifo" + , "Interface FIFO Buffer Errors" + , "errors" + , 7004 + , update_every + , RRDSET_TYPE_LINE + ); rrdset_flag_set(d->st_fifo, RRDSET_FLAG_DETAIL); @@ -339,8 +426,8 @@ int do_proc_net_dev(int update_every, usec_t dt) { } else rrdset_next(d->st_fifo); - rrddim_set_by_pointer(d->st_fifo, d->rd_rfifo, d->rfifo); - rrddim_set_by_pointer(d->st_fifo, d->rd_tfifo, d->tfifo); + rrddim_set_by_pointer(d->st_fifo, d->rd_rfifo, (collected_number)d->rfifo); + rrddim_set_by_pointer(d->st_fifo, d->rd_tfifo, (collected_number)d->tfifo); rrdset_done(d->st_fifo); } @@ -351,11 +438,19 @@ int do_proc_net_dev(int update_every, usec_t dt) { if(d->do_compressed == CONFIG_BOOLEAN_YES) { if(unlikely(!d->st_compressed)) { - d->st_compressed = rrdset_find_bytype_localhost("net_compressed", d->name); - if(!d->st_compressed) - d->st_compressed = rrdset_create_localhost("net_compressed", d->name, NULL, d->name - , "net.compressed", "Compressed Packets", "packets/s" - , 7005, update_every, RRDSET_TYPE_LINE); + + d->st_compressed = rrdset_create_localhost( + "net_compressed" + , d->name + , NULL + , d->name + , "net.compressed" + , "Compressed Packets" + , "packets/s" + , 7005 + , update_every + , RRDSET_TYPE_LINE + ); rrdset_flag_set(d->st_compressed, RRDSET_FLAG_DETAIL); @@ -364,8 +459,8 @@ int do_proc_net_dev(int update_every, usec_t dt) { } else rrdset_next(d->st_compressed); - rrddim_set_by_pointer(d->st_compressed, d->rd_rcompressed, d->rcompressed); - rrddim_set_by_pointer(d->st_compressed, d->rd_tcompressed, d->tcompressed); + rrddim_set_by_pointer(d->st_compressed, d->rd_rcompressed, (collected_number)d->rcompressed); + rrddim_set_by_pointer(d->st_compressed, d->rd_tcompressed, (collected_number)d->tcompressed); rrdset_done(d->st_compressed); } @@ -376,11 +471,19 @@ int do_proc_net_dev(int update_every, usec_t dt) { if(d->do_events == CONFIG_BOOLEAN_YES) { if(unlikely(!d->st_events)) { - d->st_events = rrdset_find_bytype_localhost("net_events", d->name); - if(!d->st_events) - d->st_events = rrdset_create_localhost("net_events", d->name, NULL, d->name, "net.events" - , "Network Interface Events", "events/s", 7006, update_every - , RRDSET_TYPE_LINE); + + d->st_events = rrdset_create_localhost( + "net_events" + , d->name + , NULL + , d->name + , "net.events" + , "Network Interface Events" + , "events/s" + , 7006 + , update_every + , RRDSET_TYPE_LINE + ); rrdset_flag_set(d->st_events, RRDSET_FLAG_DETAIL); @@ -390,12 +493,14 @@ int do_proc_net_dev(int update_every, usec_t dt) { } else rrdset_next(d->st_events); - rrddim_set_by_pointer(d->st_events, d->rd_rframe, d->rframe); - rrddim_set_by_pointer(d->st_events, d->rd_tcollisions, d->tcollisions); - rrddim_set_by_pointer(d->st_events, d->rd_tcarrier, d->tcarrier); + rrddim_set_by_pointer(d->st_events, d->rd_rframe, (collected_number)d->rframe); + rrddim_set_by_pointer(d->st_events, d->rd_tcollisions, (collected_number)d->tcollisions); + rrddim_set_by_pointer(d->st_events, d->rd_tcarrier, (collected_number)d->tcarrier); rrdset_done(d->st_events); } } + netdev_cleanup(); + return 0; } diff --git a/src/rrd.h b/src/rrd.h index 26623a24..8c625989 100644 --- a/src/rrd.h +++ b/src/rrd.h @@ -213,7 +213,8 @@ typedef enum rrdset_flags { RRDSET_FLAG_ENABLED = 1 << 0, // enables or disables a chart RRDSET_FLAG_DETAIL = 1 << 1, // if set, the data set should be considered as a detail of another // (the master data set should be the one that has the same family and is not detail) - RRDSET_FLAG_DEBUG = 1 << 2 // enables or disables debugging for a chart + RRDSET_FLAG_DEBUG = 1 << 2, // enables or disables debugging for a chart + RRDSET_FLAG_OBSOLETE = 1 << 3 // this is marked by the collector/module as obsolete } RRDSET_FLAGS; #define rrdset_flag_check(st, flag) ((st)->flags & flag) @@ -276,7 +277,9 @@ struct rrdset { size_t counter; // the number of times we added values to this database size_t counter_done; // the number of times rrdset_done() has been called - size_t unused[10]; + + time_t last_accessed_time; // the last time this RRDSET has been accessed + size_t unused[9]; uint32_t hash; // a simple hash on the id, to speed up searching // we first compare hashes, and only if the hashes are equal we do string comparisons @@ -540,6 +543,9 @@ extern void rrdset_next_usec(RRDSET *st, usec_t microseconds); extern void rrdset_done(RRDSET *st); +// checks if the RRDSET should be offered to viewers +#define rrdset_is_available_for_viewers(st) (rrdset_flag_check(st, RRDSET_FLAG_ENABLED) && !rrdset_flag_check(st, RRDSET_FLAG_OBSOLETE) && (st)->dimensions) + // get the total duration in seconds of the round robin database #define rrdset_duration(st) ((time_t)( (((st)->counter >= ((unsigned long)(st)->entries))?(unsigned long)(st)->entries:(st)->counter) * (st)->update_every )) @@ -617,6 +623,9 @@ extern void rrdfamily_free(RRDHOST *host, RRDFAMILY *rc); #define rrdset_index_del(host, st) (RRDSET *)avl_remove_lock(&((host)->rrdset_root_index), (avl *)(st)) extern RRDSET *rrdset_index_del_name(RRDHOST *host, RRDSET *st); +extern void rrdset_save(RRDSET *st); +extern void rrdhost_cleanup(RRDHOST *host); + #endif /* NETDATA_RRD_INTERNALS */ diff --git a/src/rrd2json.c b/src/rrd2json.c index 2a46fb13..ec10e4d5 100644 --- a/src/rrd2json.c +++ b/src/rrd2json.c @@ -83,6 +83,8 @@ void rrd_stats_api_v1_charts(RRDHOST *host, BUFFER *wb) size_t c, dimensions = 0, memory = 0, alarms = 0; RRDSET *st; + time_t now = now_realtime_sec(); + buffer_sprintf(wb, "{\n" "\t\"hostname\": \"%s\"" ",\n\t\"version\": \"%s\"" @@ -100,13 +102,15 @@ void rrd_stats_api_v1_charts(RRDHOST *host, BUFFER *wb) c = 0; rrdhost_rdlock(host); rrdset_foreach_read(st, host) { - if(rrdset_flag_check(st, RRDSET_FLAG_ENABLED) && st->dimensions) { + if(rrdset_is_available_for_viewers(st)) { if(c) buffer_strcat(wb, ","); buffer_strcat(wb, "\n\t\t\""); buffer_strcat(wb, st->id); buffer_strcat(wb, "\": "); rrd_stats_api_v1_chart_with_data(st, wb, &dimensions, &memory); + c++; + st->last_accessed_time = now; } } @@ -134,14 +138,15 @@ void rrd_stats_api_v1_charts(RRDHOST *host, BUFFER *wb) if(unlikely(rrd_hosts_available > 1)) { rrd_rdlock(); RRDHOST *h; - rrdhost_foreach_read(h) + rrdhost_foreach_read(h) { buffer_sprintf(wb, - "%s\n\t\t{" - "\n\t\t\t\"hostname\": \"%s\"" - "\n\t\t}" - , (h != localhost)?",":"" - , h->hostname + "%s\n\t\t{" + "\n\t\t\t\"hostname\": \"%s\"" + "\n\t\t}" + , (h != localhost) ? "," : "" + , h->hostname ); + } rrd_unlock(); } else { @@ -189,7 +194,7 @@ void rrd_stats_api_v1_charts_allmetrics_prometheus(RRDHOST *host, BUFFER *wb) { prometheus_name_copy(chart, st->id, PROMETHEUS_ELEMENT_MAX); buffer_strcat(wb, "\n"); - if(rrdset_flag_check(st, RRDSET_FLAG_ENABLED) && st->dimensions) { + if(rrdset_is_available_for_viewers(st)) { rrdset_rdlock(st); // for each dimension @@ -261,7 +266,7 @@ void rrd_stats_api_v1_charts_allmetrics_shell(RRDHOST *host, BUFFER *wb) { shell_name_copy(chart, st->id, SHELL_ELEMENT_MAX); buffer_sprintf(wb, "\n# chart: %s (name: %s)\n", st->id, st->name); - if(rrdset_flag_check(st, RRDSET_FLAG_ENABLED) && st->dimensions) { + if(rrdset_is_available_for_viewers(st)) { rrdset_rdlock(st); // for each dimension @@ -317,157 +322,6 @@ void rrd_stats_api_v1_charts_allmetrics_shell(RRDHOST *host, BUFFER *wb) { rrdhost_unlock(host); } -// ---------------------------------------------------------------------------- - -unsigned long rrd_stats_one_json(RRDSET *st, char *options, BUFFER *wb) -{ - time_t now = now_realtime_sec(); - - rrdset_rdlock(st); - - buffer_sprintf(wb, - "\t\t{\n" - "\t\t\t\"id\": \"%s\",\n" - "\t\t\t\"name\": \"%s\",\n" - "\t\t\t\"type\": \"%s\",\n" - "\t\t\t\"family\": \"%s\",\n" - "\t\t\t\"context\": \"%s\",\n" - "\t\t\t\"title\": \"%s\",\n" - "\t\t\t\"priority\": %ld,\n" - "\t\t\t\"enabled\": %d,\n" - "\t\t\t\"units\": \"%s\",\n" - "\t\t\t\"url\": \"/data/%s/%s\",\n" - "\t\t\t\"chart_type\": \"%s\",\n" - "\t\t\t\"counter\": %lu,\n" - "\t\t\t\"entries\": %ld,\n" - "\t\t\t\"first_entry_t\": %ld,\n" - "\t\t\t\"last_entry\": %lu,\n" - "\t\t\t\"last_entry_t\": %ld,\n" - "\t\t\t\"last_entry_secs_ago\": %ld,\n" - "\t\t\t\"update_every\": %d,\n" - "\t\t\t\"isdetail\": %d,\n" - "\t\t\t\"usec_since_last_update\": %llu,\n" - "\t\t\t\"collected_total\": " TOTAL_NUMBER_FORMAT ",\n" - "\t\t\t\"last_collected_total\": " TOTAL_NUMBER_FORMAT ",\n" - "\t\t\t\"dimensions\": [\n" - , st->id - , st->name - , st->type - , st->family - , st->context - , st->title - , st->priority - , rrdset_flag_check(st, RRDSET_FLAG_ENABLED)?1:0 - , st->units - , st->name, options?options:"" - , rrdset_type_name(st->chart_type) - , st->counter - , st->entries - , rrdset_first_entry_t(st) - , rrdset_last_slot(st) - , rrdset_last_entry_t(st) - , (now < rrdset_last_entry_t(st)) ? (time_t)0 : now - rrdset_last_entry_t(st) - , st->update_every - , rrdset_flag_check(st, RRDSET_FLAG_DETAIL)?1:0 - , st->usec_since_last_update - , st->collected_total - , st->last_collected_total - ); - - unsigned long memory = st->memsize; - - RRDDIM *rd; - rrddim_foreach_read(rd, st) { - - memory += rd->memsize; - - buffer_sprintf(wb, - "\t\t\t\t{\n" - "\t\t\t\t\t\"id\": \"%s\",\n" - "\t\t\t\t\t\"name\": \"%s\",\n" - "\t\t\t\t\t\"entries\": %ld,\n" - "\t\t\t\t\t\"isHidden\": %d,\n" - "\t\t\t\t\t\"algorithm\": \"%s\",\n" - "\t\t\t\t\t\"multiplier\": " COLLECTED_NUMBER_FORMAT ",\n" - "\t\t\t\t\t\"divisor\": " COLLECTED_NUMBER_FORMAT ",\n" - "\t\t\t\t\t\"last_entry_t\": %ld,\n" - "\t\t\t\t\t\"collected_value\": " COLLECTED_NUMBER_FORMAT ",\n" - "\t\t\t\t\t\"calculated_value\": " CALCULATED_NUMBER_FORMAT ",\n" - "\t\t\t\t\t\"last_collected_value\": " COLLECTED_NUMBER_FORMAT ",\n" - "\t\t\t\t\t\"last_calculated_value\": " CALCULATED_NUMBER_FORMAT ",\n" - "\t\t\t\t\t\"memory\": %lu\n" - "\t\t\t\t}%s\n" - , rd->id - , rd->name - , rd->entries - , rrddim_flag_check(rd, RRDDIM_FLAG_HIDDEN)?1:0 - , rrd_algorithm_name(rd->algorithm) - , rd->multiplier - , rd->divisor - , rd->last_collected_time.tv_sec - , rd->collected_value - , rd->calculated_value - , rd->last_collected_value - , rd->last_calculated_value - , rd->memsize - , rd->next?",":"" - ); - } - - buffer_sprintf(wb, - "\t\t\t],\n" - "\t\t\t\"memory\" : %lu\n" - "\t\t}" - , memory - ); - - rrdset_unlock(st); - return memory; -} - -#define RRD_GRAPH_JSON_HEADER "{\n\t\"charts\": [\n" -#define RRD_GRAPH_JSON_FOOTER "\n\t]\n}\n" - -void rrd_stats_graph_json(RRDSET *st, char *options, BUFFER *wb) -{ - buffer_strcat(wb, RRD_GRAPH_JSON_HEADER); - rrd_stats_one_json(st, options, wb); - buffer_strcat(wb, RRD_GRAPH_JSON_FOOTER); -} - -void rrd_stats_all_json(RRDHOST *host, BUFFER *wb) -{ - unsigned long memory = 0; - long c = 0; - RRDSET *st; - - buffer_strcat(wb, RRD_GRAPH_JSON_HEADER); - - rrdhost_rdlock(host); - rrdset_foreach_read(st, host) { - if(rrdset_flag_check(st, RRDSET_FLAG_ENABLED) && st->dimensions) { - if(c) buffer_strcat(wb, ",\n"); - memory += rrd_stats_one_json(st, NULL, wb); - c++; - } - } - rrdhost_unlock(host); - - buffer_sprintf(wb, "\n\t],\n" - "\t\"hostname\": \"%s\",\n" - "\t\"update_every\": %d,\n" - "\t\"history\": %d,\n" - "\t\"memory\": %lu\n" - "}\n" - , host->hostname - , host->rrd_update_every - , host->rrd_history_entries - , memory - ); -} - - - // ---------------------------------------------------------------------------- // RRDR dimension options @@ -1665,7 +1519,7 @@ RRDR *rrd2rrdr(RRDSET *st, long points, long long after, long long before, int g ); r->group = group; - r->update_every = group * st->update_every; + r->update_every = (int)group * st->update_every; r->before = now; r->after = now; @@ -1817,8 +1671,20 @@ RRDR *rrd2rrdr(RRDSET *st, long points, long long after, long long before, int g return r; } -int rrd2value(RRDSET *st, BUFFER *wb, calculated_number *n, const char *dimensions, long points, long long after, long long before, int group_method, uint32_t options, time_t *db_after, time_t *db_before, int *value_is_null) -{ +int rrdset2value_api_v1( + RRDSET *st + , BUFFER *wb + , calculated_number *n + , const char *dimensions + , long points + , long long after + , long long before + , int group_method + , uint32_t options + , time_t *db_after + , time_t *db_before + , int *value_is_null +) { RRDR *r = rrd2rrdr(st, points, after, before, group_method, !(options & RRDR_OPTION_NOT_ALIGNED)); if(!r) { if(value_is_null) *value_is_null = 1; @@ -1855,8 +1721,20 @@ int rrd2value(RRDSET *st, BUFFER *wb, calculated_number *n, const char *dimensio return 200; } -int rrd2format(RRDSET *st, BUFFER *wb, BUFFER *dimensions, uint32_t format, long points, long long after, long long before, int group_method, uint32_t options, time_t *latest_timestamp) -{ +int rrdset2anything_api_v1( + RRDSET *st + , BUFFER *wb + , BUFFER *dimensions + , uint32_t format + , long points + , long long after + , long long before + , int group_method + , uint32_t options + , time_t *latest_timestamp +) { + st->last_accessed_time = now_realtime_sec(); + RRDR *r = rrd2rrdr(st, points, after, before, group_method, !(options & RRDR_OPTION_NOT_ALIGNED)); if(!r) { buffer_strcat(wb, "Cannot generate output with these parameters on this chart."); @@ -2028,327 +1906,3 @@ int rrd2format(RRDSET *st, BUFFER *wb, BUFFER *dimensions, uint32_t format, long rrdr_free(r); return 200; } - -time_t rrd_stats_json(int type, RRDSET *st, BUFFER *wb, long points, long group, int group_method, time_t after, time_t before, int only_non_zero) -{ - int c; - rrdset_rdlock(st); - - - // ------------------------------------------------------------------------- - // switch from JSON to google JSON - - char kq[2] = "\""; - char sq[2] = "\""; - switch(type) { - case DATASOURCE_DATATABLE_JSON: - case DATASOURCE_DATATABLE_JSONP: - kq[0] = '\0'; - sq[0] = '\''; - break; - - case DATASOURCE_JSON: - default: - break; - } - - - // ------------------------------------------------------------------------- - // validate the parameters - - if(points < 1) points = 1; - if(group < 1) group = 1; - - if(before == 0 || before > rrdset_last_entry_t(st)) before = rrdset_last_entry_t(st); - if(after == 0 || after < rrdset_first_entry_t(st)) after = rrdset_first_entry_t(st); - - // --- - - // our return value (the last timestamp printed) - // this is required to detect re-transmit in google JSONP - time_t last_timestamp = 0; - - - // ------------------------------------------------------------------------- - // find how many dimensions we have - - int dimensions = 0; - RRDDIM *rd; - rrddim_foreach_read(rd, st) dimensions++; - if(!dimensions) { - rrdset_unlock(st); - buffer_strcat(wb, "No dimensions yet."); - return 0; - } - - - // ------------------------------------------------------------------------- - // prepare various strings, to speed up the loop - - char overflow_annotation[201]; snprintfz(overflow_annotation, 200, ",{%sv%s:%sRESET OR OVERFLOW%s},{%sv%s:%sThe counters have been wrapped.%s}", kq, kq, sq, sq, kq, kq, sq, sq); - char normal_annotation[201]; snprintfz(normal_annotation, 200, ",{%sv%s:null},{%sv%s:null}", kq, kq, kq, kq); - char pre_date[51]; snprintfz(pre_date, 50, " {%sc%s:[{%sv%s:%s", kq, kq, kq, kq, sq); - char post_date[21]; snprintfz(post_date, 20, "%s}", sq); - char pre_value[21]; snprintfz(pre_value, 20, ",{%sv%s:", kq, kq); - char post_value[21]; strcpy(post_value, "}"); - - - // ------------------------------------------------------------------------- - // checks for debugging - - if(rrdset_flag_check(st, RRDSET_FLAG_DEBUG)) { - debug(D_RRD_STATS, "%s first_entry_t = %ld, last_entry_t = %ld, duration = %ld, after = %ld, before = %ld, duration = %ld, entries_to_show = %ld, group = %ld" - , st->id - , rrdset_first_entry_t(st) - , rrdset_last_entry_t(st) - , rrdset_last_entry_t(st) - rrdset_first_entry_t(st) - , after - , before - , before - after - , points - , group - ); - - if(before < after) - debug(D_RRD_STATS, "WARNING: %s The newest value in the database (%ld) is earlier than the oldest (%ld)", st->name, before, after); - - if((before - after) > st->entries * st->update_every) - debug(D_RRD_STATS, "WARNING: %s The time difference between the oldest and the newest entries (%ld) is higher than the capacity of the database (%ld)", st->name, before - after, st->entries * st->update_every); - } - - - // ------------------------------------------------------------------------- - // temp arrays for keeping values per dimension - - calculated_number group_values[dimensions]; // keep sums when grouping - int print_hidden[dimensions]; // keep hidden flags - int found_non_zero[dimensions]; - int found_non_existing[dimensions]; - - // initialize them - for( rd = st->dimensions, c = 0 ; rd && c < dimensions ; rd = rd->next, c++) { - group_values[c] = 0; - print_hidden[c] = rrddim_flag_check(rd, RRDDIM_FLAG_HIDDEN)?1:0; - found_non_zero[c] = 0; - found_non_existing[c] = 0; - } - - - // error("OLD: points=%d after=%d before=%d group=%d, duration=%d", entries_to_show, before - (st->update_every * group * entries_to_show), before, group, before - after + 1); - // rrd2array(st, entries_to_show, before - (st->update_every * group * entries_to_show), before, group_method, only_non_zero); - // rrd2rrdr(st, entries_to_show, before - (st->update_every * group * entries_to_show), before, group_method); - - // ------------------------------------------------------------------------- - // remove dimensions that contain only zeros - - int max_loop = 1; - if(only_non_zero) max_loop = 2; - - for(; max_loop ; max_loop--) { - - // ------------------------------------------------------------------------- - // print the JSON header - - buffer_sprintf(wb, "{\n %scols%s:\n [\n", kq, kq); - buffer_sprintf(wb, " {%sid%s:%s%s,%slabel%s:%stime%s,%spattern%s:%s%s,%stype%s:%sdatetime%s},\n", kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq); - buffer_sprintf(wb, " {%sid%s:%s%s,%slabel%s:%s%s,%spattern%s:%s%s,%stype%s:%sstring%s,%sp%s:{%srole%s:%sannotation%s}},\n", kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, kq, kq, sq, sq); - buffer_sprintf(wb, " {%sid%s:%s%s,%slabel%s:%s%s,%spattern%s:%s%s,%stype%s:%sstring%s,%sp%s:{%srole%s:%sannotationText%s}}", kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, kq, kq, sq, sq); - - // print the header for each dimension - // and update the print_hidden array for the dimensions that should be hidden - int pc = 0; - for( rd = st->dimensions, c = 0 ; rd && c < dimensions ; rd = rd->next, c++) { - if(!print_hidden[c]) { - pc++; - buffer_sprintf(wb, ",\n {%sid%s:%s%s,%slabel%s:%s%s%s,%spattern%s:%s%s,%stype%s:%snumber%s}", kq, kq, sq, sq, kq, kq, sq, rd->name, sq, kq, kq, sq, sq, kq, kq, sq, sq); - } - } - if(!pc) { - buffer_sprintf(wb, ",\n {%sid%s:%s%s,%slabel%s:%s%s%s,%spattern%s:%s%s,%stype%s:%snumber%s}", kq, kq, sq, sq, kq, kq, sq, "no data", sq, kq, kq, sq, sq, kq, kq, sq, sq); - } - - // print the begin of row data - buffer_sprintf(wb, "\n ],\n %srows%s:\n [\n", kq, kq); - - - // ------------------------------------------------------------------------- - // the main loop - - int annotate_reset = 0; - int annotation_count = 0; - - long t = rrdset_time2slot(st, before), - stop_at_t = rrdset_time2slot(st, after), - stop_now = 0; - - t -= t % group; - - time_t now = rrdset_slot2time(st, t), - dt = st->update_every; - - long count = 0, printed = 0, group_count = 0; - last_timestamp = 0; - - if(unlikely(rrdset_flag_check(st, RRDSET_FLAG_DEBUG))) - debug(D_RRD_STATS, "%s: REQUEST after:%u before:%u, points:%ld, group:%ld, CHART cur:%ld first: %u last:%u, CALC start_t:%ld, stop_t:%ld" - , st->id - , (uint32_t)after - , (uint32_t)before - , points - , group - , st->current_entry - , (uint32_t)rrdset_first_entry_t(st) - , (uint32_t)rrdset_last_entry_t(st) - , t - , stop_at_t - ); - - long counter = 0; - for(; !stop_now ; now -= dt, t--, counter++) { - if(t < 0) t = st->entries - 1; - if(t == stop_at_t) stop_now = counter; - - int print_this = 0; - - if(unlikely(rrdset_flag_check(st, RRDSET_FLAG_DEBUG))) - debug(D_RRD_STATS, "%s t = %ld, count = %ld, group_count = %ld, printed = %ld, now = %ld, %s %s" - , st->id - , t - , count + 1 - , group_count + 1 - , printed - , now - , (group_count + 1 == group)?"PRINT":" - " - , (now >= after && now <= before)?"RANGE":" - " - ); - - - // make sure we return data in the proper time range - if(now > before) continue; - if(now < after) break; - - //if(rrdset_slot2time(st, t) != now) - // error("%s: slot=%ld, now=%ld, slot2time=%ld, diff=%ld, last_entry_t=%ld, rrdset_last_slot=%ld", st->id, t, now, rrdset_slot2time(st,t), now - rrdset_slot2time(st,t), rrdset_last_entry_t(st), rrdset_last_slot(st)); - - count++; - group_count++; - - // check if we have to print this now - if(group_count == group) { - if(printed >= points) { - // debug(D_RRD_STATS, "Already printed all rows. Stopping."); - break; - } - - // generate the local date time - struct tm tmbuf, *tm = localtime_r(&now, &tmbuf); - if(!tm) { error("localtime() failed."); continue; } - if(now > last_timestamp) last_timestamp = now; - - if(printed) buffer_strcat(wb, "]},\n"); - buffer_strcat(wb, pre_date); - buffer_jsdate(wb, tm->tm_year + 1900, tm->tm_mon, tm->tm_mday, tm->tm_hour, tm->tm_min, tm->tm_sec); - buffer_strcat(wb, post_date); - - print_this = 1; - } - - // do the calculations - for(rd = st->dimensions, c = 0 ; rd && c < dimensions ; rd = rd->next, c++) { - storage_number n = rd->values[t]; - calculated_number value = unpack_storage_number(n); - - if(!does_storage_number_exist(n)) { - value = 0.0; - found_non_existing[c]++; - } - if(did_storage_number_reset(n)) annotate_reset = 1; - - switch(group_method) { - case GROUP_MAX: - if(abs(value) > abs(group_values[c])) group_values[c] = value; - break; - - case GROUP_SUM: - group_values[c] += value; - break; - - default: - case GROUP_AVERAGE: - group_values[c] += value; - if(print_this) group_values[c] /= ( group_count - found_non_existing[c] ); - break; - } - } - - if(print_this) { - if(annotate_reset) { - annotation_count++; - buffer_strcat(wb, overflow_annotation); - annotate_reset = 0; - } - else - buffer_strcat(wb, normal_annotation); - - pc = 0; - for(c = 0 ; c < dimensions ; c++) { - if(found_non_existing[c] == group_count) { - // all entries are non-existing - pc++; - buffer_strcat(wb, pre_value); - buffer_strcat(wb, "null"); - buffer_strcat(wb, post_value); - } - else if(!print_hidden[c]) { - pc++; - buffer_strcat(wb, pre_value); - buffer_rrd_value(wb, group_values[c]); - buffer_strcat(wb, post_value); - - if(group_values[c]) found_non_zero[c]++; - } - - // reset them for the next loop - group_values[c] = 0; - found_non_existing[c] = 0; - } - - // if all dimensions are hidden, print a null - if(!pc) { - buffer_strcat(wb, pre_value); - buffer_strcat(wb, "null"); - buffer_strcat(wb, post_value); - } - - printed++; - group_count = 0; - } - } - - if(printed) buffer_strcat(wb, "]}"); - buffer_strcat(wb, "\n ]\n}\n"); - - if(only_non_zero && max_loop > 1) { - int changed = 0; - for(rd = st->dimensions, c = 0 ; rd && c < dimensions ; rd = rd->next, c++) { - group_values[c] = 0; - found_non_existing[c] = 0; - - if(!print_hidden[c] && !found_non_zero[c]) { - changed = 1; - print_hidden[c] = 1; - } - } - - if(changed) buffer_flush(wb); - else break; - } - else break; - - } // max_loop - - debug(D_RRD_STATS, "RRD_STATS_JSON: %s total %zu bytes", st->name, wb->len); - - rrdset_unlock(st); - return last_timestamp; -} diff --git a/src/rrd2json.h b/src/rrd2json.h index 18d54b7b..2e977eb5 100644 --- a/src/rrd2json.h +++ b/src/rrd2json.h @@ -64,15 +64,11 @@ extern void rrd_stats_api_v1_charts(RRDHOST *host, BUFFER *wb); extern void rrd_stats_api_v1_charts_allmetrics_shell(RRDHOST *host, BUFFER *wb); extern void rrd_stats_api_v1_charts_allmetrics_prometheus(RRDHOST *host, BUFFER *wb); -extern unsigned long rrd_stats_one_json(RRDSET *st, char *options, BUFFER *wb); - -extern void rrd_stats_graph_json(RRDSET *st, char *options, BUFFER *wb); - -extern void rrd_stats_all_json(RRDHOST *host, BUFFER *wb); - -extern time_t rrd_stats_json(int type, RRDSET *st, BUFFER *wb, long entries_to_show, long group, int group_method, time_t after, time_t before, int only_non_zero); - -extern int rrd2format(RRDSET *st, BUFFER *out, BUFFER *dimensions, uint32_t format, long points, long long after, long long before, int group_method, uint32_t options, time_t *latest_timestamp); -extern int rrd2value(RRDSET *st, BUFFER *wb, calculated_number *n, const char *dimensions, long points, long long after, long long before, int group_method, uint32_t options, time_t *db_before, time_t *db_after, int *value_is_null); +extern int rrdset2anything_api_v1(RRDSET *st, BUFFER *out, BUFFER *dimensions, uint32_t format, long points + , long long after, long long before, int group_method, uint32_t options + , time_t *latest_timestamp); +extern int rrdset2value_api_v1(RRDSET *st, BUFFER *wb, calculated_number *n, const char *dimensions, long points + , long long after, long long before, int group_method, uint32_t options + , time_t *db_before, time_t *db_after, int *value_is_null); #endif /* NETDATA_RRD2JSON_H */ diff --git a/src/rrd2json_api_old.c b/src/rrd2json_api_old.c new file mode 100644 index 00000000..3b028308 --- /dev/null +++ b/src/rrd2json_api_old.c @@ -0,0 +1,487 @@ +#include "common.h" + +unsigned long rrdset_info2json_api_old(RRDSET *st, char *options, BUFFER *wb) { + time_t now = now_realtime_sec(); + + rrdset_rdlock(st); + + st->last_accessed_time = now; + + buffer_sprintf(wb, + "\t\t{\n" + "\t\t\t\"id\": \"%s\",\n" + "\t\t\t\"name\": \"%s\",\n" + "\t\t\t\"type\": \"%s\",\n" + "\t\t\t\"family\": \"%s\",\n" + "\t\t\t\"context\": \"%s\",\n" + "\t\t\t\"title\": \"%s\",\n" + "\t\t\t\"priority\": %ld,\n" + "\t\t\t\"enabled\": %d,\n" + "\t\t\t\"units\": \"%s\",\n" + "\t\t\t\"url\": \"/data/%s/%s\",\n" + "\t\t\t\"chart_type\": \"%s\",\n" + "\t\t\t\"counter\": %lu,\n" + "\t\t\t\"entries\": %ld,\n" + "\t\t\t\"first_entry_t\": %ld,\n" + "\t\t\t\"last_entry\": %lu,\n" + "\t\t\t\"last_entry_t\": %ld,\n" + "\t\t\t\"last_entry_secs_ago\": %ld,\n" + "\t\t\t\"update_every\": %d,\n" + "\t\t\t\"isdetail\": %d,\n" + "\t\t\t\"usec_since_last_update\": %llu,\n" + "\t\t\t\"collected_total\": " TOTAL_NUMBER_FORMAT ",\n" + "\t\t\t\"last_collected_total\": " TOTAL_NUMBER_FORMAT ",\n" + "\t\t\t\"dimensions\": [\n" + , st->id + , st->name + , st->type + , st->family + , st->context + , st->title + , st->priority + , rrdset_flag_check(st, RRDSET_FLAG_ENABLED)?1:0 + , st->units + , st->name, options?options:"" + , rrdset_type_name(st->chart_type) + , st->counter + , st->entries + , rrdset_first_entry_t(st) + , rrdset_last_slot(st) + , rrdset_last_entry_t(st) + , (now < rrdset_last_entry_t(st)) ? (time_t)0 : now - rrdset_last_entry_t(st) + , st->update_every + , rrdset_flag_check(st, RRDSET_FLAG_DETAIL)?1:0 + , st->usec_since_last_update + , st->collected_total + , st->last_collected_total + ); + + unsigned long memory = st->memsize; + + RRDDIM *rd; + rrddim_foreach_read(rd, st) { + + memory += rd->memsize; + + buffer_sprintf(wb, + "\t\t\t\t{\n" + "\t\t\t\t\t\"id\": \"%s\",\n" + "\t\t\t\t\t\"name\": \"%s\",\n" + "\t\t\t\t\t\"entries\": %ld,\n" + "\t\t\t\t\t\"isHidden\": %d,\n" + "\t\t\t\t\t\"algorithm\": \"%s\",\n" + "\t\t\t\t\t\"multiplier\": " COLLECTED_NUMBER_FORMAT ",\n" + "\t\t\t\t\t\"divisor\": " COLLECTED_NUMBER_FORMAT ",\n" + "\t\t\t\t\t\"last_entry_t\": %ld,\n" + "\t\t\t\t\t\"collected_value\": " COLLECTED_NUMBER_FORMAT ",\n" + "\t\t\t\t\t\"calculated_value\": " CALCULATED_NUMBER_FORMAT ",\n" + "\t\t\t\t\t\"last_collected_value\": " COLLECTED_NUMBER_FORMAT ",\n" + "\t\t\t\t\t\"last_calculated_value\": " CALCULATED_NUMBER_FORMAT ",\n" + "\t\t\t\t\t\"memory\": %lu\n" + "\t\t\t\t}%s\n" + , rd->id + , rd->name + , rd->entries + , rrddim_flag_check(rd, RRDDIM_FLAG_HIDDEN)?1:0 + , rrd_algorithm_name(rd->algorithm) + , rd->multiplier + , rd->divisor + , rd->last_collected_time.tv_sec + , rd->collected_value + , rd->calculated_value + , rd->last_collected_value + , rd->last_calculated_value + , rd->memsize + , rd->next?",":"" + ); + } + + buffer_sprintf(wb, + "\t\t\t],\n" + "\t\t\t\"memory\" : %lu\n" + "\t\t}" + , memory + ); + + rrdset_unlock(st); + return memory; +} + +#define RRD_GRAPH_JSON_HEADER "{\n\t\"charts\": [\n" +#define RRD_GRAPH_JSON_FOOTER "\n\t]\n}\n" + +void rrd_graph2json_api_old(RRDSET *st, char *options, BUFFER *wb) +{ + buffer_strcat(wb, RRD_GRAPH_JSON_HEADER); + rrdset_info2json_api_old(st, options, wb); + buffer_strcat(wb, RRD_GRAPH_JSON_FOOTER); +} + +void rrd_all2json_api_old(RRDHOST *host, BUFFER *wb) +{ + unsigned long memory = 0; + long c = 0; + RRDSET *st; + + time_t now = now_realtime_sec(); + + buffer_strcat(wb, RRD_GRAPH_JSON_HEADER); + + rrdhost_rdlock(host); + rrdset_foreach_read(st, host) { + if(rrdset_is_available_for_viewers(st)) { + if(c) buffer_strcat(wb, ",\n"); + memory += rrdset_info2json_api_old(st, NULL, wb); + + c++; + st->last_accessed_time = now; + } + } + rrdhost_unlock(host); + + buffer_sprintf(wb, "\n\t],\n" + "\t\"hostname\": \"%s\",\n" + "\t\"update_every\": %d,\n" + "\t\"history\": %d,\n" + "\t\"memory\": %lu\n" + "}\n" + , host->hostname + , host->rrd_update_every + , host->rrd_history_entries + , memory + ); +} + +time_t rrdset2json_api_old( + int type + , RRDSET *st + , BUFFER *wb + , long points + , long group + , int group_method + , time_t after + , time_t before + , int only_non_zero +) { + int c; + rrdset_rdlock(st); + + st->last_accessed_time = now_realtime_sec(); + + // ------------------------------------------------------------------------- + // switch from JSON to google JSON + + char kq[2] = "\""; + char sq[2] = "\""; + switch(type) { + case DATASOURCE_DATATABLE_JSON: + case DATASOURCE_DATATABLE_JSONP: + kq[0] = '\0'; + sq[0] = '\''; + break; + + case DATASOURCE_JSON: + default: + break; + } + + + // ------------------------------------------------------------------------- + // validate the parameters + + if(points < 1) points = 1; + if(group < 1) group = 1; + + if(before == 0 || before > rrdset_last_entry_t(st)) before = rrdset_last_entry_t(st); + if(after == 0 || after < rrdset_first_entry_t(st)) after = rrdset_first_entry_t(st); + + // --- + + // our return value (the last timestamp printed) + // this is required to detect re-transmit in google JSONP + time_t last_timestamp = 0; + + + // ------------------------------------------------------------------------- + // find how many dimensions we have + + int dimensions = 0; + RRDDIM *rd; + rrddim_foreach_read(rd, st) dimensions++; + if(!dimensions) { + rrdset_unlock(st); + buffer_strcat(wb, "No dimensions yet."); + return 0; + } + + + // ------------------------------------------------------------------------- + // prepare various strings, to speed up the loop + + char overflow_annotation[201]; snprintfz(overflow_annotation, 200, ",{%sv%s:%sRESET OR OVERFLOW%s},{%sv%s:%sThe counters have been wrapped.%s}", kq, kq, sq, sq, kq, kq, sq, sq); + char normal_annotation[201]; snprintfz(normal_annotation, 200, ",{%sv%s:null},{%sv%s:null}", kq, kq, kq, kq); + char pre_date[51]; snprintfz(pre_date, 50, " {%sc%s:[{%sv%s:%s", kq, kq, kq, kq, sq); + char post_date[21]; snprintfz(post_date, 20, "%s}", sq); + char pre_value[21]; snprintfz(pre_value, 20, ",{%sv%s:", kq, kq); + char post_value[21]; strcpy(post_value, "}"); + + + // ------------------------------------------------------------------------- + // checks for debugging + + if(rrdset_flag_check(st, RRDSET_FLAG_DEBUG)) { + debug(D_RRD_STATS, "%s first_entry_t = %ld, last_entry_t = %ld, duration = %ld, after = %ld, before = %ld, duration = %ld, entries_to_show = %ld, group = %ld" + , st->id + , rrdset_first_entry_t(st) + , rrdset_last_entry_t(st) + , rrdset_last_entry_t(st) - rrdset_first_entry_t(st) + , after + , before + , before - after + , points + , group + ); + + if(before < after) + debug(D_RRD_STATS, "WARNING: %s The newest value in the database (%ld) is earlier than the oldest (%ld)", st->name, before, after); + + if((before - after) > st->entries * st->update_every) + debug(D_RRD_STATS, "WARNING: %s The time difference between the oldest and the newest entries (%ld) is higher than the capacity of the database (%ld)", st->name, before - after, st->entries * st->update_every); + } + + + // ------------------------------------------------------------------------- + // temp arrays for keeping values per dimension + + calculated_number group_values[dimensions]; // keep sums when grouping + int print_hidden[dimensions]; // keep hidden flags + int found_non_zero[dimensions]; + int found_non_existing[dimensions]; + + // initialize them + for( rd = st->dimensions, c = 0 ; rd && c < dimensions ; rd = rd->next, c++) { + group_values[c] = 0; + print_hidden[c] = rrddim_flag_check(rd, RRDDIM_FLAG_HIDDEN)?1:0; + found_non_zero[c] = 0; + found_non_existing[c] = 0; + } + + + // error("OLD: points=%d after=%d before=%d group=%d, duration=%d", entries_to_show, before - (st->update_every * group * entries_to_show), before, group, before - after + 1); + // rrd2array(st, entries_to_show, before - (st->update_every * group * entries_to_show), before, group_method, only_non_zero); + // rrd2rrdr(st, entries_to_show, before - (st->update_every * group * entries_to_show), before, group_method); + + // ------------------------------------------------------------------------- + // remove dimensions that contain only zeros + + int max_loop = 1; + if(only_non_zero) max_loop = 2; + + for(; max_loop ; max_loop--) { + + // ------------------------------------------------------------------------- + // print the JSON header + + buffer_sprintf(wb, "{\n %scols%s:\n [\n", kq, kq); + buffer_sprintf(wb, " {%sid%s:%s%s,%slabel%s:%stime%s,%spattern%s:%s%s,%stype%s:%sdatetime%s},\n", kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq); + buffer_sprintf(wb, " {%sid%s:%s%s,%slabel%s:%s%s,%spattern%s:%s%s,%stype%s:%sstring%s,%sp%s:{%srole%s:%sannotation%s}},\n", kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, kq, kq, sq, sq); + buffer_sprintf(wb, " {%sid%s:%s%s,%slabel%s:%s%s,%spattern%s:%s%s,%stype%s:%sstring%s,%sp%s:{%srole%s:%sannotationText%s}}", kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, kq, kq, sq, sq); + + // print the header for each dimension + // and update the print_hidden array for the dimensions that should be hidden + int pc = 0; + for( rd = st->dimensions, c = 0 ; rd && c < dimensions ; rd = rd->next, c++) { + if(!print_hidden[c]) { + pc++; + buffer_sprintf(wb, ",\n {%sid%s:%s%s,%slabel%s:%s%s%s,%spattern%s:%s%s,%stype%s:%snumber%s}", kq, kq, sq, sq, kq, kq, sq, rd->name, sq, kq, kq, sq, sq, kq, kq, sq, sq); + } + } + if(!pc) { + buffer_sprintf(wb, ",\n {%sid%s:%s%s,%slabel%s:%s%s%s,%spattern%s:%s%s,%stype%s:%snumber%s}", kq, kq, sq, sq, kq, kq, sq, "no data", sq, kq, kq, sq, sq, kq, kq, sq, sq); + } + + // print the begin of row data + buffer_sprintf(wb, "\n ],\n %srows%s:\n [\n", kq, kq); + + + // ------------------------------------------------------------------------- + // the main loop + + int annotate_reset = 0; + int annotation_count = 0; + + long t = rrdset_time2slot(st, before), + stop_at_t = rrdset_time2slot(st, after), + stop_now = 0; + + t -= t % group; + + time_t now = rrdset_slot2time(st, t), + dt = st->update_every; + + long count = 0, printed = 0, group_count = 0; + last_timestamp = 0; + + if(unlikely(rrdset_flag_check(st, RRDSET_FLAG_DEBUG))) + debug(D_RRD_STATS, "%s: REQUEST after:%u before:%u, points:%ld, group:%ld, CHART cur:%ld first: %u last:%u, CALC start_t:%ld, stop_t:%ld" + , st->id + , (uint32_t)after + , (uint32_t)before + , points + , group + , st->current_entry + , (uint32_t)rrdset_first_entry_t(st) + , (uint32_t)rrdset_last_entry_t(st) + , t + , stop_at_t + ); + + long counter = 0; + for(; !stop_now ; now -= dt, t--, counter++) { + if(t < 0) t = st->entries - 1; + if(t == stop_at_t) stop_now = counter; + + int print_this = 0; + + if(unlikely(rrdset_flag_check(st, RRDSET_FLAG_DEBUG))) + debug(D_RRD_STATS, "%s t = %ld, count = %ld, group_count = %ld, printed = %ld, now = %ld, %s %s" + , st->id + , t + , count + 1 + , group_count + 1 + , printed + , now + , (group_count + 1 == group)?"PRINT":" - " + , (now >= after && now <= before)?"RANGE":" - " + ); + + + // make sure we return data in the proper time range + if(now > before) continue; + if(now < after) break; + + //if(rrdset_slot2time(st, t) != now) + // error("%s: slot=%ld, now=%ld, slot2time=%ld, diff=%ld, last_entry_t=%ld, rrdset_last_slot=%ld", st->id, t, now, rrdset_slot2time(st,t), now - rrdset_slot2time(st,t), rrdset_last_entry_t(st), rrdset_last_slot(st)); + + count++; + group_count++; + + // check if we have to print this now + if(group_count == group) { + if(printed >= points) { + // debug(D_RRD_STATS, "Already printed all rows. Stopping."); + break; + } + + // generate the local date time + struct tm tmbuf, *tm = localtime_r(&now, &tmbuf); + if(!tm) { error("localtime() failed."); continue; } + if(now > last_timestamp) last_timestamp = now; + + if(printed) buffer_strcat(wb, "]},\n"); + buffer_strcat(wb, pre_date); + buffer_jsdate(wb, tm->tm_year + 1900, tm->tm_mon, tm->tm_mday, tm->tm_hour, tm->tm_min, tm->tm_sec); + buffer_strcat(wb, post_date); + + print_this = 1; + } + + // do the calculations + for(rd = st->dimensions, c = 0 ; rd && c < dimensions ; rd = rd->next, c++) { + storage_number n = rd->values[t]; + calculated_number value = unpack_storage_number(n); + + if(!does_storage_number_exist(n)) { + value = 0.0; + found_non_existing[c]++; + } + if(did_storage_number_reset(n)) annotate_reset = 1; + + switch(group_method) { + case GROUP_MAX: + if(abs(value) > abs(group_values[c])) group_values[c] = value; + break; + + case GROUP_SUM: + group_values[c] += value; + break; + + default: + case GROUP_AVERAGE: + group_values[c] += value; + if(print_this) group_values[c] /= ( group_count - found_non_existing[c] ); + break; + } + } + + if(print_this) { + if(annotate_reset) { + annotation_count++; + buffer_strcat(wb, overflow_annotation); + annotate_reset = 0; + } + else + buffer_strcat(wb, normal_annotation); + + pc = 0; + for(c = 0 ; c < dimensions ; c++) { + if(found_non_existing[c] == group_count) { + // all entries are non-existing + pc++; + buffer_strcat(wb, pre_value); + buffer_strcat(wb, "null"); + buffer_strcat(wb, post_value); + } + else if(!print_hidden[c]) { + pc++; + buffer_strcat(wb, pre_value); + buffer_rrd_value(wb, group_values[c]); + buffer_strcat(wb, post_value); + + if(group_values[c]) found_non_zero[c]++; + } + + // reset them for the next loop + group_values[c] = 0; + found_non_existing[c] = 0; + } + + // if all dimensions are hidden, print a null + if(!pc) { + buffer_strcat(wb, pre_value); + buffer_strcat(wb, "null"); + buffer_strcat(wb, post_value); + } + + printed++; + group_count = 0; + } + } + + if(printed) buffer_strcat(wb, "]}"); + buffer_strcat(wb, "\n ]\n}\n"); + + if(only_non_zero && max_loop > 1) { + int changed = 0; + for(rd = st->dimensions, c = 0 ; rd && c < dimensions ; rd = rd->next, c++) { + group_values[c] = 0; + found_non_existing[c] = 0; + + if(!print_hidden[c] && !found_non_zero[c]) { + changed = 1; + print_hidden[c] = 1; + } + } + + if(changed) buffer_flush(wb); + else break; + } + else break; + + } // max_loop + + debug(D_RRD_STATS, "RRD_STATS_JSON: %s total %zu bytes", st->name, wb->len); + + rrdset_unlock(st); + return last_timestamp; +} diff --git a/src/rrd2json_api_old.h b/src/rrd2json_api_old.h new file mode 100644 index 00000000..f8c63814 --- /dev/null +++ b/src/rrd2json_api_old.h @@ -0,0 +1,14 @@ +#ifndef NETDATA_RRD2JSON_API_OLD_H +#define NETDATA_RRD2JSON_API_OLD_H + +extern unsigned long rrdset_info2json_api_old(RRDSET *st, char *options, BUFFER *wb); + +extern void rrd_graph2json_api_old(RRDSET *st, char *options, BUFFER *wb); + +extern void rrd_all2json_api_old(RRDHOST *host, BUFFER *wb); + +extern time_t rrdset2json_api_old(int type, RRDSET *st, BUFFER *wb, long entries_to_show, long group, int group_method + , time_t after, time_t before, int only_non_zero); + + +#endif //NETDATA_RRD2JSON_API_OLD_H diff --git a/src/rrdhost.c b/src/rrdhost.c index de342bd3..7002c482 100644 --- a/src/rrdhost.c +++ b/src/rrdhost.c @@ -5,6 +5,7 @@ RRDHOST *localhost = NULL; size_t rrd_hosts_available = 0; pthread_rwlock_t rrd_rwlock = PTHREAD_RWLOCK_INITIALIZER; +time_t rrdset_free_obsolete_time = 3600; time_t rrdhost_free_orphan_time = 3600; // ---------------------------------------------------------------------------- @@ -315,18 +316,22 @@ RRDHOST *rrdhost_find_or_create( } void rrdhost_cleanup_remote_stale(RRDHOST *protected) { + time_t now = now_realtime_sec(); + rrd_wrlock(); RRDHOST *h; + +restart_after_removal: rrdhost_foreach_write(h) { if(h != protected && h != localhost && !h->connected_senders - && h->senders_disconnected_time + rrdhost_free_orphan_time > now_realtime_sec()) { + && h->senders_disconnected_time + rrdhost_free_orphan_time < now) { info("Host '%s' with machine guid '%s' is obsolete - cleaning up.", h->hostname, h->machine_guid); rrdhost_save(h); rrdhost_free(h); - break; + goto restart_after_removal; } } @@ -337,6 +342,8 @@ void rrdhost_cleanup_remote_stale(RRDHOST *protected) { // RRDHOST global / startup initialization void rrd_init(char *hostname) { + rrdset_free_obsolete_time = config_get_number(CONFIG_SECTION_GLOBAL, "cleanup obsolete charts after seconds", rrdset_free_obsolete_time); + health_init(); registry_init(); rrdpush_init(); @@ -472,7 +479,6 @@ void rrdhost_save(RRDHOST *host) { info("Saving database of host '%s'...", host->hostname); RRDSET *st; - RRDDIM *rd; // we get a write lock // to ensure only one thread is saving the database @@ -480,19 +486,7 @@ void rrdhost_save(RRDHOST *host) { rrdset_foreach_write(st, host) { rrdset_rdlock(st); - - if(st->rrd_memory_mode == RRD_MEMORY_MODE_SAVE) { - debug(D_RRD_STATS, "Saving stats '%s' to '%s'.", st->name, st->cache_filename); - savememory(st->cache_filename, st, st->memsize); - } - - rrddim_foreach_read(rd, st) { - if(likely(rd->rrd_memory_mode == RRD_MEMORY_MODE_SAVE)) { - debug(D_RRD_STATS, "Saving dimension '%s' to '%s'.", rd->name, rd->cache_filename); - savememory(rd->cache_filename, rd, rd->memsize); - } - } - + rrdset_save(st); rrdset_unlock(st); } @@ -510,3 +504,26 @@ void rrdhost_save_all(void) { rrd_unlock(); } + +void rrdhost_cleanup(RRDHOST *host) { + time_t now = now_realtime_sec(); + + RRDSET *st; + +restart_after_removal: + rrdset_foreach_write(st, host) { + if(unlikely(rrdset_flag_check(st, RRDSET_FLAG_OBSOLETE) + && st->last_accessed_time + rrdset_free_obsolete_time < now + && st->last_updated.tv_sec + rrdset_free_obsolete_time < now + && st->last_collected_time.tv_sec + rrdset_free_obsolete_time < now + )) { + + rrdset_rdlock(st); + rrdset_save(st); + rrdset_unlock(st); + + rrdset_free(st); + goto restart_after_removal; + } + } +} diff --git a/src/rrdset.c b/src/rrdset.c index 13d6014e..63aeaceb 100644 --- a/src/rrdset.c +++ b/src/rrdset.c @@ -234,14 +234,7 @@ void rrdset_free(RRDSET *st) { rrdhost_check_wrlock(st->rrdhost); // make sure we have a write lock on the host rrdset_wrlock(st); // lock this RRDSET - // ------------------------------------------------------------------------ - // free its children structures - - while(st->variables) rrdsetvar_free(st->variables); - while(st->alarms) rrdsetcalc_unlink(st->alarms); - while(st->dimensions) rrddim_free(st, st->dimensions); - - rrdfamily_free(st->rrdhost, st->rrdfamily); + // info("Removing chart '%s' ('%s')", st->id, st->name); // ------------------------------------------------------------------------ // remove it from the indexes @@ -251,6 +244,15 @@ void rrdset_free(RRDSET *st) { rrdset_index_del_name(st->rrdhost, st); + // ------------------------------------------------------------------------ + // free its children structures + + while(st->variables) rrdsetvar_free(st->variables); + while(st->alarms) rrdsetcalc_unlink(st->alarms); + while(st->dimensions) rrddim_free(st, st->dimensions); + + rrdfamily_free(st->rrdhost, st->rrdfamily); + // ------------------------------------------------------------------------ // unlink it from the host @@ -283,13 +285,42 @@ void rrdset_free(RRDSET *st) { freez(st); } +void rrdset_save(RRDSET *st) { + RRDDIM *rd; + + rrdset_check_rdlock(st); + + // info("Saving chart '%s' ('%s')", st->id, st->name); + + if(st->rrd_memory_mode == RRD_MEMORY_MODE_SAVE) { + debug(D_RRD_STATS, "Saving stats '%s' to '%s'.", st->name, st->cache_filename); + savememory(st->cache_filename, st, st->memsize); + } + + rrddim_foreach_read(rd, st) { + if(likely(rd->rrd_memory_mode == RRD_MEMORY_MODE_SAVE)) { + debug(D_RRD_STATS, "Saving dimension '%s' to '%s'.", rd->name, rd->cache_filename); + savememory(rd->cache_filename, rd, rd->memsize); + } + } +} + // ---------------------------------------------------------------------------- // RRDSET - create a chart -RRDSET *rrdset_create(RRDHOST *host, const char *type, const char *id, const char *name, const char *family - , const char *context, const char *title, const char *units, long priority - , int update_every, RRDSET_TYPE chart_type) { - +RRDSET *rrdset_create( + RRDHOST *host + , const char *type + , const char *id + , const char *name + , const char *family + , const char *context + , const char *title + , const char *units + , long priority + , int update_every + , RRDSET_TYPE chart_type +) { if(!type || !type[0]) { fatal("Cannot create rrd stats without a type."); return NULL; @@ -308,6 +339,7 @@ RRDSET *rrdset_create(RRDHOST *host, const char *type, const char *id, const cha RRDSET *st = rrdset_find(host, fullid); if(st) { + rrdset_flag_clear(st, RRDSET_FLAG_OBSOLETE); debug(D_RRD_CALLS, "RRDSET '%s', already exists.", fullid); return st; } @@ -449,6 +481,7 @@ RRDSET *rrdset_create(RRDHOST *host, const char *type, const char *id, const cha rrdset_flag_clear(st, RRDSET_FLAG_DETAIL); rrdset_flag_clear(st, RRDSET_FLAG_DEBUG); + rrdset_flag_clear(st, RRDSET_FLAG_OBSOLETE); // if(!strcmp(st->id, "disk_util.dm-0")) { // st->debug = 1; @@ -502,6 +535,8 @@ RRDSET *rrdset_create(RRDHOST *host, const char *type, const char *id, const cha rrdsetcalc_link_matching(st); rrdcalctemplate_link_matching(st); + rrdhost_cleanup(host); + rrdhost_unlock(host); return(st); @@ -677,6 +712,11 @@ void rrdset_done(RRDSET *st) { st->enabled = 1; */ + if(unlikely(rrdset_flag_check(st, RRDSET_FLAG_OBSOLETE))) { + error("Chart '%s' has the OBSOLETE flag set, but it is collected.", st->id); + rrdset_flag_clear(st, RRDSET_FLAG_OBSOLETE); + } + // check if the chart has a long time to be updated if(unlikely(st->usec_since_last_update > st->entries * update_every_ut)) { info("%s: took too long to be updated (%0.3Lf secs). Resetting it.", st->name, (long double)(st->usec_since_last_update / 1000000.0)); diff --git a/src/sys_fs_cgroup.c b/src/sys_fs_cgroup.c index 7981804a..248f3d62 100644 --- a/src/sys_fs_cgroup.c +++ b/src/sys_fs_cgroup.c @@ -881,6 +881,21 @@ static inline struct cgroup *cgroup_add(const char *id) { static inline void cgroup_free(struct cgroup *cg) { debug(D_CGROUP, "Removing cgroup '%s' with chart id '%s' (was %s and %s)", cg->id, cg->chart_id, (cg->enabled)?"enabled":"disabled", (cg->available)?"available":"not available"); + if(cg->st_cpu) rrdset_flag_set(cg->st_cpu, RRDSET_FLAG_OBSOLETE); + if(cg->st_cpu_per_core) rrdset_flag_set(cg->st_cpu_per_core, RRDSET_FLAG_OBSOLETE); + if(cg->st_mem) rrdset_flag_set(cg->st_mem, RRDSET_FLAG_OBSOLETE); + if(cg->st_writeback) rrdset_flag_set(cg->st_writeback, RRDSET_FLAG_OBSOLETE); + if(cg->st_mem_activity) rrdset_flag_set(cg->st_mem_activity, RRDSET_FLAG_OBSOLETE); + if(cg->st_pgfaults) rrdset_flag_set(cg->st_pgfaults, RRDSET_FLAG_OBSOLETE); + if(cg->st_mem_usage) rrdset_flag_set(cg->st_mem_usage, RRDSET_FLAG_OBSOLETE); + if(cg->st_mem_failcnt) rrdset_flag_set(cg->st_mem_failcnt, RRDSET_FLAG_OBSOLETE); + if(cg->st_io) rrdset_flag_set(cg->st_io, RRDSET_FLAG_OBSOLETE); + if(cg->st_serviced_ops) rrdset_flag_set(cg->st_serviced_ops, RRDSET_FLAG_OBSOLETE); + if(cg->st_throttle_io) rrdset_flag_set(cg->st_throttle_io, RRDSET_FLAG_OBSOLETE); + if(cg->st_throttle_serviced_ops) rrdset_flag_set(cg->st_throttle_serviced_ops, RRDSET_FLAG_OBSOLETE); + if(cg->st_queued_ops) rrdset_flag_set(cg->st_queued_ops, RRDSET_FLAG_OBSOLETE); + if(cg->st_merged_ops) rrdset_flag_set(cg->st_merged_ops, RRDSET_FLAG_OBSOLETE); + freez(cg->cpuacct_usage.cpu_percpu); freez(cg->cpuacct_stat.filename); @@ -1266,18 +1281,19 @@ static inline void find_all_cgroups() { #define CHART_TITLE_MAX 300 -void update_services_charts(int update_every, - int do_cpu, - int do_mem_usage, - int do_mem_detailed, - int do_mem_failcnt, - int do_swap_usage, - int do_io, - int do_io_ops, - int do_throttle_io, - int do_throttle_ops, - int do_queued_ops, - int do_merged_ops +void update_systemd_services_charts( + int update_every + , int do_cpu + , int do_mem_usage + , int do_mem_detailed + , int do_mem_failcnt + , int do_swap_usage + , int do_io + , int do_io_ops + , int do_throttle_io + , int do_throttle_ops + , int do_queued_ops + , int do_merged_ops ) { static RRDSET *st_cpu = NULL, @@ -1313,13 +1329,21 @@ void update_services_charts(int update_every, if(likely(do_cpu)) { if(unlikely(!st_cpu)) { char title[CHART_TITLE_MAX + 1]; + snprintfz(title, CHART_TITLE_MAX, "Systemd Services CPU utilization (%d%% = %d core%s)", (processors * 100), processors, (processors > 1) ? "s" : ""); + + st_cpu = rrdset_create_localhost( + "services" + , "cpu" + , NULL + , "cpu" + , "services.cpu" + , title + , "%" + , CHART_PRIORITY_SYSTEMD_SERVICES + , update_every + , RRDSET_TYPE_STACKED + ); - st_cpu = rrdset_find_bytype_localhost("services", "cpu"); - if(likely(!st_cpu)) { - snprintfz(title, CHART_TITLE_MAX, "Systemd Services CPU utilization (%d%% = %d core%s)", (processors * 100), processors, (processors > 1) ? "s" : ""); - st_cpu = rrdset_create_localhost("services", "cpu", NULL, "cpu", "services.cpu", title, "%" - , CHART_PRIORITY_SYSTEMD_SERVICES, update_every, RRDSET_TYPE_STACKED); - } } else rrdset_next(st_cpu); @@ -1327,13 +1351,21 @@ void update_services_charts(int update_every, if(likely(do_mem_usage)) { if(unlikely(!st_mem_usage)) { - st_mem_usage = rrdset_find_bytype_localhost("services", "mem_usage"); - if(likely(!st_mem_usage)) - st_mem_usage = rrdset_create_localhost("services", "mem_usage", NULL, "mem", "services.mem_usage" - , (cgroup_used_memory_without_cache) - ? "Systemd Services Used Memory without Cache" - : "Systemd Services Used Memory", "MB", - CHART_PRIORITY_SYSTEMD_SERVICES + 10, update_every, RRDSET_TYPE_STACKED); + + st_mem_usage = rrdset_create_localhost( + "services" + , "mem_usage" + , NULL + , "mem" + , "services.mem_usage" + , (cgroup_used_memory_without_cache) ? "Systemd Services Used Memory without Cache" + : "Systemd Services Used Memory" + , "MB" + , CHART_PRIORITY_SYSTEMD_SERVICES + 10 + , update_every + , RRDSET_TYPE_STACKED + ); + } else rrdset_next(st_mem_usage); @@ -1341,90 +1373,152 @@ void update_services_charts(int update_every, if(likely(do_mem_detailed)) { if(unlikely(!st_mem_detailed_rss)) { - st_mem_detailed_rss = rrdset_find_bytype_localhost("services", "mem_rss"); - if(likely(!st_mem_detailed_rss)) - st_mem_detailed_rss = rrdset_create_localhost("services", "mem_rss", NULL, "mem", "services.mem_rss" - , "Systemd Services RSS Memory", "MB", - CHART_PRIORITY_SYSTEMD_SERVICES + 20, update_every, RRDSET_TYPE_STACKED); + + st_mem_detailed_rss = rrdset_create_localhost( + "services" + , "mem_rss" + , NULL + , "mem" + , "services.mem_rss" + , "Systemd Services RSS Memory" + , "MB" + , CHART_PRIORITY_SYSTEMD_SERVICES + 20 + , update_every + , RRDSET_TYPE_STACKED + ); + } else rrdset_next(st_mem_detailed_rss); if(unlikely(!st_mem_detailed_mapped)) { - st_mem_detailed_mapped = rrdset_find_bytype_localhost("services", "mem_mapped"); - if(likely(!st_mem_detailed_mapped)) - st_mem_detailed_mapped = rrdset_create_localhost("services", "mem_mapped", NULL, "mem" - , "services.mem_mapped" - , "Systemd Services Mapped Memory", "MB", - CHART_PRIORITY_SYSTEMD_SERVICES + 30, update_every, RRDSET_TYPE_STACKED); + + st_mem_detailed_mapped = rrdset_create_localhost( + "services" + , "mem_mapped" + , NULL + , "mem" + , "services.mem_mapped" + , "Systemd Services Mapped Memory" + , "MB" + , CHART_PRIORITY_SYSTEMD_SERVICES + 30 + , update_every + , RRDSET_TYPE_STACKED + ); + } else rrdset_next(st_mem_detailed_mapped); if(unlikely(!st_mem_detailed_cache)) { - st_mem_detailed_cache = rrdset_find_bytype_localhost("services", "mem_cache"); - if(likely(!st_mem_detailed_cache)) - st_mem_detailed_cache = rrdset_create_localhost("services", "mem_cache", NULL, "mem" - , "services.mem_cache", "Systemd Services Cache Memory" - , "MB", CHART_PRIORITY_SYSTEMD_SERVICES + 40 - , update_every, RRDSET_TYPE_STACKED); + + st_mem_detailed_cache = rrdset_create_localhost( + "services" + , "mem_cache" + , NULL + , "mem" + , "services.mem_cache" + , "Systemd Services Cache Memory" + , "MB" + , CHART_PRIORITY_SYSTEMD_SERVICES + 40 + , update_every + , RRDSET_TYPE_STACKED + ); + } else rrdset_next(st_mem_detailed_cache); if(unlikely(!st_mem_detailed_writeback)) { - st_mem_detailed_writeback = rrdset_find_bytype_localhost("services", "mem_writeback"); - if(likely(!st_mem_detailed_writeback)) - st_mem_detailed_writeback = rrdset_create_localhost("services", "mem_writeback", NULL, "mem" - , "services.mem_writeback" - , "Systemd Services Writeback Memory", "MB", - CHART_PRIORITY_SYSTEMD_SERVICES + 50, update_every, RRDSET_TYPE_STACKED); + + st_mem_detailed_writeback = rrdset_create_localhost( + "services" + , "mem_writeback" + , NULL + , "mem" + , "services.mem_writeback" + , "Systemd Services Writeback Memory" + , "MB" + , CHART_PRIORITY_SYSTEMD_SERVICES + 50 + , update_every + , RRDSET_TYPE_STACKED + ); + } else rrdset_next(st_mem_detailed_writeback); if(unlikely(!st_mem_detailed_pgfault)) { - st_mem_detailed_pgfault = rrdset_find_bytype_localhost("services", "mem_pgfault"); - if(likely(!st_mem_detailed_pgfault)) - st_mem_detailed_pgfault = rrdset_create_localhost("services", "mem_pgfault", NULL, "mem" - , "services.mem_pgfault" - , "Systemd Services Memory Minor Page Faults", "MB/s", - CHART_PRIORITY_SYSTEMD_SERVICES + 60, update_every, RRDSET_TYPE_STACKED); + + st_mem_detailed_pgfault = rrdset_create_localhost( + "services" + , "mem_pgfault" + , NULL + , "mem" + , "services.mem_pgfault" + , "Systemd Services Memory Minor Page Faults" + , "MB/s" + , CHART_PRIORITY_SYSTEMD_SERVICES + 60 + , update_every + , RRDSET_TYPE_STACKED + ); } else rrdset_next(st_mem_detailed_pgfault); if(unlikely(!st_mem_detailed_pgmajfault)) { - st_mem_detailed_pgmajfault = rrdset_find_bytype_localhost("services", "mem_pgmajfault"); - if(likely(!st_mem_detailed_pgmajfault)) - st_mem_detailed_pgmajfault = rrdset_create_localhost("services", "mem_pgmajfault", NULL, "mem" - , "services.mem_pgmajfault" - , "Systemd Services Memory Major Page Faults" - , "MB/s", CHART_PRIORITY_SYSTEMD_SERVICES + 70 - , update_every, RRDSET_TYPE_STACKED); + + st_mem_detailed_pgmajfault = rrdset_create_localhost( + "services" + , "mem_pgmajfault" + , NULL + , "mem" + , "services.mem_pgmajfault" + , "Systemd Services Memory Major Page Faults" + , "MB/s" + , CHART_PRIORITY_SYSTEMD_SERVICES + 70 + , update_every + , RRDSET_TYPE_STACKED + ); + } else rrdset_next(st_mem_detailed_pgmajfault); if(unlikely(!st_mem_detailed_pgpgin)) { - st_mem_detailed_pgpgin = rrdset_find_bytype_localhost("services", "mem_pgpgin"); - if(likely(!st_mem_detailed_pgpgin)) - st_mem_detailed_pgpgin = rrdset_create_localhost("services", "mem_pgpgin", NULL, "mem" - , "services.mem_pgpgin" - , "Systemd Services Memory Charging Activity", "MB/s", - CHART_PRIORITY_SYSTEMD_SERVICES + 80, update_every, RRDSET_TYPE_STACKED); + + st_mem_detailed_pgpgin = rrdset_create_localhost( + "services" + , "mem_pgpgin" + , NULL + , "mem" + , "services.mem_pgpgin" + , "Systemd Services Memory Charging Activity" + , "MB/s" + , CHART_PRIORITY_SYSTEMD_SERVICES + 80 + , update_every + , RRDSET_TYPE_STACKED + ); + } else rrdset_next(st_mem_detailed_pgpgin); if(unlikely(!st_mem_detailed_pgpgout)) { - st_mem_detailed_pgpgout = rrdset_find_bytype_localhost("services", "mem_pgpgout"); - if(likely(!st_mem_detailed_pgpgout)) - st_mem_detailed_pgpgout = rrdset_create_localhost("services", "mem_pgpgout", NULL, "mem" - , "services.mem_pgpgout" - , "Systemd Services Memory Uncharging Activity" - , "MB/s", CHART_PRIORITY_SYSTEMD_SERVICES + 90 - , update_every, RRDSET_TYPE_STACKED); + + st_mem_detailed_pgpgout = rrdset_create_localhost( + "services" + , "mem_pgpgout" + , NULL + , "mem" + , "services.mem_pgpgout" + , "Systemd Services Memory Uncharging Activity" + , "MB/s" + , CHART_PRIORITY_SYSTEMD_SERVICES + 90 + , update_every + , RRDSET_TYPE_STACKED + ); + } else rrdset_next(st_mem_detailed_pgpgout); @@ -1432,11 +1526,20 @@ void update_services_charts(int update_every, if(likely(do_mem_failcnt)) { if(unlikely(!st_mem_failcnt)) { - st_mem_failcnt = rrdset_find_bytype_localhost("services", "mem_failcnt"); - if(likely(!st_mem_failcnt)) - st_mem_failcnt = rrdset_create_localhost("services", "mem_failcnt", NULL, "mem", "services.mem_failcnt" - , "Systemd Services Memory Limit Failures", "MB", - CHART_PRIORITY_SYSTEMD_SERVICES + 110, update_every, RRDSET_TYPE_STACKED); + + st_mem_failcnt = rrdset_create_localhost( + "services" + , "mem_failcnt" + , NULL + , "mem" + , "services.mem_failcnt" + , "Systemd Services Memory Limit Failures" + , "MB" + , CHART_PRIORITY_SYSTEMD_SERVICES + 110 + , update_every + , RRDSET_TYPE_STACKED + ); + } else rrdset_next(st_mem_failcnt); @@ -1444,11 +1547,20 @@ void update_services_charts(int update_every, if(likely(do_swap_usage)) { if(unlikely(!st_swap_usage)) { - st_swap_usage = rrdset_find_bytype_localhost("services", "swap_usage"); - if(likely(!st_swap_usage)) - st_swap_usage = rrdset_create_localhost("services", "swap_usage", NULL, "swap", "services.swap_usage" - , "Systemd Services Swap Memory Used", "MB", - CHART_PRIORITY_SYSTEMD_SERVICES + 100, update_every, RRDSET_TYPE_STACKED); + + st_swap_usage = rrdset_create_localhost( + "services" + , "swap_usage" + , NULL + , "swap" + , "services.swap_usage" + , "Systemd Services Swap Memory Used" + , "MB" + , CHART_PRIORITY_SYSTEMD_SERVICES + 100 + , update_every + , RRDSET_TYPE_STACKED + ); + } else rrdset_next(st_swap_usage); @@ -1456,21 +1568,39 @@ void update_services_charts(int update_every, if(likely(do_io)) { if(unlikely(!st_io_read)) { - st_io_read = rrdset_find_bytype_localhost("services", "io_read"); - if(likely(!st_io_read)) - st_io_read = rrdset_create_localhost("services", "io_read", NULL, "disk", "services.io_read" - , "Systemd Services Disk Read Bandwidth", "KB/s", - CHART_PRIORITY_SYSTEMD_SERVICES + 120, update_every, RRDSET_TYPE_STACKED); + + st_io_read = rrdset_create_localhost( + "services" + , "io_read" + , NULL + , "disk" + , "services.io_read" + , "Systemd Services Disk Read Bandwidth" + , "KB/s" + , CHART_PRIORITY_SYSTEMD_SERVICES + 120 + , update_every + , RRDSET_TYPE_STACKED + ); + } else rrdset_next(st_io_read); if(unlikely(!st_io_write)) { - st_io_write = rrdset_find_bytype_localhost("services", "io_write"); - if(likely(!st_io_write)) - st_io_write = rrdset_create_localhost("services", "io_write", NULL, "disk", "services.io_write" - , "Systemd Services Disk Write Bandwidth", "KB/s", - CHART_PRIORITY_SYSTEMD_SERVICES + 130, update_every, RRDSET_TYPE_STACKED); + + st_io_write = rrdset_create_localhost( + "services" + , "io_write" + , NULL + , "disk" + , "services.io_write" + , "Systemd Services Disk Write Bandwidth" + , "KB/s" + , CHART_PRIORITY_SYSTEMD_SERVICES + 130 + , update_every + , RRDSET_TYPE_STACKED + ); + } else rrdset_next(st_io_write); @@ -1478,24 +1608,39 @@ void update_services_charts(int update_every, if(likely(do_io_ops)) { if(unlikely(!st_io_serviced_read)) { - st_io_serviced_read = rrdset_find_bytype_localhost("services", "io_ops_read"); - if(likely(!st_io_serviced_read)) - st_io_serviced_read = rrdset_create_localhost("services", "io_ops_read", NULL, "disk" - , "services.io_ops_read" - , "Systemd Services Disk Read Operations", "operations/s", - CHART_PRIORITY_SYSTEMD_SERVICES + 140, update_every, RRDSET_TYPE_STACKED); + + st_io_serviced_read = rrdset_create_localhost( + "services" + , "io_ops_read" + , NULL + , "disk" + , "services.io_ops_read" + , "Systemd Services Disk Read Operations" + , "operations/s" + , CHART_PRIORITY_SYSTEMD_SERVICES + 140 + , update_every + , RRDSET_TYPE_STACKED + ); + } else rrdset_next(st_io_serviced_read); if(unlikely(!st_io_serviced_write)) { - st_io_serviced_write = rrdset_find_bytype_localhost("services", "io_ops_write"); - if(likely(!st_io_serviced_write)) - st_io_serviced_write = rrdset_create_localhost("services", "io_ops_write", NULL, "disk" - , "services.io_ops_write" - , "Systemd Services Disk Write Operations" - , "operations/s", CHART_PRIORITY_SYSTEMD_SERVICES + 150 - , update_every, RRDSET_TYPE_STACKED); + + st_io_serviced_write = rrdset_create_localhost( + "services" + , "io_ops_write" + , NULL + , "disk" + , "services.io_ops_write" + , "Systemd Services Disk Write Operations" + , "operations/s" + , CHART_PRIORITY_SYSTEMD_SERVICES + 150 + , update_every + , RRDSET_TYPE_STACKED + ); + } else rrdset_next(st_io_serviced_write); @@ -1503,24 +1648,39 @@ void update_services_charts(int update_every, if(likely(do_throttle_io)) { if(unlikely(!st_throttle_io_read)) { - st_throttle_io_read = rrdset_find_bytype_localhost("services", "throttle_io_read"); - if(likely(!st_throttle_io_read)) - st_throttle_io_read = rrdset_create_localhost("services", "throttle_io_read", NULL, "disk" - , "services.throttle_io_read" - , "Systemd Services Throttle Disk Read Bandwidth", "KB/s", - CHART_PRIORITY_SYSTEMD_SERVICES + 160, update_every, RRDSET_TYPE_STACKED); + + st_throttle_io_read = rrdset_create_localhost( + "services" + , "throttle_io_read" + , NULL + , "disk" + , "services.throttle_io_read" + , "Systemd Services Throttle Disk Read Bandwidth" + , "KB/s" + , CHART_PRIORITY_SYSTEMD_SERVICES + 160 + , update_every + , RRDSET_TYPE_STACKED + ); + } else rrdset_next(st_throttle_io_read); if(unlikely(!st_throttle_io_write)) { - st_throttle_io_write = rrdset_find_bytype_localhost("services", "throttle_io_write"); - if(likely(!st_throttle_io_write)) - st_throttle_io_write = rrdset_create_localhost("services", "throttle_io_write", NULL, "disk" - , "services.throttle_io_write" - , "Systemd Services Throttle Disk Write Bandwidth" - , "KB/s", CHART_PRIORITY_SYSTEMD_SERVICES + 170 - , update_every, RRDSET_TYPE_STACKED); + + st_throttle_io_write = rrdset_create_localhost( + "services" + , "throttle_io_write" + , NULL + , "disk" + , "services.throttle_io_write" + , "Systemd Services Throttle Disk Write Bandwidth" + , "KB/s" + , CHART_PRIORITY_SYSTEMD_SERVICES + 170 + , update_every + , RRDSET_TYPE_STACKED + ); + } else rrdset_next(st_throttle_io_write); @@ -1528,25 +1688,39 @@ void update_services_charts(int update_every, if(likely(do_throttle_ops)) { if(unlikely(!st_throttle_ops_read)) { - st_throttle_ops_read = rrdset_find_bytype_localhost("services", "throttle_io_ops_read"); - if(likely(!st_throttle_ops_read)) - st_throttle_ops_read = rrdset_create_localhost("services", "throttle_io_ops_read", NULL, "disk" - , "services.throttle_io_ops_read" - , "Systemd Services Throttle Disk Read Operations" - , "operations/s", CHART_PRIORITY_SYSTEMD_SERVICES + 180 - , update_every, RRDSET_TYPE_STACKED); + + st_throttle_ops_read = rrdset_create_localhost( + "services" + , "throttle_io_ops_read" + , NULL + , "disk" + , "services.throttle_io_ops_read" + , "Systemd Services Throttle Disk Read Operations" + , "operations/s" + , CHART_PRIORITY_SYSTEMD_SERVICES + 180 + , update_every + , RRDSET_TYPE_STACKED + ); + } else rrdset_next(st_throttle_ops_read); if(unlikely(!st_throttle_ops_write)) { - st_throttle_ops_write = rrdset_find_bytype_localhost("services", "throttle_io_ops_write"); - if(likely(!st_throttle_ops_write)) - st_throttle_ops_write = rrdset_create_localhost("services", "throttle_io_ops_write", NULL, "disk" - , "services.throttle_io_ops_write" - , "Systemd Services Throttle Disk Write Operations" - , "operations/s", CHART_PRIORITY_SYSTEMD_SERVICES + 190 - , update_every, RRDSET_TYPE_STACKED); + + st_throttle_ops_write = rrdset_create_localhost( + "services" + , "throttle_io_ops_write" + , NULL + , "disk" + , "services.throttle_io_ops_write" + , "Systemd Services Throttle Disk Write Operations" + , "operations/s" + , CHART_PRIORITY_SYSTEMD_SERVICES + 190 + , update_every + , RRDSET_TYPE_STACKED + ); + } else rrdset_next(st_throttle_ops_write); @@ -1554,25 +1728,39 @@ void update_services_charts(int update_every, if(likely(do_queued_ops)) { if(unlikely(!st_queued_ops_read)) { - st_queued_ops_read = rrdset_find_bytype_localhost("services", "queued_io_ops_read"); - if(likely(!st_queued_ops_read)) - st_queued_ops_read = rrdset_create_localhost("services", "queued_io_ops_read", NULL, "disk" - , "services.queued_io_ops_read" - , "Systemd Services Queued Disk Read Operations" - , "operations/s", CHART_PRIORITY_SYSTEMD_SERVICES + 200 - , update_every, RRDSET_TYPE_STACKED); + + st_queued_ops_read = rrdset_create_localhost( + "services" + , "queued_io_ops_read" + , NULL + , "disk" + , "services.queued_io_ops_read" + , "Systemd Services Queued Disk Read Operations" + , "operations/s" + , CHART_PRIORITY_SYSTEMD_SERVICES + 200 + , update_every + , RRDSET_TYPE_STACKED + ); + } else rrdset_next(st_queued_ops_read); if(unlikely(!st_queued_ops_write)) { - st_queued_ops_write = rrdset_find_bytype_localhost("services", "queued_io_ops_write"); - if(likely(!st_queued_ops_write)) - st_queued_ops_write = rrdset_create_localhost("services", "queued_io_ops_write", NULL, "disk" - , "services.queued_io_ops_write" - , "Systemd Services Queued Disk Write Operations" - , "operations/s", CHART_PRIORITY_SYSTEMD_SERVICES + 210 - , update_every, RRDSET_TYPE_STACKED); + + st_queued_ops_write = rrdset_create_localhost( + "services" + , "queued_io_ops_write" + , NULL + , "disk" + , "services.queued_io_ops_write" + , "Systemd Services Queued Disk Write Operations" + , "operations/s" + , CHART_PRIORITY_SYSTEMD_SERVICES + 210 + , update_every + , RRDSET_TYPE_STACKED + ); + } else rrdset_next(st_queued_ops_write); @@ -1580,25 +1768,39 @@ void update_services_charts(int update_every, if(likely(do_merged_ops)) { if(unlikely(!st_merged_ops_read)) { - st_merged_ops_read = rrdset_find_bytype_localhost("services", "merged_io_ops_read"); - if(likely(!st_merged_ops_read)) - st_merged_ops_read = rrdset_create_localhost("services", "merged_io_ops_read", NULL, "disk" - , "services.merged_io_ops_read" - , "Systemd Services Merged Disk Read Operations" - , "operations/s", CHART_PRIORITY_SYSTEMD_SERVICES + 220 - , update_every, RRDSET_TYPE_STACKED); + + st_merged_ops_read = rrdset_create_localhost( + "services" + , "merged_io_ops_read" + , NULL + , "disk" + , "services.merged_io_ops_read" + , "Systemd Services Merged Disk Read Operations" + , "operations/s" + , CHART_PRIORITY_SYSTEMD_SERVICES + 220 + , update_every + , RRDSET_TYPE_STACKED + ); + } else rrdset_next(st_merged_ops_read); if(unlikely(!st_merged_ops_write)) { - st_merged_ops_write = rrdset_find_bytype_localhost("services", "merged_io_ops_write"); - if(likely(!st_merged_ops_write)) - st_merged_ops_write = rrdset_create_localhost("services", "merged_io_ops_write", NULL, "disk" - , "services.merged_io_ops_write" - , "Systemd Services Merged Disk Write Operations" - , "operations/s", CHART_PRIORITY_SYSTEMD_SERVICES + 230 - , update_every, RRDSET_TYPE_STACKED); + + st_merged_ops_write = rrdset_create_localhost( + "services" + , "merged_io_ops_write" + , NULL + , "disk" + , "services.merged_io_ops_write" + , "Systemd Services Merged Disk Write Operations" + , "operations/s" + , CHART_PRIORITY_SYSTEMD_SERVICES + 230 + , update_every + , RRDSET_TYPE_STACKED + ); + } else rrdset_next(st_merged_ops_write); @@ -1627,34 +1829,42 @@ void update_services_charts(int update_every, if(likely(do_mem_detailed && cg->memory.updated_detailed)) { if(unlikely(!cg->rd_mem_detailed_rss)) cg->rd_mem_detailed_rss = rrddim_add(st_mem_detailed_rss, cg->chart_id, cg->chart_title, 1, 1024 * 1024, RRD_ALGORITHM_ABSOLUTE); + rrddim_set_by_pointer(st_mem_detailed_rss, cg->rd_mem_detailed_rss, cg->memory.rss + cg->memory.rss_huge); if(unlikely(!cg->rd_mem_detailed_mapped)) cg->rd_mem_detailed_mapped = rrddim_add(st_mem_detailed_mapped, cg->chart_id, cg->chart_title, 1, 1024 * 1024, RRD_ALGORITHM_ABSOLUTE); + rrddim_set_by_pointer(st_mem_detailed_mapped, cg->rd_mem_detailed_mapped, cg->memory.mapped_file); if(unlikely(!cg->rd_mem_detailed_cache)) cg->rd_mem_detailed_cache = rrddim_add(st_mem_detailed_cache, cg->chart_id, cg->chart_title, 1, 1024 * 1024, RRD_ALGORITHM_ABSOLUTE); + rrddim_set_by_pointer(st_mem_detailed_cache, cg->rd_mem_detailed_cache, cg->memory.cache); if(unlikely(!cg->rd_mem_detailed_writeback)) cg->rd_mem_detailed_writeback = rrddim_add(st_mem_detailed_writeback, cg->chart_id, cg->chart_title, 1, 1024 * 1024, RRD_ALGORITHM_ABSOLUTE); + rrddim_set_by_pointer(st_mem_detailed_writeback, cg->rd_mem_detailed_writeback, cg->memory.writeback); if(unlikely(!cg->rd_mem_detailed_pgfault)) cg->rd_mem_detailed_pgfault = rrddim_add(st_mem_detailed_pgfault, cg->chart_id, cg->chart_title, system_page_size, 1024 * 1024, RRD_ALGORITHM_INCREMENTAL); + rrddim_set_by_pointer(st_mem_detailed_pgfault, cg->rd_mem_detailed_pgfault, cg->memory.pgfault); if(unlikely(!cg->rd_mem_detailed_pgmajfault)) cg->rd_mem_detailed_pgmajfault = rrddim_add(st_mem_detailed_pgmajfault, cg->chart_id, cg->chart_title, system_page_size, 1024 * 1024, RRD_ALGORITHM_INCREMENTAL); + rrddim_set_by_pointer(st_mem_detailed_pgmajfault, cg->rd_mem_detailed_pgmajfault, cg->memory.pgmajfault); if(unlikely(!cg->rd_mem_detailed_pgpgin)) cg->rd_mem_detailed_pgpgin = rrddim_add(st_mem_detailed_pgpgin, cg->chart_id, cg->chart_title, system_page_size, 1024 * 1024, RRD_ALGORITHM_INCREMENTAL); + rrddim_set_by_pointer(st_mem_detailed_pgpgin, cg->rd_mem_detailed_pgpgin, cg->memory.pgpgin); if(unlikely(!cg->rd_mem_detailed_pgpgout)) cg->rd_mem_detailed_pgpgout = rrddim_add(st_mem_detailed_pgpgout, cg->chart_id, cg->chart_title, system_page_size, 1024 * 1024, RRD_ALGORITHM_INCREMENTAL); + rrddim_set_by_pointer(st_mem_detailed_pgpgout, cg->rd_mem_detailed_pgpgout, cg->memory.pgpgout); } @@ -1856,13 +2066,21 @@ void update_cgroup_charts(int update_every) { if(likely(cg->cpuacct_stat.updated && cg->cpuacct_stat.enabled == CONFIG_BOOLEAN_YES)) { if(unlikely(!cg->st_cpu)) { - cg->st_cpu = rrdset_find_bytype_localhost(cgroup_chart_type(type, cg->chart_id, RRD_ID_LENGTH_MAX) - , "cpu"); - if(likely(!cg->st_cpu)) { - snprintfz(title, CHART_TITLE_MAX, "CPU Usage (%d%% = %d core%s) for cgroup %s", (processors * 100), processors, (processors > 1) ? "s" : "", cg->chart_title); - cg->st_cpu = rrdset_create_localhost(type, "cpu", NULL, "cpu", "cgroup.cpu", title, "%" - , CHART_PRIORITY_CONTAINERS, update_every, RRDSET_TYPE_STACKED); - } + snprintfz(title, CHART_TITLE_MAX, "CPU Usage (%d%% = %d core%s) for cgroup %s", (processors * 100), processors, (processors > 1) ? "s" : "", cg->chart_title); + + cg->st_cpu = rrdset_create_localhost( + cgroup_chart_type(type, cg->chart_id, RRD_ID_LENGTH_MAX) + , "cpu" + , NULL + , "cpu" + , "cgroup.cpu" + , title + , "%" + , CHART_PRIORITY_CONTAINERS + , update_every + , RRDSET_TYPE_STACKED + ); + rrddim_add(cg->st_cpu, "user", NULL, 100, hz, RRD_ALGORITHM_INCREMENTAL); rrddim_add(cg->st_cpu, "system", NULL, 100, hz, RRD_ALGORITHM_INCREMENTAL); } @@ -1879,14 +2097,21 @@ void update_cgroup_charts(int update_every) { unsigned int i; if(unlikely(!cg->st_cpu_per_core)) { - cg->st_cpu_per_core = rrdset_find_bytype_localhost( - cgroup_chart_type(type, cg->chart_id, RRD_ID_LENGTH_MAX), "cpu_per_core"); - if(likely(!cg->st_cpu_per_core)) { - snprintfz(title, CHART_TITLE_MAX, "CPU Usage (%d%% = %d core%s) Per Core for cgroup %s", (processors * 100), processors, (processors > 1) ? "s" : "", cg->chart_title); - cg->st_cpu_per_core = rrdset_create_localhost(type, "cpu_per_core", NULL, "cpu" - , "cgroup.cpu_per_core", title, "%", - CHART_PRIORITY_CONTAINERS + 100, update_every, RRDSET_TYPE_STACKED); - } + snprintfz(title, CHART_TITLE_MAX, "CPU Usage (%d%% = %d core%s) Per Core for cgroup %s", (processors * 100), processors, (processors > 1) ? "s" : "", cg->chart_title); + + cg->st_cpu_per_core = rrdset_create_localhost( + cgroup_chart_type(type, cg->chart_id, RRD_ID_LENGTH_MAX) + , "cpu_per_core" + , NULL + , "cpu" + , "cgroup.cpu_per_core" + , title + , "%" + , CHART_PRIORITY_CONTAINERS + 100 + , update_every + , RRDSET_TYPE_STACKED + ); + for(i = 0; i < cg->cpuacct_usage.cpus; i++) { snprintfz(id, CHART_TITLE_MAX, "cpu%u", i); rrddim_add(cg->st_cpu_per_core, id, NULL, 100, 1000000000, RRD_ALGORITHM_INCREMENTAL); @@ -1904,18 +2129,27 @@ void update_cgroup_charts(int update_every) { if(likely(cg->memory.updated_detailed && cg->memory.enabled_detailed == CONFIG_BOOLEAN_YES)) { if(unlikely(!cg->st_mem)) { - cg->st_mem = rrdset_find_bytype_localhost(cgroup_chart_type(type, cg->chart_id, RRD_ID_LENGTH_MAX) - , "mem"); - if(likely(!cg->st_mem)) { - snprintfz(title, CHART_TITLE_MAX, "Memory Usage for cgroup %s", cg->chart_title); - cg->st_mem = rrdset_create_localhost(type, "mem", NULL, "mem", "cgroup.mem", title, "MB", - CHART_PRIORITY_CONTAINERS + 210, update_every, RRDSET_TYPE_STACKED); - } + snprintfz(title, CHART_TITLE_MAX, "Memory Usage for cgroup %s", cg->chart_title); + + cg->st_mem = rrdset_create_localhost( + cgroup_chart_type(type, cg->chart_id, RRD_ID_LENGTH_MAX) + , "mem" + , NULL + , "mem" + , "cgroup.mem" + , title + , "MB" + , CHART_PRIORITY_CONTAINERS + 210 + , update_every + , RRDSET_TYPE_STACKED + ); rrddim_add(cg->st_mem, "cache", NULL, 1, 1024 * 1024, RRD_ALGORITHM_ABSOLUTE); rrddim_add(cg->st_mem, "rss", NULL, 1, 1024 * 1024, RRD_ALGORITHM_ABSOLUTE); + if(cg->memory.detailed_has_swap) rrddim_add(cg->st_mem, "swap", NULL, 1, 1024 * 1024, RRD_ALGORITHM_ABSOLUTE); + rrddim_add(cg->st_mem, "rss_huge", NULL, 1, 1024 * 1024, RRD_ALGORITHM_ABSOLUTE); rrddim_add(cg->st_mem, "mapped_file", NULL, 1, 1024 * 1024, RRD_ALGORITHM_ABSOLUTE); } @@ -1924,24 +2158,33 @@ void update_cgroup_charts(int update_every) { rrddim_set(cg->st_mem, "cache", cg->memory.cache); rrddim_set(cg->st_mem, "rss", cg->memory.rss); + if(cg->memory.detailed_has_swap) rrddim_set(cg->st_mem, "swap", cg->memory.swap); + rrddim_set(cg->st_mem, "rss_huge", cg->memory.rss_huge); rrddim_set(cg->st_mem, "mapped_file", cg->memory.mapped_file); rrdset_done(cg->st_mem); if(unlikely(!cg->st_writeback)) { - cg->st_writeback = rrdset_find_bytype_localhost(cgroup_chart_type(type, cg->chart_id, RRD_ID_LENGTH_MAX) - , "writeback"); - if(likely(!cg->st_writeback)) { - snprintfz(title, CHART_TITLE_MAX, "Writeback Memory for cgroup %s", cg->chart_title); - cg->st_writeback = rrdset_create_localhost(type, "writeback", NULL, "mem", "cgroup.writeback", title - , "MB", CHART_PRIORITY_CONTAINERS + 300, update_every - , RRDSET_TYPE_AREA); - } + snprintfz(title, CHART_TITLE_MAX, "Writeback Memory for cgroup %s", cg->chart_title); + + cg->st_writeback = rrdset_create_localhost( + cgroup_chart_type(type, cg->chart_id, RRD_ID_LENGTH_MAX) + , "writeback" + , NULL + , "mem" + , "cgroup.writeback" + , title + , "MB" + , CHART_PRIORITY_CONTAINERS + 300 + , update_every + , RRDSET_TYPE_AREA + ); if(cg->memory.detailed_has_dirty) rrddim_add(cg->st_writeback, "dirty", NULL, 1, 1024 * 1024, RRD_ALGORITHM_ABSOLUTE); + rrddim_add(cg->st_writeback, "writeback", NULL, 1, 1024 * 1024, RRD_ALGORITHM_ABSOLUTE); } else @@ -1949,18 +2192,26 @@ void update_cgroup_charts(int update_every) { if(cg->memory.detailed_has_dirty) rrddim_set(cg->st_writeback, "dirty", cg->memory.dirty); + rrddim_set(cg->st_writeback, "writeback", cg->memory.writeback); rrdset_done(cg->st_writeback); if(unlikely(!cg->st_mem_activity)) { - cg->st_mem_activity = rrdset_find_bytype_localhost( - cgroup_chart_type(type, cg->chart_id, RRD_ID_LENGTH_MAX), "mem_activity"); - if(likely(!cg->st_mem_activity)) { - snprintfz(title, CHART_TITLE_MAX, "Memory Activity for cgroup %s", cg->chart_title); - cg->st_mem_activity = rrdset_create_localhost(type, "mem_activity", NULL, "mem" - , "cgroup.mem_activity", title, "MB/s", - CHART_PRIORITY_CONTAINERS + 400, update_every, RRDSET_TYPE_LINE); - } + snprintfz(title, CHART_TITLE_MAX, "Memory Activity for cgroup %s", cg->chart_title); + + cg->st_mem_activity = rrdset_create_localhost( + cgroup_chart_type(type, cg->chart_id, RRD_ID_LENGTH_MAX) + , "mem_activity" + , NULL + , "mem" + , "cgroup.mem_activity" + , title + , "MB/s" + , CHART_PRIORITY_CONTAINERS + 400 + , update_every + , RRDSET_TYPE_LINE + ); + rrddim_add(cg->st_mem_activity, "pgpgin", "in", system_page_size, 1024 * 1024, RRD_ALGORITHM_INCREMENTAL); rrddim_add(cg->st_mem_activity, "pgpgout", "out", -system_page_size, 1024 * 1024, RRD_ALGORITHM_INCREMENTAL); } @@ -1972,14 +2223,21 @@ void update_cgroup_charts(int update_every) { rrdset_done(cg->st_mem_activity); if(unlikely(!cg->st_pgfaults)) { - cg->st_pgfaults = rrdset_find_bytype_localhost(cgroup_chart_type(type, cg->chart_id, RRD_ID_LENGTH_MAX) - , "pgfaults"); - if(likely(!cg->st_pgfaults)) { - snprintfz(title, CHART_TITLE_MAX, "Memory Page Faults for cgroup %s", cg->chart_title); - cg->st_pgfaults = rrdset_create_localhost(type, "pgfaults", NULL, "mem", "cgroup.pgfaults", title - , "MB/s", CHART_PRIORITY_CONTAINERS + 500, update_every - , RRDSET_TYPE_LINE); - } + snprintfz(title, CHART_TITLE_MAX, "Memory Page Faults for cgroup %s", cg->chart_title); + + cg->st_pgfaults = rrdset_create_localhost( + cgroup_chart_type(type, cg->chart_id, RRD_ID_LENGTH_MAX) + , "pgfaults" + , NULL + , "mem" + , "cgroup.pgfaults" + , title + , "MB/s" + , CHART_PRIORITY_CONTAINERS + 500 + , update_every + , RRDSET_TYPE_LINE + ); + rrddim_add(cg->st_pgfaults, "pgfault", NULL, system_page_size, 1024 * 1024, RRD_ALGORITHM_INCREMENTAL); rrddim_add(cg->st_pgfaults, "pgmajfault", "swap", -system_page_size, 1024 * 1024, RRD_ALGORITHM_INCREMENTAL); } @@ -1993,14 +2251,21 @@ void update_cgroup_charts(int update_every) { if(likely(cg->memory.updated_usage_in_bytes && cg->memory.enabled_usage_in_bytes == CONFIG_BOOLEAN_YES)) { if(unlikely(!cg->st_mem_usage)) { - cg->st_mem_usage = rrdset_find_bytype_localhost(cgroup_chart_type(type, cg->chart_id, RRD_ID_LENGTH_MAX) - , "mem_usage"); - if(likely(!cg->st_mem_usage)) { - snprintfz(title, CHART_TITLE_MAX, "Used Memory %sfor cgroup %s", (cgroup_used_memory_without_cache && cg->memory.updated_detailed)?"without Cache ":"", cg->chart_title); - cg->st_mem_usage = rrdset_create_localhost(type, "mem_usage", NULL, "mem", "cgroup.mem_usage", title - , "MB", CHART_PRIORITY_CONTAINERS + 200, update_every - , RRDSET_TYPE_STACKED); - } + snprintfz(title, CHART_TITLE_MAX, "Used Memory %sfor cgroup %s", (cgroup_used_memory_without_cache && cg->memory.updated_detailed)?"without Cache ":"", cg->chart_title); + + cg->st_mem_usage = rrdset_create_localhost( + cgroup_chart_type(type, cg->chart_id, RRD_ID_LENGTH_MAX) + , "mem_usage" + , NULL + , "mem" + , "cgroup.mem_usage" + , title + , "MB" + , CHART_PRIORITY_CONTAINERS + 200 + , update_every + , RRDSET_TYPE_STACKED + ); + rrddim_add(cg->st_mem_usage, "ram", NULL, 1, 1024 * 1024, RRD_ALGORITHM_ABSOLUTE); rrddim_add(cg->st_mem_usage, "swap", NULL, 1, 1024 * 1024, RRD_ALGORITHM_ABSOLUTE); } @@ -2014,14 +2279,21 @@ void update_cgroup_charts(int update_every) { if(likely(cg->memory.updated_failcnt && cg->memory.enabled_failcnt == CONFIG_BOOLEAN_YES)) { if(unlikely(!cg->st_mem_failcnt)) { - cg->st_mem_failcnt = rrdset_find_bytype_localhost( - cgroup_chart_type(type, cg->chart_id, RRD_ID_LENGTH_MAX), "mem_failcnt"); - if(likely(!cg->st_mem_failcnt)) { - snprintfz(title, CHART_TITLE_MAX, "Memory Limit Failures for cgroup %s", cg->chart_title); - cg->st_mem_failcnt = rrdset_create_localhost(type, "mem_failcnt", NULL, "mem", "cgroup.mem_failcnt" - , title, "count", CHART_PRIORITY_CONTAINERS + 250 - , update_every, RRDSET_TYPE_LINE); - } + snprintfz(title, CHART_TITLE_MAX, "Memory Limit Failures for cgroup %s", cg->chart_title); + + cg->st_mem_failcnt = rrdset_create_localhost( + cgroup_chart_type(type, cg->chart_id, RRD_ID_LENGTH_MAX) + , "mem_failcnt" + , NULL + , "mem" + , "cgroup.mem_failcnt" + , title + , "count" + , CHART_PRIORITY_CONTAINERS + 250 + , update_every + , RRDSET_TYPE_LINE + ); + rrddim_add(cg->st_mem_failcnt, "failures", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); } else @@ -2033,12 +2305,21 @@ void update_cgroup_charts(int update_every) { if(likely(cg->io_service_bytes.updated && cg->io_service_bytes.enabled == CONFIG_BOOLEAN_YES)) { if(unlikely(!cg->st_io)) { - cg->st_io = rrdset_find_bytype_localhost(cgroup_chart_type(type, cg->chart_id, RRD_ID_LENGTH_MAX), "io"); - if(likely(!cg->st_io)) { - snprintfz(title, CHART_TITLE_MAX, "I/O Bandwidth (all disks) for cgroup %s", cg->chart_title); - cg->st_io = rrdset_create_localhost(type, "io", NULL, "disk", "cgroup.io", title, "KB/s", - CHART_PRIORITY_CONTAINERS + 1200, update_every, RRDSET_TYPE_AREA); - } + snprintfz(title, CHART_TITLE_MAX, "I/O Bandwidth (all disks) for cgroup %s", cg->chart_title); + + cg->st_io = rrdset_create_localhost( + cgroup_chart_type(type, cg->chart_id, RRD_ID_LENGTH_MAX) + , "io" + , NULL + , "disk" + , "cgroup.io" + , title + , "KB/s" + , CHART_PRIORITY_CONTAINERS + 1200 + , update_every + , RRDSET_TYPE_AREA + ); + rrddim_add(cg->st_io, "read", NULL, 1, 1024, RRD_ALGORITHM_INCREMENTAL); rrddim_add(cg->st_io, "write", NULL, -1, 1024, RRD_ALGORITHM_INCREMENTAL); } @@ -2052,14 +2333,21 @@ void update_cgroup_charts(int update_every) { if(likely(cg->io_serviced.updated && cg->io_serviced.enabled == CONFIG_BOOLEAN_YES)) { if(unlikely(!cg->st_serviced_ops)) { - cg->st_serviced_ops = rrdset_find_bytype_localhost( - cgroup_chart_type(type, cg->chart_id, RRD_ID_LENGTH_MAX), "serviced_ops"); - if(likely(!cg->st_serviced_ops)) { - snprintfz(title, CHART_TITLE_MAX, "Serviced I/O Operations (all disks) for cgroup %s", cg->chart_title); - cg->st_serviced_ops = rrdset_create_localhost(type, "serviced_ops", NULL, "disk" - , "cgroup.serviced_ops", title, "operations/s", - CHART_PRIORITY_CONTAINERS + 1200, update_every, RRDSET_TYPE_LINE); - } + snprintfz(title, CHART_TITLE_MAX, "Serviced I/O Operations (all disks) for cgroup %s", cg->chart_title); + + cg->st_serviced_ops = rrdset_create_localhost( + cgroup_chart_type(type, cg->chart_id, RRD_ID_LENGTH_MAX) + , "serviced_ops" + , NULL + , "disk" + , "cgroup.serviced_ops" + , title + , "operations/s" + , CHART_PRIORITY_CONTAINERS + 1200 + , update_every + , RRDSET_TYPE_LINE + ); + rrddim_add(cg->st_serviced_ops, "read", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); rrddim_add(cg->st_serviced_ops, "write", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL); } @@ -2073,14 +2361,21 @@ void update_cgroup_charts(int update_every) { if(likely(cg->throttle_io_service_bytes.updated && cg->throttle_io_service_bytes.enabled == CONFIG_BOOLEAN_YES)) { if(unlikely(!cg->st_throttle_io)) { - cg->st_throttle_io = rrdset_find_bytype_localhost( - cgroup_chart_type(type, cg->chart_id, RRD_ID_LENGTH_MAX), "throttle_io"); - if(likely(!cg->st_throttle_io)) { - snprintfz(title, CHART_TITLE_MAX, "Throttle I/O Bandwidth (all disks) for cgroup %s", cg->chart_title); - cg->st_throttle_io = rrdset_create_localhost(type, "throttle_io", NULL, "disk", "cgroup.throttle_io" - , title, "KB/s", CHART_PRIORITY_CONTAINERS + 1200 - , update_every, RRDSET_TYPE_AREA); - } + snprintfz(title, CHART_TITLE_MAX, "Throttle I/O Bandwidth (all disks) for cgroup %s", cg->chart_title); + + cg->st_throttle_io = rrdset_create_localhost( + cgroup_chart_type(type, cg->chart_id, RRD_ID_LENGTH_MAX) + , "throttle_io" + , NULL + , "disk" + , "cgroup.throttle_io" + , title + , "KB/s" + , CHART_PRIORITY_CONTAINERS + 1200 + , update_every + , RRDSET_TYPE_AREA + ); + rrddim_add(cg->st_throttle_io, "read", NULL, 1, 1024, RRD_ALGORITHM_INCREMENTAL); rrddim_add(cg->st_throttle_io, "write", NULL, -1, 1024, RRD_ALGORITHM_INCREMENTAL); } @@ -2094,16 +2389,21 @@ void update_cgroup_charts(int update_every) { if(likely(cg->throttle_io_serviced.updated && cg->throttle_io_serviced.enabled == CONFIG_BOOLEAN_YES)) { if(unlikely(!cg->st_throttle_serviced_ops)) { - cg->st_throttle_serviced_ops = rrdset_find_bytype_localhost( - cgroup_chart_type(type, cg->chart_id, RRD_ID_LENGTH_MAX), "throttle_serviced_ops"); - if(likely(!cg->st_throttle_serviced_ops)) { - snprintfz(title, CHART_TITLE_MAX, "Throttle Serviced I/O Operations (all disks) for cgroup %s", cg->chart_title); - cg->st_throttle_serviced_ops = rrdset_create_localhost(type, "throttle_serviced_ops", NULL, "disk" - , "cgroup.throttle_serviced_ops", title - , "operations/s", CHART_PRIORITY_CONTAINERS + - 1200, update_every - , RRDSET_TYPE_LINE); - } + snprintfz(title, CHART_TITLE_MAX, "Throttle Serviced I/O Operations (all disks) for cgroup %s", cg->chart_title); + + cg->st_throttle_serviced_ops = rrdset_create_localhost( + cgroup_chart_type(type, cg->chart_id, RRD_ID_LENGTH_MAX) + , "throttle_serviced_ops" + , NULL + , "disk" + , "cgroup.throttle_serviced_ops" + , title + , "operations/s" + , CHART_PRIORITY_CONTAINERS + 1200 + , update_every + , RRDSET_TYPE_LINE + ); + rrddim_add(cg->st_throttle_serviced_ops, "read", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL); rrddim_add(cg->st_throttle_serviced_ops, "write", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL); } @@ -2117,14 +2417,21 @@ void update_cgroup_charts(int update_every) { if(likely(cg->io_queued.updated && cg->io_queued.enabled == CONFIG_BOOLEAN_YES)) { if(unlikely(!cg->st_queued_ops)) { - cg->st_queued_ops = rrdset_find_bytype_localhost( - cgroup_chart_type(type, cg->chart_id, RRD_ID_LENGTH_MAX), "queued_ops"); - if(likely(!cg->st_queued_ops)) { - snprintfz(title, CHART_TITLE_MAX, "Queued I/O Operations (all disks) for cgroup %s", cg->chart_title); - cg->st_queued_ops = rrdset_create_localhost(type, "queued_ops", NULL, "disk", "cgroup.queued_ops" - , title, "operations", CHART_PRIORITY_CONTAINERS + 2000 - , update_every, RRDSET_TYPE_LINE); - } + snprintfz(title, CHART_TITLE_MAX, "Queued I/O Operations (all disks) for cgroup %s", cg->chart_title); + + cg->st_queued_ops = rrdset_create_localhost( + cgroup_chart_type(type, cg->chart_id, RRD_ID_LENGTH_MAX) + , "queued_ops" + , NULL + , "disk" + , "cgroup.queued_ops" + , title + , "operations" + , CHART_PRIORITY_CONTAINERS + 2000 + , update_every + , RRDSET_TYPE_LINE + ); + rrddim_add(cg->st_queued_ops, "read", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); rrddim_add(cg->st_queued_ops, "write", NULL, -1, 1, RRD_ALGORITHM_ABSOLUTE); } @@ -2138,15 +2445,21 @@ void update_cgroup_charts(int update_every) { if(likely(cg->io_merged.updated && cg->io_merged.enabled == CONFIG_BOOLEAN_YES)) { if(unlikely(!cg->st_merged_ops)) { - cg->st_merged_ops = rrdset_find_bytype_localhost( - cgroup_chart_type(type, cg->chart_id, RRD_ID_LENGTH_MAX), "merged_ops"); - if(likely(!cg->st_merged_ops)) { - snprintfz(title, CHART_TITLE_MAX, "Merged I/O Operations (all disks) for cgroup %s", cg->chart_title); - cg->st_merged_ops = rrdset_create_localhost(type, "merged_ops", NULL, "disk", "cgroup.merged_ops" - , title, "operations/s", CHART_PRIORITY_CONTAINERS + - 2100, update_every - , RRDSET_TYPE_LINE); - } + snprintfz(title, CHART_TITLE_MAX, "Merged I/O Operations (all disks) for cgroup %s", cg->chart_title); + + cg->st_merged_ops = rrdset_create_localhost( + cgroup_chart_type(type, cg->chart_id, RRD_ID_LENGTH_MAX) + , "merged_ops" + , NULL + , "disk" + , "cgroup.merged_ops" + , title + , "operations/s" + , CHART_PRIORITY_CONTAINERS + 2100 + , update_every + , RRDSET_TYPE_LINE + ); + rrddim_add(cg->st_merged_ops, "read", NULL, 1, 1024, RRD_ALGORITHM_INCREMENTAL); rrddim_add(cg->st_merged_ops, "write", NULL, -1, 1024, RRD_ALGORITHM_INCREMENTAL); } @@ -2160,18 +2473,10 @@ void update_cgroup_charts(int update_every) { } if(likely(cgroup_enable_systemd_services)) - update_services_charts(update_every, - services_do_cpu, - services_do_mem_usage, - services_do_mem_detailed, - services_do_mem_failcnt, - services_do_swap_usage, - services_do_io, - services_do_io_ops, - services_do_throttle_io, - services_do_throttle_ops, - services_do_queued_ops, - services_do_merged_ops + update_systemd_services_charts(update_every, services_do_cpu, services_do_mem_usage, services_do_mem_detailed + , services_do_mem_failcnt, services_do_swap_usage, services_do_io + , services_do_io_ops, services_do_throttle_io, services_do_throttle_ops + , services_do_queued_ops, services_do_merged_ops ); debug(D_CGROUP, "done updating cgroups charts"); @@ -2227,11 +2532,19 @@ void *cgroups_main(void *ptr) { getrusage(RUSAGE_THREAD, &thread); if(unlikely(!stcpu_thread)) { - stcpu_thread = rrdset_find_localhost("netdata.plugin_cgroups_cpu"); - if(unlikely(!stcpu_thread)) - stcpu_thread = rrdset_create_localhost("netdata", "plugin_cgroups_cpu", NULL, "cgroups", NULL - , "NetData CGroups Plugin CPU usage", "milliseconds/s" - , 132000, cgroup_update_every, RRDSET_TYPE_STACKED); + + stcpu_thread = rrdset_create_localhost( + "netdata" + , "plugin_cgroups_cpu" + , NULL + , "cgroups" + , NULL + , "NetData CGroups Plugin CPU usage" + , "milliseconds/s" + , 132000 + , cgroup_update_every + , RRDSET_TYPE_STACKED + ); rrddim_add(stcpu_thread, "user", NULL, 1, 1000, RRD_ALGORITHM_INCREMENTAL); rrddim_add(stcpu_thread, "system", NULL, 1, 1000, RRD_ALGORITHM_INCREMENTAL); diff --git a/src/web_api_old.c b/src/web_api_old.c index 127536ec..373e7e9f 100644 --- a/src/web_api_old.c +++ b/src/web_api_old.c @@ -154,7 +154,9 @@ int web_client_api_old_data_request(RRDHOST *host, struct web_client *w, char *u debug(D_WEB_CLIENT_ACCESS, "%llu: Sending RRD data '%s' (id %s, %d lines, %d group, %d group_method, %ld after, %ld before).", w->id, st->name, st->id, lines, group_count, group_method, after, before); - time_t timestamp_in_data = rrd_stats_json(datasource_type, st, w->response.data, lines, group_count, group_method, (unsigned long)after, (unsigned long)before, nonzero); + time_t timestamp_in_data = rrdset2json_api_old(datasource_type, st, w->response.data, lines, group_count + , group_method, (unsigned long) after, (unsigned long) before + , nonzero); if(datasource_type == DATASOURCE_DATATABLE_JSONP) { if(timestamp_in_data > last_timestamp_in_data) @@ -195,11 +197,12 @@ inline int web_client_api_old_graph_request(RRDHOST *host, struct web_client *w, buffer_flush(w->response.data); return mysendfile(w, tok); } + st->last_accessed_time = now_realtime_sec(); debug(D_WEB_CLIENT_ACCESS, "%llu: Sending %s.json of RRD_STATS...", w->id, st->name); w->response.data->contenttype = CT_APPLICATION_JSON; buffer_flush(w->response.data); - rrd_stats_graph_json(st, url, w->response.data); + rrd_graph2json_api_old(st, url, w->response.data); return 200; } @@ -215,7 +218,10 @@ inline int web_client_api_old_list_request(RRDHOST *host, struct web_client *w, RRDSET *st; rrdhost_rdlock(host); - rrdset_foreach_read(st, host) buffer_sprintf(w->response.data, "%s\n", st->name); + rrdset_foreach_read(st, host) { + if(rrdset_is_available_for_viewers(st)) + buffer_sprintf(w->response.data, "%s\n", st->name); + } rrdhost_unlock(host); return 200; @@ -226,6 +232,6 @@ inline int web_client_api_old_all_json(RRDHOST *host, struct web_client *w, char w->response.data->contenttype = CT_APPLICATION_JSON; buffer_flush(w->response.data); - rrd_stats_all_json(host, w->response.data); + rrd_all2json_api_old(host, w->response.data); return 200; } diff --git a/src/web_api_v1.c b/src/web_api_v1.c index c180cf39..da93c894 100644 --- a/src/web_api_v1.c +++ b/src/web_api_v1.c @@ -185,6 +185,7 @@ inline int web_client_api_request_single_chart(RRDHOST *host, struct web_client } w->response.data->contenttype = CT_APPLICATION_JSON; + st->last_accessed_time = now_realtime_sec(); callback(st, w->response.data); return 200; @@ -329,6 +330,7 @@ int web_client_api_request_v1_badge(RRDHOST *host, struct web_client *w, char *u ret = 200; goto cleanup; } + st->last_accessed_time = now_realtime_sec(); RRDCALC *rc = NULL; if(alarm) { @@ -462,18 +464,8 @@ int web_client_api_request_v1_badge(RRDHOST *host, struct web_client *w, char *u // if the collected value is too old, don't calculate its value if (rrdset_last_entry_t(st) >= (now_realtime_sec() - (st->update_every * st->gap_when_lost_iterations_above))) - ret = rrd2value(st, - w->response.data, - &n, - (dimensions) ? buffer_tostring(dimensions) : NULL, - points, - after, - before, - group, - options, - NULL, - &latest_timestamp, - &value_is_null); + ret = rrdset2value_api_v1(st, w->response.data, &n, (dimensions) ? buffer_tostring(dimensions) : NULL + , points, after, before, group, options, NULL, &latest_timestamp, &value_is_null); // if the value cannot be calculated, show empty badge if (ret != 200) { @@ -613,6 +605,7 @@ inline int web_client_api_request_v1_data(RRDHOST *host, struct web_client *w, c ret = 404; goto cleanup; } + st->last_accessed_time = now_realtime_sec(); long long before = (before_str && *before_str)?str2l(before_str):0; long long after = (after_str && *after_str) ?str2l(after_str):0; @@ -655,7 +648,8 @@ inline int web_client_api_request_v1_data(RRDHOST *host, struct web_client *w, c buffer_strcat(w->response.data, "("); } - ret = rrd2format(st, w->response.data, dimensions, format, points, after, before, group, options, &last_timestamp_in_data); + ret = rrdset2anything_api_v1(st, w->response.data, dimensions, format, points, after, before, group, options + , &last_timestamp_in_data); if(format == DATASOURCE_DATATABLE_JSONP) { if(google_timestamp < last_timestamp_in_data) -- 2.39.2