]> arthur.barton.de Git - netdata.git/commitdiff
global statistics now also report max API response time; global statistics implemente...
authorCosta Tsaousis <costa@tsaousis.gr>
Fri, 12 Aug 2016 22:44:31 +0000 (01:44 +0300)
committerCosta Tsaousis <costa@tsaousis.gr>
Fri, 12 Aug 2016 22:44:31 +0000 (01:44 +0300)
12 files changed:
src/apps_plugin.c
src/common.c
src/common.h
src/global_statistics.c
src/global_statistics.h
src/plugin_checks.c
src/plugin_idlejitter.c
src/plugin_proc.c
src/rrd.c
src/sys_fs_cgroup.c
src/web_client.c
web/demosites.html

index 210b51038602810c1389df2c75529325df5a2f72..01da625b7924043e6920d70e75c18e65286311aa 100644 (file)
@@ -642,7 +642,7 @@ int read_proc_pid_stat(struct pid_stat *p) {
        if(unlikely(!ff)) goto cleanup;
 
        p->last_stat_collected_usec = p->stat_collected_usec;
-       p->stat_collected_usec = timems();
+       p->stat_collected_usec = time_usec();
        file_counter++;
 
        // p->pid                       = atol(procfile_lineword(ff, 0, 0+i));
@@ -824,7 +824,7 @@ int read_proc_pid_io(struct pid_stat *p) {
        file_counter++;
 
        p->last_io_collected_usec = p->io_collected_usec;
-       p->io_collected_usec = timems();
+       p->io_collected_usec = time_usec();
 
        unsigned long long last;
 
@@ -898,7 +898,7 @@ int read_proc_stat() {
        if(unlikely(!ff)) goto cleanup;
 
        last_collected_usec = collected_usec;
-       collected_usec = timems();
+       collected_usec = time_usec();
 
        file_counter++;
 
@@ -2205,7 +2205,7 @@ unsigned long long send_resource_usage_to_netdata() {
                gettimeofday(&now, NULL);
                getrusage(RUSAGE_SELF, &me);
 
-               usec = usecdiff(&now, &last);
+               usec = usec_dt(&now, &last);
                cpuuser = me.ru_utime.tv_sec * 1000000ULL + me.ru_utime.tv_usec;
                cpusyst = me.ru_stime.tv_sec * 1000000ULL + me.ru_stime.tv_usec;
 
@@ -2836,11 +2836,11 @@ int main(int argc, char **argv)
        for(;1; global_iterations_counter++) {
 #ifndef PROFILING_MODE
                // delay until it is our time to run
-               while((sunow = timems()) < sunext)
-                       usecsleep(sunext - sunow);
+               while((sunow = time_usec()) < sunext)
+                       sleep_usec(sunext - sunow);
 
                // find the next time we need to run
-               while(timems() > sunext)
+               while(time_usec() > sunext)
                        sunext += update_every * 1000000ULL;
 #endif /* PROFILING_MODE */
 
index 8d2c3aa8a40d1b7858ceeae32613bfbce86433f9..7d0f89016c2906401ed235dcdde17a8dff23148e 100644 (file)
@@ -43,15 +43,26 @@ void freez(void *ptr) {
 }
 
 // ----------------------------------------------------------------------------
+// time functions
 
-// time(NULL) in milliseconds
-unsigned long long timems(void) {
+inline unsigned long long timeval_usec(struct timeval *tv) {
+       return tv->tv_sec * 1000000ULL + tv->tv_usec;
+}
+
+// time(NULL) in nanoseconds
+inline unsigned long long time_usec(void) {
        struct timeval now;
        gettimeofday(&now, NULL);
-       return now.tv_sec * 1000000ULL + now.tv_usec;
+       return timeval_usec(&now);
+}
+
+inline unsigned long long usec_dt(struct timeval *now, struct timeval *old) {
+       unsigned long long tv1 = timeval_usec(now);
+       unsigned long long tv2 = timeval_usec(old);
+       return (tv1 > tv2)?(tv1 - tv2):(tv2 - tv1);
 }
 
-int usecsleep(unsigned long long usec) {
+int sleep_usec(unsigned long long usec) {
 
 #ifndef NETDATA_WITH_USLEEP
        // we expect microseconds (1.000.000 per second)
index 9793adf66f0d9bddb0d45658357b15028ac59cb4..4bb949c50430ac62e55fb026310d55f10c7e6339 100644 (file)
 #endif
 #define abs(x) ((x < 0)? -x : x)
 
-#define usecdiff(now, last) (((((now)->tv_sec * 1000000ULL) + (now)->tv_usec) - (((last)->tv_sec * 1000000ULL) + (last)->tv_usec)))
+extern unsigned long long usec_dt(struct timeval *now, struct timeval *old);
+extern unsigned long long timeval_usec(struct timeval *tv);
+
+// #define usec_dt(now, last) (((((now)->tv_sec * 1000000ULL) + (now)->tv_usec) - (((last)->tv_sec * 1000000ULL) + (last)->tv_usec)))
 
 extern void netdata_fix_chart_id(char *s);
 extern void netdata_fix_chart_name(char *s);
@@ -144,8 +147,8 @@ extern void get_HZ(void);
 
 extern pid_t gettid(void);
 
-extern unsigned long long timems(void);
-extern int usecsleep(unsigned long long usec);
+extern unsigned long long time_usec(void);
+extern int sleep_usec(unsigned long long usec);
 
 extern char *fgets_trim_len(char *buf, size_t buf_size, FILE *fp, size_t *len);
 
index e2852962a44507a30a9e57222e38ede8d0d3631a..0512c4382c570ed1e1ec4aeb21055ee11c83e8f0 100644 (file)
 #include "common.h"
 
-struct global_statistics global_statistics = { 0, 0ULL, 0ULL, 0ULL, 0ULL, 0ULL, 0ULL};
+volatile struct global_statistics global_statistics = {
+        .connected_clients = 0,
+        .web_requests = 0,
+        .web_usec = 0,
+        .bytes_received = 0,
+        .bytes_sent = 0,
+        .content_size = 0,
+        .compressed_content_size = 0
+};
 
 pthread_mutex_t global_statistics_mutex = PTHREAD_MUTEX_INITIALIZER;
 
-void global_statistics_lock(void) {
-       pthread_mutex_lock(&global_statistics_mutex);
+inline void global_statistics_lock(void) {
+    pthread_mutex_lock(&global_statistics_mutex);
 }
 
-void global_statistics_unlock(void) {
-       pthread_mutex_unlock(&global_statistics_mutex);
+inline void global_statistics_unlock(void) {
+    pthread_mutex_unlock(&global_statistics_mutex);
 }
+
+void finished_web_request_statistics(uint64_t dt,
+                                     uint64_t bytes_received,
+                                     uint64_t bytes_sent,
+                                     uint64_t content_size,
+                                     uint64_t compressed_content_size) {
+#ifndef NETDATA_NO_ATOMIC_INSTRUCTIONS
+    uint64_t old_web_usec_max = global_statistics.web_usec_max;
+    while(dt > old_web_usec_max) {
+        if(__sync_bool_compare_and_swap(&global_statistics.web_usec_max, old_web_usec_max, dt))
+            old_web_usec_max = dt;
+        else
+            old_web_usec_max = global_statistics.web_usec_max;
+    }
+
+    __sync_fetch_and_add(&global_statistics.web_requests, 1);
+    __sync_fetch_and_add(&global_statistics.web_usec, dt);
+    __sync_fetch_and_add(&global_statistics.bytes_received, bytes_received);
+    __sync_fetch_and_add(&global_statistics.bytes_sent, bytes_sent);
+    __sync_fetch_and_add(&global_statistics.content_size, content_size);
+    __sync_fetch_and_add(&global_statistics.compressed_content_size, compressed_content_size);
+#else
+    if (web_server_mode == WEB_SERVER_MODE_MULTI_THREADED)
+        global_statistics_lock();
+
+    if (dt > global_statistics.web_usec_max)
+        global_statistics.web_usec_max = dt;
+
+    global_statistics.web_requests++;
+    global_statistics.web_usec += dt;
+    global_statistics.bytes_received += bytes_received;
+    global_statistics.bytes_sent += bytes_sent;
+    global_statistics.content_size += content_size;
+    global_statistics.compressed_content_size += compressed_content_size;
+
+    if (web_server_mode == WEB_SERVER_MODE_MULTI_THREADED)
+        global_statistics_unlock();
+#endif
+}
+
+void web_client_connected(void) {
+#ifndef NETDATA_NO_ATOMIC_INSTRUCTIONS
+    __sync_fetch_and_add(&global_statistics.connected_clients, 1);
+#else
+    if (web_server_mode == WEB_SERVER_MODE_MULTI_THREADED)
+        global_statistics_lock();
+
+    global_statistics.connected_clients++;
+
+    if (web_server_mode == WEB_SERVER_MODE_MULTI_THREADED)
+        global_statistics_unlock();
+#endif
+}
+
+void web_client_disconnected(void) {
+#ifndef NETDATA_NO_ATOMIC_INSTRUCTIONS
+    __sync_fetch_and_sub(&global_statistics.connected_clients, 1);
+#else
+    if (web_server_mode == WEB_SERVER_MODE_MULTI_THREADED)
+        global_statistics_lock();
+
+    global_statistics.connected_clients--;
+
+    if (web_server_mode == WEB_SERVER_MODE_MULTI_THREADED)
+        global_statistics_unlock();
+#endif
+}
+
+
+inline void global_statistics_copy(struct global_statistics *gs, uint8_t options) {
+#ifndef NETDATA_NO_ATOMIC_INSTRUCTIONS
+    gs->connected_clients       = __sync_fetch_and_add(&global_statistics.connected_clients, 0);
+    gs->web_requests            = __sync_fetch_and_add(&global_statistics.web_requests, 0);
+    gs->web_usec                = __sync_fetch_and_add(&global_statistics.web_usec, 0);
+    gs->web_usec_max            = __sync_fetch_and_add(&global_statistics.web_usec_max, 0);
+    gs->bytes_received          = __sync_fetch_and_add(&global_statistics.bytes_received, 0);
+    gs->bytes_sent              = __sync_fetch_and_add(&global_statistics.bytes_sent, 0);
+    gs->content_size            = __sync_fetch_and_add(&global_statistics.content_size, 0);
+    gs->compressed_content_size = __sync_fetch_and_add(&global_statistics.compressed_content_size, 0);
+
+    if(options & GLOBAL_STATS_RESET_WEB_USEC_MAX)
+        __sync_bool_compare_and_swap(&global_statistics.web_usec_max, gs->web_usec_max, 0);
+#else
+    global_statistics_lock();
+
+    memcpy(gs, (const void *)&global_statistics, sizeof(struct global_statistics));
+
+    if (options & GLOBAL_STATS_RESET_WEB_USEC_MAX)
+        global_statistics.web_usec_max = 0;
+
+    global_statistics_unlock();
+#endif
+}
+
+void global_statistics_charts(void) {
+    static unsigned long long old_web_requests = 0, old_web_usec = 0,
+            old_content_size = 0, old_compressed_content_size = 0;
+
+    static collected_number compression_ratio = -1, average_response_time = -1;
+
+    static RRDSET *stcpu = NULL, *stcpu_thread = NULL, *stclients = NULL, *streqs = NULL, *stbytes = NULL, *stduration = NULL,
+            *stcompression = NULL;
+
+    struct global_statistics gs;
+    struct rusage me, thread;
+
+    global_statistics_copy(&gs, GLOBAL_STATS_RESET_WEB_USEC_MAX);
+    getrusage(RUSAGE_THREAD, &thread);
+    getrusage(RUSAGE_SELF, &me);
+
+    if (!stcpu_thread) stcpu_thread = rrdset_find("netdata.plugin_proc_cpu");
+    if (!stcpu_thread) {
+        stcpu_thread = rrdset_create("netdata", "plugin_proc_cpu", NULL, "proc.internal", NULL,
+                                     "NetData Proc Plugin CPU usage", "milliseconds/s", 132000, rrd_update_every,
+                                     RRDSET_TYPE_STACKED);
+
+        rrddim_add(stcpu_thread, "user", NULL, 1, 1000, RRDDIM_INCREMENTAL);
+        rrddim_add(stcpu_thread, "system", NULL, 1, 1000, RRDDIM_INCREMENTAL);
+    } else rrdset_next(stcpu_thread);
+
+    rrddim_set(stcpu_thread, "user", thread.ru_utime.tv_sec * 1000000ULL + thread.ru_utime.tv_usec);
+    rrddim_set(stcpu_thread, "system", thread.ru_stime.tv_sec * 1000000ULL + thread.ru_stime.tv_usec);
+    rrdset_done(stcpu_thread);
+
+    // ----------------------------------------------------------------
+
+    if (!stcpu) stcpu = rrdset_find("netdata.server_cpu");
+    if (!stcpu) {
+        stcpu = rrdset_create("netdata", "server_cpu", NULL, "netdata", NULL, "NetData CPU usage", "milliseconds/s",
+                              130000, rrd_update_every, RRDSET_TYPE_STACKED);
+
+        rrddim_add(stcpu, "user", NULL, 1, 1000, RRDDIM_INCREMENTAL);
+        rrddim_add(stcpu, "system", NULL, 1, 1000, RRDDIM_INCREMENTAL);
+    } else rrdset_next(stcpu);
+
+    rrddim_set(stcpu, "user", me.ru_utime.tv_sec * 1000000ULL + me.ru_utime.tv_usec);
+    rrddim_set(stcpu, "system", me.ru_stime.tv_sec * 1000000ULL + me.ru_stime.tv_usec);
+    rrdset_done(stcpu);
+
+    // ----------------------------------------------------------------
+
+    if (!stclients) stclients = rrdset_find("netdata.clients");
+    if (!stclients) {
+        stclients = rrdset_create("netdata", "clients", NULL, "netdata", NULL, "NetData Web Clients",
+                                  "connected clients", 130100, rrd_update_every, RRDSET_TYPE_LINE);
+
+        rrddim_add(stclients, "clients", NULL, 1, 1, RRDDIM_ABSOLUTE);
+    } else rrdset_next(stclients);
+
+    rrddim_set(stclients, "clients", gs.connected_clients);
+    rrdset_done(stclients);
+
+    // ----------------------------------------------------------------
+
+    if (!streqs) streqs = rrdset_find("netdata.requests");
+    if (!streqs) {
+        streqs = rrdset_create("netdata", "requests", NULL, "netdata", NULL, "NetData Web Requests", "requests/s",
+                               130200, rrd_update_every, RRDSET_TYPE_LINE);
+
+        rrddim_add(streqs, "requests", NULL, 1, 1, RRDDIM_INCREMENTAL);
+    } else rrdset_next(streqs);
+
+    rrddim_set(streqs, "requests", (collected_number) gs.web_requests);
+    rrdset_done(streqs);
+
+    // ----------------------------------------------------------------
+
+    if (!stbytes) stbytes = rrdset_find("netdata.net");
+    if (!stbytes) {
+        stbytes = rrdset_create("netdata", "net", NULL, "netdata", NULL, "NetData Network Traffic", "kilobits/s",
+                                130300, rrd_update_every, RRDSET_TYPE_AREA);
+
+        rrddim_add(stbytes, "in", NULL, 8, 1024, RRDDIM_INCREMENTAL);
+        rrddim_add(stbytes, "out", NULL, -8, 1024, RRDDIM_INCREMENTAL);
+    } else rrdset_next(stbytes);
+
+    rrddim_set(stbytes, "in", (collected_number) gs.bytes_received);
+    rrddim_set(stbytes, "out", (collected_number) gs.bytes_sent);
+    rrdset_done(stbytes);
+
+    // ----------------------------------------------------------------
+
+    if (!stduration) stduration = rrdset_find("netdata.response_time");
+    if (!stduration) {
+        stduration = rrdset_create("netdata", "response_time", NULL, "netdata", NULL, "NetData API Response Time",
+                                   "ms/request", 130400, rrd_update_every, RRDSET_TYPE_LINE);
+
+        rrddim_add(stduration, "average", NULL, 1, 1000, RRDDIM_ABSOLUTE);
+        rrddim_add(stduration, "max", NULL, 1, 1000, RRDDIM_ABSOLUTE);
+    } else rrdset_next(stduration);
+
+    uint64_t gweb_usec = gs.web_usec;
+    uint64_t gweb_requests = gs.web_requests;
+
+    uint64_t web_usec = (gweb_usec >= old_web_usec) ? gweb_usec - old_web_usec : 0;
+    uint64_t web_requests = (gweb_requests >= old_web_requests) ? gweb_requests - old_web_requests : 0;
+
+    old_web_usec = gweb_usec;
+    old_web_requests = gweb_requests;
+
+    if (web_requests)
+        average_response_time = (collected_number) (web_usec / web_requests);
+
+    if (unlikely(average_response_time != -1))
+        rrddim_set(stduration, "average", average_response_time);
+    else
+        rrddim_set(stduration, "average", 0);
+
+    rrddim_set(stduration, "max", ((gs.web_usec_max)?(collected_number)gs.web_usec_max:average_response_time));
+    rrdset_done(stduration);
+
+    // ----------------------------------------------------------------
+
+    if (!stcompression) stcompression = rrdset_find("netdata.compression_ratio");
+    if (!stcompression) {
+        stcompression = rrdset_create("netdata", "compression_ratio", NULL, "netdata", NULL,
+                                      "NetData API Responses Compression Savings Ratio", "percentage", 130500,
+                                      rrd_update_every, RRDSET_TYPE_LINE);
+
+        rrddim_add(stcompression, "savings", NULL, 1, 1000, RRDDIM_ABSOLUTE);
+    } else rrdset_next(stcompression);
+
+    // since we don't lock here to read the global statistics
+    // read the smaller value first
+    unsigned long long gcompressed_content_size = gs.compressed_content_size;
+    unsigned long long gcontent_size = gs.content_size;
+
+    unsigned long long compressed_content_size = gcompressed_content_size - old_compressed_content_size;
+    unsigned long long content_size = gcontent_size - old_content_size;
+
+    old_compressed_content_size = gcompressed_content_size;
+    old_content_size = gcontent_size;
+
+    if (content_size && content_size >= compressed_content_size)
+        compression_ratio = ((content_size - compressed_content_size) * 100 * 1000) / content_size;
+
+    if (compression_ratio != -1)
+        rrddim_set(stcompression, "savings", compression_ratio);
+
+    rrdset_done(stcompression);
+}
\ No newline at end of file
index 56a8118c727fb27f3aea58f7a0ee7f2d6547baa5..7c7eac3c041ad64debda4ce7e763d30a373f15fa 100644 (file)
@@ -5,18 +5,32 @@
 // global statistics
 
 struct global_statistics {
-       volatile unsigned long volatile connected_clients;
-       volatile unsigned long long volatile web_requests;
-       volatile unsigned long long volatile web_usec;
-       volatile unsigned long long volatile bytes_received;
-       volatile unsigned long long volatile bytes_sent;
-       volatile unsigned long long volatile content_size;
-       volatile unsigned long long volatile compressed_content_size;
+    volatile uint16_t connected_clients;
+
+       volatile uint64_t web_requests;
+    volatile uint64_t web_usec;
+    volatile uint64_t web_usec_max;
+       volatile uint64_t bytes_received;
+       volatile uint64_t bytes_sent;
+       volatile uint64_t content_size;
+       volatile uint64_t compressed_content_size;
 };
 
-extern struct global_statistics global_statistics;
+extern volatile struct global_statistics global_statistics;
 
 extern void global_statistics_lock(void);
 extern void global_statistics_unlock(void);
+extern void finished_web_request_statistics(uint64_t dt,
+                                     uint64_t bytes_received,
+                                     uint64_t bytes_sent,
+                                     uint64_t content_size,
+                                     uint64_t compressed_content_size);
+
+extern void web_client_connected(void);
+extern void web_client_disconnected(void);
+
+#define GLOBAL_STATS_RESET_WEB_USEC_MAX 0x01
+extern void global_statistics_copy(struct global_statistics *gs, uint8_t options);
+extern void global_statistics_charts(void);
 
 #endif /* NETDATA_GLOBAL_STATISTICS_H */
index 217cdc512ef03e9659c50f1dbc7c98fd0e0f3593..8de62b720a26ae8c8db0e90ba140ef28a0156a46 100644 (file)
@@ -36,7 +36,7 @@ void *checks_main(void *ptr)
 
                // find the time to sleep in order to wait exactly update_every seconds
                gettimeofday(&now, NULL);
-               loop_usec = usecdiff(&now, &last);
+               loop_usec = usec_dt(&now, &last);
                usec = loop_usec - susec;
                debug(D_PROCNETDEV_LOOP, "CHECK: last loop took %llu usec (worked for %llu, sleeped for %llu).", loop_usec, usec, susec);
 
@@ -72,9 +72,9 @@ void *checks_main(void *ptr)
                if(!apps_cpu) apps_cpu = rrdset_find("apps.cpu");
                if(check3->counter_done) rrdset_next_usec(check3, loop_usec);
                gettimeofday(&loop, NULL);
-               rrddim_set(check3, "caller", (long long)usecdiff(&loop, &check1->last_collected_time));
-               rrddim_set(check3, "netdata", (long long)usecdiff(&loop, &check2->last_collected_time));
-               if(apps_cpu) rrddim_set(check3, "apps.plugin", (long long)usecdiff(&loop, &apps_cpu->last_collected_time));
+               rrddim_set(check3, "caller", (long long) usec_dt(&loop, &check1->last_collected_time));
+               rrddim_set(check3, "netdata", (long long) usec_dt(&loop, &check2->last_collected_time));
+               if(apps_cpu) rrddim_set(check3, "apps.plugin", (long long) usec_dt(&loop, &apps_cpu->last_collected_time));
                rrdset_done(check3);
        }
 
index abdb03d1d91b4ffed8157334a0d17d1fd2e087b8..d0a9181ab2384e1c746c45d10648204569d34990 100644 (file)
@@ -38,7 +38,7 @@ void *cpuidlejitter_main(void *ptr)
                        gettimeofday(&after, NULL);
 
                        // calculate the time it took for a full loop
-                       usec = usecdiff(&after, &before);
+                       usec = usec_dt(&after, &before);
                        susec += usec;
                }
                usec -= (sleep_ms * 1000);
index 1f69b284bc96145f882a310d7cfc030db2d5ffa7..ac45775a855cce39a5b14eb2662e671ce41d0be4 100644 (file)
@@ -4,11 +4,6 @@ void *proc_main(void *ptr)
 {
        (void)ptr;
 
-       unsigned long long old_web_requests = 0, old_web_usec = 0,
-                       old_content_size = 0, old_compressed_content_size = 0;
-
-       collected_number compression_ratio = -1, average_response_time = -1;
-
        info("PROC Plugin thread created with task id %d", gettid());
 
        if(pthread_setcanceltype(PTHREAD_CANCEL_DEFERRED, NULL) != 0)
@@ -17,8 +12,6 @@ void *proc_main(void *ptr)
        if(pthread_setcancelstate(PTHREAD_CANCEL_ENABLE, NULL) != 0)
                error("Cannot set pthread cancel state to ENABLE.");
 
-       struct rusage me, thread;
-
        // disable (by default) various interface that are not needed
        config_get_boolean("plugin:proc:/proc/net/dev:lo", "enabled", 0);
        config_get_boolean("plugin:proc:/proc/net/dev:fireqos_monitor", "enabled", 0);
@@ -66,18 +59,15 @@ void *proc_main(void *ptr)
        unsigned long long sunext = (time(NULL) - (time(NULL) % rrd_update_every) + rrd_update_every) * 1000000ULL;
        unsigned long long sunow;
 
-       RRDSET *stcpu = NULL, *stcpu_thread = NULL, *stclients = NULL, *streqs = NULL, *stbytes = NULL, *stduration = NULL,
-                       *stcompression = NULL;
-
        for(;1;) {
                if(unlikely(netdata_exit)) break;
 
                // delay until it is our time to run
-               while((sunow = timems()) < sunext)
-                       usecsleep(sunext - sunow);
+               while((sunow = time_usec()) < sunext)
+                       sleep_usec(sunext - sunow);
 
                // find the next time we need to run
-               while(timems() > sunext)
+               while(time_usec() > sunext)
                        sunext += rrd_update_every * 1000000ULL;
 
                if(unlikely(netdata_exit)) break;
@@ -87,7 +77,7 @@ void *proc_main(void *ptr)
                if(!vdo_sys_kernel_mm_ksm) {
                        debug(D_PROCNETDEV_LOOP, "PROCNETDEV: calling do_sys_kernel_mm_ksm().");
 
-                       sunow = timems();
+                       sunow = time_usec();
                        vdo_sys_kernel_mm_ksm = do_sys_kernel_mm_ksm(rrd_update_every, (sutime_sys_kernel_mm_ksm > 0)?sunow - sutime_sys_kernel_mm_ksm:0ULL);
                        sutime_sys_kernel_mm_ksm = sunow;
                }
@@ -95,7 +85,7 @@ void *proc_main(void *ptr)
 
                if(!vdo_proc_loadavg) {
                        debug(D_PROCNETDEV_LOOP, "PROCNETDEV: calling do_proc_loadavg().");
-                       sunow = timems();
+                       sunow = time_usec();
                        vdo_proc_loadavg = do_proc_loadavg(rrd_update_every, (sutime_proc_loadavg > 0)?sunow - sutime_proc_loadavg:0ULL);
                        sutime_proc_loadavg = sunow;
                }
@@ -103,7 +93,7 @@ void *proc_main(void *ptr)
 
                if(!vdo_proc_interrupts) {
                        debug(D_PROCNETDEV_LOOP, "PROCNETDEV: calling do_proc_interrupts().");
-                       sunow = timems();
+                       sunow = time_usec();
                        vdo_proc_interrupts = do_proc_interrupts(rrd_update_every, (sutime_proc_interrupts > 0)?sunow - sutime_proc_interrupts:0ULL);
                        sutime_proc_interrupts = sunow;
                }
@@ -111,7 +101,7 @@ void *proc_main(void *ptr)
 
                if(!vdo_proc_softirqs) {
                        debug(D_PROCNETDEV_LOOP, "PROCNETDEV: calling do_proc_softirqs().");
-                       sunow = timems();
+                       sunow = time_usec();
                        vdo_proc_softirqs = do_proc_softirqs(rrd_update_every, (sutime_proc_softirqs > 0)?sunow - sutime_proc_softirqs:0ULL);
                        sutime_proc_softirqs = sunow;
                }
@@ -119,7 +109,7 @@ void *proc_main(void *ptr)
 
                if(!vdo_proc_sys_kernel_random_entropy_avail) {
                        debug(D_PROCNETDEV_LOOP, "PROCNETDEV: calling do_proc_sys_kernel_random_entropy_avail().");
-                       sunow = timems();
+                       sunow = time_usec();
                        vdo_proc_sys_kernel_random_entropy_avail = do_proc_sys_kernel_random_entropy_avail(rrd_update_every, (sutime_proc_sys_kernel_random_entropy_avail > 0)?sunow - sutime_proc_sys_kernel_random_entropy_avail:0ULL);
                        sutime_proc_sys_kernel_random_entropy_avail = sunow;
                }
@@ -127,7 +117,7 @@ void *proc_main(void *ptr)
 
                if(!vdo_proc_net_dev) {
                        debug(D_PROCNETDEV_LOOP, "PROCNETDEV: calling do_proc_net_dev().");
-                       sunow = timems();
+                       sunow = time_usec();
                        vdo_proc_net_dev = do_proc_net_dev(rrd_update_every, (sutime_proc_net_dev > 0)?sunow - sutime_proc_net_dev:0ULL);
                        sutime_proc_net_dev = sunow;
                }
@@ -135,7 +125,7 @@ void *proc_main(void *ptr)
 
                if(!vdo_proc_diskstats) {
                        debug(D_PROCNETDEV_LOOP, "PROCNETDEV: calling do_proc_diskstats().");
-                       sunow = timems();
+                       sunow = time_usec();
                        vdo_proc_diskstats = do_proc_diskstats(rrd_update_every, (sutime_proc_diskstats > 0)?sunow - sutime_proc_diskstats:0ULL);
                        sutime_proc_diskstats = sunow;
                }
@@ -143,7 +133,7 @@ void *proc_main(void *ptr)
 
                if(!vdo_proc_net_snmp) {
                        debug(D_PROCNETDEV_LOOP, "PROCNETDEV: calling do_proc_net_snmp().");
-                       sunow = timems();
+                       sunow = time_usec();
                        vdo_proc_net_snmp = do_proc_net_snmp(rrd_update_every, (sutime_proc_net_snmp > 0)?sunow - sutime_proc_net_snmp:0ULL);
                        sutime_proc_net_snmp = sunow;
                }
@@ -151,7 +141,7 @@ void *proc_main(void *ptr)
 
                if(!vdo_proc_net_snmp6) {
                        debug(D_PROCNETDEV_LOOP, "PROCNETDEV: calling do_proc_net_snmp6().");
-                       sunow = timems();
+                       sunow = time_usec();
                        vdo_proc_net_snmp6 = do_proc_net_snmp6(rrd_update_every, (sutime_proc_net_snmp6 > 0)?sunow - sutime_proc_net_snmp6:0ULL);
                        sutime_proc_net_snmp6 = sunow;
                }
@@ -159,7 +149,7 @@ void *proc_main(void *ptr)
 
                if(!vdo_proc_net_netstat) {
                        debug(D_PROCNETDEV_LOOP, "PROCNETDEV: calling do_proc_net_netstat().");
-                       sunow = timems();
+                       sunow = time_usec();
                        vdo_proc_net_netstat = do_proc_net_netstat(rrd_update_every, (sutime_proc_net_netstat > 0)?sunow - sutime_proc_net_netstat:0ULL);
                        sutime_proc_net_netstat = sunow;
                }
@@ -167,7 +157,7 @@ void *proc_main(void *ptr)
 
                if(!vdo_proc_net_stat_conntrack) {
                        debug(D_PROCNETDEV_LOOP, "PROCNETDEV: calling do_proc_net_stat_conntrack().");
-                       sunow = timems();
+                       sunow = time_usec();
                        vdo_proc_net_stat_conntrack     = do_proc_net_stat_conntrack(rrd_update_every, (sutime_proc_net_stat_conntrack > 0)?sunow - sutime_proc_net_stat_conntrack:0ULL);
                        sutime_proc_net_stat_conntrack = sunow;
                }
@@ -175,7 +165,7 @@ void *proc_main(void *ptr)
 
                if(!vdo_proc_net_ip_vs_stats) {
                        debug(D_PROCNETDEV_LOOP, "PROCNETDEV: calling vdo_proc_net_ip_vs_stats().");
-                       sunow = timems();
+                       sunow = time_usec();
                        vdo_proc_net_ip_vs_stats = do_proc_net_ip_vs_stats(rrd_update_every, (sutime_proc_net_ip_vs_stats > 0)?sunow - sutime_proc_net_ip_vs_stats:0ULL);
                        sutime_proc_net_ip_vs_stats = sunow;
                }
@@ -183,7 +173,7 @@ void *proc_main(void *ptr)
 
                if(!vdo_proc_net_stat_synproxy) {
                        debug(D_PROCNETDEV_LOOP, "PROCNETDEV: calling vdo_proc_net_stat_synproxy().");
-                       sunow = timems();
+                       sunow = time_usec();
                        vdo_proc_net_stat_synproxy = do_proc_net_stat_synproxy(rrd_update_every, (sutime_proc_net_stat_synproxy > 0)?sunow - sutime_proc_net_stat_synproxy:0ULL);
                        sutime_proc_net_stat_synproxy = sunow;
                }
@@ -191,7 +181,7 @@ void *proc_main(void *ptr)
 
                if(!vdo_proc_stat) {
                        debug(D_PROCNETDEV_LOOP, "PROCNETDEV: calling do_proc_stat().");
-                       sunow = timems();
+                       sunow = time_usec();
                        vdo_proc_stat = do_proc_stat(rrd_update_every, (sutime_proc_stat > 0)?sunow - sutime_proc_stat:0ULL);
                        sutime_proc_stat = sunow;
                }
@@ -199,7 +189,7 @@ void *proc_main(void *ptr)
 
                if(!vdo_proc_meminfo) {
                        debug(D_PROCNETDEV_LOOP, "PROCNETDEV: calling vdo_proc_meminfo().");
-                       sunow = timems();
+                       sunow = time_usec();
                        vdo_proc_meminfo = do_proc_meminfo(rrd_update_every, (sutime_proc_meminfo > 0)?sunow - sutime_proc_meminfo:0ULL);
                        sutime_proc_meminfo = sunow;
                }
@@ -207,7 +197,7 @@ void *proc_main(void *ptr)
 
                if(!vdo_proc_vmstat) {
                        debug(D_PROCNETDEV_LOOP, "PROCNETDEV: calling vdo_proc_vmstat().");
-                       sunow = timems();
+                       sunow = time_usec();
                        vdo_proc_vmstat = do_proc_vmstat(rrd_update_every, (sutime_proc_vmstat > 0)?sunow - sutime_proc_vmstat:0ULL);
                        sutime_proc_vmstat = sunow;
                }
@@ -215,7 +205,7 @@ void *proc_main(void *ptr)
 
                if(!vdo_proc_net_rpc_nfsd) {
                        debug(D_PROCNETDEV_LOOP, "PROCNETDEV: calling do_proc_net_rpc_nfsd().");
-                       sunow = timems();
+                       sunow = time_usec();
                        vdo_proc_net_rpc_nfsd = do_proc_net_rpc_nfsd(rrd_update_every, (sutime_proc_net_rpc_nfsd > 0)?sunow - sutime_proc_net_rpc_nfsd:0ULL);
                        sutime_proc_net_rpc_nfsd = sunow;
                }
@@ -226,136 +216,7 @@ void *proc_main(void *ptr)
                // --------------------------------------------------------------------
 
                if(!vdo_cpu_netdata) {
-                       getrusage(RUSAGE_THREAD, &thread);
-                       getrusage(RUSAGE_SELF, &me);
-
-                       if(!stcpu_thread) stcpu_thread = rrdset_find("netdata.plugin_proc_cpu");
-                       if(!stcpu_thread) {
-                               stcpu_thread = rrdset_create("netdata", "plugin_proc_cpu", NULL, "proc.internal", NULL, "NetData Proc Plugin CPU usage", "milliseconds/s", 132000, rrd_update_every, RRDSET_TYPE_STACKED);
-
-                               rrddim_add(stcpu_thread, "user",  NULL,  1, 1000, RRDDIM_INCREMENTAL);
-                               rrddim_add(stcpu_thread, "system", NULL, 1, 1000, RRDDIM_INCREMENTAL);
-                       }
-                       else rrdset_next(stcpu_thread);
-
-                       rrddim_set(stcpu_thread, "user"  , thread.ru_utime.tv_sec * 1000000ULL + thread.ru_utime.tv_usec);
-                       rrddim_set(stcpu_thread, "system", thread.ru_stime.tv_sec * 1000000ULL + thread.ru_stime.tv_usec);
-                       rrdset_done(stcpu_thread);
-
-                       // ----------------------------------------------------------------
-
-                       if(!stcpu) stcpu = rrdset_find("netdata.server_cpu");
-                       if(!stcpu) {
-                               stcpu = rrdset_create("netdata", "server_cpu", NULL, "netdata", NULL, "NetData CPU usage", "milliseconds/s", 130000, rrd_update_every, RRDSET_TYPE_STACKED);
-
-                               rrddim_add(stcpu, "user",  NULL,  1, 1000, RRDDIM_INCREMENTAL);
-                               rrddim_add(stcpu, "system", NULL, 1, 1000, RRDDIM_INCREMENTAL);
-                       }
-                       else rrdset_next(stcpu);
-
-                       rrddim_set(stcpu, "user"  , me.ru_utime.tv_sec * 1000000ULL + me.ru_utime.tv_usec);
-                       rrddim_set(stcpu, "system", me.ru_stime.tv_sec * 1000000ULL + me.ru_stime.tv_usec);
-                       rrdset_done(stcpu);
-
-                       // ----------------------------------------------------------------
-
-                       if(!stclients) stclients = rrdset_find("netdata.clients");
-                       if(!stclients) {
-                               stclients = rrdset_create("netdata", "clients", NULL, "netdata", NULL, "NetData Web Clients", "connected clients", 130100, rrd_update_every, RRDSET_TYPE_LINE);
-
-                               rrddim_add(stclients, "clients",  NULL,  1, 1, RRDDIM_ABSOLUTE);
-                       }
-                       else rrdset_next(stclients);
-
-                       rrddim_set(stclients, "clients", global_statistics.connected_clients);
-                       rrdset_done(stclients);
-
-                       // ----------------------------------------------------------------
-
-                       if(!streqs) streqs = rrdset_find("netdata.requests");
-                       if(!streqs) {
-                               streqs = rrdset_create("netdata", "requests", NULL, "netdata", NULL, "NetData Web Requests", "requests/s", 130200, rrd_update_every, RRDSET_TYPE_LINE);
-
-                               rrddim_add(streqs, "requests",  NULL,  1, 1, RRDDIM_INCREMENTAL);
-                       }
-                       else rrdset_next(streqs);
-
-                       rrddim_set(streqs, "requests", global_statistics.web_requests);
-                       rrdset_done(streqs);
-
-                       // ----------------------------------------------------------------
-
-                       if(!stbytes) stbytes = rrdset_find("netdata.net");
-                       if(!stbytes) {
-                               stbytes = rrdset_create("netdata", "net", NULL, "netdata", NULL, "NetData Network Traffic", "kilobits/s", 130300, rrd_update_every, RRDSET_TYPE_AREA);
-
-                               rrddim_add(stbytes, "in",  NULL,  8, 1024, RRDDIM_INCREMENTAL);
-                               rrddim_add(stbytes, "out",  NULL,  -8, 1024, RRDDIM_INCREMENTAL);
-                       }
-                       else rrdset_next(stbytes);
-
-                       rrddim_set(stbytes, "in", global_statistics.bytes_received);
-                       rrddim_set(stbytes, "out", global_statistics.bytes_sent);
-                       rrdset_done(stbytes);
-
-                       // ----------------------------------------------------------------
-
-                       if(!stduration) stduration = rrdset_find("netdata.response_time");
-                       if(!stduration) {
-                               stduration = rrdset_create("netdata", "response_time", NULL, "netdata", NULL, "NetData Average API Response Time", "ms/request", 130400, rrd_update_every, RRDSET_TYPE_LINE);
-
-                               rrddim_add(stduration, "response_time", "response time",  1, 1000, RRDDIM_ABSOLUTE);
-                       }
-                       else rrdset_next(stduration);
-
-                       unsigned long long gweb_usec     = global_statistics.web_usec;
-                       unsigned long long gweb_requests = global_statistics.web_requests;
-
-                       unsigned long long web_usec     = gweb_usec     - old_web_usec;
-                       unsigned long long web_requests = gweb_requests - old_web_requests;
-
-                       old_web_usec     = gweb_usec;
-                       old_web_requests = gweb_requests;
-
-                       if(web_requests)
-                               average_response_time =  web_usec / web_requests;
-
-                       if(average_response_time != -1)
-                               rrddim_set(stduration, "response_time", average_response_time);
-
-                       rrdset_done(stduration);
-
-                       // ----------------------------------------------------------------
-
-                       if(!stcompression) stcompression = rrdset_find("netdata.compression_ratio");
-                       if(!stcompression) {
-                               stcompression = rrdset_create("netdata", "compression_ratio", NULL, "netdata", NULL, "NetData API Responses Compression Savings Ratio", "percentage", 130500, rrd_update_every, RRDSET_TYPE_LINE);
-
-                               rrddim_add(stcompression, "savings", NULL,  1, 1000, RRDDIM_ABSOLUTE);
-                       }
-                       else rrdset_next(stcompression);
-
-                       // since we don't lock here to read the global statistics
-                       // read the smaller value first
-                       unsigned long long gcompressed_content_size = global_statistics.compressed_content_size;
-                       unsigned long long gcontent_size            = global_statistics.content_size;
-
-                       unsigned long long compressed_content_size  = gcompressed_content_size - old_compressed_content_size;
-                       unsigned long long content_size             = gcontent_size            - old_content_size;
-
-                       old_compressed_content_size = gcompressed_content_size;
-                       old_content_size            = gcontent_size;
-
-                       if(content_size && content_size >= compressed_content_size)
-                               compression_ratio = ((content_size - compressed_content_size) * 100 * 1000) / content_size;
-
-                       if(compression_ratio != -1)
-                               rrddim_set(stcompression, "savings", compression_ratio);
-
-                       rrdset_done(stcompression);
-
-                       // ----------------------------------------------------------------
-
+            global_statistics_charts();
                        registry_statistics();
                }
        }
index 92b149595aa3e7985d64c90f64ad39fcdbbc6068..f3c6f260ecbee02c504da21a8bc3c997300becf4 100644 (file)
--- a/src/rrd.c
+++ b/src/rrd.c
@@ -600,7 +600,7 @@ RRDDIM *rrddim_add(RRDSET *st, const char *id, const char *name, long multiplier
                        error("File %s does not have the same refresh frequency. Clearing it.", fullfilename);
                        bzero(rd, size);
                }
-               else if(usecdiff(&now, &rd->last_collected_time) > (rd->entries * rd->update_every * 1000000ULL)) {
+               else if(usec_dt(&now, &rd->last_collected_time) > (rd->entries * rd->update_every * 1000000ULL)) {
                        errno = 0;
                        error("File %s is too old. Clearing it.", fullfilename);
                        bzero(rd, size);
@@ -922,7 +922,7 @@ void rrdset_next(RRDSET *st)
        if(likely(st->last_collected_time.tv_sec)) {
                struct timeval now;
                gettimeofday(&now, NULL);
-               microseconds = usecdiff(&now, &st->last_collected_time);
+               microseconds = usec_dt(&now, &st->last_collected_time);
        }
        // prevent infinite loop
        else microseconds = st->update_every * 1000000ULL;
@@ -1001,7 +1001,7 @@ unsigned long long rrdset_done(RRDSET *st)
        }
 
        // check if we will re-write the entire data set
-       if(unlikely(usecdiff(&st->last_collected_time, &st->last_updated) > st->update_every * st->entries * 1000000ULL)) {
+       if(unlikely(usec_dt(&st->last_collected_time, &st->last_updated) > st->update_every * st->entries * 1000000ULL)) {
                info("%s: too old data (last updated at %ld.%ld, last collected at %ld.%ld). Reseting it. Will not store the next entry.", st->name, st->last_updated.tv_sec, st->last_updated.tv_usec, st->last_collected_time.tv_sec, st->last_collected_time.tv_usec);
                rrdset_reset(st);
 
index 522aaf1744a7c541f3369a44fea19c225f7e55c7..72d2dc263b2e24c3cf918c89d34783f55ddfd30c 100644 (file)
@@ -1287,11 +1287,11 @@ void *cgroups_main(void *ptr)
                if(unlikely(netdata_exit)) break;
 
                // delay until it is our time to run
-               while((sunow = timems()) < sunext)
-                       usecsleep(sunext - sunow);
+               while((sunow = time_usec()) < sunext)
+                       sleep_usec(sunext - sunow);
 
                // find the next time we need to run
-               while(timems() > sunext)
+               while(time_usec() > sunext)
                        sunext += rrd_update_every * 1000000ULL;
 
                if(unlikely(netdata_exit)) break;
@@ -1300,7 +1300,7 @@ void *cgroups_main(void *ptr)
 
                if(!vdo_sys_fs_cgroup) {
                        debug(D_PROCNETDEV_LOOP, "PROCNETDEV: calling do_sys_fs_cgroup().");
-                       sunow = timems();
+                       sunow = time_usec();
                        vdo_sys_fs_cgroup = do_sys_fs_cgroup(rrd_update_every, (sutime_sys_fs_cgroup > 0)?sunow - sutime_sys_fs_cgroup:0ULL);
                        sutime_sys_fs_cgroup = sunow;
                }
index 721310de6434f04125f2f5c46be029c34282a65d..bc321a6b92beb7df1c6c59d60783eb93a2519d8d 100644 (file)
@@ -113,7 +113,7 @@ struct web_client *web_client_create(int listener)
        w->next = web_clients;
        web_clients = w;
 
-       global_statistics.connected_clients++;
+    web_client_connected();
 
        return(w);
 }
@@ -136,18 +136,11 @@ void web_client_reset(struct web_client *w) {
                // --------------------------------------------------------------------
                // global statistics
 
-               if(web_server_mode == WEB_SERVER_MODE_MULTI_THREADED)
-                       global_statistics_lock();
-
-               global_statistics.web_requests++;
-               global_statistics.web_usec += usecdiff(&tv, &w->tv_in);
-               global_statistics.bytes_received += w->stats_received_bytes;
-               global_statistics.bytes_sent += w->stats_sent_bytes;
-               global_statistics.content_size += size;
-               global_statistics.compressed_content_size += sent;
-
-               if(web_server_mode == WEB_SERVER_MODE_MULTI_THREADED)
-                       global_statistics_unlock();
+        finished_web_request_statistics(usec_dt(&tv, &w->tv_in),
+                                        w->stats_received_bytes,
+                                        w->stats_sent_bytes,
+                                        size,
+                                        sent);
 
                w->stats_received_bytes = 0;
                w->stats_sent_bytes = 0;
@@ -159,9 +152,9 @@ void web_client_reset(struct web_client *w) {
                log_access("%llu: (sent/all = %zu/%zu bytes %0.0f%%, prep/sent/total = %0.2f/%0.2f/%0.2f ms) %s: %d '%s'",
                                   w->id,
                                   sent, size, -((size > 0) ? ((size - sent) / (double) size * 100.0) : 0.0),
-                                  usecdiff(&w->tv_ready, &w->tv_in) / 1000.0,
-                                  usecdiff(&tv, &w->tv_ready) / 1000.0,
-                                  usecdiff(&tv, &w->tv_in) / 1000.0,
+                                  usec_dt(&w->tv_ready, &w->tv_in) / 1000.0,
+                                  usec_dt(&tv, &w->tv_ready) / 1000.0,
+                                  usec_dt(&tv, &w->tv_in) / 1000.0,
                                   (w->mode == WEB_CLIENT_MODE_FILECOPY) ? "filecopy" : ((w->mode == WEB_CLIENT_MODE_OPTIONS)
                                                                                                                                                 ? "options" : "data"),
                                   w->response.code,
@@ -236,7 +229,7 @@ struct web_client *web_client_free(struct web_client *w) {
        if(w->ofd != -1 && w->ofd != w->ifd) close(w->ofd);
        freez(w);
 
-       global_statistics.connected_clients--;
+    web_client_disconnected();
 
        return(n);
 }
@@ -1668,7 +1661,10 @@ void web_client_process(struct web_client *w) {
        static uint32_t hash_api = 0, hash_netdata_conf = 0, hash_data = 0, hash_datasource = 0, hash_graph = 0,
                        hash_list = 0, hash_all_json = 0, hash_exit = 0, hash_debug = 0, hash_mirror = 0;
 
-       if(unlikely(!hash_api)) {
+    // start timing us
+    gettimeofday(&w->tv_in, NULL);
+
+    if(unlikely(!hash_api)) {
                hash_api = simple_hash("api");
                hash_netdata_conf = simple_hash("netdata.conf");
                hash_data = simple_hash(WEB_PATH_DATA);
@@ -1703,7 +1699,7 @@ void web_client_process(struct web_client *w) {
                }
        }
        else if(what_to_do > 0) {
-               strcpy(w->last_url, "not a valid request");
+               // strcpy(w->last_url, "not a valid request");
 
                debug(D_WEB_CLIENT_ACCESS, "%llu: Cannot understand '%s'.", w->id, w->response.data->buffer);
 
@@ -1712,8 +1708,6 @@ void web_client_process(struct web_client *w) {
                buffer_strcat(w->response.data, "I don't understand you...\r\n");
        }
        else { // what_to_do == 0
-               gettimeofday(&w->tv_in, NULL);
-
                if(w->mode == WEB_CLIENT_MODE_OPTIONS) {
                        code = 200;
                        w->response.data->contenttype = CT_TEXT_PLAIN;
index 78c47034d4540b56a4448f660e36527d13ef1696..c6f5f2e7f8baa0194c7354c2acc63da64b6383d5 100644 (file)
@@ -945,8 +945,8 @@ body {
                                                data-width="100%"
                                                data-height="100%"
                                                data-after="-300"
-                                               data-colors="#558855"
-                                               data-show-value-of-response_time-at="netdata.response_time1"
+                                               data-colors="#558855 #356835"
+                                               data-show-value-of-average-at="netdata.response_time1"
                                                ></div>
                        </div>
 
@@ -964,8 +964,8 @@ body {
                                                data-width="100%"
                                                data-height="100%"
                                                data-after="-300"
-                                               data-colors="#885555"
-                                               data-show-value-of-response_time-at="netdata.response_time2"
+                                               data-colors="#885555 #683535"
+                                               data-show-value-of-average-at="netdata.response_time2"
                                                ></div>
                        </div>
 
@@ -983,8 +983,8 @@ body {
                                                data-width="100%"
                                                data-height="100%"
                                                data-after="-300"
-                                               data-colors="#555588"
-                                               data-show-value-of-response_time-at="netdata.response_time3"
+                                               data-colors="#555588 #353568"
+                                               data-show-value-of-average-at="netdata.response_time3"
                                                ></div>
                        </div>
 
@@ -1002,8 +1002,8 @@ body {
                                                data-width="100%"
                                                data-height="100%"
                                                data-after="-300"
-                                               data-colors="#885588"
-                                               data-show-value-of-response_time-at="netdata.response_time4"
+                                               data-colors="#885588 #683568"
+                                               data-show-value-of-average-at="netdata.response_time4"
                                                ></div>
                        </div>