From: Costa Tsaousis Date: Fri, 12 Aug 2016 22:44:31 +0000 (+0300) Subject: global statistics now also report max API response time; global statistics implemente... X-Git-Tag: v1.3.0~23^2~67 X-Git-Url: https://arthur.barton.de/gitweb/?p=netdata.git;a=commitdiff_plain;h=9b4f149140248792bf2ccbc6e6f90222e5e078ab global statistics now also report max API response time; global statistics implemented with alternative lock-free code; more code cleanups --- diff --git a/src/apps_plugin.c b/src/apps_plugin.c index 210b5103..01da625b 100644 --- a/src/apps_plugin.c +++ b/src/apps_plugin.c @@ -642,7 +642,7 @@ int read_proc_pid_stat(struct pid_stat *p) { if(unlikely(!ff)) goto cleanup; p->last_stat_collected_usec = p->stat_collected_usec; - p->stat_collected_usec = timems(); + p->stat_collected_usec = time_usec(); file_counter++; // p->pid = atol(procfile_lineword(ff, 0, 0+i)); @@ -824,7 +824,7 @@ int read_proc_pid_io(struct pid_stat *p) { file_counter++; p->last_io_collected_usec = p->io_collected_usec; - p->io_collected_usec = timems(); + p->io_collected_usec = time_usec(); unsigned long long last; @@ -898,7 +898,7 @@ int read_proc_stat() { if(unlikely(!ff)) goto cleanup; last_collected_usec = collected_usec; - collected_usec = timems(); + collected_usec = time_usec(); file_counter++; @@ -2205,7 +2205,7 @@ unsigned long long send_resource_usage_to_netdata() { gettimeofday(&now, NULL); getrusage(RUSAGE_SELF, &me); - usec = usecdiff(&now, &last); + usec = usec_dt(&now, &last); cpuuser = me.ru_utime.tv_sec * 1000000ULL + me.ru_utime.tv_usec; cpusyst = me.ru_stime.tv_sec * 1000000ULL + me.ru_stime.tv_usec; @@ -2836,11 +2836,11 @@ int main(int argc, char **argv) for(;1; global_iterations_counter++) { #ifndef PROFILING_MODE // delay until it is our time to run - while((sunow = timems()) < sunext) - usecsleep(sunext - sunow); + while((sunow = time_usec()) < sunext) + sleep_usec(sunext - sunow); // find the next time we need to run - while(timems() > sunext) + while(time_usec() > sunext) sunext += update_every * 1000000ULL; #endif /* PROFILING_MODE */ diff --git a/src/common.c b/src/common.c index 8d2c3aa8..7d0f8901 100644 --- a/src/common.c +++ b/src/common.c @@ -43,15 +43,26 @@ void freez(void *ptr) { } // ---------------------------------------------------------------------------- +// time functions -// time(NULL) in milliseconds -unsigned long long timems(void) { +inline unsigned long long timeval_usec(struct timeval *tv) { + return tv->tv_sec * 1000000ULL + tv->tv_usec; +} + +// time(NULL) in nanoseconds +inline unsigned long long time_usec(void) { struct timeval now; gettimeofday(&now, NULL); - return now.tv_sec * 1000000ULL + now.tv_usec; + return timeval_usec(&now); +} + +inline unsigned long long usec_dt(struct timeval *now, struct timeval *old) { + unsigned long long tv1 = timeval_usec(now); + unsigned long long tv2 = timeval_usec(old); + return (tv1 > tv2)?(tv1 - tv2):(tv2 - tv1); } -int usecsleep(unsigned long long usec) { +int sleep_usec(unsigned long long usec) { #ifndef NETDATA_WITH_USLEEP // we expect microseconds (1.000.000 per second) diff --git a/src/common.h b/src/common.h index 9793adf6..4bb949c5 100644 --- a/src/common.h +++ b/src/common.h @@ -107,7 +107,10 @@ #endif #define abs(x) ((x < 0)? -x : x) -#define usecdiff(now, last) (((((now)->tv_sec * 1000000ULL) + (now)->tv_usec) - (((last)->tv_sec * 1000000ULL) + (last)->tv_usec))) +extern unsigned long long usec_dt(struct timeval *now, struct timeval *old); +extern unsigned long long timeval_usec(struct timeval *tv); + +// #define usec_dt(now, last) (((((now)->tv_sec * 1000000ULL) + (now)->tv_usec) - (((last)->tv_sec * 1000000ULL) + (last)->tv_usec))) extern void netdata_fix_chart_id(char *s); extern void netdata_fix_chart_name(char *s); @@ -144,8 +147,8 @@ extern void get_HZ(void); extern pid_t gettid(void); -extern unsigned long long timems(void); -extern int usecsleep(unsigned long long usec); +extern unsigned long long time_usec(void); +extern int sleep_usec(unsigned long long usec); extern char *fgets_trim_len(char *buf, size_t buf_size, FILE *fp, size_t *len); diff --git a/src/global_statistics.c b/src/global_statistics.c index e2852962..0512c438 100644 --- a/src/global_statistics.c +++ b/src/global_statistics.c @@ -1,13 +1,262 @@ #include "common.h" -struct global_statistics global_statistics = { 0, 0ULL, 0ULL, 0ULL, 0ULL, 0ULL, 0ULL}; +volatile struct global_statistics global_statistics = { + .connected_clients = 0, + .web_requests = 0, + .web_usec = 0, + .bytes_received = 0, + .bytes_sent = 0, + .content_size = 0, + .compressed_content_size = 0 +}; pthread_mutex_t global_statistics_mutex = PTHREAD_MUTEX_INITIALIZER; -void global_statistics_lock(void) { - pthread_mutex_lock(&global_statistics_mutex); +inline void global_statistics_lock(void) { + pthread_mutex_lock(&global_statistics_mutex); } -void global_statistics_unlock(void) { - pthread_mutex_unlock(&global_statistics_mutex); +inline void global_statistics_unlock(void) { + pthread_mutex_unlock(&global_statistics_mutex); } + +void finished_web_request_statistics(uint64_t dt, + uint64_t bytes_received, + uint64_t bytes_sent, + uint64_t content_size, + uint64_t compressed_content_size) { +#ifndef NETDATA_NO_ATOMIC_INSTRUCTIONS + uint64_t old_web_usec_max = global_statistics.web_usec_max; + while(dt > old_web_usec_max) { + if(__sync_bool_compare_and_swap(&global_statistics.web_usec_max, old_web_usec_max, dt)) + old_web_usec_max = dt; + else + old_web_usec_max = global_statistics.web_usec_max; + } + + __sync_fetch_and_add(&global_statistics.web_requests, 1); + __sync_fetch_and_add(&global_statistics.web_usec, dt); + __sync_fetch_and_add(&global_statistics.bytes_received, bytes_received); + __sync_fetch_and_add(&global_statistics.bytes_sent, bytes_sent); + __sync_fetch_and_add(&global_statistics.content_size, content_size); + __sync_fetch_and_add(&global_statistics.compressed_content_size, compressed_content_size); +#else + if (web_server_mode == WEB_SERVER_MODE_MULTI_THREADED) + global_statistics_lock(); + + if (dt > global_statistics.web_usec_max) + global_statistics.web_usec_max = dt; + + global_statistics.web_requests++; + global_statistics.web_usec += dt; + global_statistics.bytes_received += bytes_received; + global_statistics.bytes_sent += bytes_sent; + global_statistics.content_size += content_size; + global_statistics.compressed_content_size += compressed_content_size; + + if (web_server_mode == WEB_SERVER_MODE_MULTI_THREADED) + global_statistics_unlock(); +#endif +} + +void web_client_connected(void) { +#ifndef NETDATA_NO_ATOMIC_INSTRUCTIONS + __sync_fetch_and_add(&global_statistics.connected_clients, 1); +#else + if (web_server_mode == WEB_SERVER_MODE_MULTI_THREADED) + global_statistics_lock(); + + global_statistics.connected_clients++; + + if (web_server_mode == WEB_SERVER_MODE_MULTI_THREADED) + global_statistics_unlock(); +#endif +} + +void web_client_disconnected(void) { +#ifndef NETDATA_NO_ATOMIC_INSTRUCTIONS + __sync_fetch_and_sub(&global_statistics.connected_clients, 1); +#else + if (web_server_mode == WEB_SERVER_MODE_MULTI_THREADED) + global_statistics_lock(); + + global_statistics.connected_clients--; + + if (web_server_mode == WEB_SERVER_MODE_MULTI_THREADED) + global_statistics_unlock(); +#endif +} + + +inline void global_statistics_copy(struct global_statistics *gs, uint8_t options) { +#ifndef NETDATA_NO_ATOMIC_INSTRUCTIONS + gs->connected_clients = __sync_fetch_and_add(&global_statistics.connected_clients, 0); + gs->web_requests = __sync_fetch_and_add(&global_statistics.web_requests, 0); + gs->web_usec = __sync_fetch_and_add(&global_statistics.web_usec, 0); + gs->web_usec_max = __sync_fetch_and_add(&global_statistics.web_usec_max, 0); + gs->bytes_received = __sync_fetch_and_add(&global_statistics.bytes_received, 0); + gs->bytes_sent = __sync_fetch_and_add(&global_statistics.bytes_sent, 0); + gs->content_size = __sync_fetch_and_add(&global_statistics.content_size, 0); + gs->compressed_content_size = __sync_fetch_and_add(&global_statistics.compressed_content_size, 0); + + if(options & GLOBAL_STATS_RESET_WEB_USEC_MAX) + __sync_bool_compare_and_swap(&global_statistics.web_usec_max, gs->web_usec_max, 0); +#else + global_statistics_lock(); + + memcpy(gs, (const void *)&global_statistics, sizeof(struct global_statistics)); + + if (options & GLOBAL_STATS_RESET_WEB_USEC_MAX) + global_statistics.web_usec_max = 0; + + global_statistics_unlock(); +#endif +} + +void global_statistics_charts(void) { + static unsigned long long old_web_requests = 0, old_web_usec = 0, + old_content_size = 0, old_compressed_content_size = 0; + + static collected_number compression_ratio = -1, average_response_time = -1; + + static RRDSET *stcpu = NULL, *stcpu_thread = NULL, *stclients = NULL, *streqs = NULL, *stbytes = NULL, *stduration = NULL, + *stcompression = NULL; + + struct global_statistics gs; + struct rusage me, thread; + + global_statistics_copy(&gs, GLOBAL_STATS_RESET_WEB_USEC_MAX); + getrusage(RUSAGE_THREAD, &thread); + getrusage(RUSAGE_SELF, &me); + + if (!stcpu_thread) stcpu_thread = rrdset_find("netdata.plugin_proc_cpu"); + if (!stcpu_thread) { + stcpu_thread = rrdset_create("netdata", "plugin_proc_cpu", NULL, "proc.internal", NULL, + "NetData Proc Plugin CPU usage", "milliseconds/s", 132000, rrd_update_every, + RRDSET_TYPE_STACKED); + + rrddim_add(stcpu_thread, "user", NULL, 1, 1000, RRDDIM_INCREMENTAL); + rrddim_add(stcpu_thread, "system", NULL, 1, 1000, RRDDIM_INCREMENTAL); + } else rrdset_next(stcpu_thread); + + rrddim_set(stcpu_thread, "user", thread.ru_utime.tv_sec * 1000000ULL + thread.ru_utime.tv_usec); + rrddim_set(stcpu_thread, "system", thread.ru_stime.tv_sec * 1000000ULL + thread.ru_stime.tv_usec); + rrdset_done(stcpu_thread); + + // ---------------------------------------------------------------- + + if (!stcpu) stcpu = rrdset_find("netdata.server_cpu"); + if (!stcpu) { + stcpu = rrdset_create("netdata", "server_cpu", NULL, "netdata", NULL, "NetData CPU usage", "milliseconds/s", + 130000, rrd_update_every, RRDSET_TYPE_STACKED); + + rrddim_add(stcpu, "user", NULL, 1, 1000, RRDDIM_INCREMENTAL); + rrddim_add(stcpu, "system", NULL, 1, 1000, RRDDIM_INCREMENTAL); + } else rrdset_next(stcpu); + + rrddim_set(stcpu, "user", me.ru_utime.tv_sec * 1000000ULL + me.ru_utime.tv_usec); + rrddim_set(stcpu, "system", me.ru_stime.tv_sec * 1000000ULL + me.ru_stime.tv_usec); + rrdset_done(stcpu); + + // ---------------------------------------------------------------- + + if (!stclients) stclients = rrdset_find("netdata.clients"); + if (!stclients) { + stclients = rrdset_create("netdata", "clients", NULL, "netdata", NULL, "NetData Web Clients", + "connected clients", 130100, rrd_update_every, RRDSET_TYPE_LINE); + + rrddim_add(stclients, "clients", NULL, 1, 1, RRDDIM_ABSOLUTE); + } else rrdset_next(stclients); + + rrddim_set(stclients, "clients", gs.connected_clients); + rrdset_done(stclients); + + // ---------------------------------------------------------------- + + if (!streqs) streqs = rrdset_find("netdata.requests"); + if (!streqs) { + streqs = rrdset_create("netdata", "requests", NULL, "netdata", NULL, "NetData Web Requests", "requests/s", + 130200, rrd_update_every, RRDSET_TYPE_LINE); + + rrddim_add(streqs, "requests", NULL, 1, 1, RRDDIM_INCREMENTAL); + } else rrdset_next(streqs); + + rrddim_set(streqs, "requests", (collected_number) gs.web_requests); + rrdset_done(streqs); + + // ---------------------------------------------------------------- + + if (!stbytes) stbytes = rrdset_find("netdata.net"); + if (!stbytes) { + stbytes = rrdset_create("netdata", "net", NULL, "netdata", NULL, "NetData Network Traffic", "kilobits/s", + 130300, rrd_update_every, RRDSET_TYPE_AREA); + + rrddim_add(stbytes, "in", NULL, 8, 1024, RRDDIM_INCREMENTAL); + rrddim_add(stbytes, "out", NULL, -8, 1024, RRDDIM_INCREMENTAL); + } else rrdset_next(stbytes); + + rrddim_set(stbytes, "in", (collected_number) gs.bytes_received); + rrddim_set(stbytes, "out", (collected_number) gs.bytes_sent); + rrdset_done(stbytes); + + // ---------------------------------------------------------------- + + if (!stduration) stduration = rrdset_find("netdata.response_time"); + if (!stduration) { + stduration = rrdset_create("netdata", "response_time", NULL, "netdata", NULL, "NetData API Response Time", + "ms/request", 130400, rrd_update_every, RRDSET_TYPE_LINE); + + rrddim_add(stduration, "average", NULL, 1, 1000, RRDDIM_ABSOLUTE); + rrddim_add(stduration, "max", NULL, 1, 1000, RRDDIM_ABSOLUTE); + } else rrdset_next(stduration); + + uint64_t gweb_usec = gs.web_usec; + uint64_t gweb_requests = gs.web_requests; + + uint64_t web_usec = (gweb_usec >= old_web_usec) ? gweb_usec - old_web_usec : 0; + uint64_t web_requests = (gweb_requests >= old_web_requests) ? gweb_requests - old_web_requests : 0; + + old_web_usec = gweb_usec; + old_web_requests = gweb_requests; + + if (web_requests) + average_response_time = (collected_number) (web_usec / web_requests); + + if (unlikely(average_response_time != -1)) + rrddim_set(stduration, "average", average_response_time); + else + rrddim_set(stduration, "average", 0); + + rrddim_set(stduration, "max", ((gs.web_usec_max)?(collected_number)gs.web_usec_max:average_response_time)); + rrdset_done(stduration); + + // ---------------------------------------------------------------- + + if (!stcompression) stcompression = rrdset_find("netdata.compression_ratio"); + if (!stcompression) { + stcompression = rrdset_create("netdata", "compression_ratio", NULL, "netdata", NULL, + "NetData API Responses Compression Savings Ratio", "percentage", 130500, + rrd_update_every, RRDSET_TYPE_LINE); + + rrddim_add(stcompression, "savings", NULL, 1, 1000, RRDDIM_ABSOLUTE); + } else rrdset_next(stcompression); + + // since we don't lock here to read the global statistics + // read the smaller value first + unsigned long long gcompressed_content_size = gs.compressed_content_size; + unsigned long long gcontent_size = gs.content_size; + + unsigned long long compressed_content_size = gcompressed_content_size - old_compressed_content_size; + unsigned long long content_size = gcontent_size - old_content_size; + + old_compressed_content_size = gcompressed_content_size; + old_content_size = gcontent_size; + + if (content_size && content_size >= compressed_content_size) + compression_ratio = ((content_size - compressed_content_size) * 100 * 1000) / content_size; + + if (compression_ratio != -1) + rrddim_set(stcompression, "savings", compression_ratio); + + rrdset_done(stcompression); +} \ No newline at end of file diff --git a/src/global_statistics.h b/src/global_statistics.h index 56a8118c..7c7eac3c 100644 --- a/src/global_statistics.h +++ b/src/global_statistics.h @@ -5,18 +5,32 @@ // global statistics struct global_statistics { - volatile unsigned long volatile connected_clients; - volatile unsigned long long volatile web_requests; - volatile unsigned long long volatile web_usec; - volatile unsigned long long volatile bytes_received; - volatile unsigned long long volatile bytes_sent; - volatile unsigned long long volatile content_size; - volatile unsigned long long volatile compressed_content_size; + volatile uint16_t connected_clients; + + volatile uint64_t web_requests; + volatile uint64_t web_usec; + volatile uint64_t web_usec_max; + volatile uint64_t bytes_received; + volatile uint64_t bytes_sent; + volatile uint64_t content_size; + volatile uint64_t compressed_content_size; }; -extern struct global_statistics global_statistics; +extern volatile struct global_statistics global_statistics; extern void global_statistics_lock(void); extern void global_statistics_unlock(void); +extern void finished_web_request_statistics(uint64_t dt, + uint64_t bytes_received, + uint64_t bytes_sent, + uint64_t content_size, + uint64_t compressed_content_size); + +extern void web_client_connected(void); +extern void web_client_disconnected(void); + +#define GLOBAL_STATS_RESET_WEB_USEC_MAX 0x01 +extern void global_statistics_copy(struct global_statistics *gs, uint8_t options); +extern void global_statistics_charts(void); #endif /* NETDATA_GLOBAL_STATISTICS_H */ diff --git a/src/plugin_checks.c b/src/plugin_checks.c index 217cdc51..8de62b72 100644 --- a/src/plugin_checks.c +++ b/src/plugin_checks.c @@ -36,7 +36,7 @@ void *checks_main(void *ptr) // find the time to sleep in order to wait exactly update_every seconds gettimeofday(&now, NULL); - loop_usec = usecdiff(&now, &last); + loop_usec = usec_dt(&now, &last); usec = loop_usec - susec; debug(D_PROCNETDEV_LOOP, "CHECK: last loop took %llu usec (worked for %llu, sleeped for %llu).", loop_usec, usec, susec); @@ -72,9 +72,9 @@ void *checks_main(void *ptr) if(!apps_cpu) apps_cpu = rrdset_find("apps.cpu"); if(check3->counter_done) rrdset_next_usec(check3, loop_usec); gettimeofday(&loop, NULL); - rrddim_set(check3, "caller", (long long)usecdiff(&loop, &check1->last_collected_time)); - rrddim_set(check3, "netdata", (long long)usecdiff(&loop, &check2->last_collected_time)); - if(apps_cpu) rrddim_set(check3, "apps.plugin", (long long)usecdiff(&loop, &apps_cpu->last_collected_time)); + rrddim_set(check3, "caller", (long long) usec_dt(&loop, &check1->last_collected_time)); + rrddim_set(check3, "netdata", (long long) usec_dt(&loop, &check2->last_collected_time)); + if(apps_cpu) rrddim_set(check3, "apps.plugin", (long long) usec_dt(&loop, &apps_cpu->last_collected_time)); rrdset_done(check3); } diff --git a/src/plugin_idlejitter.c b/src/plugin_idlejitter.c index abdb03d1..d0a9181a 100644 --- a/src/plugin_idlejitter.c +++ b/src/plugin_idlejitter.c @@ -38,7 +38,7 @@ void *cpuidlejitter_main(void *ptr) gettimeofday(&after, NULL); // calculate the time it took for a full loop - usec = usecdiff(&after, &before); + usec = usec_dt(&after, &before); susec += usec; } usec -= (sleep_ms * 1000); diff --git a/src/plugin_proc.c b/src/plugin_proc.c index 1f69b284..ac45775a 100644 --- a/src/plugin_proc.c +++ b/src/plugin_proc.c @@ -4,11 +4,6 @@ void *proc_main(void *ptr) { (void)ptr; - unsigned long long old_web_requests = 0, old_web_usec = 0, - old_content_size = 0, old_compressed_content_size = 0; - - collected_number compression_ratio = -1, average_response_time = -1; - info("PROC Plugin thread created with task id %d", gettid()); if(pthread_setcanceltype(PTHREAD_CANCEL_DEFERRED, NULL) != 0) @@ -17,8 +12,6 @@ void *proc_main(void *ptr) if(pthread_setcancelstate(PTHREAD_CANCEL_ENABLE, NULL) != 0) error("Cannot set pthread cancel state to ENABLE."); - struct rusage me, thread; - // disable (by default) various interface that are not needed config_get_boolean("plugin:proc:/proc/net/dev:lo", "enabled", 0); config_get_boolean("plugin:proc:/proc/net/dev:fireqos_monitor", "enabled", 0); @@ -66,18 +59,15 @@ void *proc_main(void *ptr) unsigned long long sunext = (time(NULL) - (time(NULL) % rrd_update_every) + rrd_update_every) * 1000000ULL; unsigned long long sunow; - RRDSET *stcpu = NULL, *stcpu_thread = NULL, *stclients = NULL, *streqs = NULL, *stbytes = NULL, *stduration = NULL, - *stcompression = NULL; - for(;1;) { if(unlikely(netdata_exit)) break; // delay until it is our time to run - while((sunow = timems()) < sunext) - usecsleep(sunext - sunow); + while((sunow = time_usec()) < sunext) + sleep_usec(sunext - sunow); // find the next time we need to run - while(timems() > sunext) + while(time_usec() > sunext) sunext += rrd_update_every * 1000000ULL; if(unlikely(netdata_exit)) break; @@ -87,7 +77,7 @@ void *proc_main(void *ptr) if(!vdo_sys_kernel_mm_ksm) { debug(D_PROCNETDEV_LOOP, "PROCNETDEV: calling do_sys_kernel_mm_ksm()."); - sunow = timems(); + sunow = time_usec(); vdo_sys_kernel_mm_ksm = do_sys_kernel_mm_ksm(rrd_update_every, (sutime_sys_kernel_mm_ksm > 0)?sunow - sutime_sys_kernel_mm_ksm:0ULL); sutime_sys_kernel_mm_ksm = sunow; } @@ -95,7 +85,7 @@ void *proc_main(void *ptr) if(!vdo_proc_loadavg) { debug(D_PROCNETDEV_LOOP, "PROCNETDEV: calling do_proc_loadavg()."); - sunow = timems(); + sunow = time_usec(); vdo_proc_loadavg = do_proc_loadavg(rrd_update_every, (sutime_proc_loadavg > 0)?sunow - sutime_proc_loadavg:0ULL); sutime_proc_loadavg = sunow; } @@ -103,7 +93,7 @@ void *proc_main(void *ptr) if(!vdo_proc_interrupts) { debug(D_PROCNETDEV_LOOP, "PROCNETDEV: calling do_proc_interrupts()."); - sunow = timems(); + sunow = time_usec(); vdo_proc_interrupts = do_proc_interrupts(rrd_update_every, (sutime_proc_interrupts > 0)?sunow - sutime_proc_interrupts:0ULL); sutime_proc_interrupts = sunow; } @@ -111,7 +101,7 @@ void *proc_main(void *ptr) if(!vdo_proc_softirqs) { debug(D_PROCNETDEV_LOOP, "PROCNETDEV: calling do_proc_softirqs()."); - sunow = timems(); + sunow = time_usec(); vdo_proc_softirqs = do_proc_softirqs(rrd_update_every, (sutime_proc_softirqs > 0)?sunow - sutime_proc_softirqs:0ULL); sutime_proc_softirqs = sunow; } @@ -119,7 +109,7 @@ void *proc_main(void *ptr) if(!vdo_proc_sys_kernel_random_entropy_avail) { debug(D_PROCNETDEV_LOOP, "PROCNETDEV: calling do_proc_sys_kernel_random_entropy_avail()."); - sunow = timems(); + sunow = time_usec(); vdo_proc_sys_kernel_random_entropy_avail = do_proc_sys_kernel_random_entropy_avail(rrd_update_every, (sutime_proc_sys_kernel_random_entropy_avail > 0)?sunow - sutime_proc_sys_kernel_random_entropy_avail:0ULL); sutime_proc_sys_kernel_random_entropy_avail = sunow; } @@ -127,7 +117,7 @@ void *proc_main(void *ptr) if(!vdo_proc_net_dev) { debug(D_PROCNETDEV_LOOP, "PROCNETDEV: calling do_proc_net_dev()."); - sunow = timems(); + sunow = time_usec(); vdo_proc_net_dev = do_proc_net_dev(rrd_update_every, (sutime_proc_net_dev > 0)?sunow - sutime_proc_net_dev:0ULL); sutime_proc_net_dev = sunow; } @@ -135,7 +125,7 @@ void *proc_main(void *ptr) if(!vdo_proc_diskstats) { debug(D_PROCNETDEV_LOOP, "PROCNETDEV: calling do_proc_diskstats()."); - sunow = timems(); + sunow = time_usec(); vdo_proc_diskstats = do_proc_diskstats(rrd_update_every, (sutime_proc_diskstats > 0)?sunow - sutime_proc_diskstats:0ULL); sutime_proc_diskstats = sunow; } @@ -143,7 +133,7 @@ void *proc_main(void *ptr) if(!vdo_proc_net_snmp) { debug(D_PROCNETDEV_LOOP, "PROCNETDEV: calling do_proc_net_snmp()."); - sunow = timems(); + sunow = time_usec(); vdo_proc_net_snmp = do_proc_net_snmp(rrd_update_every, (sutime_proc_net_snmp > 0)?sunow - sutime_proc_net_snmp:0ULL); sutime_proc_net_snmp = sunow; } @@ -151,7 +141,7 @@ void *proc_main(void *ptr) if(!vdo_proc_net_snmp6) { debug(D_PROCNETDEV_LOOP, "PROCNETDEV: calling do_proc_net_snmp6()."); - sunow = timems(); + sunow = time_usec(); vdo_proc_net_snmp6 = do_proc_net_snmp6(rrd_update_every, (sutime_proc_net_snmp6 > 0)?sunow - sutime_proc_net_snmp6:0ULL); sutime_proc_net_snmp6 = sunow; } @@ -159,7 +149,7 @@ void *proc_main(void *ptr) if(!vdo_proc_net_netstat) { debug(D_PROCNETDEV_LOOP, "PROCNETDEV: calling do_proc_net_netstat()."); - sunow = timems(); + sunow = time_usec(); vdo_proc_net_netstat = do_proc_net_netstat(rrd_update_every, (sutime_proc_net_netstat > 0)?sunow - sutime_proc_net_netstat:0ULL); sutime_proc_net_netstat = sunow; } @@ -167,7 +157,7 @@ void *proc_main(void *ptr) if(!vdo_proc_net_stat_conntrack) { debug(D_PROCNETDEV_LOOP, "PROCNETDEV: calling do_proc_net_stat_conntrack()."); - sunow = timems(); + sunow = time_usec(); vdo_proc_net_stat_conntrack = do_proc_net_stat_conntrack(rrd_update_every, (sutime_proc_net_stat_conntrack > 0)?sunow - sutime_proc_net_stat_conntrack:0ULL); sutime_proc_net_stat_conntrack = sunow; } @@ -175,7 +165,7 @@ void *proc_main(void *ptr) if(!vdo_proc_net_ip_vs_stats) { debug(D_PROCNETDEV_LOOP, "PROCNETDEV: calling vdo_proc_net_ip_vs_stats()."); - sunow = timems(); + sunow = time_usec(); vdo_proc_net_ip_vs_stats = do_proc_net_ip_vs_stats(rrd_update_every, (sutime_proc_net_ip_vs_stats > 0)?sunow - sutime_proc_net_ip_vs_stats:0ULL); sutime_proc_net_ip_vs_stats = sunow; } @@ -183,7 +173,7 @@ void *proc_main(void *ptr) if(!vdo_proc_net_stat_synproxy) { debug(D_PROCNETDEV_LOOP, "PROCNETDEV: calling vdo_proc_net_stat_synproxy()."); - sunow = timems(); + sunow = time_usec(); vdo_proc_net_stat_synproxy = do_proc_net_stat_synproxy(rrd_update_every, (sutime_proc_net_stat_synproxy > 0)?sunow - sutime_proc_net_stat_synproxy:0ULL); sutime_proc_net_stat_synproxy = sunow; } @@ -191,7 +181,7 @@ void *proc_main(void *ptr) if(!vdo_proc_stat) { debug(D_PROCNETDEV_LOOP, "PROCNETDEV: calling do_proc_stat()."); - sunow = timems(); + sunow = time_usec(); vdo_proc_stat = do_proc_stat(rrd_update_every, (sutime_proc_stat > 0)?sunow - sutime_proc_stat:0ULL); sutime_proc_stat = sunow; } @@ -199,7 +189,7 @@ void *proc_main(void *ptr) if(!vdo_proc_meminfo) { debug(D_PROCNETDEV_LOOP, "PROCNETDEV: calling vdo_proc_meminfo()."); - sunow = timems(); + sunow = time_usec(); vdo_proc_meminfo = do_proc_meminfo(rrd_update_every, (sutime_proc_meminfo > 0)?sunow - sutime_proc_meminfo:0ULL); sutime_proc_meminfo = sunow; } @@ -207,7 +197,7 @@ void *proc_main(void *ptr) if(!vdo_proc_vmstat) { debug(D_PROCNETDEV_LOOP, "PROCNETDEV: calling vdo_proc_vmstat()."); - sunow = timems(); + sunow = time_usec(); vdo_proc_vmstat = do_proc_vmstat(rrd_update_every, (sutime_proc_vmstat > 0)?sunow - sutime_proc_vmstat:0ULL); sutime_proc_vmstat = sunow; } @@ -215,7 +205,7 @@ void *proc_main(void *ptr) if(!vdo_proc_net_rpc_nfsd) { debug(D_PROCNETDEV_LOOP, "PROCNETDEV: calling do_proc_net_rpc_nfsd()."); - sunow = timems(); + sunow = time_usec(); vdo_proc_net_rpc_nfsd = do_proc_net_rpc_nfsd(rrd_update_every, (sutime_proc_net_rpc_nfsd > 0)?sunow - sutime_proc_net_rpc_nfsd:0ULL); sutime_proc_net_rpc_nfsd = sunow; } @@ -226,136 +216,7 @@ void *proc_main(void *ptr) // -------------------------------------------------------------------- if(!vdo_cpu_netdata) { - getrusage(RUSAGE_THREAD, &thread); - getrusage(RUSAGE_SELF, &me); - - if(!stcpu_thread) stcpu_thread = rrdset_find("netdata.plugin_proc_cpu"); - if(!stcpu_thread) { - stcpu_thread = rrdset_create("netdata", "plugin_proc_cpu", NULL, "proc.internal", NULL, "NetData Proc Plugin CPU usage", "milliseconds/s", 132000, rrd_update_every, RRDSET_TYPE_STACKED); - - rrddim_add(stcpu_thread, "user", NULL, 1, 1000, RRDDIM_INCREMENTAL); - rrddim_add(stcpu_thread, "system", NULL, 1, 1000, RRDDIM_INCREMENTAL); - } - else rrdset_next(stcpu_thread); - - rrddim_set(stcpu_thread, "user" , thread.ru_utime.tv_sec * 1000000ULL + thread.ru_utime.tv_usec); - rrddim_set(stcpu_thread, "system", thread.ru_stime.tv_sec * 1000000ULL + thread.ru_stime.tv_usec); - rrdset_done(stcpu_thread); - - // ---------------------------------------------------------------- - - if(!stcpu) stcpu = rrdset_find("netdata.server_cpu"); - if(!stcpu) { - stcpu = rrdset_create("netdata", "server_cpu", NULL, "netdata", NULL, "NetData CPU usage", "milliseconds/s", 130000, rrd_update_every, RRDSET_TYPE_STACKED); - - rrddim_add(stcpu, "user", NULL, 1, 1000, RRDDIM_INCREMENTAL); - rrddim_add(stcpu, "system", NULL, 1, 1000, RRDDIM_INCREMENTAL); - } - else rrdset_next(stcpu); - - rrddim_set(stcpu, "user" , me.ru_utime.tv_sec * 1000000ULL + me.ru_utime.tv_usec); - rrddim_set(stcpu, "system", me.ru_stime.tv_sec * 1000000ULL + me.ru_stime.tv_usec); - rrdset_done(stcpu); - - // ---------------------------------------------------------------- - - if(!stclients) stclients = rrdset_find("netdata.clients"); - if(!stclients) { - stclients = rrdset_create("netdata", "clients", NULL, "netdata", NULL, "NetData Web Clients", "connected clients", 130100, rrd_update_every, RRDSET_TYPE_LINE); - - rrddim_add(stclients, "clients", NULL, 1, 1, RRDDIM_ABSOLUTE); - } - else rrdset_next(stclients); - - rrddim_set(stclients, "clients", global_statistics.connected_clients); - rrdset_done(stclients); - - // ---------------------------------------------------------------- - - if(!streqs) streqs = rrdset_find("netdata.requests"); - if(!streqs) { - streqs = rrdset_create("netdata", "requests", NULL, "netdata", NULL, "NetData Web Requests", "requests/s", 130200, rrd_update_every, RRDSET_TYPE_LINE); - - rrddim_add(streqs, "requests", NULL, 1, 1, RRDDIM_INCREMENTAL); - } - else rrdset_next(streqs); - - rrddim_set(streqs, "requests", global_statistics.web_requests); - rrdset_done(streqs); - - // ---------------------------------------------------------------- - - if(!stbytes) stbytes = rrdset_find("netdata.net"); - if(!stbytes) { - stbytes = rrdset_create("netdata", "net", NULL, "netdata", NULL, "NetData Network Traffic", "kilobits/s", 130300, rrd_update_every, RRDSET_TYPE_AREA); - - rrddim_add(stbytes, "in", NULL, 8, 1024, RRDDIM_INCREMENTAL); - rrddim_add(stbytes, "out", NULL, -8, 1024, RRDDIM_INCREMENTAL); - } - else rrdset_next(stbytes); - - rrddim_set(stbytes, "in", global_statistics.bytes_received); - rrddim_set(stbytes, "out", global_statistics.bytes_sent); - rrdset_done(stbytes); - - // ---------------------------------------------------------------- - - if(!stduration) stduration = rrdset_find("netdata.response_time"); - if(!stduration) { - stduration = rrdset_create("netdata", "response_time", NULL, "netdata", NULL, "NetData Average API Response Time", "ms/request", 130400, rrd_update_every, RRDSET_TYPE_LINE); - - rrddim_add(stduration, "response_time", "response time", 1, 1000, RRDDIM_ABSOLUTE); - } - else rrdset_next(stduration); - - unsigned long long gweb_usec = global_statistics.web_usec; - unsigned long long gweb_requests = global_statistics.web_requests; - - unsigned long long web_usec = gweb_usec - old_web_usec; - unsigned long long web_requests = gweb_requests - old_web_requests; - - old_web_usec = gweb_usec; - old_web_requests = gweb_requests; - - if(web_requests) - average_response_time = web_usec / web_requests; - - if(average_response_time != -1) - rrddim_set(stduration, "response_time", average_response_time); - - rrdset_done(stduration); - - // ---------------------------------------------------------------- - - if(!stcompression) stcompression = rrdset_find("netdata.compression_ratio"); - if(!stcompression) { - stcompression = rrdset_create("netdata", "compression_ratio", NULL, "netdata", NULL, "NetData API Responses Compression Savings Ratio", "percentage", 130500, rrd_update_every, RRDSET_TYPE_LINE); - - rrddim_add(stcompression, "savings", NULL, 1, 1000, RRDDIM_ABSOLUTE); - } - else rrdset_next(stcompression); - - // since we don't lock here to read the global statistics - // read the smaller value first - unsigned long long gcompressed_content_size = global_statistics.compressed_content_size; - unsigned long long gcontent_size = global_statistics.content_size; - - unsigned long long compressed_content_size = gcompressed_content_size - old_compressed_content_size; - unsigned long long content_size = gcontent_size - old_content_size; - - old_compressed_content_size = gcompressed_content_size; - old_content_size = gcontent_size; - - if(content_size && content_size >= compressed_content_size) - compression_ratio = ((content_size - compressed_content_size) * 100 * 1000) / content_size; - - if(compression_ratio != -1) - rrddim_set(stcompression, "savings", compression_ratio); - - rrdset_done(stcompression); - - // ---------------------------------------------------------------- - + global_statistics_charts(); registry_statistics(); } } diff --git a/src/rrd.c b/src/rrd.c index 92b14959..f3c6f260 100644 --- a/src/rrd.c +++ b/src/rrd.c @@ -600,7 +600,7 @@ RRDDIM *rrddim_add(RRDSET *st, const char *id, const char *name, long multiplier error("File %s does not have the same refresh frequency. Clearing it.", fullfilename); bzero(rd, size); } - else if(usecdiff(&now, &rd->last_collected_time) > (rd->entries * rd->update_every * 1000000ULL)) { + else if(usec_dt(&now, &rd->last_collected_time) > (rd->entries * rd->update_every * 1000000ULL)) { errno = 0; error("File %s is too old. Clearing it.", fullfilename); bzero(rd, size); @@ -922,7 +922,7 @@ void rrdset_next(RRDSET *st) if(likely(st->last_collected_time.tv_sec)) { struct timeval now; gettimeofday(&now, NULL); - microseconds = usecdiff(&now, &st->last_collected_time); + microseconds = usec_dt(&now, &st->last_collected_time); } // prevent infinite loop else microseconds = st->update_every * 1000000ULL; @@ -1001,7 +1001,7 @@ unsigned long long rrdset_done(RRDSET *st) } // check if we will re-write the entire data set - if(unlikely(usecdiff(&st->last_collected_time, &st->last_updated) > st->update_every * st->entries * 1000000ULL)) { + if(unlikely(usec_dt(&st->last_collected_time, &st->last_updated) > st->update_every * st->entries * 1000000ULL)) { info("%s: too old data (last updated at %ld.%ld, last collected at %ld.%ld). Reseting it. Will not store the next entry.", st->name, st->last_updated.tv_sec, st->last_updated.tv_usec, st->last_collected_time.tv_sec, st->last_collected_time.tv_usec); rrdset_reset(st); diff --git a/src/sys_fs_cgroup.c b/src/sys_fs_cgroup.c index 522aaf17..72d2dc26 100644 --- a/src/sys_fs_cgroup.c +++ b/src/sys_fs_cgroup.c @@ -1287,11 +1287,11 @@ void *cgroups_main(void *ptr) if(unlikely(netdata_exit)) break; // delay until it is our time to run - while((sunow = timems()) < sunext) - usecsleep(sunext - sunow); + while((sunow = time_usec()) < sunext) + sleep_usec(sunext - sunow); // find the next time we need to run - while(timems() > sunext) + while(time_usec() > sunext) sunext += rrd_update_every * 1000000ULL; if(unlikely(netdata_exit)) break; @@ -1300,7 +1300,7 @@ void *cgroups_main(void *ptr) if(!vdo_sys_fs_cgroup) { debug(D_PROCNETDEV_LOOP, "PROCNETDEV: calling do_sys_fs_cgroup()."); - sunow = timems(); + sunow = time_usec(); vdo_sys_fs_cgroup = do_sys_fs_cgroup(rrd_update_every, (sutime_sys_fs_cgroup > 0)?sunow - sutime_sys_fs_cgroup:0ULL); sutime_sys_fs_cgroup = sunow; } diff --git a/src/web_client.c b/src/web_client.c index 721310de..bc321a6b 100644 --- a/src/web_client.c +++ b/src/web_client.c @@ -113,7 +113,7 @@ struct web_client *web_client_create(int listener) w->next = web_clients; web_clients = w; - global_statistics.connected_clients++; + web_client_connected(); return(w); } @@ -136,18 +136,11 @@ void web_client_reset(struct web_client *w) { // -------------------------------------------------------------------- // global statistics - if(web_server_mode == WEB_SERVER_MODE_MULTI_THREADED) - global_statistics_lock(); - - global_statistics.web_requests++; - global_statistics.web_usec += usecdiff(&tv, &w->tv_in); - global_statistics.bytes_received += w->stats_received_bytes; - global_statistics.bytes_sent += w->stats_sent_bytes; - global_statistics.content_size += size; - global_statistics.compressed_content_size += sent; - - if(web_server_mode == WEB_SERVER_MODE_MULTI_THREADED) - global_statistics_unlock(); + finished_web_request_statistics(usec_dt(&tv, &w->tv_in), + w->stats_received_bytes, + w->stats_sent_bytes, + size, + sent); w->stats_received_bytes = 0; w->stats_sent_bytes = 0; @@ -159,9 +152,9 @@ void web_client_reset(struct web_client *w) { log_access("%llu: (sent/all = %zu/%zu bytes %0.0f%%, prep/sent/total = %0.2f/%0.2f/%0.2f ms) %s: %d '%s'", w->id, sent, size, -((size > 0) ? ((size - sent) / (double) size * 100.0) : 0.0), - usecdiff(&w->tv_ready, &w->tv_in) / 1000.0, - usecdiff(&tv, &w->tv_ready) / 1000.0, - usecdiff(&tv, &w->tv_in) / 1000.0, + usec_dt(&w->tv_ready, &w->tv_in) / 1000.0, + usec_dt(&tv, &w->tv_ready) / 1000.0, + usec_dt(&tv, &w->tv_in) / 1000.0, (w->mode == WEB_CLIENT_MODE_FILECOPY) ? "filecopy" : ((w->mode == WEB_CLIENT_MODE_OPTIONS) ? "options" : "data"), w->response.code, @@ -236,7 +229,7 @@ struct web_client *web_client_free(struct web_client *w) { if(w->ofd != -1 && w->ofd != w->ifd) close(w->ofd); freez(w); - global_statistics.connected_clients--; + web_client_disconnected(); return(n); } @@ -1668,7 +1661,10 @@ void web_client_process(struct web_client *w) { static uint32_t hash_api = 0, hash_netdata_conf = 0, hash_data = 0, hash_datasource = 0, hash_graph = 0, hash_list = 0, hash_all_json = 0, hash_exit = 0, hash_debug = 0, hash_mirror = 0; - if(unlikely(!hash_api)) { + // start timing us + gettimeofday(&w->tv_in, NULL); + + if(unlikely(!hash_api)) { hash_api = simple_hash("api"); hash_netdata_conf = simple_hash("netdata.conf"); hash_data = simple_hash(WEB_PATH_DATA); @@ -1703,7 +1699,7 @@ void web_client_process(struct web_client *w) { } } else if(what_to_do > 0) { - strcpy(w->last_url, "not a valid request"); + // strcpy(w->last_url, "not a valid request"); debug(D_WEB_CLIENT_ACCESS, "%llu: Cannot understand '%s'.", w->id, w->response.data->buffer); @@ -1712,8 +1708,6 @@ void web_client_process(struct web_client *w) { buffer_strcat(w->response.data, "I don't understand you...\r\n"); } else { // what_to_do == 0 - gettimeofday(&w->tv_in, NULL); - if(w->mode == WEB_CLIENT_MODE_OPTIONS) { code = 200; w->response.data->contenttype = CT_TEXT_PLAIN; diff --git a/web/demosites.html b/web/demosites.html index 78c47034..c6f5f2e7 100644 --- a/web/demosites.html +++ b/web/demosites.html @@ -945,8 +945,8 @@ body { data-width="100%" data-height="100%" data-after="-300" - data-colors="#558855" - data-show-value-of-response_time-at="netdata.response_time1" + data-colors="#558855 #356835" + data-show-value-of-average-at="netdata.response_time1" > @@ -964,8 +964,8 @@ body { data-width="100%" data-height="100%" data-after="-300" - data-colors="#885555" - data-show-value-of-response_time-at="netdata.response_time2" + data-colors="#885555 #683535" + data-show-value-of-average-at="netdata.response_time2" > @@ -983,8 +983,8 @@ body { data-width="100%" data-height="100%" data-after="-300" - data-colors="#555588" - data-show-value-of-response_time-at="netdata.response_time3" + data-colors="#555588 #353568" + data-show-value-of-average-at="netdata.response_time3" > @@ -1002,8 +1002,8 @@ body { data-width="100%" data-height="100%" data-after="-300" - data-colors="#885588" - data-show-value-of-response_time-at="netdata.response_time4" + data-colors="#885588 #683568" + data-show-value-of-average-at="netdata.response_time4" >