From ef0c0432a2425e086712b53f4b417fb89764aa29 Mon Sep 17 00:00:00 2001 From: Costa Tsaousis Date: Tue, 31 May 2016 22:57:13 +0300 Subject: [PATCH] added configuration options for controlling web compression level and strategy and chart to show the savings of the compression used --- src/global_statistics.c | 2 +- src/global_statistics.h | 12 ++++--- src/main.c | 31 +++++++++++++++++- src/plugin_proc.c | 43 ++++++++++++++++++++++--- src/web_client.c | 69 ++++++++++++++++++++++------------------- src/web_client.h | 7 +++-- 6 files changed, 119 insertions(+), 45 deletions(-) diff --git a/src/global_statistics.c b/src/global_statistics.c index d813f66c..40d3c3e3 100644 --- a/src/global_statistics.c +++ b/src/global_statistics.c @@ -5,7 +5,7 @@ #include "global_statistics.h" -struct global_statistics global_statistics = { 0, 0ULL, 0ULL, 0ULL, 0ULL}; +struct global_statistics global_statistics = { 0, 0ULL, 0ULL, 0ULL, 0ULL, 0ULL, 0ULL}; pthread_mutex_t global_statistics_mutex = PTHREAD_MUTEX_INITIALIZER; diff --git a/src/global_statistics.h b/src/global_statistics.h index b6181916..56a8118c 100644 --- a/src/global_statistics.h +++ b/src/global_statistics.h @@ -5,11 +5,13 @@ // global statistics struct global_statistics { - unsigned long volatile connected_clients; - unsigned long long volatile web_requests; - unsigned long long volatile web_usec; - unsigned long long volatile bytes_received; - unsigned long long volatile bytes_sent; + volatile unsigned long volatile connected_clients; + volatile unsigned long long volatile web_requests; + volatile unsigned long long volatile web_usec; + volatile unsigned long long volatile bytes_received; + volatile unsigned long long volatile bytes_sent; + volatile unsigned long long volatile content_size; + volatile unsigned long long volatile compressed_content_size; }; extern struct global_statistics global_statistics; diff --git a/src/main.c b/src/main.c index 945728b9..f11460b5 100644 --- a/src/main.c +++ b/src/main.c @@ -101,7 +101,36 @@ void web_server_threading_selection(void) { } web_client_timeout = (int) config_get_number("global", "disconnect idle web clients after seconds", DEFAULT_DISCONNECT_IDLE_WEB_CLIENTS_AFTER_SECONDS); + +#ifdef NETDATA_WITH_ZLIB web_enable_gzip = config_get_boolean("global", "enable web responses gzip compression", web_enable_gzip); + + char *s = config_get("global", "web compression strategy", "default"); + if(!strcmp(s, "default")) + web_gzip_strategy = Z_DEFAULT_STRATEGY; + else if(!strcmp(s, "filtered")) + web_gzip_strategy = Z_FILTERED; + else if(!strcmp(s, "huffman only")) + web_gzip_strategy = Z_HUFFMAN_ONLY; + else if(!strcmp(s, "rle")) + web_gzip_strategy = Z_RLE; + else if(!strcmp(s, "fixed")) + web_gzip_strategy = Z_FIXED; + else { + error("Invalid compression strategy '%s'. Valid strategies are 'default', 'filtered', 'huffman only', 'rle' and 'fixed'. Proceeding with 'default'."); + web_gzip_strategy = Z_DEFAULT_STRATEGY; + } + + web_gzip_level = (int)config_get_number("global", "web compression level", 3); + if(web_gzip_level < 1) { + error("Invalid compression level %d. Valid levels are 1 (fastest) to 9 (best ratio). Proceeding with level 1 (fastest compression)."); + web_gzip_level = 1; + } + else if(web_gzip_level > 9) { + error("Invalid compression level %d. Valid levels are 1 (fastest) to 9 (best ratio). Proceeding with level 9 (best compression)."); + web_gzip_level = 9; + } +#endif /* NETDATA_WITH_ZLIB */ } @@ -481,7 +510,7 @@ int main(int argc, char **argv) error_log_throttle_period = config_get_number("global", "errors flood protection period", error_log_throttle_period); setenv("NETDATA_ERRORS_THROTTLE_PERIOD", config_get("global", "errors flood protection period" , ""), 1); - error_log_errors_per_period = config_get_number("global", "errors to trigger flood protection", error_log_errors_per_period); + error_log_errors_per_period = (unsigned long)config_get_number("global", "errors to trigger flood protection", error_log_errors_per_period); setenv("NETDATA_ERRORS_PER_PERIOD" , config_get("global", "errors to trigger flood protection", ""), 1); // -------------------------------------------------------------------- diff --git a/src/plugin_proc.c b/src/plugin_proc.c index 5ecd7b9e..ffcc32c4 100644 --- a/src/plugin_proc.c +++ b/src/plugin_proc.c @@ -18,9 +18,13 @@ void *proc_main(void *ptr) { - static unsigned long long old_web_requests = 0, old_web_usec = 0; (void)ptr; + unsigned long long old_web_requests = 0, old_web_usec = 0, + old_content_size = 0, old_compressed_content_size = 0; + + collected_number compression_ratio = -1, average_response_time = -1; + info("PROC Plugin thread created with task id %d", gettid()); if(pthread_setcanceltype(PTHREAD_CANCEL_DEFERRED, NULL) != 0) @@ -78,7 +82,8 @@ void *proc_main(void *ptr) unsigned long long sunext = (time(NULL) - (time(NULL) % rrd_update_every) + rrd_update_every) * 1000000ULL; unsigned long long sunow; - RRDSET *stcpu = NULL, *stcpu_thread = NULL, *stclients = NULL, *streqs = NULL, *stbytes = NULL, *stduration = NULL; + RRDSET *stcpu = NULL, *stcpu_thread = NULL, *stclients = NULL, *streqs = NULL, *stbytes = NULL, *stduration = NULL, + *stcompression = NULL; for(;1;) { if(unlikely(netdata_exit)) break; @@ -328,13 +333,43 @@ void *proc_main(void *ptr) old_web_usec = gweb_usec; old_web_requests = gweb_requests; - if(!web_requests) web_requests = 1; + if(web_requests) + average_response_time = web_usec / web_requests; + + if(average_response_time != -1) + rrddim_set(stduration, "response_time", average_response_time); - rrddim_set(stduration, "response_time", web_usec / web_requests); rrdset_done(stduration); // ---------------------------------------------------------------- + if(!stcompression) stcompression = rrdset_find("netdata.compression_ratio"); + if(!stcompression) { + stcompression = rrdset_create("netdata", "compression_ratio", NULL, "netdata", NULL, "NetData API Responses Compression Savings Ratio", "percentage", 130500, rrd_update_every, RRDSET_TYPE_LINE); + + rrddim_add(stcompression, "savings", NULL, 1, 1000, RRDDIM_ABSOLUTE); + } + else rrdset_next(stcompression); + + unsigned long long gcontent_size = global_statistics.content_size; + unsigned long long gcompressed_content_size = global_statistics.compressed_content_size; + + unsigned long long content_size = gcontent_size - old_content_size; + unsigned long long compressed_content_size = gcompressed_content_size - old_compressed_content_size; + + old_content_size = gcontent_size; + old_compressed_content_size = gcompressed_content_size; + + if(content_size) + compression_ratio = (content_size - compressed_content_size) * 100 * 1000 / content_size; + + if(compression_ratio != -1) + rrddim_set(stcompression, "savings", compression_ratio); + + rrdset_done(stcompression); + + // ---------------------------------------------------------------- + registry_statistics(); } } diff --git a/src/web_client.c b/src/web_client.c index 42355551..3a3471eb 100644 --- a/src/web_client.c +++ b/src/web_client.c @@ -36,7 +36,10 @@ #define TOO_BIG_REQUEST 16384 int web_client_timeout = DEFAULT_DISCONNECT_IDLE_WEB_CLIENTS_AFTER_SECONDS; -int web_enable_gzip = 1; + +#ifdef NETDATA_WITH_ZLIB +int web_enable_gzip = 1, web_gzip_level = 3, web_gzip_strategy = Z_DEFAULT_STRATEGY; +#endif /* NETDATA_WITH_ZLIB */ extern int netdata_exit; @@ -178,16 +181,24 @@ struct web_client *web_client_create(int listener) return(w); } -void web_client_reset(struct web_client *w) -{ +void web_client_reset(struct web_client *w) { web_client_uncrock_socket(w); - struct timeval tv; - gettimeofday(&tv, NULL); - debug(D_WEB_CLIENT, "%llu: Reseting client.", w->id); - if(w->stats_received_bytes || w->stats_sent_bytes) { + if(likely(w->last_url[0])) { + struct timeval tv; + gettimeofday(&tv, NULL); + + size_t size = (w->mode == WEB_CLIENT_MODE_FILECOPY)?w->response.rlen:w->response.data->len; + size_t sent = size; +#ifdef NETDATA_WITH_ZLIB + if(likely(w->response.zoutput)) sent = (size_t)w->response.zstream.total_out; +#endif + + // -------------------------------------------------------------------- + // global statistics + if(web_server_mode == WEB_SERVER_MODE_MULTI_THREADED) global_statistics_lock(); @@ -195,32 +206,31 @@ void web_client_reset(struct web_client *w) global_statistics.web_usec += usecdiff(&tv, &w->tv_in); global_statistics.bytes_received += w->stats_received_bytes; global_statistics.bytes_sent += w->stats_sent_bytes; + global_statistics.content_size += size; + global_statistics.compressed_content_size += sent; if(web_server_mode == WEB_SERVER_MODE_MULTI_THREADED) global_statistics_unlock(); - } - w->stats_received_bytes = 0; - w->stats_sent_bytes = 0; - size_t sent = (w->mode == WEB_CLIENT_MODE_FILECOPY)?w->response.rlen:w->response.data->len; + w->stats_received_bytes = 0; + w->stats_sent_bytes = 0; -#ifdef NETDATA_WITH_ZLIB - if(likely(w->response.zoutput)) sent = (size_t)w->response.zstream.total_out; -#endif - size_t size = (w->mode == WEB_CLIENT_MODE_FILECOPY)?w->response.rlen:w->response.data->len; + // -------------------------------------------------------------------- + // access log - if(likely(w->last_url[0])) log_access("%llu: (sent/all = %zu/%zu bytes %0.0f%%, prep/sent/total = %0.2f/%0.2f/%0.2f ms) %s: %d '%s'", - w->id, - sent, size, -((size>0)?((float)(size-sent)/(float)size * 100.0):0.0), - (float)usecdiff(&w->tv_ready, &w->tv_in) / 1000.0, - (float)usecdiff(&tv, &w->tv_ready) / 1000.0, - (float)usecdiff(&tv, &w->tv_in) / 1000.0, - (w->mode == WEB_CLIENT_MODE_FILECOPY)?"filecopy":((w->mode == WEB_CLIENT_MODE_OPTIONS)?"options":"data"), - w->response.code, - w->last_url + w->id, + sent, size, -((size > 0) ? ((float) (size - sent) / (float) size * 100.0) : 0.0), + (float) usecdiff(&w->tv_ready, &w->tv_in) / 1000.0, + (float) usecdiff(&tv, &w->tv_ready) / 1000.0, + (float) usecdiff(&tv, &w->tv_in) / 1000.0, + (w->mode == WEB_CLIENT_MODE_FILECOPY) ? "filecopy" : ((w->mode == WEB_CLIENT_MODE_OPTIONS) + ? "options" : "data"), + w->response.code, + w->last_url ); + } if(unlikely(w->mode == WEB_CLIENT_MODE_FILECOPY)) { if(w->ifd != w->ofd) { @@ -256,7 +266,7 @@ void web_client_reset(struct web_client *w) // if we had enabled compression, release it #ifdef NETDATA_WITH_ZLIB if(w->response.zinitialized) { - debug(D_DEFLATE, "%llu: Reseting compression.", w->id); + debug(D_DEFLATE, "%llu: Freeing compression resources.", w->id); deflateEnd(&w->response.zstream); w->response.zsent = 0; w->response.zhave = 0; @@ -270,18 +280,13 @@ void web_client_reset(struct web_client *w) } struct web_client *web_client_free(struct web_client *w) { - web_client_uncrock_socket(w); + web_client_reset(w); struct web_client *n = w->next; if(w == web_clients) web_clients = n; debug(D_WEB_CLIENT_ACCESS, "%llu: Closing web client from %s port %s.", w->id, w->client_ip, w->client_port); -#ifdef NETDATA_WITH_ZLIB - if(w->response.zinitialized) - deflateEnd(&w->response.zstream); -#endif // NETDATA_WITH_ZLIB - if(w->prev) w->prev->next = w->next; if(w->next) w->next->prev = w->prev; if(w->response.header_output) buffer_free(w->response.header_output); @@ -505,7 +510,7 @@ void web_client_enable_deflate(struct web_client *w, int gzip) { // } // Select GZIP compression: windowbits = 15 + 16 = 31 - if(deflateInit2(&w->response.zstream, Z_DEFAULT_COMPRESSION, Z_DEFLATED, 15 + ((gzip)?16:0), 8, Z_DEFAULT_STRATEGY) != Z_OK) { + if(deflateInit2(&w->response.zstream, web_gzip_level, Z_DEFLATED, 15 + ((gzip)?16:0), 8, web_gzip_strategy) != Z_OK) { error("%llu: Failed to initialize zlib. Proceeding without compression.", w->id); return; } diff --git a/src/web_client.h b/src/web_client.h index c5a448e0..31849fab 100644 --- a/src/web_client.h +++ b/src/web_client.h @@ -16,7 +16,10 @@ #define DEFAULT_DISCONNECT_IDLE_WEB_CLIENTS_AFTER_SECONDS 60 extern int web_client_timeout; -extern int web_enable_gzip; + +#ifdef NETDATA_WITH_ZLIB +extern int web_enable_gzip, web_gzip_level, web_gzip_strategy; +#endif /* NETDATA_WITH_ZLIB */ #ifndef NETDATA_WEB_CLIENT_H #define NETDATA_WEB_CLIENT_H 1 @@ -48,7 +51,7 @@ struct response { size_t zsent; // the compressed bytes we have sent to the client size_t zhave; // the compressed bytes that we have received from zlib int zinitialized:1; -#endif +#endif /* NETDATA_WITH_ZLIB */ }; -- 2.39.2