From 299a229c67fc74730317a4cf9a066de14ccf102c Mon Sep 17 00:00:00 2001 From: "Costa Tsaousis (ktsaou)" Date: Fri, 24 Feb 2017 19:36:06 +0200 Subject: [PATCH 1/1] improved configuration and fixes for proxy mode --- src/appconfig.c | 75 +++++--- src/appconfig.h | 12 +- src/backends.c | 18 +- src/common.c | 1 + src/common.h | 1 + src/daemon.c | 8 +- src/health.c | 8 +- src/health_log.c | 2 +- src/main.c | 422 +++++++++++++++++++++++--------------------- src/plugins_d.c | 8 +- src/registry_init.c | 34 ++-- src/rrd.h | 1 - src/rrd2json.c | 2 +- src/rrdhost.c | 11 +- src/rrdpush.c | 137 ++++++++------ src/rrdpush.h | 2 +- src/rrdset.c | 18 +- src/web_client.c | 4 +- src/web_server.c | 40 +++-- src/web_server.h | 14 +- 20 files changed, 459 insertions(+), 359 deletions(-) diff --git a/src/appconfig.c b/src/appconfig.c index 3a71b099..4a0eb4de 100644 --- a/src/appconfig.c +++ b/src/appconfig.c @@ -195,38 +195,58 @@ int appconfig_exists(struct config *root, const char *section, const char *name) return 1; } -int appconfig_rename(struct config *root, const char *section, const char *old, const char *new) { - struct config_option *cv, *cv2; +int appconfig_move(struct config *root, const char *section_old, const char *name_old, const char *section_new, const char *name_new) { + struct config_option *cv_old, *cv_new; + int ret = -1; - debug(D_CONFIG, "request to rename config in section '%s', old name '%s', new name '%s'", section, old, new); + debug(D_CONFIG, "request to rename config in section '%s', old name '%s', to section '%s', new name '%s'", section_old, name_old, section_new, name_new); - struct section *co = appconfig_section_find(root, section); - if(!co) return -1; + struct section *co_old = appconfig_section_find(root, section_old); + if(!co_old) return -1; - config_section_wrlock(co); + struct section *co_new = appconfig_section_find(root, section_new); + if(!co_new) co_new = appconfig_section_create(root, section_new); - cv = appconfig_option_index_find(co, old, 0); - if(!cv) goto cleanup; + config_section_wrlock(co_old); + config_section_wrlock(co_new); - cv2 = appconfig_option_index_find(co, new, 0); - if(cv2) goto cleanup; + cv_old = appconfig_option_index_find(co_old, name_old, 0); + if(!cv_old) goto cleanup; - if(unlikely(appconfig_option_index_del(co, cv) != cv)) - error("INTERNAL ERROR: deletion of config '%s' from section '%s', deleted tge wrong config entry.", cv->name, co->name); + cv_new = appconfig_option_index_find(co_new, name_new, 0); + if(cv_new) goto cleanup; - freez(cv->name); - cv->name = strdupz(new); - cv->hash = simple_hash(cv->name); - if(unlikely(appconfig_option_index_add(co, cv) != cv)) - error("INTERNAL ERROR: indexing of config '%s' in section '%s', already exists.", cv->name, co->name); + if(unlikely(appconfig_option_index_del(co_old, cv_old) != cv_old)) + error("INTERNAL ERROR: deletion of config '%s' from section '%s', deleted tge wrong config entry.", cv_old->name, co_old->name); - config_section_unlock(co); + if(co_old->values == cv_old) { + co_old->values = cv_old->next; + } + else { + struct config_option *t; + for(t = co_old->values; t && t->next != cv_old ;t = t->next) ; + if(!t || t->next != cv_old) + error("INTERNAL ERROR: cannot find variable '%s' in section '%s' of the config - but it should be there.", cv_old->name, co_old->name); + else + t->next = cv_old->next; + } - return 0; + freez(cv_old->name); + cv_old->name = strdupz(name_new); + cv_old->hash = simple_hash(cv_old->name); + + cv_new->next = co_new->values; + co_new->values = cv_new; + + if(unlikely(appconfig_option_index_add(co_new, cv_old) != cv_old)) + error("INTERNAL ERROR: indexing of config '%s' in section '%s', already exists.", cv_old->name, co_new->name); + + ret = 0; cleanup: - config_section_unlock(co); - return -1; + config_section_unlock(co_new); + config_section_unlock(co_old); + return ret; } char *appconfig_get(struct config *root, const char *section, const char *name, const char *default_value) @@ -503,12 +523,13 @@ void appconfig_generate(struct config *root, BUFFER *wb, int only_changed) appconfig_wrlock(root); for(co = root->sections; co ; co = co->next) { - if(!strcmp(co->name, "global") - || !strcmp(co->name, "plugins") - || !strcmp(co->name, "registry") - || !strcmp(co->name, "health") - || !strcmp(co->name, "backend") - || !strcmp(co->name, "stream") + if(!strcmp(co->name, CONFIG_SECTION_GLOBAL) + || !strcmp(co->name, CONFIG_SECTION_API) + || !strcmp(co->name, CONFIG_SECTION_PLUGINS) + || !strcmp(co->name, CONFIG_SECTION_REGISTRY) + || !strcmp(co->name, CONFIG_SECTION_HEALTH) + || !strcmp(co->name, CONFIG_SECTION_BACKEND) + || !strcmp(co->name, CONFIG_SECTION_STREAM) ) pri = 0; else if(!strncmp(co->name, "plugin:", 7)) pri = 1; diff --git a/src/appconfig.h b/src/appconfig.h index 051736dd..cf41fb3f 100644 --- a/src/appconfig.h +++ b/src/appconfig.h @@ -3,6 +3,14 @@ #define CONFIG_FILENAME "netdata.conf" +#define CONFIG_SECTION_GLOBAL "global" +#define CONFIG_SECTION_API "api" +#define CONFIG_SECTION_PLUGINS "plugins" +#define CONFIG_SECTION_REGISTRY "registry" +#define CONFIG_SECTION_HEALTH "health" +#define CONFIG_SECTION_BACKEND "backend" +#define CONFIG_SECTION_STREAM "stream" + // these are used to limit the configuration names and values lengths // they are not enforced by config.c functions (they will strdup() all strings, no matter of their length) #define CONFIG_MAX_NAME 1024 @@ -35,7 +43,7 @@ extern long long appconfig_set_number(struct config *root, const char *section, extern int appconfig_set_boolean(struct config *root, const char *section, const char *name, int value); extern int appconfig_exists(struct config *root, const char *section, const char *name); -extern int appconfig_rename(struct config *root, const char *section, const char *old, const char *new); +extern int appconfig_move(struct config *root, const char *section_old, const char *name_old, const char *section_new, const char *name_new); extern void appconfig_generate(struct config *root, BUFFER *wb, int only_changed); @@ -54,7 +62,7 @@ extern void appconfig_generate(struct config *root, BUFFER *wb, int only_changed #define config_set_boolean(section, name, value) appconfig_set_boolean(&netdata_config, section, name, value) #define config_exists(section, name) appconfig_exists(&netdata_config, section, name) -#define config_rename(section, old, new) appconfig_rename(&netdata_config, section, old, new) +#define config_move(section_old, name_old, section_new, name_new) appconfig_move(&netdata_config, section_old, name_old, section_new, name_new) #define config_generate(buffer, only_changed) appconfig_generate(&netdata_config, buffer, only_changed) diff --git a/src/backends.c b/src/backends.c index c5b57dbe..94e8f5f6 100644 --- a/src/backends.c +++ b/src/backends.c @@ -152,15 +152,15 @@ void *backends_main(void *ptr) { .tv_usec = 0 }; uint32_t options; - int enabled = config_get_boolean("backend", "enabled", 0); - const char *source = config_get("backend", "data source", "average"); - const char *type = config_get("backend", "type", "graphite"); - const char *destination = config_get("backend", "destination", "localhost"); - const char *prefix = config_get("backend", "prefix", "netdata"); - const char *hostname = config_get("backend", "hostname", localhost->hostname); - int frequency = (int)config_get_number("backend", "update every", 10); - int buffer_on_failures = (int)config_get_number("backend", "buffer on failures", 10); - long timeoutms = config_get_number("backend", "timeout ms", frequency * 2 * 1000); + int enabled = config_get_boolean(CONFIG_SECTION_BACKEND, "enabled", 0); + const char *source = config_get(CONFIG_SECTION_BACKEND, "data source", "average"); + const char *type = config_get(CONFIG_SECTION_BACKEND, "type", "graphite"); + const char *destination = config_get(CONFIG_SECTION_BACKEND, "destination", "localhost"); + const char *prefix = config_get(CONFIG_SECTION_BACKEND, "prefix", "netdata"); + const char *hostname = config_get(CONFIG_SECTION_BACKEND, "hostname", localhost->hostname); + int frequency = (int)config_get_number(CONFIG_SECTION_BACKEND, "update every", 10); + int buffer_on_failures = (int)config_get_number(CONFIG_SECTION_BACKEND, "buffer on failures", 10); + long timeoutms = config_get_number(CONFIG_SECTION_BACKEND, "timeout ms", frequency * 2 * 1000); // ------------------------------------------------------------------------ // validate configuration options diff --git a/src/common.c b/src/common.c index 5243e407..e46397c6 100644 --- a/src/common.c +++ b/src/common.c @@ -8,6 +8,7 @@ # define MADV_DONTFORK INHERIT_NONE #endif /* __FreeBSD__ || __APPLE__*/ +char *netdata_configured_hostname = NULL; char *netdata_configured_config_dir = NULL; char *netdata_configured_log_dir = NULL; char *netdata_configured_plugins_dir = NULL; diff --git a/src/common.h b/src/common.h index e2c0a520..3d864f42 100644 --- a/src/common.h +++ b/src/common.h @@ -220,6 +220,7 @@ #include "web_api_v1.h" #include "web_api_old.h" +extern char *netdata_configured_hostname; extern char *netdata_configured_config_dir; extern char *netdata_configured_log_dir; extern char *netdata_configured_plugins_dir; diff --git a/src/daemon.c b/src/daemon.c index 835c014a..42b04c40 100644 --- a/src/daemon.c +++ b/src/daemon.c @@ -156,7 +156,7 @@ int become_user(const char *username, int pid_fd) } static void oom_score_adj(void) { - int score = (int)config_get_number("global", "OOM score", 1000); + int score = (int)config_get_number(CONFIG_SECTION_GLOBAL, "OOM score", 1000); int done = 0; int fd = open("/proc/self/oom_score_adj", O_WRONLY); @@ -175,7 +175,7 @@ static void oom_score_adj(void) { static void process_nice_level(void) { #ifdef HAVE_NICE - int nice_level = (int)config_get_number("global", "process nice level", 19); + int nice_level = (int)config_get_number(CONFIG_SECTION_GLOBAL, "process nice level", 19); if(nice(nice_level) == -1) error("Cannot set netdata CPU nice level to %d.", nice_level); else debug(D_SYSTEM, "Set netdata nice level to %d.", nice_level); #endif // HAVE_NICE @@ -239,7 +239,7 @@ static void sched_setscheduler_set(void) { int found = 0; // read the configuration - name = config_get("global", "process scheduling policy", name); + name = config_get(CONFIG_SECTION_GLOBAL, "process scheduling policy", name); int i; for(i = 0 ; scheduler_defaults[i].name ; i++) { if(!strcmp(name, scheduler_defaults[i].name)) { @@ -251,7 +251,7 @@ static void sched_setscheduler_set(void) { return; if(flags & SCHED_FLAG_PRIORITY_CONFIGURABLE) - priority = (int)config_get_number("global", "process scheduling priority", priority); + priority = (int)config_get_number(CONFIG_SECTION_GLOBAL, "process scheduling priority", priority); #ifdef HAVE_SCHED_GET_PRIORITY_MIN if(priority < sched_get_priority_min(policy)) { diff --git a/src/health.c b/src/health.c index 2ebbe065..0c7983ec 100644 --- a/src/health.c +++ b/src/health.c @@ -9,13 +9,13 @@ int default_health_enabled = 1; inline char *health_config_dir(void) { char buffer[FILENAME_MAX + 1]; snprintfz(buffer, FILENAME_MAX, "%s/health.d", netdata_configured_config_dir); - return config_get("health", "health configuration directory", buffer); + return config_get(CONFIG_SECTION_HEALTH, "health configuration directory", buffer); } void health_init(void) { debug(D_HEALTH, "Health configuration initializing"); - if(!(default_health_enabled = config_get_boolean("health", "enabled", 1))) { + if(!(default_health_enabled = config_get_boolean(CONFIG_SECTION_HEALTH, "enabled", 1))) { debug(D_HEALTH, "Health is disabled."); return; } @@ -324,7 +324,7 @@ void *health_main(void *ptr) { if(pthread_setcancelstate(PTHREAD_CANCEL_ENABLE, NULL) != 0) error("Cannot set pthread cancel state to ENABLE."); - int min_run_every = (int)config_get_number("health", "run at least every seconds", 10); + int min_run_every = (int)config_get_number(CONFIG_SECTION_HEALTH, "run at least every seconds", 10); if(min_run_every < 1) min_run_every = 1; BUFFER *wb = buffer_create(100); @@ -333,7 +333,7 @@ void *health_main(void *ptr) { time_t now_boottime = now_boottime_sec(); time_t last_now = now; time_t last_now_boottime = now_boottime; - time_t hibernation_delay = config_get_number("health", "postpone alarms during hibernation for seconds", 60); + time_t hibernation_delay = config_get_number(CONFIG_SECTION_HEALTH, "postpone alarms during hibernation for seconds", 60); unsigned int loop = 0; while(!netdata_exit) { diff --git a/src/health_log.c b/src/health_log.c index eb13169b..5097496e 100644 --- a/src/health_log.c +++ b/src/health_log.c @@ -32,7 +32,7 @@ inline void health_log_rotate(RRDHOST *host) { static size_t rotate_every = 0; if(unlikely(rotate_every == 0)) { - rotate_every = (size_t)config_get_number("health", "rotate log every lines", 2000); + rotate_every = (size_t)config_get_number(CONFIG_SECTION_HEALTH, "rotate log every lines", 2000); if(rotate_every < 100) rotate_every = 100; } diff --git a/src/main.c b/src/main.c index 77ffe1de..6f8766e0 100644 --- a/src/main.c +++ b/src/main.c @@ -35,41 +35,35 @@ struct netdata_static_thread static_threads[] = { #ifdef INTERNAL_PLUGIN_NFACCT // nfacct requires root access // so, we build it as an external plugin with setuid to root - {"nfacct", "plugins", "nfacct", 1, NULL, NULL, nfacct_main}, + {"nfacct", CONFIG_SECTION_PLUGINS, "nfacct", 1, NULL, NULL, nfacct_main}, #endif - {"tc", "plugins", "tc", 1, NULL, NULL, tc_main}, - {"idlejitter", "plugins", "idlejitter", 1, NULL, NULL, cpuidlejitter_main}, + {"tc", CONFIG_SECTION_PLUGINS, "tc", 1, NULL, NULL, tc_main}, + {"idlejitter", CONFIG_SECTION_PLUGINS, "idlejitter", 1, NULL, NULL, cpuidlejitter_main}, #if defined(__FreeBSD__) - {"freebsd", "plugins", "freebsd", 1, NULL, NULL, freebsd_main}, + {"freebsd", CONFIG_SECTION_PLUGINS, "freebsd", 1, NULL, NULL, freebsd_main}, #elif defined(__APPLE__) - {"macos", "plugins", "macos", 1, NULL, NULL, macos_main}, + {"macos", CONFIG_SECTION_PLUGINS, "macos", 1, NULL, NULL, macos_main}, #else - {"proc", "plugins", "proc", 1, NULL, NULL, proc_main}, - {"diskspace", "plugins", "diskspace", 1, NULL, NULL, proc_diskspace_main}, - {"cgroups", "plugins", "cgroups", 1, NULL, NULL, cgroups_main}, + {"proc", CONFIG_SECTION_PLUGINS, "proc", 1, NULL, NULL, proc_main}, + {"diskspace", CONFIG_SECTION_PLUGINS, "diskspace", 1, NULL, NULL, proc_diskspace_main}, + {"cgroups", CONFIG_SECTION_PLUGINS, "cgroups", 1, NULL, NULL, cgroups_main}, #endif /* __FreeBSD__, __APPLE__*/ - {"check", "plugins", "checks", 0, NULL, NULL, checks_main}, - {"backends", NULL, NULL, 1, NULL, NULL, backends_main}, - {"health", NULL, NULL, 1, NULL, NULL, health_main}, - {"plugins.d", NULL, NULL, 1, NULL, NULL, pluginsd_main}, - {"web", NULL, NULL, 1, NULL, NULL, socket_listen_main_multi_threaded}, - {"web-single-threaded", NULL, NULL, 0, NULL, NULL, socket_listen_main_single_threaded}, - {"push-metrics", NULL, NULL, 0, NULL, NULL, rrdpush_sender_thread}, - {NULL, NULL, NULL, 0, NULL, NULL, NULL} + {"check", CONFIG_SECTION_PLUGINS, "checks", 0, NULL, NULL, checks_main}, + {"backends", NULL, NULL, 1, NULL, NULL, backends_main}, + {"health", NULL, NULL, 1, NULL, NULL, health_main}, + {"plugins.d", NULL, NULL, 1, NULL, NULL, pluginsd_main}, + {"web", NULL, NULL, 1, NULL, NULL, socket_listen_main_multi_threaded}, + {"web-single-threaded", NULL, NULL, 0, NULL, NULL, socket_listen_main_single_threaded}, + {"push-metrics", NULL, NULL, 0, NULL, NULL, rrdpush_sender_thread}, + {NULL, NULL, NULL, 0, NULL, NULL, NULL} }; void web_server_threading_selection(void) { - int multi_threaded = 0; - int single_threaded = 0; + web_server_mode = web_server_mode_id(config_get(CONFIG_SECTION_API, "mode", web_server_mode_name(web_server_mode))); - if(default_rrdpush_exclusive) { - info("Web server is disabled - use the remote netdata."); - } - else { - multi_threaded = config_get_boolean("global", "multi threaded web server", 1); - single_threaded = !multi_threaded; - } + int multi_threaded = (web_server_mode == WEB_SERVER_MODE_MULTI_THREADED); + int single_threaded = (web_server_mode == WEB_SERVER_MODE_SINGLE_THREADED); int i; for(i = 0; static_threads[i].name ; i++) { @@ -80,19 +74,16 @@ void web_server_threading_selection(void) { static_threads[i].enabled = single_threaded; } - if(default_rrdpush_exclusive) - return; - - web_client_timeout = (int) config_get_number("global", "disconnect idle web clients after seconds", DEFAULT_DISCONNECT_IDLE_WEB_CLIENTS_AFTER_SECONDS); + web_client_timeout = (int) config_get_number(CONFIG_SECTION_API, "disconnect idle clients after seconds", DEFAULT_DISCONNECT_IDLE_WEB_CLIENTS_AFTER_SECONDS); - respect_web_browser_do_not_track_policy = config_get_boolean("global", "respect web browser do not track policy", respect_web_browser_do_not_track_policy); - web_x_frame_options = config_get("global", "web x-frame-options header", ""); + respect_web_browser_do_not_track_policy = config_get_boolean(CONFIG_SECTION_API, "respect do not track policy", respect_web_browser_do_not_track_policy); + web_x_frame_options = config_get("web", "x-frame-options header", ""); if(!*web_x_frame_options) web_x_frame_options = NULL; #ifdef NETDATA_WITH_ZLIB - web_enable_gzip = config_get_boolean("global", "enable web responses gzip compression", web_enable_gzip); + web_enable_gzip = config_get_boolean(CONFIG_SECTION_API, "gzip compression", web_enable_gzip); - char *s = config_get("global", "web compression strategy", "default"); + char *s = config_get(CONFIG_SECTION_API, "compression strategy", "default"); if(!strcmp(s, "default")) web_gzip_strategy = Z_DEFAULT_STRATEGY; else if(!strcmp(s, "filtered")) @@ -108,7 +99,7 @@ void web_server_threading_selection(void) { web_gzip_strategy = Z_DEFAULT_STRATEGY; } - web_gzip_level = (int)config_get_number("global", "web compression level", 3); + web_gzip_level = (int)config_get_number(CONFIG_SECTION_API, "compression level", 3); if(web_gzip_level < 1) { error("Invalid compression level %d. Valid levels are 1 (fastest) to 9 (best ratio). Proceeding with level 1 (fastest compression).", web_gzip_level); web_gzip_level = 1; @@ -348,18 +339,180 @@ static const char *verify_required_directory(const char *dir) { return dir; } -static void get_netdata_configured_directories() { - netdata_configured_config_dir = config_get("global", "config directory", CONFIG_DIR); - netdata_configured_log_dir = config_get("global", "log directory", LOG_DIR); - netdata_configured_plugins_dir = config_get("global", "plugins directory", PLUGINS_DIR); - netdata_configured_web_dir = config_get("global", "web files directory", WEB_DIR); - netdata_configured_cache_dir = config_get("global", "cache directory", CACHE_DIR); - netdata_configured_varlib_dir = config_get("global", "lib directory", VARLIB_DIR); - netdata_configured_home_dir = config_get("global", "home directory", CACHE_DIR); +void log_init(void) { + char filename[FILENAME_MAX + 1]; + snprintfz(filename, FILENAME_MAX, "%s/debug.log", netdata_configured_log_dir); + stdout_filename = config_get(CONFIG_SECTION_GLOBAL, "debug log", filename); + + snprintfz(filename, FILENAME_MAX, "%s/error.log", netdata_configured_log_dir); + stderr_filename = config_get(CONFIG_SECTION_GLOBAL, "error log", filename); + + snprintfz(filename, FILENAME_MAX, "%s/access.log", netdata_configured_log_dir); + stdaccess_filename = config_get(CONFIG_SECTION_GLOBAL, "access log", filename); + + error_log_throttle_period_backup = + error_log_throttle_period = config_get_number(CONFIG_SECTION_GLOBAL, "errors flood protection period", error_log_throttle_period); + error_log_errors_per_period = (unsigned long)config_get_number(CONFIG_SECTION_GLOBAL, "errors to trigger flood protection", (long long int)error_log_errors_per_period); + + setenv("NETDATA_ERRORS_THROTTLE_PERIOD", config_get(CONFIG_SECTION_GLOBAL, "errors flood protection period" , ""), 1); + setenv("NETDATA_ERRORS_PER_PERIOD", config_get(CONFIG_SECTION_GLOBAL, "errors to trigger flood protection", ""), 1); +} + +static void backwards_compatible_config() { + // allow existing configurations to work with the current version of netdata + + if(config_exists(CONFIG_SECTION_GLOBAL, "multi threaded web server")) { + int mode = config_get_boolean(CONFIG_SECTION_GLOBAL, "multi threaded web server", 1); + web_server_mode = (mode)?WEB_SERVER_MODE_MULTI_THREADED:WEB_SERVER_MODE_SINGLE_THREADED; + } + + if(config_exists(CONFIG_SECTION_GLOBAL, "bind socket to IP") && !config_exists(CONFIG_SECTION_API, "bind to")) + config_move(CONFIG_SECTION_GLOBAL, "bind socket to IP", CONFIG_SECTION_API, "bind to"); + + if(config_exists(CONFIG_SECTION_GLOBAL, "bind to") && !config_exists(CONFIG_SECTION_API, "bind to")) + config_move(CONFIG_SECTION_GLOBAL, "bind to", CONFIG_SECTION_API, "bind to"); + + if(config_exists(CONFIG_SECTION_GLOBAL, "port") && !config_exists(CONFIG_SECTION_API, "default port")) + config_move(CONFIG_SECTION_GLOBAL, "port", CONFIG_SECTION_API, "default port"); + + if(config_exists(CONFIG_SECTION_GLOBAL, "default port") && !config_exists(CONFIG_SECTION_API, "default port")) + config_move(CONFIG_SECTION_GLOBAL, "default port", CONFIG_SECTION_API, "default port"); + + if(config_exists(CONFIG_SECTION_GLOBAL, "disconnect idle clients after seconds") && !config_exists(CONFIG_SECTION_API, "disconnect idle clients after seconds")) + config_move(CONFIG_SECTION_GLOBAL, "disconnect idle clients after seconds", CONFIG_SECTION_API, "disconnect idle clients after seconds"); + + if(config_exists(CONFIG_SECTION_GLOBAL, "respect web browser do not track policy") && !config_exists(CONFIG_SECTION_API, "respect do not track policy")) + config_move(CONFIG_SECTION_GLOBAL, "respect web browser do not track policy", CONFIG_SECTION_API, "respect do not track policy"); + + if(config_exists(CONFIG_SECTION_GLOBAL, "enable web responses gzip compression") && !config_exists(CONFIG_SECTION_API, "gzip compression")) + config_move(CONFIG_SECTION_GLOBAL, "enable web responses gzip compression", CONFIG_SECTION_API, "gzip compression"); + + if(config_exists(CONFIG_SECTION_GLOBAL, "web compression strategy") && !config_exists(CONFIG_SECTION_API, "compression strategy")) + config_move(CONFIG_SECTION_GLOBAL, "web compression strategy", CONFIG_SECTION_API, "compression strategy"); + + if(config_exists(CONFIG_SECTION_GLOBAL, "web compression level") && !config_exists(CONFIG_SECTION_API, "compression level")) + config_move(CONFIG_SECTION_GLOBAL, "web compression level", CONFIG_SECTION_API, "compression level"); + + if(config_exists(CONFIG_SECTION_GLOBAL, "web files owner") && !config_exists(CONFIG_SECTION_API, "web files owner")) + config_move(CONFIG_SECTION_GLOBAL, "web files owner", CONFIG_SECTION_API, "web files owner"); + + if(config_exists(CONFIG_SECTION_GLOBAL, "web files group") && !config_exists(CONFIG_SECTION_API, "web files group")) + config_move(CONFIG_SECTION_GLOBAL, "web files group", CONFIG_SECTION_API, "web files group"); +} + +static void get_netdata_configured_variables() { + backwards_compatible_config(); + + // ------------------------------------------------------------------------ + // get the hostname + + char buf[HOSTNAME_MAX + 1]; + if(gethostname(buf, HOSTNAME_MAX) == -1) + error("WARNING: Cannot get machine hostname."); + + netdata_configured_hostname = config_get(CONFIG_SECTION_GLOBAL, "hostname", buf); + debug(D_OPTIONS, "hostname set to '%s'", netdata_configured_hostname); + + netdata_configured_hostname = config_get(CONFIG_SECTION_GLOBAL, "hostname", CONFIG_DIR); + + // ------------------------------------------------------------------------ + // get default database size + + default_rrd_history_entries = (int) config_get_number(CONFIG_SECTION_GLOBAL, "history", align_entries_to_pagesize(default_rrd_memory_mode, RRD_DEFAULT_HISTORY_ENTRIES)); + + long h = align_entries_to_pagesize(default_rrd_memory_mode, default_rrd_history_entries); + if(h != default_rrd_history_entries) { + config_set_number(CONFIG_SECTION_GLOBAL, "history", h); + default_rrd_history_entries = (int)h; + } + + if(default_rrd_history_entries < 5 || default_rrd_history_entries > RRD_HISTORY_ENTRIES_MAX) { + error("Invalid history entries %d given. Defaulting to %d.", default_rrd_history_entries, RRD_DEFAULT_HISTORY_ENTRIES); + default_rrd_history_entries = RRD_DEFAULT_HISTORY_ENTRIES; + } + + // ------------------------------------------------------------------------ + // get default database update frequency + + default_rrd_update_every = (int) config_get_number(CONFIG_SECTION_GLOBAL, "update every", UPDATE_EVERY); + if(default_rrd_update_every < 1 || default_rrd_update_every > 600) { + error("Invalid data collection frequency (update every) %d given. Defaulting to %d.", default_rrd_update_every, UPDATE_EVERY_MAX); + default_rrd_update_every = UPDATE_EVERY; + } + + // ------------------------------------------------------------------------ + // let the plugins know the min update_every + + // get system paths + netdata_configured_config_dir = config_get(CONFIG_SECTION_GLOBAL, "config directory", CONFIG_DIR); + netdata_configured_log_dir = config_get(CONFIG_SECTION_GLOBAL, "log directory", LOG_DIR); + netdata_configured_plugins_dir = config_get(CONFIG_SECTION_GLOBAL, "plugins directory", PLUGINS_DIR); + netdata_configured_web_dir = config_get(CONFIG_SECTION_GLOBAL, "web files directory", WEB_DIR); + netdata_configured_cache_dir = config_get(CONFIG_SECTION_GLOBAL, "cache directory", CACHE_DIR); + netdata_configured_varlib_dir = config_get(CONFIG_SECTION_GLOBAL, "lib directory", VARLIB_DIR); + netdata_configured_home_dir = config_get(CONFIG_SECTION_GLOBAL, "home directory", CACHE_DIR); + + // ------------------------------------------------------------------------ + // get default memory mode for the database + + default_rrd_memory_mode = rrd_memory_mode_id(config_get(CONFIG_SECTION_GLOBAL, "memory mode", rrd_memory_mode_name(default_rrd_memory_mode))); + + // ------------------------------------------------------------------------ + + netdata_configured_host_prefix = config_get(CONFIG_SECTION_GLOBAL, "host access prefix", ""); + + // -------------------------------------------------------------------- + // get KSM settings + +#ifdef MADV_MERGEABLE + enable_ksm = config_get_boolean(CONFIG_SECTION_GLOBAL, "memory deduplication (ksm)", enable_ksm); +#endif + + // -------------------------------------------------------------------- + // get various system parameters + + get_system_HZ(); + get_system_cpus(); + get_system_pid_max(); +} + +void set_global_environment() { + { + char b[16]; + snprintfz(b, 15, "%d", default_rrd_update_every); + setenv("NETDATA_UPDATE_EVERY", b, 1); + } + + setenv("NETDATA_HOSTNAME" , netdata_configured_hostname, 1); + setenv("NETDATA_CONFIG_DIR" , verify_required_directory(netdata_configured_config_dir), 1); + setenv("NETDATA_PLUGINS_DIR", verify_required_directory(netdata_configured_plugins_dir), 1); + setenv("NETDATA_WEB_DIR" , verify_required_directory(netdata_configured_web_dir), 1); + setenv("NETDATA_CACHE_DIR" , verify_required_directory(netdata_configured_cache_dir), 1); + setenv("NETDATA_LIB_DIR" , verify_required_directory(netdata_configured_varlib_dir), 1); + setenv("NETDATA_LOG_DIR" , verify_required_directory(netdata_configured_log_dir), 1); + setenv("HOME" , verify_required_directory(netdata_configured_home_dir), 1); + setenv("NETDATA_HOST_PREFIX", netdata_configured_host_prefix, 1); + + // avoid flood calls to stat(/etc/localtime) + // http://stackoverflow.com/questions/4554271/how-to-avoid-excessive-stat-etc-localtime-calls-in-strftime-on-linux + setenv("TZ", ":/etc/localtime", 0); + + // set the path we need + char path[1024 + 1], *p = getenv("PATH"); + if(!p) p = "/bin:/usr/bin"; + snprintfz(path, 1024, "%s:%s", p, "/sbin:/usr/sbin:/usr/local/bin:/usr/local/sbin"); + setenv("PATH", config_get(CONFIG_SECTION_PLUGINS, "PATH environment variable", path), 1); + + // python options + p = getenv("PYTHONPATH"); + if(!p) p = ""; + setenv("PYTHONPATH", config_get(CONFIG_SECTION_PLUGINS, "PYTHONPATH environment variable", p), 1); + + // disable buffering for python plugins + setenv("PYTHONUNBUFFERED", "1", 1); } int main(int argc, char **argv) { - char *hostname = "localhost"; int i, check_config = 0; int config_loaded = 0; int dont_fork = 0; @@ -385,12 +538,12 @@ int main(int argc, char **argv) { remove_option(i, &argc, argv); } else if(strcmp(argv[i], "-ch") == 0 && (i+1) < argc) { - config_set("global", "host access prefix", argv[i+1]); + config_set(CONFIG_SECTION_GLOBAL, "host access prefix", argv[i+1]); fprintf(stderr, "%s: deprecated option -- %s -- please use -s instead.\n", argv[0], argv[i]); remove_option(i, &argc, argv); } else if(strcmp(argv[i], "-l") == 0 && (i+1) < argc) { - config_set("global", "history", argv[i+1]); + config_set(CONFIG_SECTION_GLOBAL, "history", argv[i+1]); fprintf(stderr, "%s: deprecated option -- %s -- This option will be removed with V2.*.\n", argv[0], argv[i]); remove_option(i, &argc, argv); } @@ -436,7 +589,7 @@ int main(int argc, char **argv) { help(0); break; case 'i': - config_set("global", "bind to", optarg); + config_set(CONFIG_SECTION_API, "bind to", optarg); break; case 'k': dont_fork = 1; @@ -447,16 +600,16 @@ int main(int argc, char **argv) { pidfile[FILENAME_MAX] = '\0'; break; case 'p': - config_set("global", "default port", optarg); + config_set(CONFIG_SECTION_GLOBAL, "default port", optarg); break; case 's': - config_set("global", "host access prefix", optarg); + config_set(CONFIG_SECTION_GLOBAL, "host access prefix", optarg); break; case 't': - config_set("global", "update every", optarg); + config_set(CONFIG_SECTION_GLOBAL, "update every", optarg); break; case 'u': - config_set("global", "run as user", optarg); + config_set(CONFIG_SECTION_GLOBAL, "run as user", optarg); break; case 'v': printf("%s %s\n", program_name, program_version); @@ -468,7 +621,7 @@ int main(int argc, char **argv) { if(strcmp(optarg, "unittest") == 0) { default_rrd_update_every = 1; if(!config_loaded) config_load(NULL, 0); - get_netdata_configured_directories(); + get_netdata_configured_variables(); registry_init(); rrd_init("unittest"); if(run_all_mockup_tests()) exit(1); @@ -520,11 +673,11 @@ int main(int argc, char **argv) { } else if(strncmp(optarg, stacksize_string, strlen(stacksize_string)) == 0) { optarg += strlen(stacksize_string); - config_set("global", "pthread stack size", optarg); + config_set(CONFIG_SECTION_GLOBAL, "pthread stack size", optarg); } else if(strncmp(optarg, debug_flags_string, strlen(debug_flags_string)) == 0) { optarg += strlen(debug_flags_string); - config_set("global", "debug flags", optarg); + config_set(CONFIG_SECTION_GLOBAL, "debug flags", optarg); debug_flags = strtoull(optarg, NULL, 0); } } @@ -550,52 +703,26 @@ int main(int argc, char **argv) { config_load(NULL, 0); { - char *pmax = config_get("global", "glibc malloc arena max for plugins", "1"); + char *pmax = config_get(CONFIG_SECTION_GLOBAL, "glibc malloc arena max for plugins", "1"); if(pmax && *pmax) setenv("MALLOC_ARENA_MAX", pmax, 1); #if defined(HAVE_C_MALLOPT) - i = (int)config_get_number("global", "glibc malloc arena max for netdata", 1); + i = (int)config_get_number(CONFIG_SECTION_GLOBAL, "glibc malloc arena max for netdata", 1); if(i > 0) mallopt(M_ARENA_MAX, 1); #endif // prepare configuration environment variables for the plugins - get_netdata_configured_directories(); - - setenv("NETDATA_CONFIG_DIR" , verify_required_directory(netdata_configured_config_dir), 1); - setenv("NETDATA_PLUGINS_DIR", verify_required_directory(netdata_configured_plugins_dir), 1); - setenv("NETDATA_WEB_DIR" , verify_required_directory(netdata_configured_web_dir), 1); - setenv("NETDATA_CACHE_DIR" , verify_required_directory(netdata_configured_cache_dir), 1); - setenv("NETDATA_LIB_DIR" , verify_required_directory(netdata_configured_varlib_dir), 1); - setenv("NETDATA_LOG_DIR" , verify_required_directory(netdata_configured_log_dir), 1); - setenv("HOME" , verify_required_directory(netdata_configured_home_dir), 1); - - netdata_configured_host_prefix = config_get("global", "host access prefix", ""); - setenv("NETDATA_HOST_PREFIX", netdata_configured_host_prefix, 1); - - // disable buffering for python plugins - setenv("PYTHONUNBUFFERED", "1", 1); - - // avoid flood calls to stat(/etc/localtime) - // http://stackoverflow.com/questions/4554271/how-to-avoid-excessive-stat-etc-localtime-calls-in-strftime-on-linux - setenv("TZ", ":/etc/localtime", 0); + get_netdata_configured_variables(); + set_global_environment(); // work while we are cd into config_dir // to allow the plugins refer to their config // files using relative filenames if(chdir(netdata_configured_config_dir) == -1) fatal("Cannot cd to '%s'", netdata_configured_config_dir); - - char path[1024 + 1], *p = getenv("PATH"); - if(!p) p = "/bin:/usr/bin"; - snprintfz(path, 1024, "%s:%s", p, "/sbin:/usr/sbin:/usr/local/bin:/usr/local/sbin"); - setenv("PATH", config_get("plugins", "PATH environment variable", path), 1); - - p = getenv("PYTHONPATH"); - if(!p) p = ""; - setenv("PYTHONPATH", config_get("plugins", "PYTHONPATH environment variable", p), 1); } char *user = NULL; @@ -604,7 +731,7 @@ int main(int argc, char **argv) { // -------------------------------------------------------------------- // get the debugging flags from the configuration file - char *flags = config_get("global", "debug flags", "0x0000000000000000"); + char *flags = config_get(CONFIG_SECTION_GLOBAL, "debug flags", "0x0000000000000000"); setenv("NETDATA_DEBUG_FLAGS", flags, 1); debug_flags = strtoull(flags, NULL, 0); @@ -624,26 +751,7 @@ int main(int argc, char **argv) { // -------------------------------------------------------------------- // get log filenames and settings - { - char filename[FILENAME_MAX + 1]; - snprintfz(filename, FILENAME_MAX, "%s/debug.log", netdata_configured_log_dir); - stdout_filename = config_get("global", "debug log", filename); - - snprintfz(filename, FILENAME_MAX, "%s/error.log", netdata_configured_log_dir); - stderr_filename = config_get("global", "error log", filename); - - snprintfz(filename, FILENAME_MAX, "%s/access.log", netdata_configured_log_dir); - stdaccess_filename = config_get("global", "access log", filename); - } - - error_log_throttle_period_backup = - error_log_throttle_period = config_get_number("global", "errors flood protection period", error_log_throttle_period); - - setenv("NETDATA_ERRORS_THROTTLE_PERIOD", config_get("global", "errors flood protection period" , ""), 1); - - error_log_errors_per_period = (unsigned long)config_get_number("global", "errors to trigger flood protection", (long long int)error_log_errors_per_period); - setenv("NETDATA_ERRORS_PER_PERIOD", config_get("global", "errors to trigger flood protection", ""), 1); - + log_init(); if(check_config) { stdout_filename = stderr_filename = stdaccess_filename = "system"; error_log_throttle_period = 0; @@ -651,98 +759,6 @@ int main(int argc, char **argv) { } error_log_limit_unlimited(); - - // -------------------------------------------------------------------- - // get KSM settings - -#ifdef MADV_MERGEABLE - enable_ksm = config_get_boolean("global", "memory deduplication (ksm)", enable_ksm); -#else -#warning "Kernel memory deduplication (KSM) is not available" -#endif - - // -------------------------------------------------------------------- - // get various system parameters - - get_system_HZ(); - get_system_cpus(); - get_system_pid_max(); - - - // -------------------------------------------------------------------- - // find the system hostname - - { - char hostnamebuf[HOSTNAME_MAX + 1]; - if(gethostname(hostnamebuf, HOSTNAME_MAX) == -1) - error("WARNING: Cannot get machine hostname."); - - hostname = config_get("global", "hostname", hostnamebuf); - debug(D_OPTIONS, "hostname set to '%s'", hostname); - - setenv("NETDATA_HOSTNAME", hostname, 1); - } - - - // -------------------------------------------------------------------- - // find we need to send data to another netdata - - rrdpush_init(); - - - // -------------------------------------------------------------------- - // get default memory mode for the database - - if(default_rrdpush_exclusive) { - default_rrd_memory_mode = RRD_MEMORY_MODE_NONE; - config_set("global", "memory mode", rrd_memory_mode_name(default_rrd_memory_mode)); - } - else - default_rrd_memory_mode = rrd_memory_mode_id(config_get("global", "memory mode", rrd_memory_mode_name(default_rrd_memory_mode))); - - - // -------------------------------------------------------------------- - // get default database size - - if(default_rrdpush_exclusive) { - default_rrd_history_entries = 10; - config_set_number("global", "history", default_rrd_history_entries); - } - else - default_rrd_history_entries = (int) config_get_number("global", "history", align_entries_to_pagesize(default_rrd_memory_mode, RRD_DEFAULT_HISTORY_ENTRIES)); - - long h = align_entries_to_pagesize(default_rrd_memory_mode, default_rrd_history_entries); - if(h != default_rrd_history_entries) { - config_set_number("global", "history", h); - default_rrd_history_entries = (int)h; - } - - if(default_rrd_history_entries < 5 || default_rrd_history_entries > RRD_HISTORY_ENTRIES_MAX) { - error("Invalid history entries %d given. Defaulting to %d.", default_rrd_history_entries, RRD_DEFAULT_HISTORY_ENTRIES); - default_rrd_history_entries = RRD_DEFAULT_HISTORY_ENTRIES; - } - else - debug(D_OPTIONS, "save lines set to %d.", default_rrd_history_entries); - - - // -------------------------------------------------------------------- - // get default database update frequency - - default_rrd_update_every = (int) config_get_number("global", "update every", UPDATE_EVERY); - if(default_rrd_update_every < 1 || default_rrd_update_every > 600) { - error("Invalid data collection frequency (update every) %d given. Defaulting to %d.", default_rrd_update_every, UPDATE_EVERY_MAX); - default_rrd_update_every = UPDATE_EVERY; - } - else debug(D_OPTIONS, "update timer set to %d.", default_rrd_update_every); - - // let the plugins know the min update_every - { - char buf[16]; - snprintfz(buf, 15, "%d", default_rrd_update_every); - setenv("NETDATA_UPDATE_EVERY", buf, 1); - } - - // -------------------------------------------------------------------- // setup process signals @@ -804,7 +820,7 @@ int main(int argc, char **argv) { else debug(D_OPTIONS, "initial pthread stack size is %zu bytes", stacksize); - wanted_stacksize = (size_t)config_get_number("global", "pthread stack size", (long)stacksize); + wanted_stacksize = (size_t)config_get_number(CONFIG_SECTION_GLOBAL, "pthread stack size", (long)stacksize); // -------------------------------------------------------------------- @@ -825,7 +841,7 @@ int main(int argc, char **argv) { // get the user we should run // IMPORTANT: this is required before web_files_uid() - user = config_get("global", "run as user" , (getuid() == 0)?NETDATA_USER:""); + user = config_get(CONFIG_SECTION_GLOBAL, "run as user" , (getuid() == 0)?NETDATA_USER:""); // IMPORTANT: these have to run once, while single threaded web_files_uid(); // IMPORTANT: web_files_uid() before web_files_gid() @@ -835,7 +851,7 @@ int main(int argc, char **argv) { // -------------------------------------------------------------------- // create the listening sockets - if(!check_config && !default_rrdpush_exclusive) + if(!check_config && web_server_mode != WEB_SERVER_MODE_NONE) create_listen_sockets(); @@ -882,6 +898,12 @@ int main(int argc, char **argv) { } + // -------------------------------------------------------------------- + // find we need to send data to another netdata + + rrdpush_init(); + + // ------------------------------------------------------------------------ // initialize health monitoring @@ -897,7 +919,7 @@ int main(int argc, char **argv) { // ------------------------------------------------------------------------ // initialize rrd host - rrd_init(hostname); + rrd_init(netdata_configured_hostname); if(check_config) diff --git a/src/plugins_d.c b/src/plugins_d.c index 81f17a92..7fa19eaf 100644 --- a/src/plugins_d.c +++ b/src/plugins_d.c @@ -455,14 +455,14 @@ void *pluginsd_main(void *ptr) { if(pthread_setcancelstate(PTHREAD_CANCEL_ENABLE, NULL) != 0) error("Cannot set pthread cancel state to ENABLE."); - int automatic_run = config_get_boolean("plugins", "enable running new plugins", 1); - int scan_frequency = (int) config_get_number("plugins", "check for new plugins every", 60); + int automatic_run = config_get_boolean(CONFIG_SECTION_PLUGINS, "enable running new plugins", 1); + int scan_frequency = (int) config_get_number(CONFIG_SECTION_PLUGINS, "check for new plugins every", 60); DIR *dir = NULL; struct dirent *file = NULL; struct plugind *cd; // enable the apps plugin by default - // config_get_boolean("plugins", "apps", 1); + // config_get_boolean(CONFIG_SECTION_PLUGINS, "apps", 1); if(scan_frequency < 1) scan_frequency = 1; @@ -491,7 +491,7 @@ void *pluginsd_main(void *ptr) { char pluginname[CONFIG_MAX_NAME + 1]; snprintfz(pluginname, CONFIG_MAX_NAME, "%.*s", (int)(len - PLUGINSD_FILE_SUFFIX_LEN), file->d_name); - int enabled = config_get_boolean("plugins", pluginname, automatic_run); + int enabled = config_get_boolean(CONFIG_SECTION_PLUGINS, pluginname, automatic_run); if(unlikely(!enabled)) { debug(D_PLUGINSD, "PLUGINSD: plugin '%s' is not enabled", file->d_name); diff --git a/src/registry_init.c b/src/registry_init.c index 2d8eb570..09167260 100644 --- a/src/registry_init.c +++ b/src/registry_init.c @@ -4,52 +4,52 @@ int registry_init(void) { char filename[FILENAME_MAX + 1]; // registry enabled? - if(!default_rrdpush_exclusive) { - registry.enabled = config_get_boolean("registry", "enabled", 0); + if(web_server_mode != WEB_SERVER_MODE_NONE) { + registry.enabled = config_get_boolean(CONFIG_SECTION_REGISTRY, "enabled", 0); } else { info("Registry is disabled - use the central netdata"); - config_set_boolean("registry", "enabled", 0); + config_set_boolean(CONFIG_SECTION_REGISTRY, "enabled", 0); registry.enabled = 0; } // pathnames snprintfz(filename, FILENAME_MAX, "%s/registry", netdata_configured_varlib_dir); - registry.pathname = config_get("registry", "registry db directory", filename); + registry.pathname = config_get(CONFIG_SECTION_REGISTRY, "registry db directory", filename); if(mkdir(registry.pathname, 0770) == -1 && errno != EEXIST) fatal("Cannot create directory '%s'.", registry.pathname); // filenames snprintfz(filename, FILENAME_MAX, "%s/netdata.public.unique.id", registry.pathname); - registry.machine_guid_filename = config_get("registry", "netdata unique id file", filename); + registry.machine_guid_filename = config_get(CONFIG_SECTION_REGISTRY, "netdata unique id file", filename); snprintfz(filename, FILENAME_MAX, "%s/registry.db", registry.pathname); - registry.db_filename = config_get("registry", "registry db file", filename); + registry.db_filename = config_get(CONFIG_SECTION_REGISTRY, "registry db file", filename); snprintfz(filename, FILENAME_MAX, "%s/registry-log.db", registry.pathname); - registry.log_filename = config_get("registry", "registry log file", filename); + registry.log_filename = config_get(CONFIG_SECTION_REGISTRY, "registry log file", filename); // configuration options - registry.save_registry_every_entries = (unsigned long long)config_get_number("registry", "registry save db every new entries", 1000000); - registry.persons_expiration = config_get_number("registry", "registry expire idle persons days", 365) * 86400; - registry.registry_domain = config_get("registry", "registry domain", ""); - registry.registry_to_announce = config_get("registry", "registry to announce", "https://registry.my-netdata.io"); - registry.hostname = config_get("registry", "registry hostname", config_get("global", "hostname", "localhost")); - registry.verify_cookies_redirects = config_get_boolean("registry", "verify browser cookies support", 1); + registry.save_registry_every_entries = (unsigned long long)config_get_number(CONFIG_SECTION_REGISTRY, "registry save db every new entries", 1000000); + registry.persons_expiration = config_get_number(CONFIG_SECTION_REGISTRY, "registry expire idle persons days", 365) * 86400; + registry.registry_domain = config_get(CONFIG_SECTION_REGISTRY, "registry domain", ""); + registry.registry_to_announce = config_get(CONFIG_SECTION_REGISTRY, "registry to announce", "https://registry.my-netdata.io"); + registry.hostname = config_get(CONFIG_SECTION_REGISTRY, "registry hostname", config_get(CONFIG_SECTION_GLOBAL, "hostname", "localhost")); + registry.verify_cookies_redirects = config_get_boolean(CONFIG_SECTION_REGISTRY, "verify browser cookies support", 1); setenv("NETDATA_REGISTRY_HOSTNAME", registry.hostname, 1); setenv("NETDATA_REGISTRY_URL", registry.registry_to_announce, 1); - registry.max_url_length = (size_t)config_get_number("registry", "max URL length", 1024); + registry.max_url_length = (size_t)config_get_number(CONFIG_SECTION_REGISTRY, "max URL length", 1024); if(registry.max_url_length < 10) { registry.max_url_length = 10; - config_set_number("registry", "max URL length", (long long)registry.max_url_length); + config_set_number(CONFIG_SECTION_REGISTRY, "max URL length", (long long)registry.max_url_length); } - registry.max_name_length = (size_t)config_get_number("registry", "max URL name length", 50); + registry.max_name_length = (size_t)config_get_number(CONFIG_SECTION_REGISTRY, "max URL name length", 50); if(registry.max_name_length < 10) { registry.max_name_length = 10; - config_set_number("registry", "max URL name length", (long long)registry.max_name_length); + config_set_number(CONFIG_SECTION_REGISTRY, "max URL name length", (long long)registry.max_name_length); } // initialize entries counters diff --git a/src/rrd.h b/src/rrd.h index cf141cc5..2cb06607 100644 --- a/src/rrd.h +++ b/src/rrd.h @@ -342,7 +342,6 @@ struct rrdhost { int rrd_history_entries; // the number of history entries for the host's charts int rrdpush_enabled; // 1 when this host sends metrics to another netdata - int rrdpush_exclusive; // 1 when this host is exclusively sending metrics without a database volatile int rrdpush_connected; // 1 when the sender is ready to push metrics volatile int rrdpush_spawn; // 1 when the sender thread has been spawn volatile int rrdpush_error_shown; // 1 when we have logged a communication error diff --git a/src/rrd2json.c b/src/rrd2json.c index d066bcb7..99528a7d 100644 --- a/src/rrd2json.c +++ b/src/rrd2json.c @@ -154,7 +154,7 @@ void rrd_stats_api_v1_charts_allmetrics_prometheus(RRDHOST *host, BUFFER *wb) { rrdhost_rdlock(host); char hostname[PROMETHEUS_ELEMENT_MAX + 1]; - prometheus_name_copy(hostname, config_get("global", "hostname", "localhost"), PROMETHEUS_ELEMENT_MAX); + prometheus_name_copy(hostname, host->hostname, PROMETHEUS_ELEMENT_MAX); // for each chart RRDSET *st; diff --git a/src/rrdhost.c b/src/rrdhost.c index fffa6758..5fc7b6b9 100644 --- a/src/rrdhost.c +++ b/src/rrdhost.c @@ -75,7 +75,6 @@ RRDHOST *rrdhost_create(const char *hostname, host->rrd_memory_mode = memory_mode; host->health_enabled = (memory_mode == RRD_MEMORY_MODE_NONE)? 0 : health_enabled; host->rrdpush_enabled = default_rrdpush_enabled; - host->rrdpush_exclusive = default_rrdpush_exclusive; host->rrdpush_pipe[0] = -1; host->rrdpush_pipe[1] = -1; @@ -102,10 +101,10 @@ RRDHOST *rrdhost_create(const char *hostname, host->health_log.next_log_id = host->health_log.next_alarm_id = (uint32_t)now_realtime_sec(); - long n = config_get_number("health", "in memory max health log entries", host->health_log.max); + long n = config_get_number(CONFIG_SECTION_HEALTH, "in memory max health log entries", host->health_log.max); if(n < 10) { error("Host '%s': health configuration has invalid max log entries %ld. Using default %u", host->hostname, n, host->health_log.max); - config_set_number("health", "in memory max health log entries", (long)host->health_log.max); + config_set_number(CONFIG_SECTION_HEALTH, "in memory max health log entries", (long)host->health_log.max); } else host->health_log.max = (unsigned int)n; @@ -150,10 +149,10 @@ RRDHOST *rrdhost_create(const char *hostname, } snprintfz(filename, FILENAME_MAX, "%s/health/health-log.db", host->varlib_dir); - host->health_log_filename = strdupz(config_get("health", "health db file", filename)); + host->health_log_filename = strdupz(config_get(CONFIG_SECTION_HEALTH, "health db file", filename)); snprintfz(filename, FILENAME_MAX, "%s/alarm-notify.sh", netdata_configured_plugins_dir); - host->health_default_exec = strdupz(config_get("health", "script to execute on alarm", filename)); + host->health_default_exec = strdupz(config_get(CONFIG_SECTION_HEALTH, "script to execute on alarm", filename)); host->health_default_recipient = strdup("root"); @@ -320,7 +319,7 @@ void rrdhost_free(RRDHOST *host) { if(host->rrdpush_spawn) { pthread_cancel(host->rrdpush_thread); - rrdpush_sender_cleanup(host); + rrdpush_sender_thread_cleanup(host); } freez(host->os); diff --git a/src/rrdpush.c b/src/rrdpush.c index 9bcd232a..82640a8f 100644 --- a/src/rrdpush.c +++ b/src/rrdpush.c @@ -1,11 +1,45 @@ #include "common.h" -int default_rrdpush_enabled = 0; -int default_rrdpush_exclusive = 1; - +/* + * rrdpush + * + * 3 threads are involved for all stream operations + * + * 1. a random data collection thread, calling rrdset_done_push() + * this is called for each chart. + * + * the output of this work is kept in a BUFFER in RRDHOST + * the sender thread is signalled via a pipe (also in RRDHOST) + * + * 2. a sender thread running at the sending netdata + * this is spawned automatically on the first chart to be pushed + * + * It tries to push the metrics to the remote netdata, as fast + * as possible (i.e. immediately after they are collected). + * + * 3. a receiver thread, running at the receiving netdata + * this is spawned automatically when the sender connects to + * the receiver. + * + */ + +int default_rrdpush_enabled = 0; static char *remote_netdata_config = NULL; static char *api_key = NULL; +int rrdpush_init() { + default_rrdpush_enabled = config_get_boolean(CONFIG_SECTION_STREAM, "enabled", default_rrdpush_enabled); + remote_netdata_config = config_get(CONFIG_SECTION_STREAM, "stream metrics to", ""); + api_key = config_get(CONFIG_SECTION_STREAM, "api key", ""); + + if(!default_rrdpush_enabled || !remote_netdata_config || !*remote_netdata_config || !api_key || !*api_key) { + error("STREAM [send]: cannot enable sending thread - information is missing."); + default_rrdpush_enabled = 0; + } + + return default_rrdpush_enabled; +} + #define CONNECTED_TO_SIZE 100 // data collection happens from multiple threads @@ -80,30 +114,6 @@ static inline void send_chart_metrics(RRDSET *st) { buffer_strcat(st->rrdhost->rrdpush_buffer, "END\n"); } -// resets all the chart, so that their definitions -// will be resent to the central netdata -static void reset_all_charts(RRDHOST *host) { - rrdhost_rdlock(host); - - RRDSET *st; - rrdset_foreach_read(st, host) { - - // make it re-align the current time - // on the remote host - st->counter_done = 0; - - rrdset_rdlock(st); - - RRDDIM *rd; - rrddim_foreach_read(rd, st) - rrddim_flag_clear(rd, RRDDIM_FLAG_EXPOSED); - - rrdset_unlock(st); - } - - rrdhost_unlock(host); -} - void rrdpush_sender_thread_spawn(RRDHOST *host); void rrdset_done_push(RRDSET *st) { @@ -131,12 +141,10 @@ void rrdset_done_push(RRDSET *st) { host->rrdpush_error_shown = 0; } - rrdset_rdlock(st); if(need_to_send_chart_definition(st)) send_chart_definition(st); send_chart_metrics(st); - rrdset_unlock(st); // signal the sender there are more data if(write(host->rrdpush_pipe[PIPE_WRITE], " ", 1) == -1) @@ -145,45 +153,58 @@ void rrdset_done_push(RRDSET *st) { rrdpush_unlock(host); } -static inline void rrdpush_flush(RRDHOST *host) { +// ---------------------------------------------------------------------------- +// rrdpush sender thread + +// resets all the chart, so that their definitions +// will be resent to the central netdata +static void rrdpush_sender_thread_reset_all_charts(RRDHOST *host) { + rrdhost_rdlock(host); + + RRDSET *st; + rrdset_foreach_read(st, host) { + + // make it re-align the current time + // on the remote host + st->counter_done = 0; + + rrdset_rdlock(st); + + RRDDIM *rd; + rrddim_foreach_read(rd, st) + rrddim_flag_clear(rd, RRDDIM_FLAG_EXPOSED); + + rrdset_unlock(st); + } + + rrdhost_unlock(host); +} + +static inline void rrdpush_sender_thread_data_flush(RRDHOST *host) { rrdpush_lock(host); if(buffer_strlen(host->rrdpush_buffer)) error("STREAM [send]: discarding %zu bytes of metrics already in the buffer.", buffer_strlen(host->rrdpush_buffer)); buffer_flush(host->rrdpush_buffer); - reset_all_charts(host); + rrdpush_sender_thread_reset_all_charts(host); rrdpush_unlock(host); } -int rrdpush_init() { - default_rrdpush_enabled = config_get_boolean("stream", "enabled", default_rrdpush_enabled); - default_rrdpush_exclusive = config_get_boolean("stream", "exclusive", default_rrdpush_exclusive); - remote_netdata_config = config_get("stream", "stream metrics to", ""); - api_key = config_get("stream", "api key", ""); - - if(!default_rrdpush_enabled || !remote_netdata_config || !*remote_netdata_config || !api_key || !*api_key) { - default_rrdpush_enabled = 0; - default_rrdpush_exclusive = 0; - } - - return default_rrdpush_enabled; -} - -static inline void rrdpush_sender_lock(RRDHOST *host) { +static inline void rrdpush_sender_thread_lock(RRDHOST *host) { if(pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, NULL) != 0) error("STREAM [send]: cannot set pthread cancel state to DISABLE."); rrdpush_lock(host); } -static inline void rrdpush_sender_unlock(RRDHOST *host) { +static inline void rrdpush_sender_thread_unlock(RRDHOST *host) { if(pthread_setcancelstate(PTHREAD_CANCEL_ENABLE, NULL) != 0) error("STREAM [send]: cannot set pthread cancel state to DISABLE."); rrdpush_unlock(host); } -void rrdpush_sender_cleanup(RRDHOST *host) { +void rrdpush_sender_thread_cleanup(RRDHOST *host) { rrdpush_lock(host); host->rrdpush_connected = 0; @@ -216,11 +237,11 @@ void *rrdpush_sender_thread(void *ptr) { if(pthread_setcancelstate(PTHREAD_CANCEL_ENABLE, NULL) != 0) error("STREAM [send]: cannot set pthread cancel state to ENABLE."); - int timeout = (int)config_get_number("stream", "timeout seconds", 60); - int default_port = (int)config_get_number("stream", "default port", 19999); - size_t max_size = (size_t)config_get_number("stream", "buffer size bytes", 1024 * 1024); - unsigned int reconnect_delay = (unsigned int)config_get_number("stream", "reconnect delay seconds", 5); - remote_clock_resync_iterations = (unsigned int)config_get_number("stream", "initial clock resync iterations", remote_clock_resync_iterations); + int timeout = (int)config_get_number(CONFIG_SECTION_STREAM, "timeout seconds", 60); + int default_port = (int)config_get_number(CONFIG_SECTION_STREAM, "default port", 19999); + size_t max_size = (size_t)config_get_number(CONFIG_SECTION_STREAM, "buffer size bytes", 1024 * 1024); + unsigned int reconnect_delay = (unsigned int)config_get_number(CONFIG_SECTION_STREAM, "reconnect delay seconds", 5); + remote_clock_resync_iterations = (unsigned int)config_get_number(CONFIG_SECTION_STREAM, "initial clock resync iterations", remote_clock_resync_iterations); char connected_to[CONNECTED_TO_SIZE + 1] = ""; if(!host->rrdpush_enabled || !remote_netdata_config || !*remote_netdata_config || !api_key || !*api_key) @@ -310,7 +331,7 @@ void *rrdpush_sender_thread(void *ptr) { if(fcntl(host->rrdpush_socket, F_SETFL, O_NONBLOCK) < 0) error("STREAM [send to %s]: cannot set non-blocking mode for socket.", connected_to); - rrdpush_flush(host); + rrdpush_sender_thread_data_flush(host); sent_connection = 0; // allow appending data into rrdpush_buffer @@ -357,7 +378,7 @@ void *rrdpush_sender_thread(void *ptr) { } if(ofd->revents & POLLOUT && begin < buffer_strlen(host->rrdpush_buffer)) { - rrdpush_sender_lock(host); + rrdpush_sender_thread_lock(host); ssize_t ret = send(host->rrdpush_socket, &host->rrdpush_buffer->buffer[begin], buffer_strlen(host->rrdpush_buffer) - begin, MSG_DONTWAIT); if(ret == -1) { if(errno != EAGAIN && errno != EINTR) { @@ -375,7 +396,7 @@ void *rrdpush_sender_thread(void *ptr) { begin = 0; } } - rrdpush_sender_unlock(host); + rrdpush_sender_thread_unlock(host); } // protection from overflow @@ -392,7 +413,7 @@ void *rrdpush_sender_thread(void *ptr) { cleanup: debug(D_WEB_CLIENT, "STREAM [send]: sending thread exits."); - rrdpush_sender_cleanup(host); + rrdpush_sender_thread_cleanup(host); pthread_exit(NULL); return NULL; @@ -400,7 +421,7 @@ cleanup: // ---------------------------------------------------------------------------- -// STREAM receiver +// rrdpush receiver thread int rrdpush_receive(int fd, const char *key, const char *hostname, const char *machine_guid, const char *os, int update_every, char *client_ip, char *client_port) { RRDHOST *host; diff --git a/src/rrdpush.h b/src/rrdpush.h index 8188d597..ab3fa15d 100644 --- a/src/rrdpush.h +++ b/src/rrdpush.h @@ -9,6 +9,6 @@ extern void rrdset_done_push(RRDSET *st); extern void *rrdpush_sender_thread(void *ptr); extern int rrdpush_receiver_thread_spawn(RRDHOST *host, struct web_client *w, char *url); -extern void rrdpush_sender_cleanup(RRDHOST *host); +extern void rrdpush_sender_thread_cleanup(RRDHOST *host); #endif //NETDATA_RRDPUSH_H diff --git a/src/rrdset.c b/src/rrdset.c index 89f87390..e2962336 100644 --- a/src/rrdset.c +++ b/src/rrdset.c @@ -607,7 +607,7 @@ static inline void rrdset_init_last_updated_time(RRDSET *st) { st->last_updated.tv_usec = 0; } -static inline void rrdset_done_push_int(RRDSET *st) { +static inline void rrdset_done_push_exclusive(RRDSET *st) { if(unlikely(!st->last_collected_time.tv_sec)) { // it is the first entry // set the last_collected_time to now @@ -619,17 +619,20 @@ static inline void rrdset_done_push_int(RRDSET *st) { rrdset_update_last_collected_time(st); } - st->counter++; st->counter_done++; + rrdset_rdlock(st); rrdset_done_push(st); + rrdset_unlock(st); } void rrdset_done(RRDSET *st) { if(unlikely(netdata_exit)) return; - if(unlikely(st->rrdhost->rrdpush_exclusive)) { - rrdset_done_push_int(st); + if(unlikely(st->rrd_memory_mode == RRD_MEMORY_MODE_NONE)) { + if(unlikely(st->rrdhost->rrdpush_enabled)) + rrdset_done_push_exclusive(st); + return; } @@ -748,6 +751,9 @@ void rrdset_done(RRDSET *st) { } st->counter_done++; + if(unlikely(st->rrdhost->rrdpush_enabled)) + rrdset_done_push(st); + // calculate totals and count the dimensions int dimensions = 0; st->collected_total = 0; @@ -1217,8 +1223,4 @@ void rrdset_done(RRDSET *st) { if(unlikely(pthread_setcancelstate(pthreadoldcancelstate, NULL) != 0)) error("Cannot set pthread cancel state to RESTORE (%d).", pthreadoldcancelstate); - - if(unlikely(st->rrdhost->rrdpush_enabled)) - rrdset_done_push_int(st); } - diff --git a/src/web_client.c b/src/web_client.c index feccde6b..a7ea3c56 100644 --- a/src/web_client.c +++ b/src/web_client.c @@ -239,7 +239,7 @@ uid_t web_files_uid(void) { static uid_t owner_uid = 0; if(unlikely(!web_owner)) { - web_owner = config_get("global", "web files owner", config_get("global", "run as user", "")); + web_owner = config_get(CONFIG_SECTION_API, "web files owner", config_get(CONFIG_SECTION_GLOBAL, "run as user", "")); if(!web_owner || !*web_owner) owner_uid = geteuid(); else { @@ -266,7 +266,7 @@ gid_t web_files_gid(void) { static gid_t owner_gid = 0; if(unlikely(!web_group)) { - web_group = config_get("global", "web files group", config_get("global", "web files owner", "")); + web_group = config_get(CONFIG_SECTION_API, "web files group", config_get(CONFIG_SECTION_API, "web files owner", "")); if(!web_group || !*web_group) owner_gid = getegid(); else { diff --git a/src/web_server.c b/src/web_server.c index e1153e31..4a18cc8e 100644 --- a/src/web_server.c +++ b/src/web_server.c @@ -5,7 +5,8 @@ size_t listen_fds_count = 0; int listen_fds[MAX_LISTEN_FDS] = { [0 ... 99] = -1 }; char *listen_fds_names[MAX_LISTEN_FDS] = { [0 ... 99] = NULL }; int listen_port = LISTEN_PORT; -int web_server_mode = WEB_SERVER_MODE_MULTI_THREADED; + +WEB_SERVER_MODE web_server_mode = WEB_SERVER_MODE_MULTI_THREADED; static int shown_server_socket_error = 0; @@ -83,6 +84,29 @@ int accept4(int sock, struct sockaddr *addr, socklen_t *addrlen, int flags) { } #endif +WEB_SERVER_MODE web_server_mode_id(const char *mode) { + if(!strcmp(mode, "none")) + return WEB_SERVER_MODE_NONE; + else if(!strcmp(mode, "single") || !strcmp(mode, "single-threaded")) + return WEB_SERVER_MODE_SINGLE_THREADED; + else // if(!strcmp(mode, "multi") || !strcmp(mode, "multi-threaded")) + return WEB_SERVER_MODE_MULTI_THREADED; +} + +const char *web_server_mode_name(WEB_SERVER_MODE id) { + switch(id) { + case WEB_SERVER_MODE_NONE: + return "none"; + + case WEB_SERVER_MODE_SINGLE_THREADED: + return "single-threaded"; + + default: + case WEB_SERVER_MODE_MULTI_THREADED: + return "multi-threaded"; + } +} + int create_listen_socket4(const char *ip, int port, int listen_backlog) { int sock; int sockopt = 1; @@ -316,22 +340,16 @@ static inline int bind_to_one(const char *definition, int default_port, int list int create_listen_sockets(void) { shown_server_socket_error = 0; - listen_backlog = (int) config_get_number("global", "http port listen backlog", LISTEN_BACKLOG); - - if(config_exists("global", "bind socket to IP") && !config_exists("global", "bind to")) - config_rename("global", "bind socket to IP", "bind to"); - - if(config_exists("global", "port") && !config_exists("global", "default port")) - config_rename("global", "port", "default port"); + listen_backlog = (int) config_get_number(CONFIG_SECTION_API, "listen backlog", LISTEN_BACKLOG); - listen_port = (int) config_get_number("global", "default port", LISTEN_PORT); + listen_port = (int) config_get_number(CONFIG_SECTION_API, "default port", LISTEN_PORT); if(listen_port < 1 || listen_port > 65535) { error("Invalid listen port %d given. Defaulting to %d.", listen_port, LISTEN_PORT); - listen_port = (int) config_set_number("global", "default port", LISTEN_PORT); + listen_port = (int) config_set_number(CONFIG_SECTION_API, "default port", LISTEN_PORT); } debug(D_OPTIONS, "Default listen port set to %d.", listen_port); - char *s = config_get("global", "bind to", "*"); + char *s = config_get(CONFIG_SECTION_API, "bind to", "*"); while(*s) { char *e = s; diff --git a/src/web_server.h b/src/web_server.h index 93adc5b2..41dcfcf0 100644 --- a/src/web_server.h +++ b/src/web_server.h @@ -13,9 +13,17 @@ #define MAX_LISTEN_FDS 100 #endif -#define WEB_SERVER_MODE_MULTI_THREADED 0 -#define WEB_SERVER_MODE_SINGLE_THREADED 1 -extern int web_server_mode; +typedef enum web_server_mode { + WEB_SERVER_MODE_SINGLE_THREADED, + WEB_SERVER_MODE_MULTI_THREADED, + WEB_SERVER_MODE_NONE +} WEB_SERVER_MODE; + +extern WEB_SERVER_MODE web_server_mode; + +extern WEB_SERVER_MODE web_server_mode_id(const char *mode); +extern const char *web_server_mode_name(WEB_SERVER_MODE id); + extern void *socket_listen_main_multi_threaded(void *ptr); extern void *socket_listen_main_single_threaded(void *ptr); -- 2.39.2