From 842ada7cd5bcf8d018df718efc629571ba2103fa Mon Sep 17 00:00:00 2001 From: "Costa Tsaousis (ktsaou)" Date: Wed, 22 Feb 2017 03:40:17 +0200 Subject: [PATCH] implemented API authentication --- conf.d/Makefile.am | 1 + conf.d/aggregated_hosts.conf | 75 ++++++++++++++++++++++++++++++++++++ src/main.c | 6 ++- src/plugins_d.c | 6 +-- src/rrd.h | 2 +- src/rrdhost.c | 32 ++++++++++----- src/rrdpush.c | 20 +++++----- src/web_client.c | 69 ++++++++++++++++++++++++++++++--- 8 files changed, 182 insertions(+), 29 deletions(-) create mode 100644 conf.d/aggregated_hosts.conf diff --git a/conf.d/Makefile.am b/conf.d/Makefile.am index c746a909..dd6c2cbf 100644 --- a/conf.d/Makefile.am +++ b/conf.d/Makefile.am @@ -4,6 +4,7 @@ MAINTAINERCLEANFILES= $(srcdir)/Makefile.in dist_config_DATA = \ + aggregated_hosts.conf \ apps_groups.conf \ charts.d.conf \ fping.conf \ diff --git a/conf.d/aggregated_hosts.conf b/conf.d/aggregated_hosts.conf new file mode 100644 index 00000000..7dd5eaa0 --- /dev/null +++ b/conf.d/aggregated_hosts.conf @@ -0,0 +1,75 @@ +# netdata configuration for aggregating data from remote hosts +# +# 1. You need an API key: API_KEY_GENERATED_BY_UUIDGEN +# +# You can generate one with the command: uuidgen +# You can add many API key sections, for different API keys +# +# 2. All options below are used in this order: +# +# a) MACHINE_GUID (settings for each machine) +# b) API_KEY (settings for the API key) +# c) this netdata defaults (as in netdata.conf) +# +# You can combine the above (the more specific will be used). +# +# 3. At the remote host that will be sending metrics, you need +# to add in netdata.conf, the following: +# +# [global] +# central netdata to send all data = 10.11.12.1:19999 +# central netdata api key = API_KEY_GENERATED_BY_UUIDGEN +# +# Of course, the same API_KEY_GENERATED_BY_UUIDGEN key must +# given there (the remote host) and here (the central netdata). +# +# ----------------------------------------------------------------------------- + +[API_KEY_GENERATED_BY_UUIDGEN] + # You can disable the API key, by setting this to: no + # The default (for unknown API keys) is also: no +# enabled = yes + + # The default history in entries, for all hosts using this API key. + # You can also set it per host below. + # If you don't set it here, the history size of the central netdata + # will be used +# default history = 3600 + + # The default memory mode to be used for all hosts using this API key. + # You can also set it per host below. + # If you don't set it here, the memory mode of the central netdata + # will be used +# default memory mode = save + + # Shall we enable health monitoring for the hosts using this API key? + # 3 values: + # yes enable alarms + # no do not enable alarms + # auto enable alarms, only when the host is streaming metrics + # You can also set it per host, below. + # The default is the same as the central netdata +# health enabled by default = auto + + +# ----------------------------------------------------------------------------- +# Each netdata has a unique GUID - generated the first time netdata starts. +# You can find it at /var/lib/netdata/registry/netdata.public.unique.id +# The host sending data will have one. If it is static and you can find it, +# you can give settings for each specific host here. + +[MACHINE_GUID] + # This can be used to stop receiving data + # THIS IS NOT A SECURITY MECHANISM - AN ATTACKER CAN SET ANY OTHER GUID. + # Use only the API key for security. +# enabled = yes + + # The number of entries in the database +# history = 3600 + + # The memory mode of the database +# memory mode = save + + # Health / alarms control +# health enabled = yes + diff --git a/src/main.c b/src/main.c index 944d01d2..3c9c16d5 100644 --- a/src/main.c +++ b/src/main.c @@ -844,8 +844,12 @@ int main(int argc, char **argv) { // -------------------------------------------------------------------- // create the listening sockets - if(!check_config && !central_netdata_to_push_data) + if(!check_config && !central_netdata_to_push_data) { + char filename[FILENAME_MAX + 1]; + snprintfz(filename, FILENAME_MAX, "%s/aggregated_hosts.conf", netdata_configured_config_dir); + appconfig_load(&stream_config, filename, 0); create_listen_sockets(); + } } // initialize the log files diff --git a/src/plugins_d.c b/src/plugins_d.c index dedb3d72..81f17a92 100644 --- a/src/plugins_d.c +++ b/src/plugins_d.c @@ -96,7 +96,7 @@ inline size_t pluginsd_process(RRDHOST *host, struct plugind *cd, FILE *fp, int char line[PLUGINSD_LINE_MAX + 1]; char *words[MAX_WORDS] = { NULL }; - uint32_t HOST_HASH = simple_hash("HOST"); + /* uint32_t HOST_HASH = simple_hash("HOST"); */ uint32_t BEGIN_HASH = simple_hash("BEGIN"); uint32_t END_HASH = simple_hash("END"); uint32_t FLUSH_HASH = simple_hash("FLUSH"); @@ -205,7 +205,7 @@ inline size_t pluginsd_process(RRDHOST *host, struct plugind *cd, FILE *fp, int count++; } - else if(likely(hash == HOST_HASH && !strcmp(s, "HOST"))) { +/* else if(likely(hash == HOST_HASH && !strcmp(s, "HOST"))) { char *guid = words[1]; char *hostname = words[2]; @@ -221,7 +221,7 @@ inline size_t pluginsd_process(RRDHOST *host, struct plugind *cd, FILE *fp, int } host = rrdhost_find_or_create(hostname, guid); - } + } */ else if(likely(hash == FLUSH_HASH && !strcmp(s, "FLUSH"))) { debug(D_PLUGINSD, "PLUGINSD: '%s' is requesting a FLUSH", cd->fullfilename); st = NULL; diff --git a/src/rrd.h b/src/rrd.h index 3a10f219..c531c4f9 100644 --- a/src/rrd.h +++ b/src/rrd.h @@ -406,7 +406,7 @@ extern pthread_rwlock_t rrd_rwlock; extern void rrd_init(char *hostname); extern RRDHOST *rrdhost_find(const char *guid, uint32_t hash); -extern RRDHOST *rrdhost_find_or_create(const char *hostname, const char *guid); +extern RRDHOST *rrdhost_find_or_create(const char *hostname, const char *guid, int update_every, int history, RRD_MEMORY_MODE mode, int health_enabled); #ifdef NETDATA_INTERNAL_CHECKS extern void rrdhost_check_wrlock_int(RRDHOST *host, const char *file, const char *function, const unsigned long line); diff --git a/src/rrdhost.c b/src/rrdhost.c index 02ccf6bf..96500852 100644 --- a/src/rrdhost.c +++ b/src/rrdhost.c @@ -176,18 +176,32 @@ RRDHOST *rrdhost_create(const char *hostname, return host; } -RRDHOST *rrdhost_find_or_create(const char *hostname, const char *guid) { +RRDHOST *rrdhost_find_or_create(const char *hostname, const char *guid, int update_every, int history, RRD_MEMORY_MODE mode, int health_enabled) { debug(D_RRDHOST, "Searching for host '%s' with guid '%s'", hostname, guid); RRDHOST *host = rrdhost_find(guid, 0); - if(!host) - host = rrdhost_create(hostname, - guid, - default_rrd_update_every, - default_rrd_history_entries, - default_rrd_memory_mode, - default_health_enabled - ); + if(!host) { + host = rrdhost_create(hostname, guid, update_every, history, mode, health_enabled); + } + else { + host->health_enabled = health_enabled; + + if(strcmp(host->hostname, hostname)) { + char *t = host->hostname; + char *n = strdupz(hostname); + host->hostname = n; + freez(t); + } + + if(host->rrd_update_every != update_every) + error("Host '%s' has an update frequency of %d seconds, but the wanted one is %d seconds.", host->hostname, host->rrd_update_every, update_every); + + if(host->rrd_history_entries != history) + error("Host '%s' has history of %d entries, but the wanted one is %d entries.", host->hostname, host->rrd_history_entries, history); + + if(host->rrd_memory_mode != mode) + error("Host '%s' has memory mode '%s', but the wanted one is '%s'.", host->hostname, rrd_memory_mode_name(host->rrd_memory_mode), rrd_memory_mode_name(mode)); + } return host; } diff --git a/src/rrdpush.c b/src/rrdpush.c index bc1c027b..8a5e462d 100644 --- a/src/rrdpush.c +++ b/src/rrdpush.c @@ -7,7 +7,6 @@ int rrdpush_pipe[2]; static BUFFER *rrdpush_buffer = NULL; static pthread_mutex_t rrdpush_mutex = PTHREAD_MUTEX_INITIALIZER; -static volatile RRDHOST *last_host = NULL; static volatile int rrdpush_connected = 0; static inline void rrdpush_lock() { @@ -99,8 +98,6 @@ static void reset_all_charts(void) { rrdhost_unlock(host); } rrd_unlock(); - - last_host = NULL; } void rrdset_done_push(RRDSET *st) { @@ -123,11 +120,6 @@ void rrdset_done_push(RRDSET *st) { } error_shown = 0; - if(st->rrdhost != last_host) { - buffer_sprintf(rrdpush_buffer, "HOST '%s' '%s'\n", st->rrdhost->machine_guid, st->rrdhost->hostname); - last_host = st->rrdhost; - } - rrdset_rdlock(st); if(need_to_send_chart_definition(st)) send_chart_definition(st); @@ -149,7 +141,6 @@ static inline void rrdpush_flush(void) { buffer_flush(rrdpush_buffer); reset_all_charts(); - last_host = NULL; rrdpush_unlock(); } @@ -209,7 +200,16 @@ void *central_netdata_push_thread(void *ptr) { info("STREAM: initializing communication to central netdata at: %s", central_netdata_to_push_data); char http[1000 + 1]; - snprintfz(http, 1000, "GET /stream?key=%s HTTP/1.1\r\nUser-Agent: netdata-push-service/%s\r\nAccept: */*\r\n\r\n", config_get("global", "central netdata api key", ""), program_version); + snprintfz(http, 1000, "GET /stream?key=%s&hostname=%s&machine_guid=%s&update_every=%d HTTP/1.1\r\n" + "User-Agent: netdata-push-service/%s\r\n" + "Accept: */*\r\n\r\n" + , config_get("global", "central netdata api key", "") + , localhost->hostname + , localhost->machine_guid + , default_rrd_update_every + , program_version + ); + if(send_timeout(sock, http, strlen(http), 0, 60) == -1) { close(sock); sock = -1; diff --git a/src/web_client.c b/src/web_client.c index 48b8c468..453b61d9 100644 --- a/src/web_client.c +++ b/src/web_client.c @@ -1668,13 +1668,18 @@ int web_client_api_old_data_request(RRDHOST *host, struct web_client *w, char *u } int validate_stream_api_key(const char *key) { + if(appconfig_get(&stream_config, key, "enabled", 0)) + return 1; + return 0; } int web_client_stream_request(RRDHOST *host, struct web_client *w, char *url) { - info("STREAM request from client '%s:%s', starting as host '%s'", w->client_ip, w->client_port, host->hostname); - - char *key = NULL; + char *key = NULL, *hostname = NULL, *machine_guid = NULL; + int update_every = default_rrd_update_every; + int history = default_rrd_history_entries; + RRD_MEMORY_MODE mode = default_rrd_memory_mode; + int health_enabled = default_health_enabled; while(url) { char *value = mystrsep(&url, "?&"); @@ -1686,6 +1691,12 @@ int web_client_stream_request(RRDHOST *host, struct web_client *w, char *url) { if(!strcmp(name, "key")) key = value; + else if(!strcmp(name, "hostname")) + hostname = value; + else if(!strcmp(name, "machine_guid")) + machine_guid = value; + else if(!strcmp(name, "update_every")) + update_every = (int)strtoul(value, NULL, 0); } if(!key || !*key) { @@ -1695,6 +1706,20 @@ int web_client_stream_request(RRDHOST *host, struct web_client *w, char *url) { return 401; } + if(!hostname || !*hostname) { + error("STREAM [%s]:%s: request without a hostname. Forbidding access.", w->client_ip, w->client_port); + buffer_flush(w->response.data); + buffer_sprintf(w->response.data, "You need to send a hostname too."); + return 400; + } + + if(!machine_guid || !*machine_guid) { + error("STREAM [%s]:%s: request without a machine GUID. Forbidding access.", w->client_ip, w->client_port); + buffer_flush(w->response.data); + buffer_sprintf(w->response.data, "You need to send a machine GUID too."); + return 400; + } + if(!validate_stream_api_key(key)) { error("STREAM [%s]:%s: API key '%s' is not allowed. Forbidding access.", w->client_ip, w->client_port, key); buffer_flush(w->response.data); @@ -1702,6 +1727,38 @@ int web_client_stream_request(RRDHOST *host, struct web_client *w, char *url) { return 401; } + if(!appconfig_get_boolean(&stream_config, machine_guid, "enabled", 1)) { + error("STREAM [%s]:%s: machine GUID '%s' is not allowed. Forbidding access.", w->client_ip, w->client_port, machine_guid); + buffer_flush(w->response.data); + buffer_sprintf(w->response.data, "Your machine guide is not permitted access."); + return 404; + } + + // update_every = (int)appconfig_get_number(&stream_config, key, "default update every", update_every); + update_every = (int)appconfig_get_number(&stream_config, machine_guid, "update every", update_every); + if(update_every < 0) update_every = 1; + + history = (int)appconfig_get_number(&stream_config, key, "default history", history); + history = (int)appconfig_get_number(&stream_config, machine_guid, "history", history); + if(history < 5) history = 5; + + mode = rrd_memory_mode_id(appconfig_get(&stream_config, key, "default memory mode", rrd_memory_mode_name(mode))); + mode = rrd_memory_mode_id(appconfig_get(&stream_config, machine_guid, "memory mode", rrd_memory_mode_name(mode))); + + health_enabled = appconfig_get_boolean_ondemand(&stream_config, key, "health enabled by default", health_enabled); + health_enabled = appconfig_get_boolean_ondemand(&stream_config, machine_guid, "health enabled", health_enabled); + + host = rrdhost_find_or_create(hostname, machine_guid, update_every, history, mode, health_enabled?1:0); + + info("STREAM request from client '%s:%s' for host '%s' with machine_guid '%s': update every = %d, history = %d, memory mode = %s, health %s", + w->client_ip, w->client_port, + hostname, machine_guid, + update_every, + history, + rrd_memory_mode_name(mode), + (health_enabled == CONFIG_BOOLEAN_NO)?"disabled":((health_enabled == CONFIG_BOOLEAN_YES)?"enabled":"auto") + ); + struct plugind cd = { .enabled = 1, .update_every = default_rrd_update_every, @@ -1750,9 +1807,11 @@ int web_client_stream_request(RRDHOST *host, struct web_client *w, char *url) { } // call the plugins.d processor to receive the metrics - info("STREAM [%s]:%s: connecting client to plugins.d.", w->client_ip, w->client_port); + info("STREAM [%s]:%s: connecting client to plugins.d on host '%s' with machine GUID '%s'.", w->client_ip, w->client_port, host->hostname, host->machine_guid); size_t count = pluginsd_process(host, &cd, fp, 1); - error("STREAM [%s]:%s: client disconnected.", w->client_ip, w->client_port); + error("STREAM [%s]:%s: client disconnected (host '%s', machine GUID '%s').", w->client_ip, w->client_port, host->hostname, host->machine_guid); + if(health_enabled == CONFIG_BOOLEAN_AUTO) + host->health_enabled = 0; // close all sockets, to let the socket worker we are done fclose(fp); -- 2.39.2