- if(!given_up)
- *delay_up_duration = 0;
-
- if(!given_down)
- *delay_down_duration = 0;
-
- if(!given_multiplier)
- *delay_multiplier = 1.0;
-
- if(!given_max) {
- if((*delay_max_duration) < (*delay_up_duration) * (*delay_multiplier))
- *delay_max_duration = (*delay_up_duration) * (*delay_multiplier);
-
- if((*delay_max_duration) < (*delay_down_duration) * (*delay_multiplier))
- *delay_max_duration = (*delay_down_duration) * (*delay_multiplier);
- }
-
- return 1;
-}
-
-static inline uint32_t health_parse_options(const char *s) {
- uint32_t options = 0;
- char buf[100+1] = "";
-
- while(*s) {
- buf[0] = '\0';
-
- // skip spaces
- while(*s && isspace(*s))
- s++;
-
- // find the next space
- size_t count = 0;
- while(*s && count < 100 && !isspace(*s))
- buf[count++] = *s++;
-
- if(buf[0]) {
- buf[count] = '\0';
-
- if(!strcasecmp(buf, "no-clear-notification") || !strcasecmp(buf, "no-clear"))
- options |= RRDCALC_FLAG_NO_CLEAR_NOTIFICATION;
- else
- error("Ignoring unknown alarm option '%s'", buf);
- }
- }
-
- return options;
-}
-
-static inline int health_parse_db_lookup(
- size_t line, const char *path, const char *file, char *string,
- int *group_method, int *after, int *before, int *every,
- uint32_t *options, char **dimensions
-) {
- debug(D_HEALTH, "Health configuration parsing database lookup %zu@%s/%s: %s", line, path, file, string);
-
- if(*dimensions) freez(*dimensions);
- *dimensions = NULL;
- *after = 0;
- *before = 0;
- *every = 0;
- *options = 0;
-
- char *s = string, *key;
-
- // first is the group method
- key = s;
- while(*s && !isspace(*s)) s++;
- while(*s && isspace(*s)) *s++ = '\0';
- if(!*s) {
- error("Health configuration invalid chart calculation at line %zu of file '%s/%s': expected group method followed by the 'after' time, but got '%s'",
- line, path, file, key);
- return 0;
- }
-
- if((*group_method = web_client_api_request_v1_data_group(key, -1)) == -1) {
- error("Health configuration at line %zu of file '%s/%s': invalid group method '%s'",
- line, path, file, key);
- return 0;
- }
-
- // then is the 'after' time
- key = s;
- while(*s && !isspace(*s)) s++;
- while(*s && isspace(*s)) *s++ = '\0';
-
- if(!health_parse_duration(key, after)) {
- error("Health configuration at line %zu of file '%s/%s': invalid duration '%s' after group method",
- line, path, file, key);
- return 0;
- }
-
- // sane defaults
- *every = abs(*after);
-
- // now we may have optional parameters
- while(*s) {
- key = s;
- while(*s && !isspace(*s)) s++;
- while(*s && isspace(*s)) *s++ = '\0';
- if(!*key) break;
-
- if(!strcasecmp(key, "at")) {
- char *value = s;
- while(*s && !isspace(*s)) s++;
- while(*s && isspace(*s)) *s++ = '\0';
-
- if (!health_parse_duration(value, before)) {
- error("Health configuration at line %zu of file '%s/%s': invalid duration '%s' for '%s' keyword",
- line, path, file, value, key);
- }
- }
- else if(!strcasecmp(key, HEALTH_EVERY_KEY)) {
- char *value = s;
- while(*s && !isspace(*s)) s++;
- while(*s && isspace(*s)) *s++ = '\0';
-
- if (!health_parse_duration(value, every)) {
- error("Health configuration at line %zu of file '%s/%s': invalid duration '%s' for '%s' keyword",
- line, path, file, value, key);
- }
- }
- else if(!strcasecmp(key, "absolute") || !strcasecmp(key, "abs") || !strcasecmp(key, "absolute_sum")) {
- *options |= RRDR_OPTION_ABSOLUTE;
- }
- else if(!strcasecmp(key, "min2max")) {
- *options |= RRDR_OPTION_MIN2MAX;
- }
- else if(!strcasecmp(key, "null2zero")) {
- *options |= RRDR_OPTION_NULL2ZERO;
- }
- else if(!strcasecmp(key, "percentage")) {
- *options |= RRDR_OPTION_PERCENTAGE;
- }
- else if(!strcasecmp(key, "unaligned")) {
- *options |= RRDR_OPTION_NOT_ALIGNED;
- }
- else if(!strcasecmp(key, "of")) {
- if(*s && strcasecmp(s, "all"))
- *dimensions = strdupz(s);
- break;
- }
- else {
- error("Health configuration at line %zu of file '%s/%s': unknown keyword '%s'",
- line, path, file, key);
- }
- }
-
- return 1;
-}
-
-static inline char *tabs2spaces(char *s) {
- char *t = s;
- while(*t) {
- if(unlikely(*t == '\t')) *t = ' ';
- t++;
- }
-
- return s;
-}
-
-static inline char *health_source_file(size_t line, const char *path, const char *filename) {
- char buffer[FILENAME_MAX + 1];
- snprintfz(buffer, FILENAME_MAX, "%zu@%s/%s", line, path, filename);
- return strdupz(buffer);
-}
-
-static inline void strip_quotes(char *s) {
- while(*s) {
- if(*s == '\'' || *s == '"') *s = ' ';
- s++;
- }
-}
-
-int health_readfile(const char *path, const char *filename) {
- debug(D_HEALTH, "Health configuration reading file '%s/%s'", path, filename);
-
- static uint32_t
- hash_alarm = 0,
- hash_template = 0,
- hash_on = 0,
- hash_families = 0,
- hash_calc = 0,
- hash_green = 0,
- hash_red = 0,
- hash_warn = 0,
- hash_crit = 0,
- hash_exec = 0,
- hash_every = 0,
- hash_lookup = 0,
- hash_units = 0,
- hash_info = 0,
- hash_recipient = 0,
- hash_delay = 0,
- hash_options = 0;
-
- char buffer[HEALTH_CONF_MAX_LINE + 1];
-
- if(unlikely(!hash_alarm)) {
- hash_alarm = simple_uhash(HEALTH_ALARM_KEY);
- hash_template = simple_uhash(HEALTH_TEMPLATE_KEY);
- hash_on = simple_uhash(HEALTH_ON_KEY);
- hash_families = simple_uhash(HEALTH_FAMILIES_KEY);
- hash_calc = simple_uhash(HEALTH_CALC_KEY);
- hash_lookup = simple_uhash(HEALTH_LOOKUP_KEY);
- hash_green = simple_uhash(HEALTH_GREEN_KEY);
- hash_red = simple_uhash(HEALTH_RED_KEY);
- hash_warn = simple_uhash(HEALTH_WARN_KEY);
- hash_crit = simple_uhash(HEALTH_CRIT_KEY);
- hash_exec = simple_uhash(HEALTH_EXEC_KEY);
- hash_every = simple_uhash(HEALTH_EVERY_KEY);
- hash_units = simple_hash(HEALTH_UNITS_KEY);
- hash_info = simple_hash(HEALTH_INFO_KEY);
- hash_recipient = simple_hash(HEALTH_RECIPIENT_KEY);
- hash_delay = simple_uhash(HEALTH_DELAY_KEY);
- hash_options = simple_uhash(HEALTH_OPTIONS_KEY);
- }
-
- snprintfz(buffer, HEALTH_CONF_MAX_LINE, "%s/%s", path, filename);
- FILE *fp = fopen(buffer, "r");
- if(!fp) {
- error("Health configuration cannot read file '%s'.", buffer);
- return 0;
- }
-
- RRDCALC *rc = NULL;
- RRDCALCTEMPLATE *rt = NULL;
-
- size_t line = 0, append = 0;
- char *s;
- while((s = fgets(&buffer[append], (int)(HEALTH_CONF_MAX_LINE - append), fp)) || append) {
- int stop_appending = !s;
- line++;
- s = trim(buffer);
- if(!s) continue;
-
- append = strlen(s);
- if(!stop_appending && s[append - 1] == '\\') {
- s[append - 1] = ' ';
- append = &s[append] - buffer;
- if(append < HEALTH_CONF_MAX_LINE)
- continue;
- else {
- error("Health configuration has too long muli-line at line %zu of file '%s/%s'.", line, path, filename);
- }
- }
- append = 0;
-
- char *key = s;
- while(*s && *s != ':') s++;
- if(!*s) {
- error("Health configuration has invalid line %zu of file '%s/%s'. It does not contain a ':'. Ignoring it.", line, path, filename);
- continue;
- }
- *s = '\0';
- s++;
-
- char *value = s;
- key = trim(key);
- value = trim(value);
-
- if(!key) {
- error("Health configuration has invalid line %zu of file '%s/%s'. Keyword is empty. Ignoring it.", line, path, filename);
- continue;
- }
-
- if(!value) {
- error("Health configuration has invalid line %zu of file '%s/%s'. value is empty. Ignoring it.", line, path, filename);
- continue;
- }
-
- uint32_t hash = simple_uhash(key);
-
- if(hash == hash_alarm && !strcasecmp(key, HEALTH_ALARM_KEY)) {
- if(rc && !rrdcalc_add_alarm_from_config(&localhost, rc))
- rrdcalc_free(&localhost, rc);
-
- if(rt) {
- if (!rrdcalctemplate_add_template_from_config(&localhost, rt))
- rrdcalctemplate_free(&localhost, rt);
- rt = NULL;
- }
-
- rc = callocz(1, sizeof(RRDCALC));
- rc->next_event_id = 1;
- rc->name = tabs2spaces(strdupz(value));
- rc->hash = simple_hash(rc->name);
- rc->source = health_source_file(line, path, filename);
- rc->green = NAN;
- rc->red = NAN;
- rc->value = NAN;
- rc->old_value = NAN;
- rc->delay_multiplier = 1.0;
-
- if(rrdvar_fix_name(rc->name))
- error("Health configuration renamed alarm '%s' to '%s'", value, rc->name);
- }
- else if(hash == hash_template && !strcasecmp(key, HEALTH_TEMPLATE_KEY)) {
- if(rc) {
- if(!rrdcalc_add_alarm_from_config(&localhost, rc))
- rrdcalc_free(&localhost, rc);
- rc = NULL;
- }
-
- if(rt && !rrdcalctemplate_add_template_from_config(&localhost, rt))
- rrdcalctemplate_free(&localhost, rt);
-
- rt = callocz(1, sizeof(RRDCALCTEMPLATE));
- rt->name = tabs2spaces(strdupz(value));
- rt->hash_name = simple_hash(rt->name);
- rt->source = health_source_file(line, path, filename);
- rt->green = NAN;
- rt->red = NAN;
- rt->delay_multiplier = 1.0;
-
- if(rrdvar_fix_name(rt->name))
- error("Health configuration renamed template '%s' to '%s'", value, rt->name);
- }
- else if(rc) {
- if(hash == hash_on && !strcasecmp(key, HEALTH_ON_KEY)) {
- if(rc->chart) {
- if(strcmp(rc->chart, value))
- error("Health configuration at line %zu of file '%s/%s' for alarm '%s' has key '%s' twice, once with value '%s' and later with value '%s'. Using ('%s').",
- line, path, filename, rc->name, key, rc->chart, value, value);
-
- freez(rc->chart);
- }
- rc->chart = tabs2spaces(strdupz(value));
- rc->hash_chart = simple_hash(rc->chart);
- }
- else if(hash == hash_lookup && !strcasecmp(key, HEALTH_LOOKUP_KEY)) {
- health_parse_db_lookup(line, path, filename, value, &rc->group, &rc->after, &rc->before,
- &rc->update_every,
- &rc->options, &rc->dimensions);
- }
- else if(hash == hash_every && !strcasecmp(key, HEALTH_EVERY_KEY)) {
- if(!health_parse_duration(value, &rc->update_every))
- error("Health configuration at line %zu of file '%s/%s' for alarm '%s' at key '%s' cannot parse duration: '%s'.",
- line, path, filename, rc->name, key, value);
- }
- else if(hash == hash_green && !strcasecmp(key, HEALTH_GREEN_KEY)) {
- char *e;
- rc->green = strtold(value, &e);
- if(e && *e) {
- error("Health configuration at line %zu of file '%s/%s' for alarm '%s' at key '%s' leaves this string unmatched: '%s'.",
- line, path, filename, rc->name, key, e);
- }
- }
- else if(hash == hash_red && !strcasecmp(key, HEALTH_RED_KEY)) {
- char *e;
- rc->red = strtold(value, &e);
- if(e && *e) {
- error("Health configuration at line %zu of file '%s/%s' for alarm '%s' at key '%s' leaves this string unmatched: '%s'.",
- line, path, filename, rc->name, key, e);
- }
- }
- else if(hash == hash_calc && !strcasecmp(key, HEALTH_CALC_KEY)) {
- const char *failed_at = NULL;
- int error = 0;
- rc->calculation = expression_parse(value, &failed_at, &error);
- if(!rc->calculation) {
- error("Health configuration at line %zu of file '%s/%s' for alarm '%s' at key '%s' has unparse-able expression '%s': %s at '%s'",
- line, path, filename, rc->name, key, value, expression_strerror(error), failed_at);
- }
- }
- else if(hash == hash_warn && !strcasecmp(key, HEALTH_WARN_KEY)) {
- const char *failed_at = NULL;
- int error = 0;
- rc->warning = expression_parse(value, &failed_at, &error);
- if(!rc->warning) {
- error("Health configuration at line %zu of file '%s/%s' for alarm '%s' at key '%s' has unparse-able expression '%s': %s at '%s'",
- line, path, filename, rc->name, key, value, expression_strerror(error), failed_at);
- }
- }
- else if(hash == hash_crit && !strcasecmp(key, HEALTH_CRIT_KEY)) {
- const char *failed_at = NULL;
- int error = 0;
- rc->critical = expression_parse(value, &failed_at, &error);
- if(!rc->critical) {
- error("Health configuration at line %zu of file '%s/%s' for alarm '%s' at key '%s' has unparse-able expression '%s': %s at '%s'",
- line, path, filename, rc->name, key, value, expression_strerror(error), failed_at);
- }
- }
- else if(hash == hash_exec && !strcasecmp(key, HEALTH_EXEC_KEY)) {
- if(rc->exec) {
- if(strcmp(rc->exec, value))
- error("Health configuration at line %zu of file '%s/%s' for alarm '%s' has key '%s' twice, once with value '%s' and later with value '%s'. Using ('%s').",
- line, path, filename, rc->name, key, rc->exec, value, value);
-
- freez(rc->exec);
- }
- rc->exec = tabs2spaces(strdupz(value));
- }
- else if(hash == hash_recipient && !strcasecmp(key, HEALTH_RECIPIENT_KEY)) {
- if(rc->recipient) {
- if(strcmp(rc->recipient, value))
- error("Health configuration at line %zu of file '%s/%s' for alarm '%s' has key '%s' twice, once with value '%s' and later with value '%s'. Using ('%s').",
- line, path, filename, rc->name, key, rc->recipient, value, value);
-
- freez(rc->recipient);
- }
- rc->recipient = tabs2spaces(strdupz(value));
- }
- else if(hash == hash_units && !strcasecmp(key, HEALTH_UNITS_KEY)) {
- if(rc->units) {
- if(strcmp(rc->units, value))
- error("Health configuration at line %zu of file '%s/%s' for alarm '%s' has key '%s' twice, once with value '%s' and later with value '%s'. Using ('%s').",
- line, path, filename, rc->name, key, rc->units, value, value);
-
- freez(rc->units);
- }
- rc->units = tabs2spaces(strdupz(value));
- strip_quotes(rc->units);
- }
- else if(hash == hash_info && !strcasecmp(key, HEALTH_INFO_KEY)) {
- if(rc->info) {
- if(strcmp(rc->info, value))
- error("Health configuration at line %zu of file '%s/%s' for alarm '%s' has key '%s' twice, once with value '%s' and later with value '%s'. Using ('%s').",
- line, path, filename, rc->name, key, rc->info, value, value);
-
- freez(rc->info);
- }
- rc->info = tabs2spaces(strdupz(value));
- strip_quotes(rc->info);
- }
- else if(hash == hash_delay && !strcasecmp(key, HEALTH_DELAY_KEY)) {
- health_parse_delay(line, path, filename, value, &rc->delay_up_duration, &rc->delay_down_duration, &rc->delay_max_duration, &rc->delay_multiplier);
- }
- else if(hash == hash_options && !strcasecmp(key, HEALTH_OPTIONS_KEY)) {
- rc->options |= health_parse_options(value);
- }
- else {
- error("Health configuration at line %zu of file '%s/%s' for alarm '%s' has unknown key '%s'.",
- line, path, filename, rc->name, key);
- }
- }
- else if(rt) {
- if(hash == hash_on && !strcasecmp(key, HEALTH_ON_KEY)) {
- if(rt->context) {
- if(strcmp(rt->context, value))
- error("Health configuration at line %zu of file '%s/%s' for template '%s' has key '%s' twice, once with value '%s' and later with value '%s'. Using ('%s').",
- line, path, filename, rt->name, key, rt->context, value, value);
-
- freez(rt->context);
- }
- rt->context = tabs2spaces(strdupz(value));
- rt->hash_context = simple_hash(rt->context);
- }
- else if(hash == hash_families && !strcasecmp(key, HEALTH_FAMILIES_KEY)) {
- freez(rt->family_match);
- simple_pattern_free(rt->family_pattern);
-
- rt->family_match = tabs2spaces(strdupz(value));
- rt->family_pattern = simple_pattern_create(rt->family_match, SIMPLE_PATTERN_EXACT);
- }
- else if(hash == hash_lookup && !strcasecmp(key, HEALTH_LOOKUP_KEY)) {
- health_parse_db_lookup(line, path, filename, value, &rt->group, &rt->after, &rt->before,
- &rt->update_every, &rt->options, &rt->dimensions);
- }
- else if(hash == hash_every && !strcasecmp(key, HEALTH_EVERY_KEY)) {
- if(!health_parse_duration(value, &rt->update_every))
- error("Health configuration at line %zu of file '%s/%s' for template '%s' at key '%s' cannot parse duration: '%s'.",
- line, path, filename, rt->name, key, value);
- }
- else if(hash == hash_green && !strcasecmp(key, HEALTH_GREEN_KEY)) {
- char *e;
- rt->green = strtold(value, &e);
- if(e && *e) {
- error("Health configuration at line %zu of file '%s/%s' for template '%s' at key '%s' leaves this string unmatched: '%s'.",
- line, path, filename, rt->name, key, e);
- }
- }
- else if(hash == hash_red && !strcasecmp(key, HEALTH_RED_KEY)) {
- char *e;
- rt->red = strtold(value, &e);
- if(e && *e) {
- error("Health configuration at line %zu of file '%s/%s' for template '%s' at key '%s' leaves this string unmatched: '%s'.",
- line, path, filename, rt->name, key, e);
- }
- }
- else if(hash == hash_calc && !strcasecmp(key, HEALTH_CALC_KEY)) {
- const char *failed_at = NULL;
- int error = 0;
- rt->calculation = expression_parse(value, &failed_at, &error);
- if(!rt->calculation) {
- error("Health configuration at line %zu of file '%s/%s' for template '%s' at key '%s' has unparse-able expression '%s': %s at '%s'",
- line, path, filename, rt->name, key, value, expression_strerror(error), failed_at);
- }
- }
- else if(hash == hash_warn && !strcasecmp(key, HEALTH_WARN_KEY)) {
- const char *failed_at = NULL;
- int error = 0;
- rt->warning = expression_parse(value, &failed_at, &error);
- if(!rt->warning) {
- error("Health configuration at line %zu of file '%s/%s' for template '%s' at key '%s' has unparse-able expression '%s': %s at '%s'",
- line, path, filename, rt->name, key, value, expression_strerror(error), failed_at);
- }
- }
- else if(hash == hash_crit && !strcasecmp(key, HEALTH_CRIT_KEY)) {
- const char *failed_at = NULL;
- int error = 0;
- rt->critical = expression_parse(value, &failed_at, &error);
- if(!rt->critical) {
- error("Health configuration at line %zu of file '%s/%s' for template '%s' at key '%s' has unparse-able expression '%s': %s at '%s'",
- line, path, filename, rt->name, key, value, expression_strerror(error), failed_at);
- }
- }
- else if(hash == hash_exec && !strcasecmp(key, HEALTH_EXEC_KEY)) {
- if(rt->exec) {
- if(strcmp(rt->exec, value))
- error("Health configuration at line %zu of file '%s/%s' for template '%s' has key '%s' twice, once with value '%s' and later with value '%s'. Using ('%s').",
- line, path, filename, rt->name, key, rt->exec, value, value);
-
- freez(rt->exec);
- }
- rt->exec = tabs2spaces(strdupz(value));
- }
- else if(hash == hash_recipient && !strcasecmp(key, HEALTH_RECIPIENT_KEY)) {
- if(rt->recipient) {
- if(strcmp(rt->recipient, value))
- error("Health configuration at line %zu of file '%s/%s' for template '%s' has key '%s' twice, once with value '%s' and later with value '%s'. Using ('%s').",
- line, path, filename, rt->name, key, rt->recipient, value, value);
-
- freez(rt->recipient);
- }
- rt->recipient = tabs2spaces(strdupz(value));
- }
- else if(hash == hash_units && !strcasecmp(key, HEALTH_UNITS_KEY)) {
- if(rt->units) {
- if(strcmp(rt->units, value))
- error("Health configuration at line %zu of file '%s/%s' for template '%s' has key '%s' twice, once with value '%s' and later with value '%s'. Using ('%s').",
- line, path, filename, rt->name, key, rt->units, value, value);
-
- freez(rt->units);
- }
- rt->units = tabs2spaces(strdupz(value));
- strip_quotes(rt->units);
- }
- else if(hash == hash_info && !strcasecmp(key, HEALTH_INFO_KEY)) {
- if(rt->info) {
- if(strcmp(rt->info, value))
- error("Health configuration at line %zu of file '%s/%s' for template '%s' has key '%s' twice, once with value '%s' and later with value '%s'. Using ('%s').",
- line, path, filename, rt->name, key, rt->info, value, value);
-
- freez(rt->info);
- }
- rt->info = tabs2spaces(strdupz(value));
- strip_quotes(rt->info);
- }
- else if(hash == hash_delay && !strcasecmp(key, HEALTH_DELAY_KEY)) {
- health_parse_delay(line, path, filename, value, &rt->delay_up_duration, &rt->delay_down_duration, &rt->delay_max_duration, &rt->delay_multiplier);
- }
- else if(hash == hash_options && !strcasecmp(key, HEALTH_OPTIONS_KEY)) {
- rt->options |= health_parse_options(value);
- }
- else {
- error("Health configuration at line %zu of file '%s/%s' for template '%s' has unknown key '%s'.",
- line, path, filename, rt->name, key);
- }
- }
- else {
- error("Health configuration at line %zu of file '%s/%s' has unknown key '%s'. Expected either '" HEALTH_ALARM_KEY "' or '" HEALTH_TEMPLATE_KEY "'.",
- line, path, filename, key);
- }
- }
-
- if(rc && !rrdcalc_add_alarm_from_config(&localhost, rc))
- rrdcalc_free(&localhost, rc);
-
- if(rt && !rrdcalctemplate_add_template_from_config(&localhost, rt))
- rrdcalctemplate_free(&localhost, rt);
-
- fclose(fp);
- return 1;
-}
-
-void health_readdir(const char *path) {
- size_t pathlen = strlen(path);
-
- debug(D_HEALTH, "Health configuration reading directory '%s'", path);
-
- DIR *dir = opendir(path);
- if (!dir) {
- error("Health configuration cannot open directory '%s'.", path);
- return;
- }
-
- struct dirent *de = NULL;
- while ((de = readdir(dir))) {
- size_t len = strlen(de->d_name);
-
- if(de->d_type == DT_DIR
- && (
- (de->d_name[0] == '.' && de->d_name[1] == '\0')
- || (de->d_name[0] == '.' && de->d_name[1] == '.' && de->d_name[2] == '\0')
- )) {
- debug(D_HEALTH, "Ignoring directory '%s'", de->d_name);
- continue;
- }
-
- else if(de->d_type == DT_DIR) {
- char *s = mallocz(pathlen + strlen(de->d_name) + 2);
- strcpy(s, path);
- strcat(s, "/");
- strcat(s, de->d_name);
- health_readdir(s);
- freez(s);
- continue;
- }
-
- else if((de->d_type == DT_LNK || de->d_type == DT_REG || de->d_type == DT_UNKNOWN) &&
- len > 5 && !strcmp(&de->d_name[len - 5], ".conf")) {
- health_readfile(path, de->d_name);
- }
-
- else debug(D_HEALTH, "Ignoring file '%s'", de->d_name);
- }
-
- closedir(dir);
-}
-
-static inline char *health_config_dir(void) {
- char buffer[FILENAME_MAX + 1];
- snprintfz(buffer, FILENAME_MAX, "%s/health.d", config_get("global", "config directory", CONFIG_DIR));
- return config_get("health", "health configuration directory", buffer);
-}
-
-void health_init(void) {
- debug(D_HEALTH, "Health configuration initializing");
-
- if(!(health_enabled = config_get_boolean("health", "enabled", 1))) {
- debug(D_HEALTH, "Health is disabled.");
- return;
- }
-
- char *pathname = config_get("health", "health db directory", VARLIB_DIR "/health");
- if(mkdir(pathname, 0770) == -1 && errno != EEXIST)
- fatal("Cannot create directory '%s'.", pathname);
-
- char filename[FILENAME_MAX + 1];
- snprintfz(filename, FILENAME_MAX, "%s/health-log.db", pathname);
- health.log_filename = config_get("health", "health db file", filename);
-
- health_alarm_log_load(&localhost);
- health_alarm_log_open();
-
- char *path = health_config_dir();
-
- {
- char buffer[FILENAME_MAX + 1];
- snprintfz(buffer, FILENAME_MAX, "%s/alarm-notify.sh", config_get("global", "plugins directory", PLUGINS_DIR));
- health.health_default_exec = config_get("health", "script to execute on alarm", buffer);
- }
-
- long n = config_get_number("health", "in memory max health log entries", (long)localhost.health_log.max);
- if(n < 10) {
- error("Health configuration has invalid max log entries %ld. Using default %u", n, localhost.health_log.max);
- config_set_number("health", "in memory max health log entries", (long)localhost.health_log.max);
- }
- else localhost.health_log.max = (unsigned int)n;
-
- rrdhost_rwlock(&localhost);
- health_readdir(path);
- rrdhost_unlock(&localhost);
-}
-
-// ----------------------------------------------------------------------------
-// JSON generation
-
-static inline void health_string2json(BUFFER *wb, const char *prefix, const char *label, const char *value, const char *suffix) {
- if(value && *value)
- buffer_sprintf(wb, "%s\"%s\":\"%s\"%s", prefix, label, value, suffix);
- else
- buffer_sprintf(wb, "%s\"%s\":null%s", prefix, label, suffix);
-}
-
-static inline void health_alarm_entry2json_nolock(BUFFER *wb, ALARM_ENTRY *ae, RRDHOST *host) {
- buffer_sprintf(wb,
- "\n\t{\n"
- "\t\t\"hostname\": \"%s\",\n"
- "\t\t\"unique_id\": %u,\n"
- "\t\t\"alarm_id\": %u,\n"
- "\t\t\"alarm_event_id\": %u,\n"
- "\t\t\"name\": \"%s\",\n"
- "\t\t\"chart\": \"%s\",\n"
- "\t\t\"family\": \"%s\",\n"
- "\t\t\"processed\": %s,\n"
- "\t\t\"updated\": %s,\n"
- "\t\t\"exec_run\": %lu,\n"
- "\t\t\"exec_failed\": %s,\n"
- "\t\t\"exec\": \"%s\",\n"
- "\t\t\"recipient\": \"%s\",\n"
- "\t\t\"exec_code\": %d,\n"
- "\t\t\"source\": \"%s\",\n"
- "\t\t\"units\": \"%s\",\n"
- "\t\t\"info\": \"%s\",\n"
- "\t\t\"when\": %lu,\n"
- "\t\t\"duration\": %lu,\n"
- "\t\t\"non_clear_duration\": %lu,\n"
- "\t\t\"status\": \"%s\",\n"
- "\t\t\"old_status\": \"%s\",\n"
- "\t\t\"delay\": %d,\n"
- "\t\t\"delay_up_to_timestamp\": %lu,\n"
- "\t\t\"updated_by_id\": %u,\n"
- "\t\t\"updates_id\": %u,\n"
- "\t\t\"value_string\": \"%s\",\n"
- "\t\t\"old_value_string\": \"%s\",\n"
- , host->hostname
- , ae->unique_id
- , ae->alarm_id
- , ae->alarm_event_id
- , ae->name
- , ae->chart
- , ae->family
- , (ae->flags & HEALTH_ENTRY_FLAG_PROCESSED)?"true":"false"
- , (ae->flags & HEALTH_ENTRY_FLAG_UPDATED)?"true":"false"
- , (unsigned long)ae->exec_run_timestamp
- , (ae->flags & HEALTH_ENTRY_FLAG_EXEC_FAILED)?"true":"false"
- , ae->exec?ae->exec:health.health_default_exec
- , ae->recipient?ae->recipient:health.health_default_recipient
- , ae->exec_code
- , ae->source
- , ae->units?ae->units:""
- , ae->info?ae->info:""
- , (unsigned long)ae->when
- , (unsigned long)ae->duration
- , (unsigned long)ae->non_clear_duration
- , rrdcalc_status2string(ae->new_status)
- , rrdcalc_status2string(ae->old_status)
- , ae->delay
- , (unsigned long)ae->delay_up_to_timestamp
- , ae->updated_by_id
- , ae->updates_id
- , ae->new_value_string
- , ae->old_value_string
- );
-
- if(unlikely(ae->flags & HEALTH_ENTRY_FLAG_NO_CLEAR_NOTIFICATION)) {
- buffer_strcat(wb, "\t\t\"no_clear_notification\": true,\n");
- }
-
- buffer_strcat(wb, "\t\t\"value\":");
- buffer_rrd_value(wb, ae->new_value);
- buffer_strcat(wb, ",\n");
-
- buffer_strcat(wb, "\t\t\"old_value\":");
- buffer_rrd_value(wb, ae->old_value);
- buffer_strcat(wb, "\n");
-
- buffer_strcat(wb, "\t}");
-}
-
-void health_alarm_log2json(RRDHOST *host, BUFFER *wb, uint32_t after) {
- pthread_rwlock_rdlock(&host->health_log.alarm_log_rwlock);
-
- buffer_strcat(wb, "[");
-
- unsigned int max = host->health_log.max;
- unsigned int count = 0;
- ALARM_ENTRY *ae;
- for(ae = host->health_log.alarms; ae && count < max ; count++, ae = ae->next) {
- if(ae->unique_id > after) {
- if(likely(count)) buffer_strcat(wb, ",");
- health_alarm_entry2json_nolock(wb, ae, host);
- }
- }
-
- buffer_strcat(wb, "\n]\n");
-
- pthread_rwlock_unlock(&host->health_log.alarm_log_rwlock);
-}
-
-static inline void health_rrdcalc2json_nolock(BUFFER *wb, RRDCALC *rc) {
- char value_string[100 + 1];
- format_value_and_unit(value_string, 100, rc->value, rc->units, -1);
-
- buffer_sprintf(wb,
- "\t\t\"%s.%s\": {\n"
- "\t\t\t\"id\": %lu,\n"
- "\t\t\t\"name\": \"%s\",\n"
- "\t\t\t\"chart\": \"%s\",\n"
- "\t\t\t\"family\": \"%s\",\n"
- "\t\t\t\"active\": %s,\n"
- "\t\t\t\"exec\": \"%s\",\n"
- "\t\t\t\"recipient\": \"%s\",\n"
- "\t\t\t\"source\": \"%s\",\n"
- "\t\t\t\"units\": \"%s\",\n"
- "\t\t\t\"info\": \"%s\",\n"
- "\t\t\t\"status\": \"%s\",\n"
- "\t\t\t\"last_status_change\": %lu,\n"
- "\t\t\t\"last_updated\": %lu,\n"
- "\t\t\t\"next_update\": %lu,\n"
- "\t\t\t\"update_every\": %d,\n"
- "\t\t\t\"delay_up_duration\": %d,\n"
- "\t\t\t\"delay_down_duration\": %d,\n"
- "\t\t\t\"delay_max_duration\": %d,\n"
- "\t\t\t\"delay_multiplier\": %f,\n"
- "\t\t\t\"delay\": %d,\n"
- "\t\t\t\"delay_up_to_timestamp\": %lu,\n"
- "\t\t\t\"value_string\": \"%s\",\n"
- , rc->chart, rc->name
- , (unsigned long)rc->id
- , rc->name
- , rc->chart
- , (rc->rrdset && rc->rrdset->family)?rc->rrdset->family:""
- , (rc->rrdset)?"true":"false"
- , rc->exec?rc->exec:health.health_default_exec
- , rc->recipient?rc->recipient:health.health_default_recipient
- , rc->source
- , rc->units?rc->units:""
- , rc->info?rc->info:""
- , rrdcalc_status2string(rc->status)
- , (unsigned long)rc->last_status_change
- , (unsigned long)rc->last_updated
- , (unsigned long)rc->next_update
- , rc->update_every
- , rc->delay_up_duration
- , rc->delay_down_duration
- , rc->delay_max_duration
- , rc->delay_multiplier
- , rc->delay_last
- , (unsigned long)rc->delay_up_to_timestamp
- , value_string
- );
-
- if(unlikely(rc->options & RRDCALC_FLAG_NO_CLEAR_NOTIFICATION)) {
- buffer_strcat(wb, "\t\t\t\"no_clear_notification\": true,\n");
- }
-
- if(RRDCALC_HAS_DB_LOOKUP(rc)) {
- if(rc->dimensions && *rc->dimensions)
- health_string2json(wb, "\t\t\t", "lookup_dimensions", rc->dimensions, ",\n");
-
- buffer_sprintf(wb,
- "\t\t\t\"db_after\": %lu,\n"
- "\t\t\t\"db_before\": %lu,\n"
- "\t\t\t\"lookup_method\": \"%s\",\n"
- "\t\t\t\"lookup_after\": %d,\n"
- "\t\t\t\"lookup_before\": %d,\n"
- "\t\t\t\"lookup_options\": \"",
- (unsigned long) rc->db_after,
- (unsigned long) rc->db_before,
- group_method2string(rc->group),
- rc->after,
- rc->before
- );
- buffer_data_options2string(wb, rc->options);
- buffer_strcat(wb, "\",\n");
- }
-
- if(rc->calculation) {
- health_string2json(wb, "\t\t\t", "calc", rc->calculation->source, ",\n");
- health_string2json(wb, "\t\t\t", "calc_parsed", rc->calculation->parsed_as, ",\n");
- }
-
- if(rc->warning) {
- health_string2json(wb, "\t\t\t", "warn", rc->warning->source, ",\n");
- health_string2json(wb, "\t\t\t", "warn_parsed", rc->warning->parsed_as, ",\n");
- }
-
- if(rc->critical) {
- health_string2json(wb, "\t\t\t", "crit", rc->critical->source, ",\n");
- health_string2json(wb, "\t\t\t", "crit_parsed", rc->critical->parsed_as, ",\n");
- }
-
- buffer_strcat(wb, "\t\t\t\"green\":");
- buffer_rrd_value(wb, rc->green);
- buffer_strcat(wb, ",\n");
-
- buffer_strcat(wb, "\t\t\t\"red\":");
- buffer_rrd_value(wb, rc->red);
- buffer_strcat(wb, ",\n");
-
- buffer_strcat(wb, "\t\t\t\"value\":");
- buffer_rrd_value(wb, rc->value);
- buffer_strcat(wb, "\n");
-
- buffer_strcat(wb, "\t\t}");
-}
-
-//void health_rrdcalctemplate2json_nolock(BUFFER *wb, RRDCALCTEMPLATE *rt) {
-//
-//}
-
-void health_alarms2json(RRDHOST *host, BUFFER *wb, int all) {
- int i;
-
- rrdhost_rdlock(&localhost);
- buffer_sprintf(wb, "{\n\t\"hostname\": \"%s\","
- "\n\t\"latest_alarm_log_unique_id\": %u,"
- "\n\t\"status\": %s,"
- "\n\t\"now\": %lu,"
- "\n\t\"alarms\": {\n",
- host->hostname,
- (host->health_log.next_log_id > 0)?(host->health_log.next_log_id - 1):0,
- health_enabled?"true":"false",
- (unsigned long)now_realtime_sec());
-
- RRDCALC *rc;
- for(i = 0, rc = host->alarms; rc ; rc = rc->next) {
- if(unlikely(!rc->rrdset || !rc->rrdset->last_collected_time.tv_sec))
- continue;
-
- if(likely(!all && !(rc->status == RRDCALC_STATUS_WARNING || rc->status == RRDCALC_STATUS_CRITICAL)))
- continue;
-
- if(likely(i)) buffer_strcat(wb, ",\n");
- health_rrdcalc2json_nolock(wb, rc);
- i++;
- }
-
-// buffer_strcat(wb, "\n\t},\n\t\"templates\": {");
-// RRDCALCTEMPLATE *rt;
-// for(rt = host->templates; rt ; rt = rt->next)
-// health_rrdcalctemplate2json_nolock(wb, rt);
-
- buffer_strcat(wb, "\n\t}\n}\n");
- rrdhost_unlock(&localhost);
-}
-
-
-// ----------------------------------------------------------------------------
-// re-load health configuration
-
-static inline void health_free_all_nolock(RRDHOST *host) {
- while(host->templates)
- rrdcalctemplate_free(host, host->templates);
-
- while(host->alarms)
- rrdcalc_free(host, host->alarms);
-}
-
-void health_reload(void) {
- if(!health_enabled) {
- error("Health reload is requested, but health is not enabled.");
- return;
- }
-
- char *path = health_config_dir();
-
- // free all running alarms
- rrdhost_rwlock(&localhost);
- health_free_all_nolock(&localhost);
- rrdhost_unlock(&localhost);
-
- // invalidate all previous entries in the alarm log
- ALARM_ENTRY *t;
- for(t = localhost.health_log.alarms ; t ; t = t->next) {
- if(t->new_status != RRDCALC_STATUS_REMOVED)
- t->flags |= HEALTH_ENTRY_FLAG_UPDATED;
- }
-
- // reset all thresholds to all charts
- RRDSET *st;
- for(st = localhost.rrdset_root; st ; st = st->next) {
- st->green = NAN;
- st->red = NAN;
- }
-
- // load the new alarms
- rrdhost_rwlock(&localhost);
- health_readdir(path);
- rrdhost_unlock(&localhost);
-
- // link the loaded alarms to their charts
- for(st = localhost.rrdset_root; st ; st = st->next) {
- rrdhost_rwlock(&localhost);
-
- rrdsetcalc_link_matching(st);
- rrdcalctemplate_link_matching(st);
-
- rrdhost_unlock(&localhost);
- }
-}
-
-// ----------------------------------------------------------------------------
-// health main thread and friends
-
-static inline int rrdcalc_value2status(calculated_number n) {
- if(isnan(n) || isinf(n)) return RRDCALC_STATUS_UNDEFINED;
- if(n) return RRDCALC_STATUS_RAISED;
- return RRDCALC_STATUS_CLEAR;
-}
-
-#define ALARM_EXEC_COMMAND_LENGTH 8192
-
-static inline void health_alarm_execute(RRDHOST *host, ALARM_ENTRY *ae) {
- ae->flags |= HEALTH_ENTRY_FLAG_PROCESSED;
-
- if(unlikely(ae->new_status < RRDCALC_STATUS_CLEAR)) {
- // do not send notifications for internal statuses
- debug(D_HEALTH, "Health not sending notification for alarm '%s.%s' status %s (internal statuses)", ae->chart, ae->name, rrdcalc_status2string(ae->new_status));
- goto done;
- }
-
- if(unlikely(ae->new_status <= RRDCALC_STATUS_CLEAR && (ae->flags & HEALTH_ENTRY_FLAG_NO_CLEAR_NOTIFICATION))) {
- // do not send notifications for disabled statuses
- debug(D_HEALTH, "Health not sending notification for alarm '%s.%s' status %s (it has no-clear-notification enabled)", ae->chart, ae->name, rrdcalc_status2string(ae->new_status));
- // mark it as run, so that we will send the same alarm if it happens again
- goto done;
- }