3 #define RRDVAR_MAX_LENGTH 1024
5 struct health_options {
6 const char *health_default_exec;
7 const char *health_default_recipient;
8 const char *log_filename;
9 size_t log_entries_written;
13 static struct health_options health = {
14 .health_default_exec = PLUGINS_DIR "/alarm-notify.sh",
15 .health_default_recipient = "root",
16 .log_filename = VARLIB_DIR "/health/alarm_log.db",
17 .log_entries_written = 0,
21 int health_enabled = 1;
23 // ----------------------------------------------------------------------------
24 // health alarm log load/save
25 // no need for locking - only one thread is reading / writing the alarms log
27 static inline int health_alarm_log_open(void) {
29 fclose(health.log_fp);
31 health.log_fp = fopen(health.log_filename, "a");
34 if (setvbuf(health.log_fp, NULL, _IOLBF, 0) != 0)
35 error("Health: cannot set line buffering on health log file.");
39 error("Health: cannot open health log file '%s'. Health data will be lost in case of netdata or server crash.", health.log_filename);
43 static inline void health_alarm_log_close(void) {
45 fclose(health.log_fp);
50 static inline void health_log_rotate(void) {
51 static size_t rotate_every = 0;
53 if(unlikely(rotate_every == 0)) {
54 rotate_every = (size_t)config_get_number("health", "rotate log every lines", 2000);
55 if(rotate_every < 100) rotate_every = 100;
58 if(unlikely(health.log_entries_written > rotate_every)) {
59 health_alarm_log_close();
61 char old_filename[FILENAME_MAX + 1];
62 snprintfz(old_filename, FILENAME_MAX, "%s.old", health.log_filename);
64 if(unlink(old_filename) == -1 && errno != ENOENT)
65 error("Health: cannot remove old alarms log file '%s'", old_filename);
67 if(link(health.log_filename, old_filename) == -1 && errno != ENOENT)
68 error("Health: cannot move file '%s' to '%s'.", health.log_filename, old_filename);
70 if(unlink(health.log_filename) == -1 && errno != ENOENT)
71 error("Health: cannot remove old alarms log file '%s'", health.log_filename);
73 // open it with truncate
74 health.log_fp = fopen(health.log_filename, "w");
77 fclose(health.log_fp);
79 error("Health: cannot truncate health log '%s'", health.log_filename);
83 health.log_entries_written = 0;
84 health_alarm_log_open();
88 static inline void health_alarm_log_save(RRDHOST *host, ALARM_ENTRY *ae) {
91 if(likely(health.log_fp)) {
92 if(unlikely(fprintf(health.log_fp
94 "\t%08x\t%08x\t%08x\t%08x\t%08x"
97 "\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s"
101 , (ae->flags & HEALTH_ENTRY_FLAG_SAVED)?'U':'A'
111 , (uint32_t)ae->duration
112 , (uint32_t)ae->non_clear_duration
113 , (uint32_t)ae->flags
114 , (uint32_t)ae->exec_run_timestamp
115 , (uint32_t)ae->delay_up_to_timestamp
117 , (ae->name)?ae->name:""
118 , (ae->chart)?ae->chart:""
119 , (ae->family)?ae->family:""
120 , (ae->exec)?ae->exec:""
121 , (ae->recipient)?ae->recipient:""
122 , (ae->source)?ae->source:""
123 , (ae->units)?ae->units:""
124 , (ae->info)?ae->info:""
131 , (long double)ae->new_value
132 , (long double)ae->old_value
134 error("Health: failed to save alarm log entry. Health data may be lost in case of abnormal restart.");
136 ae->flags |= HEALTH_ENTRY_FLAG_SAVED;
137 health.log_entries_written++;
142 static inline ssize_t health_alarm_log_read(RRDHOST *host, FILE *fp, const char *filename) {
143 static uint32_t max_unique_id = 0, max_alarm_id = 0;
144 ssize_t loaded = -1, updated = -1, errored = -1, duplicate = -1;
148 char *s, *buf = mallocz(65536 + 1);
149 size_t line = 0, len = 0;
150 loaded = updated = errored = duplicate = 0;
152 pthread_rwlock_rdlock(&host->health_log.alarm_log_rwlock);
154 while((s = fgets_trim_len(buf, 65536, fp, &len))) {
155 health.log_entries_written++;
158 int max_entries = 30, entries = 0;
159 char *pointers[max_entries];
161 pointers[entries++] = s++;
163 if(unlikely(*s == '\t')) {
165 pointers[entries++] = ++s;
166 if(entries >= max_entries) {
167 error("Health: line %zu of file '%s' has more than %d entries. Ignoring excessive entries.", line, filename, max_entries);
174 if(likely(*pointers[0] == 'U' || *pointers[0] == 'A')) {
175 ALARM_ENTRY *ae = NULL;
178 error("Health: line %zu of file '%s' should have at least 26 entries, but it has %d. Ignoring it.", line, filename, entries);
183 // check that we have valid ids
184 uint32_t unique_id = (uint32_t)strtoul(pointers[2], NULL, 16);
186 error("Health: line %zu of file '%s' states alarm entry with invalid unique id %u (%s). Ignoring it.", line, filename, unique_id, pointers[2]);
191 uint32_t alarm_id = (uint32_t)strtoul(pointers[3], NULL, 16);
193 error("Health: line %zu of file '%s' states alarm entry for invalid alarm id %u (%s). Ignoring it.", line, filename, alarm_id, pointers[3]);
198 if(unlikely(*pointers[0] == 'A')) {
199 // make sure it is properly numbered
200 if(unlikely(host->health_log.alarms && unique_id < host->health_log.alarms->unique_id)) {
201 error("Health: line %zu of file '%s' has alarm log entry with %u in wrong order. Ignoring it.", line, filename, unique_id);
206 ae = callocz(1, sizeof(ALARM_ENTRY));
208 else if(unlikely(*pointers[0] == 'U')) {
210 for(ae = host->health_log.alarms; ae; ae = ae->next) {
211 if(unlikely(unique_id == ae->unique_id)) {
212 if(unlikely(*pointers[0] == 'A')) {
213 error("Health: line %zu of file '%s' adds duplicate alarm log entry with unique id %u. Using the later."
214 , line, filename, unique_id);
220 else if(unlikely(unique_id > ae->unique_id)) {
221 // no need to continue
222 // the linked list is sorted
228 // if not found, skip this line
230 // error("Health: line %zu of file '%s' updates alarm log entry with unique id %u, but it is not found.", line, filename, unique_id);
235 // check for a possible host missmatch
236 //if(strcmp(pointers[1], host->hostname))
237 // error("Health: line %zu of file '%s' provides an alarm for host '%s' but this is named '%s'.", line, filename, pointers[1], host->hostname);
239 ae->unique_id = unique_id;
240 ae->alarm_id = alarm_id;
241 ae->alarm_event_id = (uint32_t)strtoul(pointers[4], NULL, 16);
242 ae->updated_by_id = (uint32_t)strtoul(pointers[5], NULL, 16);
243 ae->updates_id = (uint32_t)strtoul(pointers[6], NULL, 16);
245 ae->when = (uint32_t)strtoul(pointers[7], NULL, 16);
246 ae->duration = (uint32_t)strtoul(pointers[8], NULL, 16);
247 ae->non_clear_duration = (uint32_t)strtoul(pointers[9], NULL, 16);
249 ae->flags = (uint32_t)strtoul(pointers[10], NULL, 16);
250 ae->flags |= HEALTH_ENTRY_FLAG_SAVED;
252 ae->exec_run_timestamp = (uint32_t)strtoul(pointers[11], NULL, 16);
253 ae->delay_up_to_timestamp = (uint32_t)strtoul(pointers[12], NULL, 16);
255 if(unlikely(ae->name)) freez(ae->name);
256 ae->name = strdupz(pointers[13]);
257 ae->hash_name = simple_hash(ae->name);
259 if(unlikely(ae->chart)) freez(ae->chart);
260 ae->chart = strdupz(pointers[14]);
261 ae->hash_chart = simple_hash(ae->chart);
263 if(unlikely(ae->family)) freez(ae->family);
264 ae->family = strdupz(pointers[15]);
266 if(unlikely(ae->exec)) freez(ae->exec);
267 ae->exec = strdupz(pointers[16]);
268 if(!*ae->exec) { freez(ae->exec); ae->exec = NULL; }
270 if(unlikely(ae->recipient)) freez(ae->recipient);
271 ae->recipient = strdupz(pointers[17]);
272 if(!*ae->recipient) { freez(ae->recipient); ae->recipient = NULL; }
274 if(unlikely(ae->source)) freez(ae->source);
275 ae->source = strdupz(pointers[18]);
276 if(!*ae->source) { freez(ae->source); ae->source = NULL; }
278 if(unlikely(ae->units)) freez(ae->units);
279 ae->units = strdupz(pointers[19]);
280 if(!*ae->units) { freez(ae->units); ae->units = NULL; }
282 if(unlikely(ae->info)) freez(ae->info);
283 ae->info = strdupz(pointers[20]);
284 if(!*ae->info) { freez(ae->info); ae->info = NULL; }
286 ae->exec_code = atoi(pointers[21]);
287 ae->new_status = atoi(pointers[22]);
288 ae->old_status = atoi(pointers[23]);
289 ae->delay = atoi(pointers[24]);
291 ae->new_value = strtold(pointers[25], NULL);
292 ae->old_value = strtold(pointers[26], NULL);
294 // add it to host if not already there
295 if(unlikely(*pointers[0] == 'A')) {
296 ae->next = host->health_log.alarms;
297 host->health_log.alarms = ae;
302 if(unlikely(ae->unique_id > max_unique_id))
303 max_unique_id = ae->unique_id;
305 if(unlikely(ae->alarm_id >= max_alarm_id))
306 max_alarm_id = ae->alarm_id;
309 error("Health: line %zu of file '%s' is invalid (unrecognized entry type '%s').", line, filename, pointers[0]);
314 pthread_rwlock_unlock(&host->health_log.alarm_log_rwlock);
318 if(!max_unique_id) max_unique_id = (uint32_t)time(NULL);
319 if(!max_alarm_id) max_alarm_id = (uint32_t)time(NULL);
321 host->health_log.next_log_id = max_unique_id + 1;
322 host->health_log.next_alarm_id = max_alarm_id + 1;
324 debug(D_HEALTH, "Health: loaded file '%s' with %zd new alarm entries, updated %zd alarms, errors %zd entries, duplicate %zd", filename, loaded, updated, errored, duplicate);
328 static inline void health_alarm_log_load(RRDHOST *host) {
329 health_alarm_log_close();
331 char filename[FILENAME_MAX + 1];
332 snprintfz(filename, FILENAME_MAX, "%s.old", health.log_filename);
333 FILE *fp = fopen(filename, "r");
335 error("Health: cannot open health file: %s", filename);
337 health_alarm_log_read(host, fp, filename);
341 health.log_entries_written = 0;
342 fp = fopen(health.log_filename, "r");
344 error("Health: cannot open health file: %s", health.log_filename);
346 health_alarm_log_read(host, fp, health.log_filename);
350 health_alarm_log_open();
354 // ----------------------------------------------------------------------------
355 // health alarm log management
357 static inline void health_alarm_log(RRDHOST *host,
358 uint32_t alarm_id, uint32_t alarm_event_id,
360 const char *name, const char *chart, const char *family,
361 const char *exec, const char *recipient, time_t duration,
362 calculated_number old_value, calculated_number new_value,
363 int old_status, int new_status,
369 debug(D_HEALTH, "Health adding alarm log entry with id: %u", host->health_log.next_log_id);
371 ALARM_ENTRY *ae = callocz(1, sizeof(ALARM_ENTRY));
372 ae->name = strdupz(name);
373 ae->hash_name = simple_hash(ae->name);
376 ae->chart = strdupz(chart);
377 ae->hash_chart = simple_hash(ae->chart);
381 ae->family = strdupz(family);
383 if(exec) ae->exec = strdupz(exec);
384 if(recipient) ae->recipient = strdupz(recipient);
385 if(source) ae->source = strdupz(source);
386 if(units) ae->units = strdupz(units);
387 if(info) ae->info = strdupz(info);
389 ae->unique_id = host->health_log.next_log_id++;
390 ae->alarm_id = alarm_id;
391 ae->alarm_event_id = alarm_event_id;
393 ae->old_value = old_value;
394 ae->new_value = new_value;
395 ae->old_status = old_status;
396 ae->new_status = new_status;
397 ae->duration = duration;
399 ae->delay_up_to_timestamp = when + delay;
401 if(ae->old_status == RRDCALC_STATUS_WARNING || ae->old_status == RRDCALC_STATUS_CRITICAL)
402 ae->non_clear_duration += ae->duration;
405 pthread_rwlock_wrlock(&host->health_log.alarm_log_rwlock);
406 ae->next = host->health_log.alarms;
407 host->health_log.alarms = ae;
408 host->health_log.count++;
409 pthread_rwlock_unlock(&host->health_log.alarm_log_rwlock);
411 // match previous alarms
412 pthread_rwlock_rdlock(&host->health_log.alarm_log_rwlock);
414 for(t = host->health_log.alarms ; t ; t = t->next) {
415 if(t != ae && t->alarm_id == ae->alarm_id) {
416 if(!(t->flags & HEALTH_ENTRY_FLAG_UPDATED) && !t->updated_by_id) {
417 t->flags |= HEALTH_ENTRY_FLAG_UPDATED;
418 t->updated_by_id = ae->unique_id;
419 ae->updates_id = t->unique_id;
421 if((t->new_status == RRDCALC_STATUS_WARNING || t->new_status == RRDCALC_STATUS_CRITICAL) &&
422 (t->old_status == RRDCALC_STATUS_WARNING || t->old_status == RRDCALC_STATUS_CRITICAL))
423 ae->non_clear_duration += t->non_clear_duration;
425 health_alarm_log_save(host, t);
428 // no need to continue
432 pthread_rwlock_unlock(&host->health_log.alarm_log_rwlock);
434 health_alarm_log_save(host, ae);
437 // ----------------------------------------------------------------------------
440 static inline int rrdvar_fix_name(char *variable) {
443 if (!isalnum(*variable) && *variable != '.' && *variable != '_') {
454 int rrdvar_compare(void* a, void* b) {
455 if(((RRDVAR *)a)->hash < ((RRDVAR *)b)->hash) return -1;
456 else if(((RRDVAR *)a)->hash > ((RRDVAR *)b)->hash) return 1;
457 else return strcmp(((RRDVAR *)a)->name, ((RRDVAR *)b)->name);
460 static inline RRDVAR *rrdvar_index_add(avl_tree_lock *tree, RRDVAR *rv) {
461 RRDVAR *ret = (RRDVAR *)avl_insert_lock(tree, (avl *)(rv));
463 debug(D_VARIABLES, "Request to insert RRDVAR '%s' into index failed. Already exists.", rv->name);
468 static inline RRDVAR *rrdvar_index_del(avl_tree_lock *tree, RRDVAR *rv) {
469 RRDVAR *ret = (RRDVAR *)avl_remove_lock(tree, (avl *)(rv));
471 error("Request to remove RRDVAR '%s' from index failed. Not Found.", rv->name);
476 static inline RRDVAR *rrdvar_index_find(avl_tree_lock *tree, const char *name, uint32_t hash) {
478 tmp.name = (char *)name;
479 tmp.hash = (hash)?hash:simple_hash(tmp.name);
481 return (RRDVAR *)avl_search_lock(tree, (avl *)&tmp);
484 static inline void rrdvar_free(RRDHOST *host, avl_tree_lock *tree, RRDVAR *rv) {
490 rrdvar_index_del(tree, rv);
496 static inline RRDVAR *rrdvar_create_and_index(const char *scope, avl_tree_lock *tree, const char *name, int type, calculated_number *value) {
497 char *variable = strdupz(name);
498 rrdvar_fix_name(variable);
499 uint32_t hash = simple_hash(variable);
501 RRDVAR *rv = rrdvar_index_find(tree, variable, hash);
503 debug(D_VARIABLES, "Variable '%s' not found in scope '%s'. Creating a new one.", variable, scope);
505 rv = callocz(1, sizeof(RRDVAR));
511 RRDVAR *ret = rrdvar_index_add(tree, rv);
512 if(unlikely(ret != rv)) {
513 debug(D_VARIABLES, "Variable '%s' in scope '%s' already exists", variable, scope);
514 rrdvar_free(NULL, NULL, rv);
518 debug(D_VARIABLES, "Variable '%s' created in scope '%s'", variable, scope);
529 // ----------------------------------------------------------------------------
532 calculated_number rrdvar2number(RRDVAR *rv) {
534 case RRDVAR_TYPE_CALCULATED: {
535 calculated_number *n = (calculated_number *)rv->value;
539 case RRDVAR_TYPE_TIME_T: {
540 time_t *n = (time_t *)rv->value;
544 case RRDVAR_TYPE_COLLECTED: {
545 collected_number *n = (collected_number *)rv->value;
549 case RRDVAR_TYPE_TOTAL: {
550 total_number *n = (total_number *)rv->value;
554 case RRDVAR_TYPE_INT: {
555 int *n = (int *)rv->value;
560 error("I don't know how to convert RRDVAR type %d to calculated_number", rv->type);
565 void dump_variable(void *data) {
566 RRDVAR *rv = (RRDVAR *)data;
567 debug(D_HEALTH, "%50s : %20.5Lf", rv->name, rrdvar2number(rv));
570 int health_variable_lookup(const char *variable, uint32_t hash, RRDCALC *rc, calculated_number *result) {
571 RRDSET *st = rc->rrdset;
576 rv = rrdvar_index_find(&st->variables_root_index, variable, hash);
578 *result = rrdvar2number(rv);
582 rv = rrdvar_index_find(&st->rrdfamily->variables_root_index, variable, hash);
584 *result = rrdvar2number(rv);
588 rv = rrdvar_index_find(&st->rrdhost->variables_root_index, variable, hash);
590 *result = rrdvar2number(rv);
594 debug(D_HEALTH, "Available local chart '%s' variables:", st->id);
595 avl_traverse_lock(&st->variables_root_index, dump_variable);
597 debug(D_HEALTH, "Available family '%s' variables:", st->rrdfamily->family);
598 avl_traverse_lock(&st->rrdfamily->variables_root_index, dump_variable);
600 debug(D_HEALTH, "Available host '%s' variables:", st->rrdhost->hostname);
601 avl_traverse_lock(&st->rrdhost->variables_root_index, dump_variable);
606 // ----------------------------------------------------------------------------
607 // RRDSETVAR management
609 RRDSETVAR *rrdsetvar_create(RRDSET *st, const char *variable, int type, void *value, uint32_t options) {
610 debug(D_VARIABLES, "RRDVARSET create for chart id '%s' name '%s' with variable name '%s'", st->id, st->name, variable);
611 RRDSETVAR *rs = (RRDSETVAR *)callocz(1, sizeof(RRDSETVAR));
613 char buffer[RRDVAR_MAX_LENGTH + 1];
614 snprintfz(buffer, RRDVAR_MAX_LENGTH, "%s.%s", st->id, variable);
615 rs->fullid = strdupz(buffer);
617 snprintfz(buffer, RRDVAR_MAX_LENGTH, "%s.%s", st->name, variable);
618 rs->fullname = strdupz(buffer);
620 rs->variable = strdupz(variable);
624 rs->options = options;
627 rs->local = rrdvar_create_and_index("local", &st->variables_root_index, rs->variable, rs->type, rs->value);
628 rs->family = rrdvar_create_and_index("family", &st->rrdfamily->variables_root_index, rs->fullid, rs->type, rs->value);
629 rs->host = rrdvar_create_and_index("host", &st->rrdhost->variables_root_index, rs->fullid, rs->type, rs->value);
630 rs->family_name = rrdvar_create_and_index("family", &st->rrdfamily->variables_root_index, rs->fullname, rs->type, rs->value);
631 rs->host_name = rrdvar_create_and_index("host", &st->rrdhost->variables_root_index, rs->fullname, rs->type, rs->value);
633 rs->next = st->variables;
639 void rrdsetvar_rename_all(RRDSET *st) {
640 debug(D_VARIABLES, "RRDSETVAR rename for chart id '%s' name '%s'", st->id, st->name);
642 // only these 2 can change name
646 char buffer[RRDVAR_MAX_LENGTH + 1];
647 RRDSETVAR *rs, *next = st->variables;
651 snprintfz(buffer, RRDVAR_MAX_LENGTH, "%s.%s", st->name, rs->variable);
653 if (strcmp(buffer, rs->fullname)) {
655 rrdvar_free(st->rrdhost, &st->rrdfamily->variables_root_index, rs->family_name);
656 rrdvar_free(st->rrdhost, &st->rrdhost->variables_root_index, rs->host_name);
659 rs->fullname = strdupz(st->name);
660 rs->family_name = rrdvar_create_and_index("family", &st->rrdfamily->variables_root_index, rs->fullname, rs->type, rs->value);
661 rs->host_name = rrdvar_create_and_index("host", &st->rrdhost->variables_root_index, rs->fullname, rs->type, rs->value);
665 rrdsetcalc_link_matching(st);
668 void rrdsetvar_free(RRDSETVAR *rs) {
669 RRDSET *st = rs->rrdset;
670 debug(D_VARIABLES, "RRDSETVAR free for chart id '%s' name '%s', variable '%s'", st->id, st->name, rs->variable);
672 rrdvar_free(st->rrdhost, &st->variables_root_index, rs->local);
673 rrdvar_free(st->rrdhost, &st->rrdfamily->variables_root_index, rs->family);
674 rrdvar_free(st->rrdhost, &st->rrdhost->variables_root_index, rs->host);
675 rrdvar_free(st->rrdhost, &st->rrdfamily->variables_root_index, rs->family_name);
676 rrdvar_free(st->rrdhost, &st->rrdhost->variables_root_index, rs->host_name);
678 if(st->variables == rs) {
679 st->variables = rs->next;
683 for (t = st->variables; t && t->next != rs; t = t->next);
684 if(!t) error("RRDSETVAR '%s' not found in chart '%s' variables linked list", rs->fullname, st->id);
685 else t->next = rs->next;
694 // ----------------------------------------------------------------------------
695 // RRDDIMVAR management
697 #define RRDDIMVAR_ID_MAX 1024
699 RRDDIMVAR *rrddimvar_create(RRDDIM *rd, int type, const char *prefix, const char *suffix, void *value, uint32_t options) {
700 RRDSET *st = rd->rrdset;
702 debug(D_VARIABLES, "RRDDIMSET create for chart id '%s' name '%s', dimension id '%s', name '%s%s%s'", st->id, st->name, rd->id, (prefix)?prefix:"", rd->name, (suffix)?suffix:"");
704 if(!prefix) prefix = "";
705 if(!suffix) suffix = "";
707 char buffer[RRDDIMVAR_ID_MAX + 1];
708 RRDDIMVAR *rs = (RRDDIMVAR *)callocz(1, sizeof(RRDDIMVAR));
710 rs->prefix = strdupz(prefix);
711 rs->suffix = strdupz(suffix);
713 snprintfz(buffer, RRDDIMVAR_ID_MAX, "%s%s%s", rs->prefix, rd->id, rs->suffix);
714 rs->id = strdupz(buffer);
716 snprintfz(buffer, RRDDIMVAR_ID_MAX, "%s%s%s", rs->prefix, rd->name, rs->suffix);
717 rs->name = strdupz(buffer);
719 snprintfz(buffer, RRDDIMVAR_ID_MAX, "%s.%s", rd->rrdset->id, rs->id);
720 rs->fullidid = strdupz(buffer);
722 snprintfz(buffer, RRDDIMVAR_ID_MAX, "%s.%s", rd->rrdset->id, rs->name);
723 rs->fullidname = strdupz(buffer);
725 snprintfz(buffer, RRDDIMVAR_ID_MAX, "%s.%s", rd->rrdset->name, rs->id);
726 rs->fullnameid = strdupz(buffer);
728 snprintfz(buffer, RRDDIMVAR_ID_MAX, "%s.%s", rd->rrdset->name, rs->name);
729 rs->fullnamename = strdupz(buffer);
733 rs->options = options;
736 rs->local_id = rrdvar_create_and_index("local", &st->variables_root_index, rs->id, rs->type, rs->value);
737 rs->local_name = rrdvar_create_and_index("local", &st->variables_root_index, rs->name, rs->type, rs->value);
739 rs->family_id = rrdvar_create_and_index("family", &st->rrdfamily->variables_root_index, rs->id, rs->type, rs->value);
740 rs->family_name = rrdvar_create_and_index("family", &st->rrdfamily->variables_root_index, rs->name, rs->type, rs->value);
742 rs->host_fullidid = rrdvar_create_and_index("host", &st->rrdhost->variables_root_index, rs->fullidid, rs->type, rs->value);
743 rs->host_fullidname = rrdvar_create_and_index("host", &st->rrdhost->variables_root_index, rs->fullidname, rs->type, rs->value);
744 rs->host_fullnameid = rrdvar_create_and_index("host", &st->rrdhost->variables_root_index, rs->fullnameid, rs->type, rs->value);
745 rs->host_fullnamename = rrdvar_create_and_index("host", &st->rrdhost->variables_root_index, rs->fullnamename, rs->type, rs->value);
747 rs->next = rd->variables;
753 void rrddimvar_rename_all(RRDDIM *rd) {
754 RRDSET *st = rd->rrdset;
755 debug(D_VARIABLES, "RRDDIMSET rename for chart id '%s' name '%s', dimension id '%s', name '%s'", st->id, st->name, rd->id, rd->name);
757 RRDDIMVAR *rs, *next = rd->variables;
761 if (strcmp(rd->name, rs->name)) {
762 char buffer[RRDDIMVAR_ID_MAX + 1];
766 rrdvar_free(st->rrdhost, &st->variables_root_index, rs->local_name);
768 snprintfz(buffer, RRDDIMVAR_ID_MAX, "%s%s%s", rs->prefix, rd->name, rs->suffix);
769 rs->name = strdupz(buffer);
770 rs->local_name = rrdvar_create_and_index("local", &st->variables_root_index, rs->name, rs->type, rs->value);
772 rrdvar_free(st->rrdhost, &st->rrdhost->variables_root_index, rs->host_fullidname);
773 freez(rs->fullidname);
774 snprintfz(buffer, RRDDIMVAR_ID_MAX, "%s.%s", st->id, rs->name);
775 rs->fullidname = strdupz(buffer);
776 rs->host_fullidname = rrdvar_create_and_index("host", &st->rrdhost->variables_root_index,
777 rs->fullidname, rs->type, rs->value);
780 rrdvar_free(st->rrdhost, &st->rrdhost->variables_root_index, rs->host_fullnameid);
781 freez(rs->fullnameid);
782 snprintfz(buffer, RRDDIMVAR_ID_MAX, "%s.%s", st->name, rs->id);
783 rs->fullnameid = strdupz(buffer);
784 rs->host_fullnameid = rrdvar_create_and_index("host", &st->rrdhost->variables_root_index,
785 rs->fullnameid, rs->type, rs->value);
788 rrdvar_free(st->rrdhost, &st->rrdhost->variables_root_index, rs->host_fullnamename);
789 freez(rs->fullnamename);
790 snprintfz(buffer, RRDDIMVAR_ID_MAX, "%s.%s", st->name, rs->name);
791 rs->fullnamename = strdupz(buffer);
792 rs->host_fullnamename = rrdvar_create_and_index("host", &st->rrdhost->variables_root_index,
793 rs->fullnamename, rs->type, rs->value);
798 void rrddimvar_free(RRDDIMVAR *rs) {
799 RRDDIM *rd = rs->rrddim;
800 RRDSET *st = rd->rrdset;
801 debug(D_VARIABLES, "RRDDIMSET free for chart id '%s' name '%s', dimension id '%s', name '%s', prefix='%s', suffix='%s'", st->id, st->name, rd->id, rd->name, rs->prefix, rs->suffix);
803 rrdvar_free(st->rrdhost, &st->variables_root_index, rs->local_id);
804 rrdvar_free(st->rrdhost, &st->variables_root_index, rs->local_name);
806 rrdvar_free(st->rrdhost, &st->rrdfamily->variables_root_index, rs->family_id);
807 rrdvar_free(st->rrdhost, &st->rrdfamily->variables_root_index, rs->family_name);
809 rrdvar_free(st->rrdhost, &st->rrdhost->variables_root_index, rs->host_fullidid);
810 rrdvar_free(st->rrdhost, &st->rrdhost->variables_root_index, rs->host_fullidname);
811 rrdvar_free(st->rrdhost, &st->rrdhost->variables_root_index, rs->host_fullnameid);
812 rrdvar_free(st->rrdhost, &st->rrdhost->variables_root_index, rs->host_fullnamename);
814 if(rd->variables == rs) {
815 debug(D_VARIABLES, "RRDDIMSET removing first entry for chart id '%s' name '%s', dimension id '%s', name '%s'", st->id, st->name, rd->id, rd->name);
816 rd->variables = rs->next;
819 debug(D_VARIABLES, "RRDDIMSET removing non-first entry for chart id '%s' name '%s', dimension id '%s', name '%s'", st->id, st->name, rd->id, rd->name);
821 for (t = rd->variables; t && t->next != rs; t = t->next) ;
822 if(!t) error("RRDDIMVAR '%s' not found in dimension '%s/%s' variables linked list", rs->name, st->id, rd->id);
823 else t->next = rs->next;
831 freez(rs->fullidname);
832 freez(rs->fullnameid);
833 freez(rs->fullnamename);
837 // ----------------------------------------------------------------------------
838 // RRDCALC management
840 static inline const char *rrdcalc_status2string(int status) {
842 case RRDCALC_STATUS_REMOVED:
845 case RRDCALC_STATUS_UNDEFINED:
848 case RRDCALC_STATUS_UNINITIALIZED:
849 return "UNINITIALIZED";
851 case RRDCALC_STATUS_CLEAR:
854 case RRDCALC_STATUS_RAISED:
857 case RRDCALC_STATUS_WARNING:
860 case RRDCALC_STATUS_CRITICAL:
864 error("Unknown alarm status %d", status);
869 static void rrdsetcalc_link(RRDSET *st, RRDCALC *rc) {
870 debug(D_HEALTH, "Health linking alarm '%s.%s' to chart '%s' of host '%s'", rc->chart?rc->chart:"NOCHART", rc->name, st->id, st->rrdhost->hostname);
872 rc->last_status_change = time(NULL);
875 rc->rrdset_next = st->alarms;
876 rc->rrdset_prev = NULL;
879 rc->rrdset_next->rrdset_prev = rc;
883 if(rc->update_every < rc->rrdset->update_every) {
884 error("Health alarm '%s.%s' has update every %d, less than chart update every %d. Setting alarm update frequency to %d.", rc->rrdset->id, rc->name, rc->update_every, rc->rrdset->update_every, rc->rrdset->update_every);
885 rc->update_every = rc->rrdset->update_every;
888 if(!isnan(rc->green) && isnan(st->green)) {
889 debug(D_HEALTH, "Health alarm '%s.%s' green threshold set from %Lf to %Lf.", rc->rrdset->id, rc->name, rc->rrdset->green, rc->green);
890 st->green = rc->green;
893 if(!isnan(rc->red) && isnan(st->red)) {
894 debug(D_HEALTH, "Health alarm '%s.%s' red threshold set from %Lf to %Lf.", rc->rrdset->id, rc->name, rc->rrdset->red, rc->red);
898 rc->local = rrdvar_create_and_index("local", &st->variables_root_index, rc->name, RRDVAR_TYPE_CALCULATED, &rc->value);
899 rc->family = rrdvar_create_and_index("family", &st->rrdfamily->variables_root_index, rc->name, RRDVAR_TYPE_CALCULATED, &rc->value);
901 char fullname[RRDVAR_MAX_LENGTH + 1];
902 snprintfz(fullname, RRDVAR_MAX_LENGTH, "%s.%s", st->id, rc->name);
903 rc->hostid = rrdvar_create_and_index("host", &st->rrdhost->variables_root_index, fullname, RRDVAR_TYPE_CALCULATED, &rc->value);
905 snprintfz(fullname, RRDVAR_MAX_LENGTH, "%s.%s", st->name, rc->name);
906 rc->hostname = rrdvar_create_and_index("host", &st->rrdhost->variables_root_index, fullname, RRDVAR_TYPE_CALCULATED, &rc->value);
908 if(!rc->units) rc->units = strdupz(st->units);
911 time_t now = time(NULL);
912 health_alarm_log(st->rrdhost, rc->id, rc->next_event_id++, now, rc->name, rc->rrdset->id, rc->rrdset->family, rc->exec, rc->recipient, now - rc->last_status_change, rc->old_value, rc->value, rc->status, RRDCALC_STATUS_UNINITIALIZED, rc->source, rc->units, rc->info, 0);
916 static inline int rrdcalc_is_matching_this_rrdset(RRDCALC *rc, RRDSET *st) {
917 if( (rc->hash_chart == st->hash && !strcmp(rc->chart, st->id)) ||
918 (rc->hash_chart == st->hash_name && !strcmp(rc->chart, st->name)))
924 // this has to be called while the RRDHOST is locked
925 inline void rrdsetcalc_link_matching(RRDSET *st) {
926 // debug(D_HEALTH, "find matching alarms for chart '%s'", st->id);
929 for(rc = st->rrdhost->alarms; rc ; rc = rc->next) {
930 if(unlikely(rc->rrdset))
933 if(unlikely(rrdcalc_is_matching_this_rrdset(rc, st)))
934 rrdsetcalc_link(st, rc);
938 // this has to be called while the RRDHOST is locked
939 inline void rrdsetcalc_unlink(RRDCALC *rc) {
940 RRDSET *st = rc->rrdset;
943 debug(D_HEALTH, "Requested to unlink RRDCALC '%s.%s' which is not linked to any RRDSET", rc->chart?rc->chart:"NOCHART", rc->name);
944 error("Requested to unlink RRDCALC '%s.%s' which is not linked to any RRDSET", rc->chart?rc->chart:"NOCHART", rc->name);
949 time_t now = time(NULL);
950 health_alarm_log(st->rrdhost, rc->id, rc->next_event_id++, now, rc->name, rc->rrdset->id, rc->rrdset->family, rc->exec, rc->recipient, now - rc->last_status_change, rc->old_value, rc->value, rc->status, RRDCALC_STATUS_REMOVED, rc->source, rc->units, rc->info, 0);
953 RRDHOST *host = st->rrdhost;
955 debug(D_HEALTH, "Health unlinking alarm '%s.%s' from chart '%s' of host '%s'", rc->chart?rc->chart:"NOCHART", rc->name, st->id, host->hostname);
959 rc->rrdset_prev->rrdset_next = rc->rrdset_next;
962 rc->rrdset_next->rrdset_prev = rc->rrdset_prev;
965 st->alarms = rc->rrdset_next;
967 rc->rrdset_prev = rc->rrdset_next = NULL;
969 rrdvar_free(st->rrdhost, &st->variables_root_index, rc->local);
972 rrdvar_free(st->rrdhost, &st->rrdfamily->variables_root_index, rc->family);
975 rrdvar_free(st->rrdhost, &st->rrdhost->variables_root_index, rc->hostid);
978 rrdvar_free(st->rrdhost, &st->rrdhost->variables_root_index, rc->hostname);
983 // RRDCALC will remain in RRDHOST
984 // so that if the matching chart is found in the future
985 // it will be applied automatically
988 RRDCALC *rrdcalc_find(RRDSET *st, const char *name) {
990 uint32_t hash = simple_hash(name);
992 for( rc = st->alarms; rc ; rc = rc->rrdset_next ) {
993 if(unlikely(rc->hash == hash && !strcmp(rc->name, name)))
1000 static inline int rrdcalc_exists(RRDHOST *host, const char *chart, const char *name, uint32_t hash_chart, uint32_t hash_name) {
1003 if(unlikely(!chart)) {
1004 error("attempt to find RRDCALC '%s' without giving a chart name", name);
1008 if(unlikely(!hash_chart)) hash_chart = simple_hash(chart);
1009 if(unlikely(!hash_name)) hash_name = simple_hash(name);
1011 // make sure it does not already exist
1012 for(rc = host->alarms; rc ; rc = rc->next) {
1013 if (unlikely(rc->chart && rc->hash == hash_name && rc->hash_chart == hash_chart && !strcmp(name, rc->name) && !strcmp(chart, rc->chart))) {
1014 debug(D_HEALTH, "Health alarm '%s.%s' already exists in host '%s'.", chart, name, host->hostname);
1015 error("Health alarm '%s.%s' already exists in host '%s'.", chart, name, host->hostname);
1023 static inline uint32_t rrdcalc_get_unique_id(RRDHOST *host, const char *chart, const char *name, uint32_t *next_event_id) {
1025 uint32_t hash_chart = simple_hash(chart);
1026 uint32_t hash_name = simple_hash(name);
1028 // re-use old IDs, by looking them up in the alarm log
1030 for(ae = host->health_log.alarms; ae ;ae = ae->next) {
1031 if(unlikely(ae->hash_name == hash_name && ae->hash_chart == hash_chart && !strcmp(name, ae->name) && !strcmp(chart, ae->chart))) {
1032 if(next_event_id) *next_event_id = ae->alarm_event_id + 1;
1033 return ae->alarm_id;
1038 return host->health_log.next_alarm_id++;
1041 static inline void rrdcalc_create_part2(RRDHOST *host, RRDCALC *rc) {
1042 rrdhost_check_rdlock(host);
1044 if(rc->calculation) {
1045 rc->calculation->status = &rc->status;
1046 rc->calculation->this = &rc->value;
1047 rc->calculation->after = &rc->db_after;
1048 rc->calculation->before = &rc->db_before;
1049 rc->calculation->rrdcalc = rc;
1053 rc->warning->status = &rc->status;
1054 rc->warning->this = &rc->value;
1055 rc->warning->after = &rc->db_after;
1056 rc->warning->before = &rc->db_before;
1057 rc->warning->rrdcalc = rc;
1061 rc->critical->status = &rc->status;
1062 rc->critical->this = &rc->value;
1063 rc->critical->after = &rc->db_after;
1064 rc->critical->before = &rc->db_before;
1065 rc->critical->rrdcalc = rc;
1068 // link it to the host
1069 if(likely(host->alarms)) {
1072 for(t = host->alarms; t && t->next ; t = t->next) ;
1079 // link it to its chart
1081 for(st = host->rrdset_root; st ; st = st->next) {
1082 if(rrdcalc_is_matching_this_rrdset(rc, st)) {
1083 rrdsetcalc_link(st, rc);
1089 static inline RRDCALC *rrdcalc_create(RRDHOST *host, RRDCALCTEMPLATE *rt, const char *chart) {
1091 debug(D_HEALTH, "Health creating dynamic alarm (from template) '%s.%s'", chart, rt->name);
1093 if(rrdcalc_exists(host, chart, rt->name, 0, 0))
1096 RRDCALC *rc = callocz(1, sizeof(RRDCALC));
1097 rc->next_event_id = 1;
1098 rc->id = rrdcalc_get_unique_id(host, chart, rt->name, &rc->next_event_id);
1099 rc->name = strdupz(rt->name);
1100 rc->hash = simple_hash(rc->name);
1101 rc->chart = strdupz(chart);
1102 rc->hash_chart = simple_hash(rc->chart);
1104 if(rt->dimensions) rc->dimensions = strdupz(rt->dimensions);
1106 rc->green = rt->green;
1109 rc->old_value = NAN;
1111 rc->delay_up_duration = rt->delay_up_duration;
1112 rc->delay_down_duration = rt->delay_down_duration;
1113 rc->delay_max_duration = rt->delay_max_duration;
1114 rc->delay_multiplier = rt->delay_multiplier;
1116 rc->group = rt->group;
1117 rc->after = rt->after;
1118 rc->before = rt->before;
1119 rc->update_every = rt->update_every;
1120 rc->options = rt->options;
1122 if(rt->exec) rc->exec = strdupz(rt->exec);
1123 if(rt->recipient) rc->recipient = strdupz(rt->recipient);
1124 if(rt->source) rc->source = strdupz(rt->source);
1125 if(rt->units) rc->units = strdupz(rt->units);
1126 if(rt->info) rc->info = strdupz(rt->info);
1128 if(rt->calculation) {
1129 rc->calculation = expression_parse(rt->calculation->source, NULL, NULL);
1130 if(!rc->calculation)
1131 error("Health alarm '%s.%s': failed to parse calculation expression '%s'", chart, rt->name, rt->calculation->source);
1134 rc->warning = expression_parse(rt->warning->source, NULL, NULL);
1136 error("Health alarm '%s.%s': failed to re-parse warning expression '%s'", chart, rt->name, rt->warning->source);
1139 rc->critical = expression_parse(rt->critical->source, NULL, NULL);
1141 error("Health alarm '%s.%s': failed to re-parse critical expression '%s'", chart, rt->name, rt->critical->source);
1144 debug(D_HEALTH, "Health runtime added alarm '%s.%s': exec '%s', recipient '%s', green %Lf, red %Lf, lookup: group %d, after %d, before %d, options %u, dimensions '%s', update every %d, calculation '%s', warning '%s', critical '%s', source '%s', delay up %d, delay down %d, delay max %d, delay_multiplier %f",
1145 (rc->chart)?rc->chart:"NOCHART",
1147 (rc->exec)?rc->exec:"DEFAULT",
1148 (rc->recipient)?rc->recipient:"DEFAULT",
1155 (rc->dimensions)?rc->dimensions:"NONE",
1157 (rc->calculation)?rc->calculation->parsed_as:"NONE",
1158 (rc->warning)?rc->warning->parsed_as:"NONE",
1159 (rc->critical)?rc->critical->parsed_as:"NONE",
1161 rc->delay_up_duration,
1162 rc->delay_down_duration,
1163 rc->delay_max_duration,
1164 rc->delay_multiplier
1167 rrdcalc_create_part2(host, rc);
1171 void rrdcalc_free(RRDHOST *host, RRDCALC *rc) {
1174 debug(D_HEALTH, "Health removing alarm '%s.%s' of host '%s'", rc->chart?rc->chart:"NOCHART", rc->name, host->hostname);
1176 // unlink it from RRDSET
1177 if(rc->rrdset) rrdsetcalc_unlink(rc);
1179 // unlink it from RRDHOST
1180 if(unlikely(rc == host->alarms))
1181 host->alarms = rc->next;
1183 else if(likely(host->alarms)) {
1184 RRDCALC *t, *last = host->alarms;
1185 for(t = last->next; t && t != rc; last = t, t = t->next) ;
1186 if(last->next == rc)
1187 last->next = rc->next;
1189 error("Cannot unlink alarm '%s.%s' from host '%s': not found", rc->chart?rc->chart:"NOCHART", rc->name, host->hostname);
1192 error("Cannot unlink unlink '%s.%s' from host '%s': This host does not have any calculations", rc->chart?rc->chart:"NOCHART", rc->name, host->hostname);
1194 expression_free(rc->calculation);
1195 expression_free(rc->warning);
1196 expression_free(rc->critical);
1201 freez(rc->dimensions);
1203 freez(rc->recipient);
1210 // ----------------------------------------------------------------------------
1211 // RRDCALCTEMPLATE management
1213 void rrdcalctemplate_link_matching(RRDSET *st) {
1214 RRDCALCTEMPLATE *rt;
1216 for(rt = st->rrdhost->templates; rt ; rt = rt->next) {
1217 if(rt->hash_context == st->hash_context && !strcmp(rt->context, st->context)) {
1218 RRDCALC *rc = rrdcalc_create(st->rrdhost, rt, st->id);
1220 error("Health tried to create alarm from template '%s', but it failed", rt->name);
1222 #ifdef NETDATA_INTERNAL_CHECKS
1223 else if(rc->rrdset != st)
1224 error("Health alarm '%s.%s' should be linked to chart '%s', but it is not", rc->chart?rc->chart:"NOCHART", rc->name, st->id);
1230 static inline void rrdcalctemplate_free(RRDHOST *host, RRDCALCTEMPLATE *rt) {
1231 debug(D_HEALTH, "Health removing template '%s' of host '%s'", rt->name, host->hostname);
1233 if(host->templates) {
1234 if(host->templates == rt) {
1235 host->templates = rt->next;
1238 RRDCALCTEMPLATE *t, *last = host->templates;
1239 for (t = last->next; t && t != rt; last = t, t = t->next ) ;
1240 if(last && last->next == rt) {
1241 last->next = rt->next;
1245 error("Cannot find RRDCALCTEMPLATE '%s' linked in host '%s'", rt->name, host->hostname);
1249 expression_free(rt->calculation);
1250 expression_free(rt->warning);
1251 expression_free(rt->critical);
1255 freez(rt->recipient);
1260 freez(rt->dimensions);
1264 // ----------------------------------------------------------------------------
1265 // load health configuration
1267 #define HEALTH_CONF_MAX_LINE 4096
1269 #define HEALTH_ALARM_KEY "alarm"
1270 #define HEALTH_TEMPLATE_KEY "template"
1271 #define HEALTH_ON_KEY "on"
1272 #define HEALTH_LOOKUP_KEY "lookup"
1273 #define HEALTH_CALC_KEY "calc"
1274 #define HEALTH_EVERY_KEY "every"
1275 #define HEALTH_GREEN_KEY "green"
1276 #define HEALTH_RED_KEY "red"
1277 #define HEALTH_WARN_KEY "warn"
1278 #define HEALTH_CRIT_KEY "crit"
1279 #define HEALTH_EXEC_KEY "exec"
1280 #define HEALTH_RECIPIENT_KEY "to"
1281 #define HEALTH_UNITS_KEY "units"
1282 #define HEALTH_INFO_KEY "info"
1283 #define HEALTH_DELAY_KEY "delay"
1285 static inline int rrdcalc_add_alarm_from_config(RRDHOST *host, RRDCALC *rc) {
1287 error("Health configuration for alarm '%s' does not have a chart", rc->name);
1291 if(!rc->update_every) {
1292 error("Health configuration for alarm '%s.%s' has no frequency (parameter 'every'). Ignoring it.", rc->chart?rc->chart:"NOCHART", rc->name);
1296 if(!RRDCALC_HAS_DB_LOOKUP(rc) && !rc->warning && !rc->critical) {
1297 error("Health configuration for alarm '%s.%s' is useless (no calculation, no warning and no critical evaluation)", rc->chart?rc->chart:"NOCHART", rc->name);
1301 if (rrdcalc_exists(host, rc->chart, rc->name, rc->hash_chart, rc->hash))
1304 rc->id = rrdcalc_get_unique_id(&localhost, rc->chart, rc->name, &rc->next_event_id);
1306 debug(D_HEALTH, "Health configuration adding alarm '%s.%s' (%u): exec '%s', recipient '%s', green %Lf, red %Lf, lookup: group %d, after %d, before %d, options %u, dimensions '%s', update every %d, calculation '%s', warning '%s', critical '%s', source '%s', delay up %d, delay down %d, delay max %d, delay_multiplier %f",
1307 rc->chart?rc->chart:"NOCHART",
1310 (rc->exec)?rc->exec:"DEFAULT",
1311 (rc->recipient)?rc->recipient:"DEFAULT",
1318 (rc->dimensions)?rc->dimensions:"NONE",
1320 (rc->calculation)?rc->calculation->parsed_as:"NONE",
1321 (rc->warning)?rc->warning->parsed_as:"NONE",
1322 (rc->critical)?rc->critical->parsed_as:"NONE",
1324 rc->delay_up_duration,
1325 rc->delay_down_duration,
1326 rc->delay_max_duration,
1327 rc->delay_multiplier
1330 rrdcalc_create_part2(host, rc);
1334 static inline int rrdcalctemplate_add_template_from_config(RRDHOST *host, RRDCALCTEMPLATE *rt) {
1335 if(unlikely(!rt->context)) {
1336 error("Health configuration for template '%s' does not have a context", rt->name);
1340 if(unlikely(!rt->update_every)) {
1341 error("Health configuration for template '%s' has no frequency (parameter 'every'). Ignoring it.", rt->name);
1345 if(unlikely(!RRDCALCTEMPLATE_HAS_CALCULATION(rt) && !rt->warning && !rt->critical)) {
1346 error("Health configuration for template '%s' is useless (no calculation, no warning and no critical evaluation)", rt->name);
1350 RRDCALCTEMPLATE *t, *last = NULL;
1351 for (t = host->templates; t ; last = t, t = t->next) {
1352 if(unlikely(t->hash_name == rt->hash_name && !strcmp(t->name, rt->name))) {
1353 error("Health configuration template '%s' already exists for host '%s'.", rt->name, host->hostname);
1358 debug(D_HEALTH, "Health configuration adding template '%s': context '%s', exec '%s', recipient '%s', green %Lf, red %Lf, lookup: group %d, after %d, before %d, options %u, dimensions '%s', update every %d, calculation '%s', warning '%s', critical '%s', source '%s', delay up %d, delay down %d, delay max %d, delay_multiplier %f",
1360 (rt->context)?rt->context:"NONE",
1361 (rt->exec)?rt->exec:"DEFAULT",
1362 (rt->recipient)?rt->recipient:"DEFAULT",
1369 (rt->dimensions)?rt->dimensions:"NONE",
1371 (rt->calculation)?rt->calculation->parsed_as:"NONE",
1372 (rt->warning)?rt->warning->parsed_as:"NONE",
1373 (rt->critical)?rt->critical->parsed_as:"NONE",
1375 rt->delay_up_duration,
1376 rt->delay_down_duration,
1377 rt->delay_max_duration,
1378 rt->delay_multiplier
1385 rt->next = host->templates;
1386 host->templates = rt;
1392 static inline int health_parse_duration(char *string, int *result) {
1393 // make sure it is a number
1394 if(!*string || !(isdigit(*string) || *string == '+' || *string == '-')) {
1400 calculated_number n = strtold(string, &e);
1404 *result = (int) (n * 86400 * 365);
1407 *result = (int) (n * 86400 * 30);
1410 *result = (int) (n * 86400 * 7);
1413 *result = (int) (n * 86400);
1416 *result = (int) (n * 3600);
1419 *result = (int) (n * 60);
1424 *result = (int) (n);
1434 static inline int health_parse_delay(
1435 size_t line, const char *path, const char *file, char *string,
1436 int *delay_up_duration,
1437 int *delay_down_duration,
1438 int *delay_max_duration,
1439 float *delay_multiplier) {
1442 char given_down = 0;
1444 char given_multiplier = 0;
1450 while(*s && !isspace(*s)) s++;
1451 while(*s && isspace(*s)) *s++ = '\0';
1456 while(*s && !isspace(*s)) s++;
1457 while(*s && isspace(*s)) *s++ = '\0';
1459 if(!strcasecmp(key, "up")) {
1460 if (!health_parse_duration(value, delay_up_duration)) {
1461 error("Health configuration at line %zu of file '%s/%s': invalid value '%s' for '%s' keyword",
1462 line, path, file, value, key);
1466 else if(!strcasecmp(key, "down")) {
1467 if (!health_parse_duration(value, delay_down_duration)) {
1468 error("Health configuration at line %zu of file '%s/%s': invalid value '%s' for '%s' keyword",
1469 line, path, file, value, key);
1471 else given_down = 1;
1473 else if(!strcasecmp(key, "multiplier")) {
1474 *delay_multiplier = strtof(value, NULL);
1475 if(isnan(*delay_multiplier) || isinf(*delay_multiplier) || islessequal(*delay_multiplier, 0)) {
1476 error("Health configuration at line %zu of file '%s/%s': invalid value '%s' for '%s' keyword",
1477 line, path, file, value, key);
1479 else given_multiplier = 1;
1481 else if(!strcasecmp(key, "max")) {
1482 if (!health_parse_duration(value, delay_max_duration)) {
1483 error("Health configuration at line %zu of file '%s/%s': invalid value '%s' for '%s' keyword",
1484 line, path, file, value, key);
1489 error("Health configuration at line %zu of file '%s/%s': unknown keyword '%s'",
1490 line, path, file, key);
1495 *delay_up_duration = 0;
1498 *delay_down_duration = 0;
1500 if(!given_multiplier)
1501 *delay_multiplier = 1.0;
1504 if((*delay_max_duration) < (*delay_up_duration) * (*delay_multiplier))
1505 *delay_max_duration = (*delay_up_duration) * (*delay_multiplier);
1507 if((*delay_max_duration) < (*delay_down_duration) * (*delay_multiplier))
1508 *delay_max_duration = (*delay_down_duration) * (*delay_multiplier);
1514 static inline int health_parse_db_lookup(
1515 size_t line, const char *path, const char *file, char *string,
1516 int *group_method, int *after, int *before, int *every,
1517 uint32_t *options, char **dimensions
1519 debug(D_HEALTH, "Health configuration parsing database lookup %zu@%s/%s: %s", line, path, file, string);
1521 if(*dimensions) freez(*dimensions);
1528 char *s = string, *key;
1530 // first is the group method
1532 while(*s && !isspace(*s)) s++;
1533 while(*s && isspace(*s)) *s++ = '\0';
1535 error("Health configuration invalid chart calculation at line %zu of file '%s/%s': expected group method followed by the 'after' time, but got '%s'",
1536 line, path, file, key);
1540 if((*group_method = web_client_api_request_v1_data_group(key, -1)) == -1) {
1541 error("Health configuration at line %zu of file '%s/%s': invalid group method '%s'",
1542 line, path, file, key);
1546 // then is the 'after' time
1548 while(*s && !isspace(*s)) s++;
1549 while(*s && isspace(*s)) *s++ = '\0';
1551 if(!health_parse_duration(key, after)) {
1552 error("Health configuration at line %zu of file '%s/%s': invalid duration '%s' after group method",
1553 line, path, file, key);
1558 *every = abs(*after);
1560 // now we may have optional parameters
1563 while(*s && !isspace(*s)) s++;
1564 while(*s && isspace(*s)) *s++ = '\0';
1567 if(!strcasecmp(key, "at")) {
1569 while(*s && !isspace(*s)) s++;
1570 while(*s && isspace(*s)) *s++ = '\0';
1572 if (!health_parse_duration(value, before)) {
1573 error("Health configuration at line %zu of file '%s/%s': invalid duration '%s' for '%s' keyword",
1574 line, path, file, value, key);
1577 else if(!strcasecmp(key, HEALTH_EVERY_KEY)) {
1579 while(*s && !isspace(*s)) s++;
1580 while(*s && isspace(*s)) *s++ = '\0';
1582 if (!health_parse_duration(value, every)) {
1583 error("Health configuration at line %zu of file '%s/%s': invalid duration '%s' for '%s' keyword",
1584 line, path, file, value, key);
1587 else if(!strcasecmp(key, "absolute") || !strcasecmp(key, "abs") || !strcasecmp(key, "absolute_sum")) {
1588 *options |= RRDR_OPTION_ABSOLUTE;
1590 else if(!strcasecmp(key, "min2max")) {
1591 *options |= RRDR_OPTION_MIN2MAX;
1593 else if(!strcasecmp(key, "null2zero")) {
1594 *options |= RRDR_OPTION_NULL2ZERO;
1596 else if(!strcasecmp(key, "percentage")) {
1597 *options |= RRDR_OPTION_PERCENTAGE;
1599 else if(!strcasecmp(key, "unaligned")) {
1600 *options |= RRDR_OPTION_NOT_ALIGNED;
1602 else if(!strcasecmp(key, "of")) {
1603 if(*s && strcasecmp(s, "all"))
1604 *dimensions = strdupz(s);
1608 error("Health configuration at line %zu of file '%s/%s': unknown keyword '%s'",
1609 line, path, file, key);
1616 static inline char *tabs2spaces(char *s) {
1619 if(unlikely(*t == '\t')) *t = ' ';
1626 static inline char *health_source_file(size_t line, const char *path, const char *filename) {
1627 char buffer[FILENAME_MAX + 1];
1628 snprintfz(buffer, FILENAME_MAX, "%zu@%s/%s", line, path, filename);
1629 return strdupz(buffer);
1632 static inline void strip_quotes(char *s) {
1634 if(*s == '\'' || *s == '"') *s = ' ';
1639 int health_readfile(const char *path, const char *filename) {
1640 debug(D_HEALTH, "Health configuration reading file '%s/%s'", path, filename);
1642 static uint32_t hash_alarm = 0, hash_template = 0, hash_on = 0, hash_calc = 0, hash_green = 0, hash_red = 0, hash_warn = 0, hash_crit = 0, hash_exec = 0, hash_every = 0, hash_lookup = 0, hash_units = 0, hash_info = 0, hash_recipient = 0, hash_delay = 0;
1643 char buffer[HEALTH_CONF_MAX_LINE + 1];
1645 if(unlikely(!hash_alarm)) {
1646 hash_alarm = simple_uhash(HEALTH_ALARM_KEY);
1647 hash_template = simple_uhash(HEALTH_TEMPLATE_KEY);
1648 hash_on = simple_uhash(HEALTH_ON_KEY);
1649 hash_calc = simple_uhash(HEALTH_CALC_KEY);
1650 hash_lookup = simple_uhash(HEALTH_LOOKUP_KEY);
1651 hash_green = simple_uhash(HEALTH_GREEN_KEY);
1652 hash_red = simple_uhash(HEALTH_RED_KEY);
1653 hash_warn = simple_uhash(HEALTH_WARN_KEY);
1654 hash_crit = simple_uhash(HEALTH_CRIT_KEY);
1655 hash_exec = simple_uhash(HEALTH_EXEC_KEY);
1656 hash_every = simple_uhash(HEALTH_EVERY_KEY);
1657 hash_units = simple_hash(HEALTH_UNITS_KEY);
1658 hash_info = simple_hash(HEALTH_INFO_KEY);
1659 hash_recipient = simple_hash(HEALTH_RECIPIENT_KEY);
1660 hash_delay = simple_uhash(HEALTH_DELAY_KEY);
1663 snprintfz(buffer, HEALTH_CONF_MAX_LINE, "%s/%s", path, filename);
1664 FILE *fp = fopen(buffer, "r");
1666 error("Health configuration cannot read file '%s'.", buffer);
1671 RRDCALCTEMPLATE *rt = NULL;
1673 size_t line = 0, append = 0;
1675 while((s = fgets(&buffer[append], (int)(HEALTH_CONF_MAX_LINE - append), fp)) || append) {
1676 int stop_appending = !s;
1682 if(!stop_appending && s[append - 1] == '\\') {
1683 s[append - 1] = ' ';
1684 append = &s[append] - buffer;
1685 if(append < HEALTH_CONF_MAX_LINE)
1688 error("Health configuration has too long muli-line at line %zu of file '%s/%s'.", line, path, filename);
1694 while(*s && *s != ':') s++;
1696 error("Health configuration has invalid line %zu of file '%s/%s'. It does not contain a ':'. Ignoring it.", line, path, filename);
1704 value = trim(value);
1707 error("Health configuration has invalid line %zu of file '%s/%s'. Keyword is empty. Ignoring it.", line, path, filename);
1712 error("Health configuration has invalid line %zu of file '%s/%s'. value is empty. Ignoring it.", line, path, filename);
1716 uint32_t hash = simple_uhash(key);
1718 if(hash == hash_alarm && !strcasecmp(key, HEALTH_ALARM_KEY)) {
1719 if(rc && !rrdcalc_add_alarm_from_config(&localhost, rc))
1720 rrdcalc_free(&localhost, rc);
1723 if (!rrdcalctemplate_add_template_from_config(&localhost, rt))
1724 rrdcalctemplate_free(&localhost, rt);
1728 rc = callocz(1, sizeof(RRDCALC));
1729 rc->next_event_id = 1;
1730 rc->name = tabs2spaces(strdupz(value));
1731 rc->hash = simple_hash(rc->name);
1732 rc->source = health_source_file(line, path, filename);
1736 rc->old_value = NAN;
1737 rc->delay_multiplier = 1.0;
1739 if(rrdvar_fix_name(rc->name))
1740 error("Health configuration renamed alarm '%s' to '%s'", value, rc->name);
1742 else if(hash == hash_template && !strcasecmp(key, HEALTH_TEMPLATE_KEY)) {
1744 if(!rrdcalc_add_alarm_from_config(&localhost, rc))
1745 rrdcalc_free(&localhost, rc);
1749 if(rt && !rrdcalctemplate_add_template_from_config(&localhost, rt))
1750 rrdcalctemplate_free(&localhost, rt);
1752 rt = callocz(1, sizeof(RRDCALCTEMPLATE));
1753 rt->name = tabs2spaces(strdupz(value));
1754 rt->hash_name = simple_hash(rt->name);
1755 rt->source = health_source_file(line, path, filename);
1758 rt->delay_multiplier = 1.0;
1760 if(rrdvar_fix_name(rt->name))
1761 error("Health configuration renamed template '%s' to '%s'", value, rt->name);
1764 if(hash == hash_on && !strcasecmp(key, HEALTH_ON_KEY)) {
1766 if(strcmp(rc->chart, value))
1767 error("Health configuration at line %zu of file '%s/%s' for alarm '%s' has key '%s' twice, once with value '%s' and later with value '%s'. Using ('%s').",
1768 line, path, filename, rc->name, key, rc->chart, value, value);
1772 rc->chart = tabs2spaces(strdupz(value));
1773 rc->hash_chart = simple_hash(rc->chart);
1775 else if(hash == hash_lookup && !strcasecmp(key, HEALTH_LOOKUP_KEY)) {
1776 health_parse_db_lookup(line, path, filename, value, &rc->group, &rc->after, &rc->before,
1778 &rc->options, &rc->dimensions);
1780 else if(hash == hash_every && !strcasecmp(key, HEALTH_EVERY_KEY)) {
1781 if(!health_parse_duration(value, &rc->update_every))
1782 error("Health configuration at line %zu of file '%s/%s' for alarm '%s' at key '%s' cannot parse duration: '%s'.",
1783 line, path, filename, rc->name, key, value);
1785 else if(hash == hash_green && !strcasecmp(key, HEALTH_GREEN_KEY)) {
1787 rc->green = strtold(value, &e);
1789 error("Health configuration at line %zu of file '%s/%s' for alarm '%s' at key '%s' leaves this string unmatched: '%s'.",
1790 line, path, filename, rc->name, key, e);
1793 else if(hash == hash_red && !strcasecmp(key, HEALTH_RED_KEY)) {
1795 rc->red = strtold(value, &e);
1797 error("Health configuration at line %zu of file '%s/%s' for alarm '%s' at key '%s' leaves this string unmatched: '%s'.",
1798 line, path, filename, rc->name, key, e);
1801 else if(hash == hash_calc && !strcasecmp(key, HEALTH_CALC_KEY)) {
1802 const char *failed_at = NULL;
1804 rc->calculation = expression_parse(value, &failed_at, &error);
1805 if(!rc->calculation) {
1806 error("Health configuration at line %zu of file '%s/%s' for alarm '%s' at key '%s' has unparse-able expression '%s': %s at '%s'",
1807 line, path, filename, rc->name, key, value, expression_strerror(error), failed_at);
1810 else if(hash == hash_warn && !strcasecmp(key, HEALTH_WARN_KEY)) {
1811 const char *failed_at = NULL;
1813 rc->warning = expression_parse(value, &failed_at, &error);
1815 error("Health configuration at line %zu of file '%s/%s' for alarm '%s' at key '%s' has unparse-able expression '%s': %s at '%s'",
1816 line, path, filename, rc->name, key, value, expression_strerror(error), failed_at);
1819 else if(hash == hash_crit && !strcasecmp(key, HEALTH_CRIT_KEY)) {
1820 const char *failed_at = NULL;
1822 rc->critical = expression_parse(value, &failed_at, &error);
1824 error("Health configuration at line %zu of file '%s/%s' for alarm '%s' at key '%s' has unparse-able expression '%s': %s at '%s'",
1825 line, path, filename, rc->name, key, value, expression_strerror(error), failed_at);
1828 else if(hash == hash_exec && !strcasecmp(key, HEALTH_EXEC_KEY)) {
1830 if(strcmp(rc->exec, value))
1831 error("Health configuration at line %zu of file '%s/%s' for alarm '%s' has key '%s' twice, once with value '%s' and later with value '%s'. Using ('%s').",
1832 line, path, filename, rc->name, key, rc->exec, value, value);
1836 rc->exec = tabs2spaces(strdupz(value));
1838 else if(hash == hash_recipient && !strcasecmp(key, HEALTH_RECIPIENT_KEY)) {
1840 if(strcmp(rc->recipient, value))
1841 error("Health configuration at line %zu of file '%s/%s' for alarm '%s' has key '%s' twice, once with value '%s' and later with value '%s'. Using ('%s').",
1842 line, path, filename, rc->name, key, rc->recipient, value, value);
1844 freez(rc->recipient);
1846 rc->recipient = tabs2spaces(strdupz(value));
1848 else if(hash == hash_units && !strcasecmp(key, HEALTH_UNITS_KEY)) {
1850 if(strcmp(rc->units, value))
1851 error("Health configuration at line %zu of file '%s/%s' for alarm '%s' has key '%s' twice, once with value '%s' and later with value '%s'. Using ('%s').",
1852 line, path, filename, rc->name, key, rc->units, value, value);
1856 rc->units = tabs2spaces(strdupz(value));
1857 strip_quotes(rc->units);
1859 else if(hash == hash_info && !strcasecmp(key, HEALTH_INFO_KEY)) {
1861 if(strcmp(rc->info, value))
1862 error("Health configuration at line %zu of file '%s/%s' for alarm '%s' has key '%s' twice, once with value '%s' and later with value '%s'. Using ('%s').",
1863 line, path, filename, rc->name, key, rc->info, value, value);
1867 rc->info = tabs2spaces(strdupz(value));
1868 strip_quotes(rc->info);
1870 else if(hash == hash_delay && !strcasecmp(key, HEALTH_DELAY_KEY)) {
1871 health_parse_delay(line, path, filename, value, &rc->delay_up_duration, &rc->delay_down_duration, &rc->delay_max_duration, &rc->delay_multiplier);
1874 error("Health configuration at line %zu of file '%s/%s' for alarm '%s' has unknown key '%s'.",
1875 line, path, filename, rc->name, key);
1879 if(hash == hash_on && !strcasecmp(key, HEALTH_ON_KEY)) {
1881 if(strcmp(rt->context, value))
1882 error("Health configuration at line %zu of file '%s/%s' for template '%s' has key '%s' twice, once with value '%s' and later with value '%s'. Using ('%s').",
1883 line, path, filename, rt->name, key, rt->context, value, value);
1887 rt->context = tabs2spaces(strdupz(value));
1888 rt->hash_context = simple_hash(rt->context);
1890 else if(hash == hash_lookup && !strcasecmp(key, HEALTH_LOOKUP_KEY)) {
1891 health_parse_db_lookup(line, path, filename, value, &rt->group, &rt->after, &rt->before,
1893 &rt->options, &rt->dimensions);
1895 else if(hash == hash_every && !strcasecmp(key, HEALTH_EVERY_KEY)) {
1896 if(!health_parse_duration(value, &rt->update_every))
1897 error("Health configuration at line %zu of file '%s/%s' for template '%s' at key '%s' cannot parse duration: '%s'.",
1898 line, path, filename, rt->name, key, value);
1900 else if(hash == hash_green && !strcasecmp(key, HEALTH_GREEN_KEY)) {
1902 rt->green = strtold(value, &e);
1904 error("Health configuration at line %zu of file '%s/%s' for template '%s' at key '%s' leaves this string unmatched: '%s'.",
1905 line, path, filename, rt->name, key, e);
1908 else if(hash == hash_red && !strcasecmp(key, HEALTH_RED_KEY)) {
1910 rt->red = strtold(value, &e);
1912 error("Health configuration at line %zu of file '%s/%s' for template '%s' at key '%s' leaves this string unmatched: '%s'.",
1913 line, path, filename, rt->name, key, e);
1916 else if(hash == hash_calc && !strcasecmp(key, HEALTH_CALC_KEY)) {
1917 const char *failed_at = NULL;
1919 rt->calculation = expression_parse(value, &failed_at, &error);
1920 if(!rt->calculation) {
1921 error("Health configuration at line %zu of file '%s/%s' for template '%s' at key '%s' has unparse-able expression '%s': %s at '%s'",
1922 line, path, filename, rt->name, key, value, expression_strerror(error), failed_at);
1925 else if(hash == hash_warn && !strcasecmp(key, HEALTH_WARN_KEY)) {
1926 const char *failed_at = NULL;
1928 rt->warning = expression_parse(value, &failed_at, &error);
1930 error("Health configuration at line %zu of file '%s/%s' for template '%s' at key '%s' has unparse-able expression '%s': %s at '%s'",
1931 line, path, filename, rt->name, key, value, expression_strerror(error), failed_at);
1934 else if(hash == hash_crit && !strcasecmp(key, HEALTH_CRIT_KEY)) {
1935 const char *failed_at = NULL;
1937 rt->critical = expression_parse(value, &failed_at, &error);
1939 error("Health configuration at line %zu of file '%s/%s' for template '%s' at key '%s' has unparse-able expression '%s': %s at '%s'",
1940 line, path, filename, rt->name, key, value, expression_strerror(error), failed_at);
1943 else if(hash == hash_exec && !strcasecmp(key, HEALTH_EXEC_KEY)) {
1945 if(strcmp(rt->exec, value))
1946 error("Health configuration at line %zu of file '%s/%s' for template '%s' has key '%s' twice, once with value '%s' and later with value '%s'. Using ('%s').",
1947 line, path, filename, rt->name, key, rt->exec, value, value);
1951 rt->exec = tabs2spaces(strdupz(value));
1953 else if(hash == hash_recipient && !strcasecmp(key, HEALTH_RECIPIENT_KEY)) {
1955 if(strcmp(rt->recipient, value))
1956 error("Health configuration at line %zu of file '%s/%s' for template '%s' has key '%s' twice, once with value '%s' and later with value '%s'. Using ('%s').",
1957 line, path, filename, rt->name, key, rt->recipient, value, value);
1959 freez(rt->recipient);
1961 rt->recipient = tabs2spaces(strdupz(value));
1963 else if(hash == hash_units && !strcasecmp(key, HEALTH_UNITS_KEY)) {
1965 if(strcmp(rt->units, value))
1966 error("Health configuration at line %zu of file '%s/%s' for template '%s' has key '%s' twice, once with value '%s' and later with value '%s'. Using ('%s').",
1967 line, path, filename, rt->name, key, rt->units, value, value);
1971 rt->units = tabs2spaces(strdupz(value));
1972 strip_quotes(rt->units);
1974 else if(hash == hash_info && !strcasecmp(key, HEALTH_INFO_KEY)) {
1976 if(strcmp(rt->info, value))
1977 error("Health configuration at line %zu of file '%s/%s' for template '%s' has key '%s' twice, once with value '%s' and later with value '%s'. Using ('%s').",
1978 line, path, filename, rt->name, key, rt->info, value, value);
1982 rt->info = tabs2spaces(strdupz(value));
1983 strip_quotes(rt->info);
1985 else if(hash == hash_delay && !strcasecmp(key, HEALTH_DELAY_KEY)) {
1986 health_parse_delay(line, path, filename, value, &rt->delay_up_duration, &rt->delay_down_duration, &rt->delay_max_duration, &rt->delay_multiplier);
1989 error("Health configuration at line %zu of file '%s/%s' for template '%s' has unknown key '%s'.",
1990 line, path, filename, rt->name, key);
1994 error("Health configuration at line %zu of file '%s/%s' has unknown key '%s'. Expected either '" HEALTH_ALARM_KEY "' or '" HEALTH_TEMPLATE_KEY "'.",
1995 line, path, filename, key);
1999 if(rc && !rrdcalc_add_alarm_from_config(&localhost, rc))
2000 rrdcalc_free(&localhost, rc);
2002 if(rt && !rrdcalctemplate_add_template_from_config(&localhost, rt))
2003 rrdcalctemplate_free(&localhost, rt);
2009 void health_readdir(const char *path) {
2010 size_t pathlen = strlen(path);
2012 debug(D_HEALTH, "Health configuration reading directory '%s'", path);
2014 DIR *dir = opendir(path);
2016 error("Health configuration cannot open directory '%s'.", path);
2020 struct dirent *de = NULL;
2021 while ((de = readdir(dir))) {
2022 size_t len = strlen(de->d_name);
2024 if(de->d_type == DT_DIR
2026 (de->d_name[0] == '.' && de->d_name[1] == '\0')
2027 || (de->d_name[0] == '.' && de->d_name[1] == '.' && de->d_name[2] == '\0')
2029 debug(D_HEALTH, "Ignoring directory '%s'", de->d_name);
2033 else if(de->d_type == DT_DIR) {
2034 char *s = mallocz(pathlen + strlen(de->d_name) + 2);
2037 strcat(s, de->d_name);
2043 else if((de->d_type == DT_LNK || de->d_type == DT_REG || de->d_type == DT_UNKNOWN) &&
2044 len > 5 && !strcmp(&de->d_name[len - 5], ".conf")) {
2045 health_readfile(path, de->d_name);
2048 else debug(D_HEALTH, "Ignoring file '%s'", de->d_name);
2054 static inline char *health_config_dir(void) {
2055 char buffer[FILENAME_MAX + 1];
2056 snprintfz(buffer, FILENAME_MAX, "%s/health.d", config_get("global", "config directory", CONFIG_DIR));
2057 return config_get("health", "health configuration directory", buffer);
2060 void health_init(void) {
2061 debug(D_HEALTH, "Health configuration initializing");
2063 if(!(health_enabled = config_get_boolean("health", "enabled", 1))) {
2064 debug(D_HEALTH, "Health is disabled.");
2068 char *pathname = config_get("health", "health db directory", VARLIB_DIR "/health");
2069 if(mkdir(pathname, 0770) == -1 && errno != EEXIST)
2070 fatal("Cannot create directory '%s'.", pathname);
2072 char filename[FILENAME_MAX + 1];
2073 snprintfz(filename, FILENAME_MAX, "%s/health-log.db", pathname);
2074 health.log_filename = config_get("health", "health db file", filename);
2076 health_alarm_log_load(&localhost);
2077 health_alarm_log_open();
2079 char *path = health_config_dir();
2082 char buffer[FILENAME_MAX + 1];
2083 snprintfz(buffer, FILENAME_MAX, "%s/alarm-notify.sh", config_get("global", "plugins directory", PLUGINS_DIR));
2084 health.health_default_exec = config_get("health", "script to execute on alarm", buffer);
2087 long n = config_get_number("health", "in memory max health log entries", (long)localhost.health_log.max);
2089 error("Health configuration has invalid max log entries %ld. Using default %u", n, localhost.health_log.max);
2090 config_set_number("health", "in memory max health log entries", (long)localhost.health_log.max);
2092 else localhost.health_log.max = (unsigned int)n;
2094 rrdhost_rwlock(&localhost);
2095 health_readdir(path);
2096 rrdhost_unlock(&localhost);
2099 // ----------------------------------------------------------------------------
2102 static inline void health_string2json(BUFFER *wb, const char *prefix, const char *label, const char *value, const char *suffix) {
2104 buffer_sprintf(wb, "%s\"%s\":\"%s\"%s", prefix, label, value, suffix);
2106 buffer_sprintf(wb, "%s\"%s\":null%s", prefix, label, suffix);
2109 static inline void health_alarm_entry2json_nolock(BUFFER *wb, ALARM_ENTRY *ae, RRDHOST *host) {
2110 buffer_sprintf(wb, "\n\t{\n"
2111 "\t\t\"hostname\": \"%s\",\n"
2112 "\t\t\"unique_id\": %u,\n"
2113 "\t\t\"alarm_id\": %u,\n"
2114 "\t\t\"alarm_event_id\": %u,\n"
2115 "\t\t\"name\": \"%s\",\n"
2116 "\t\t\"chart\": \"%s\",\n"
2117 "\t\t\"family\": \"%s\",\n"
2118 "\t\t\"processed\": %s,\n"
2119 "\t\t\"updated\": %s,\n"
2120 "\t\t\"exec_run\": %lu,\n"
2121 "\t\t\"exec_failed\": %s,\n"
2122 "\t\t\"exec\": \"%s\",\n"
2123 "\t\t\"recipient\": \"%s\",\n"
2124 "\t\t\"exec_code\": %d,\n"
2125 "\t\t\"source\": \"%s\",\n"
2126 "\t\t\"units\": \"%s\",\n"
2127 "\t\t\"info\": \"%s\",\n"
2128 "\t\t\"when\": %lu,\n"
2129 "\t\t\"duration\": %lu,\n"
2130 "\t\t\"non_clear_duration\": %lu,\n"
2131 "\t\t\"status\": \"%s\",\n"
2132 "\t\t\"old_status\": \"%s\",\n"
2133 "\t\t\"delay\": %d,\n"
2134 "\t\t\"delay_up_to_timestamp\": %lu,\n"
2135 "\t\t\"updated_by_id\": %u,\n"
2136 "\t\t\"updates_id\": %u,\n",
2144 (ae->flags & HEALTH_ENTRY_FLAG_PROCESSED)?"true":"false",
2145 (ae->flags & HEALTH_ENTRY_FLAG_UPDATED)?"true":"false",
2146 (unsigned long)ae->exec_run_timestamp,
2147 (ae->flags & HEALTH_ENTRY_FLAG_EXEC_FAILED)?"true":"false",
2148 ae->exec?ae->exec:health.health_default_exec,
2149 ae->recipient?ae->recipient:health.health_default_recipient,
2152 ae->units?ae->units:"",
2153 ae->info?ae->info:"",
2154 (unsigned long)ae->when,
2155 (unsigned long)ae->duration,
2156 (unsigned long)ae->non_clear_duration,
2157 rrdcalc_status2string(ae->new_status),
2158 rrdcalc_status2string(ae->old_status),
2160 (unsigned long)ae->delay_up_to_timestamp,
2165 buffer_strcat(wb, "\t\t\"value\":");
2166 buffer_rrd_value(wb, ae->new_value);
2167 buffer_strcat(wb, ",\n");
2169 buffer_strcat(wb, "\t\t\"old_value\":");
2170 buffer_rrd_value(wb, ae->old_value);
2171 buffer_strcat(wb, "\n");
2173 buffer_strcat(wb, "\t}");
2176 void health_alarm_log2json(RRDHOST *host, BUFFER *wb, uint32_t after) {
2177 pthread_rwlock_rdlock(&host->health_log.alarm_log_rwlock);
2179 buffer_strcat(wb, "[");
2181 unsigned int max = host->health_log.max;
2182 unsigned int count = 0;
2184 for(ae = host->health_log.alarms; ae && count < max ; count++, ae = ae->next) {
2185 if(ae->unique_id > after) {
2186 if(likely(count)) buffer_strcat(wb, ",");
2187 health_alarm_entry2json_nolock(wb, ae, host);
2191 buffer_strcat(wb, "\n]\n");
2193 pthread_rwlock_unlock(&host->health_log.alarm_log_rwlock);
2196 static inline void health_rrdcalc2json_nolock(BUFFER *wb, RRDCALC *rc) {
2198 "\t\t\"%s.%s\": {\n"
2199 "\t\t\t\"id\": %lu,\n"
2200 "\t\t\t\"name\": \"%s\",\n"
2201 "\t\t\t\"chart\": \"%s\",\n"
2202 "\t\t\t\"family\": \"%s\",\n"
2203 "\t\t\t\"active\": %s,\n"
2204 "\t\t\t\"exec\": \"%s\",\n"
2205 "\t\t\t\"recipient\": \"%s\",\n"
2206 "\t\t\t\"source\": \"%s\",\n"
2207 "\t\t\t\"units\": \"%s\",\n"
2208 "\t\t\t\"info\": \"%s\",\n"
2209 "\t\t\t\"status\": \"%s\",\n"
2210 "\t\t\t\"last_status_change\": %lu,\n"
2211 "\t\t\t\"last_updated\": %lu,\n"
2212 "\t\t\t\"next_update\": %lu,\n"
2213 "\t\t\t\"update_every\": %d,\n"
2214 "\t\t\t\"delay_up_duration\": %d,\n"
2215 "\t\t\t\"delay_down_duration\": %d,\n"
2216 "\t\t\t\"delay_max_duration\": %d,\n"
2217 "\t\t\t\"delay_multiplier\": %f,\n"
2218 "\t\t\t\"delay\": %d,\n"
2219 "\t\t\t\"delay_up_to_timestamp\": %lu,\n"
2220 , rc->chart, rc->name
2221 , (unsigned long)rc->id
2224 , (rc->rrdset && rc->rrdset->family)?rc->rrdset->family:""
2225 , (rc->rrdset)?"true":"false"
2226 , rc->exec?rc->exec:health.health_default_exec
2227 , rc->recipient?rc->recipient:health.health_default_recipient
2229 , rc->units?rc->units:""
2230 , rc->info?rc->info:""
2231 , rrdcalc_status2string(rc->status)
2232 , (unsigned long)rc->last_status_change
2233 , (unsigned long)rc->last_updated
2234 , (unsigned long)rc->next_update
2236 , rc->delay_up_duration
2237 , rc->delay_down_duration
2238 , rc->delay_max_duration
2239 , rc->delay_multiplier
2241 , (unsigned long)rc->delay_up_to_timestamp
2244 if(RRDCALC_HAS_DB_LOOKUP(rc)) {
2245 if(rc->dimensions && *rc->dimensions)
2246 health_string2json(wb, "\t\t\t", "lookup_dimensions", rc->dimensions, ",\n");
2249 "\t\t\t\"db_after\": %lu,\n"
2250 "\t\t\t\"db_before\": %lu,\n"
2251 "\t\t\t\"lookup_method\": \"%s\",\n"
2252 "\t\t\t\"lookup_after\": %d,\n"
2253 "\t\t\t\"lookup_before\": %d,\n"
2254 "\t\t\t\"lookup_options\": \"",
2255 (unsigned long) rc->db_after,
2256 (unsigned long) rc->db_before,
2257 group_method2string(rc->group),
2261 buffer_data_options2string(wb, rc->options);
2262 buffer_strcat(wb, "\",\n");
2265 if(rc->calculation) {
2266 health_string2json(wb, "\t\t\t", "calc", rc->calculation->source, ",\n");
2267 health_string2json(wb, "\t\t\t", "calc_parsed", rc->calculation->parsed_as, ",\n");
2271 health_string2json(wb, "\t\t\t", "warn", rc->warning->source, ",\n");
2272 health_string2json(wb, "\t\t\t", "warn_parsed", rc->warning->parsed_as, ",\n");
2276 health_string2json(wb, "\t\t\t", "crit", rc->critical->source, ",\n");
2277 health_string2json(wb, "\t\t\t", "crit_parsed", rc->critical->parsed_as, ",\n");
2280 buffer_strcat(wb, "\t\t\t\"green\":");
2281 buffer_rrd_value(wb, rc->green);
2282 buffer_strcat(wb, ",\n");
2284 buffer_strcat(wb, "\t\t\t\"red\":");
2285 buffer_rrd_value(wb, rc->red);
2286 buffer_strcat(wb, ",\n");
2288 buffer_strcat(wb, "\t\t\t\"value\":");
2289 buffer_rrd_value(wb, rc->value);
2290 buffer_strcat(wb, "\n");
2292 buffer_strcat(wb, "\t\t}");
2295 //void health_rrdcalctemplate2json_nolock(BUFFER *wb, RRDCALCTEMPLATE *rt) {
2299 void health_alarms2json(RRDHOST *host, BUFFER *wb, int all) {
2302 rrdhost_rdlock(&localhost);
2303 buffer_sprintf(wb, "{\n\t\"hostname\": \"%s\","
2304 "\n\t\"latest_alarm_log_unique_id\": %u,"
2305 "\n\t\"status\": %s,"
2307 "\n\t\"alarms\": {\n",
2309 (host->health_log.next_log_id > 0)?(host->health_log.next_log_id - 1):0,
2310 health_enabled?"true":"false",
2311 (unsigned long)time(NULL));
2314 for(i = 0, rc = host->alarms; rc ; rc = rc->next) {
2315 if(unlikely(!rc->rrdset || !rc->rrdset->last_collected_time.tv_sec))
2318 if(likely(!all && !(rc->status == RRDCALC_STATUS_WARNING || rc->status == RRDCALC_STATUS_CRITICAL)))
2321 if(likely(i)) buffer_strcat(wb, ",\n");
2322 health_rrdcalc2json_nolock(wb, rc);
2326 // buffer_strcat(wb, "\n\t},\n\t\"templates\": {");
2327 // RRDCALCTEMPLATE *rt;
2328 // for(rt = host->templates; rt ; rt = rt->next)
2329 // health_rrdcalctemplate2json_nolock(wb, rt);
2331 buffer_strcat(wb, "\n\t}\n}\n");
2332 rrdhost_unlock(&localhost);
2336 // ----------------------------------------------------------------------------
2337 // re-load health configuration
2339 static inline void health_free_all_nolock(RRDHOST *host) {
2340 while(host->templates)
2341 rrdcalctemplate_free(host, host->templates);
2344 rrdcalc_free(host, host->alarms);
2347 void health_reload(void) {
2348 if(!health_enabled) {
2349 error("Health reload is requested, but health is not enabled.");
2353 char *path = health_config_dir();
2355 // free all running alarms
2356 rrdhost_rwlock(&localhost);
2357 health_free_all_nolock(&localhost);
2358 rrdhost_unlock(&localhost);
2360 // invalidate all previous entries in the alarm log
2362 for(t = localhost.health_log.alarms ; t ; t = t->next) {
2363 if(t->new_status != RRDCALC_STATUS_REMOVED)
2364 t->flags |= HEALTH_ENTRY_FLAG_UPDATED;
2367 // reset all thresholds to all charts
2369 for(st = localhost.rrdset_root; st ; st = st->next) {
2374 // load the new alarms
2375 rrdhost_rwlock(&localhost);
2376 health_readdir(path);
2377 rrdhost_unlock(&localhost);
2379 // link the loaded alarms to their charts
2380 for(st = localhost.rrdset_root; st ; st = st->next) {
2381 rrdhost_rwlock(&localhost);
2383 rrdsetcalc_link_matching(st);
2384 rrdcalctemplate_link_matching(st);
2386 rrdhost_unlock(&localhost);
2390 // ----------------------------------------------------------------------------
2391 // health main thread and friends
2393 static inline int rrdcalc_value2status(calculated_number n) {
2394 if(isnan(n)) return RRDCALC_STATUS_UNDEFINED;
2395 if(n) return RRDCALC_STATUS_RAISED;
2396 return RRDCALC_STATUS_CLEAR;
2399 static inline void health_alarm_execute(RRDHOST *host, ALARM_ENTRY *ae) {
2400 ae->flags |= HEALTH_ENTRY_FLAG_PROCESSED;
2402 if(unlikely(ae->new_status < RRDCALC_STATUS_CLEAR)) {
2403 // do not send notifications for internal statuses
2407 // find the previous notification for the same alarm
2408 // which we have run the exec script
2410 for(t = ae->next; t ;t = t->next) {
2411 if(t->alarm_id == ae->alarm_id && t->flags & HEALTH_ENTRY_FLAG_EXEC_RUN)
2416 // we have executed this alarm notification in the past
2417 if (t && t->new_status == ae->new_status) {
2418 // don't send the same notification again
2419 debug(D_HEALTH, "Health not sending again notification for alarm '%s.%s' status %s", ae->chart, ae->name,
2420 rrdcalc_status2string(ae->new_status));
2425 // we have not executed this alarm notification in the past
2426 if(unlikely(ae->old_status == RRDCALC_STATUS_UNINITIALIZED && ae->new_status == RRDCALC_STATUS_CLEAR)) {
2427 debug(D_HEALTH, "Health not sending notification for first initialization of alarm '%s.%s' status %s", ae->chart, ae->name, rrdcalc_status2string(ae->new_status));
2432 char buffer[FILENAME_MAX + 1];
2435 const char *exec = ae->exec;
2436 if(!exec) exec = health.health_default_exec;
2438 const char *recipient = ae->recipient;
2439 if(!recipient) recipient = health.health_default_recipient;
2441 snprintfz(buffer, FILENAME_MAX, "exec %s '%s' '%s' '%u' '%u' '%u' '%lu' '%s' '%s' '%s' '%s' '%s' '%0.0Lf' '%0.0Lf' '%s' '%u' '%u' '%s' '%s'",
2448 (unsigned long)ae->when,
2450 ae->chart?ae->chart:"NOCAHRT",
2451 ae->family?ae->family:"NOFAMILY",
2452 rrdcalc_status2string(ae->new_status),
2453 rrdcalc_status2string(ae->old_status),
2456 ae->source?ae->source:"UNKNOWN",
2457 (uint32_t)ae->duration,
2458 (uint32_t)ae->non_clear_duration,
2459 ae->units?ae->units:"",
2460 ae->info?ae->info:""
2463 ae->flags |= HEALTH_ENTRY_FLAG_EXEC_RUN;
2464 ae->exec_run_timestamp = time(NULL);
2466 debug(D_HEALTH, "executing command '%s'", buffer);
2467 FILE *fp = mypopen(buffer, &command_pid);
2469 error("HEALTH: Cannot popen(\"%s\", \"r\").", buffer);
2472 debug(D_HEALTH, "HEALTH reading from command");
2473 char *s = fgets(buffer, FILENAME_MAX, fp);
2475 ae->exec_code = mypclose(fp, command_pid);
2476 debug(D_HEALTH, "done executing command - returned with code %d", ae->exec_code);
2478 if(ae->exec_code != 0)
2479 ae->flags |= HEALTH_ENTRY_FLAG_EXEC_FAILED;
2482 health_alarm_log_save(host, ae);
2486 static inline void health_process_notifications(RRDHOST *host, ALARM_ENTRY *ae) {
2487 debug(D_HEALTH, "Health alarm '%s.%s' = %0.2Lf - changed status from %s to %s",
2488 ae->chart?ae->chart:"NOCHART", ae->name,
2490 rrdcalc_status2string(ae->old_status),
2491 rrdcalc_status2string(ae->new_status)
2494 health_alarm_execute(host, ae);
2497 static inline void health_alarm_log_process(RRDHOST *host) {
2498 static uint32_t stop_at_id = 0;
2499 uint32_t first_waiting = (host->health_log.alarms)?host->health_log.alarms->unique_id:0;
2500 time_t now = time(NULL);
2502 pthread_rwlock_rdlock(&host->health_log.alarm_log_rwlock);
2505 for(ae = host->health_log.alarms; ae && ae->unique_id >= stop_at_id ; ae = ae->next) {
2507 !(ae->flags & HEALTH_ENTRY_FLAG_PROCESSED) &&
2508 !(ae->flags & HEALTH_ENTRY_FLAG_UPDATED)
2511 if(unlikely(ae->unique_id < first_waiting))
2512 first_waiting = ae->unique_id;
2514 if(likely(now >= ae->delay_up_to_timestamp))
2515 health_process_notifications(host, ae);
2519 // remember this for the next iteration
2520 stop_at_id = first_waiting;
2522 pthread_rwlock_unlock(&host->health_log.alarm_log_rwlock);
2524 if(host->health_log.count <= host->health_log.max)
2527 // cleanup excess entries in the log
2528 pthread_rwlock_wrlock(&host->health_log.alarm_log_rwlock);
2530 ALARM_ENTRY *last = NULL;
2531 unsigned int count = host->health_log.max * 2 / 3;
2532 for(ae = host->health_log.alarms; ae && count ; count--, last = ae, ae = ae->next) ;
2534 if(ae && last && last->next == ae)
2540 debug(D_HEALTH, "Health removing alarm log entry with id: %u", ae->unique_id);
2542 ALARM_ENTRY *t = ae->next;
2548 freez(ae->recipient);
2555 host->health_log.count--;
2558 pthread_rwlock_unlock(&host->health_log.alarm_log_rwlock);
2561 static inline int rrdcalc_isrunnable(RRDCALC *rc, time_t now, time_t *next_run) {
2562 if (unlikely(!rc->rrdset)) {
2563 debug(D_HEALTH, "Health not running alarm '%s.%s'. It is not linked to a chart.", rc->chart?rc->chart:"NOCHART", rc->name);
2567 if (unlikely(!rc->rrdset->last_collected_time.tv_sec || rc->rrdset->counter_done < 2)) {
2568 debug(D_HEALTH, "Health not running alarm '%s.%s'. Chart is not fully collected yet.", rc->chart?rc->chart:"NOCHART", rc->name);
2572 if (unlikely(!rc->update_every)) {
2573 debug(D_HEALTH, "Health not running alarm '%s.%s'. It does not have an update frequency", rc->chart?rc->chart:"NOCHART", rc->name);
2577 int update_every = rc->rrdset->update_every;
2578 time_t first = rrdset_first_entry_t(rc->rrdset);
2579 time_t last = rrdset_last_entry_t(rc->rrdset);
2581 if(unlikely(now + update_every < first /* || now - update_every > last */)) {
2583 , "Health not examining alarm '%s.%s' yet (wanted time is out of bounds - we need %lu but got %lu - %lu)."
2584 , rc->chart ? rc->chart : "NOCHART", rc->name, (unsigned long) now, (unsigned long) first
2585 , (unsigned long) last);
2589 if (RRDCALC_HAS_DB_LOOKUP(rc)) {
2590 time_t needed = now + rc->before + rc->after;
2592 if(needed + update_every < first || needed - update_every > last) {
2594 , "Health not examining alarm '%s.%s' yet (not enough data yet - we need %lu but got %lu - %lu)."
2595 , rc->chart ? rc->chart : "NOCHART", rc->name, (unsigned long) needed, (unsigned long) first
2596 , (unsigned long) last);
2601 if (unlikely(rc->next_update > now)) {
2602 if (unlikely(*next_run > rc->next_update))
2603 *next_run = rc->next_update;
2605 debug(D_HEALTH, "Health not examining alarm '%s.%s' yet (will do in %d secs).", rc->chart?rc->chart:"NOCHART", rc->name, (int) (rc->next_update - now));
2612 void *health_main(void *ptr) {
2615 info("HEALTH thread created with task id %d", gettid());
2617 if(pthread_setcanceltype(PTHREAD_CANCEL_DEFERRED, NULL) != 0)
2618 error("Cannot set pthread cancel type to DEFERRED.");
2620 if(pthread_setcancelstate(PTHREAD_CANCEL_ENABLE, NULL) != 0)
2621 error("Cannot set pthread cancel state to ENABLE.");
2623 int min_run_every = (int)config_get_number("health", "run at least every seconds", 10);
2624 if(min_run_every < 1) min_run_every = 1;
2626 BUFFER *wb = buffer_create(100);
2628 unsigned int loop = 0;
2629 while(health_enabled && !netdata_exit) {
2631 debug(D_HEALTH, "Health monitoring iteration no %u started", loop);
2633 int oldstate, runnable = 0;
2634 time_t now = time(NULL);
2635 time_t next_run = now + min_run_every;
2638 if (unlikely(pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, &oldstate) != 0))
2639 error("Cannot set pthread cancel state to DISABLE.");
2641 rrdhost_rdlock(&localhost);
2643 // the first loop is to lookup values from the db
2644 for (rc = localhost.alarms; rc; rc = rc->next) {
2645 if (unlikely(!rrdcalc_isrunnable(rc, now, &next_run)))
2649 rc->old_value = rc->value;
2651 // 1. if there is database lookup, do it
2652 // 2. if there is calculation expression, run it
2654 if (unlikely(RRDCALC_HAS_DB_LOOKUP(rc))) {
2655 /* time_t old_db_timestamp = rc->db_before; */
2656 int value_is_null = 0;
2658 int ret = rrd2value(rc->rrdset, wb, &rc->value,
2659 rc->dimensions, 1, rc->after, rc->before, rc->group,
2660 rc->options, &rc->db_after, &rc->db_before, &value_is_null);
2662 if (unlikely(ret != 200)) {
2663 // database lookup failed
2666 debug(D_HEALTH, "Health alarm '%s.%s': database lookup returned error %d", rc->chart?rc->chart:"NOCHART", rc->name, ret);
2668 if (unlikely(!(rc->rrdcalc_flags & RRDCALC_FLAG_DB_ERROR))) {
2669 rc->rrdcalc_flags |= RRDCALC_FLAG_DB_ERROR;
2670 error("Health alarm '%s.%s': database lookup returned error %d", rc->chart?rc->chart:"NOCHART", rc->name, ret);
2673 else if (unlikely(rc->rrdcalc_flags & RRDCALC_FLAG_DB_ERROR))
2674 rc->rrdcalc_flags &= ~RRDCALC_FLAG_DB_ERROR;
2676 /* - RRDCALC_FLAG_DB_STALE not currently used
2677 if (unlikely(old_db_timestamp == rc->db_before)) {
2678 // database is stale
2680 debug(D_HEALTH, "Health alarm '%s.%s': database is stale", rc->chart?rc->chart:"NOCHART", rc->name);
2682 if (unlikely(!(rc->rrdcalc_flags & RRDCALC_FLAG_DB_STALE))) {
2683 rc->rrdcalc_flags |= RRDCALC_FLAG_DB_STALE;
2684 error("Health alarm '%s.%s': database is stale", rc->chart?rc->chart:"NOCHART", rc->name);
2687 else if (unlikely(rc->rrdcalc_flags & RRDCALC_FLAG_DB_STALE))
2688 rc->rrdcalc_flags &= ~RRDCALC_FLAG_DB_STALE;
2691 if (unlikely(value_is_null)) {
2692 // collected value is null
2696 debug(D_HEALTH, "Health alarm '%s.%s': database lookup returned empty value (possibly value is not collected yet)",
2697 rc->chart?rc->chart:"NOCHART", rc->name);
2699 if (unlikely(!(rc->rrdcalc_flags & RRDCALC_FLAG_DB_NAN))) {
2700 rc->rrdcalc_flags |= RRDCALC_FLAG_DB_NAN;
2701 error("Health alarm '%s.%s': database lookup returned empty value (possibly value is not collected yet)",
2702 rc->chart?rc->chart:"NOCHART", rc->name);
2705 else if (unlikely(rc->rrdcalc_flags & RRDCALC_FLAG_DB_NAN))
2706 rc->rrdcalc_flags &= ~RRDCALC_FLAG_DB_NAN;
2708 debug(D_HEALTH, "Health alarm '%s.%s': database lookup gave value "
2709 CALCULATED_NUMBER_FORMAT, rc->chart?rc->chart:"NOCHART", rc->name, rc->value);
2712 if(unlikely(rc->calculation)) {
2713 if (unlikely(!expression_evaluate(rc->calculation))) {
2714 // calculation failed
2718 debug(D_HEALTH, "Health alarm '%s.%s': failed to evaluate calculation with error: %s",
2719 rc->chart?rc->chart:"NOCHART", rc->name, buffer_tostring(rc->calculation->error_msg));
2721 if (unlikely(!(rc->rrdcalc_flags & RRDCALC_FLAG_CALC_ERROR))) {
2722 rc->rrdcalc_flags |= RRDCALC_FLAG_CALC_ERROR;
2723 error("Health alarm '%s.%s': failed to evaluate calculation with error: %s",
2724 rc->chart?rc->chart:"NOCHART", rc->name, buffer_tostring(rc->calculation->error_msg));
2728 if (unlikely(rc->rrdcalc_flags & RRDCALC_FLAG_CALC_ERROR))
2729 rc->rrdcalc_flags &= ~RRDCALC_FLAG_CALC_ERROR;
2731 debug(D_HEALTH, "Health alarm '%s.%s': calculation expression gave value "
2732 CALCULATED_NUMBER_FORMAT
2733 ": %s (source: %s)",
2734 rc->chart?rc->chart:"NOCHART", rc->name,
2735 rc->calculation->result,
2736 buffer_tostring(rc->calculation->error_msg),
2740 rc->value = rc->calculation->result;
2744 rrdhost_unlock(&localhost);
2746 if (unlikely(runnable && !netdata_exit)) {
2747 rrdhost_rdlock(&localhost);
2749 for (rc = localhost.alarms; rc; rc = rc->next) {
2750 if (unlikely(!rrdcalc_isrunnable(rc, now, &next_run)))
2753 int warning_status = RRDCALC_STATUS_UNDEFINED;
2754 int critical_status = RRDCALC_STATUS_UNDEFINED;
2756 if(likely(rc->warning)) {
2757 if(unlikely(!expression_evaluate(rc->warning))) {
2758 // calculation failed
2760 debug(D_HEALTH, "Health alarm '%s.%s': warning expression failed with error: %s",
2761 rc->chart?rc->chart:"NOCHART", rc->name, buffer_tostring(rc->warning->error_msg));
2763 if (unlikely(!(rc->rrdcalc_flags & RRDCALC_FLAG_WARN_ERROR))) {
2764 rc->rrdcalc_flags |= RRDCALC_FLAG_WARN_ERROR;
2765 error("Health alarm '%s.%s': warning expression failed with error: %s",
2766 rc->chart?rc->chart:"NOCHART", rc->name, buffer_tostring(rc->warning->error_msg));
2770 if(unlikely(rc->rrdcalc_flags & RRDCALC_FLAG_WARN_ERROR))
2771 rc->rrdcalc_flags &= ~RRDCALC_FLAG_WARN_ERROR;
2773 debug(D_HEALTH, "Health alarm '%s.%s': warning expression gave value "
2774 CALCULATED_NUMBER_FORMAT
2775 ": %s (source: %s)",
2776 rc->chart?rc->chart:"NOCHART", rc->name,
2777 rc->warning->result,
2778 buffer_tostring(rc->warning->error_msg),
2782 warning_status = rrdcalc_value2status(rc->warning->result);
2786 if(likely(rc->critical)) {
2787 if(unlikely(!expression_evaluate(rc->critical))) {
2788 // calculation failed
2790 debug(D_HEALTH, "Health alarm '%s.%s': critical expression failed with error: %s",
2791 rc->chart?rc->chart:"NOCHART", rc->name, buffer_tostring(rc->critical->error_msg));
2793 if (unlikely(!(rc->rrdcalc_flags & RRDCALC_FLAG_CRIT_ERROR))) {
2794 rc->rrdcalc_flags |= RRDCALC_FLAG_CRIT_ERROR;
2795 error("Health alarm '%s.%s': critical expression failed with error: %s",
2796 rc->chart?rc->chart:"NOCHART", rc->name, buffer_tostring(rc->critical->error_msg));
2800 if(unlikely(rc->rrdcalc_flags & RRDCALC_FLAG_CRIT_ERROR))
2801 rc->rrdcalc_flags &= ~RRDCALC_FLAG_CRIT_ERROR;
2803 debug(D_HEALTH, "Health alarm '%s.%s': critical expression gave value "
2804 CALCULATED_NUMBER_FORMAT
2805 ": %s (source: %s)",
2806 rc->chart?rc->chart:"NOCHART", rc->name,
2807 rc->critical->result,
2808 buffer_tostring(rc->critical->error_msg),
2812 critical_status = rrdcalc_value2status(rc->critical->result);
2816 int status = RRDCALC_STATUS_UNDEFINED;
2818 switch(warning_status) {
2819 case RRDCALC_STATUS_CLEAR:
2820 status = RRDCALC_STATUS_CLEAR;
2823 case RRDCALC_STATUS_RAISED:
2824 status = RRDCALC_STATUS_WARNING;
2831 switch(critical_status) {
2832 case RRDCALC_STATUS_CLEAR:
2833 if(status == RRDCALC_STATUS_UNDEFINED)
2834 status = RRDCALC_STATUS_CLEAR;
2837 case RRDCALC_STATUS_RAISED:
2838 status = RRDCALC_STATUS_CRITICAL;
2845 if(status != rc->status) {
2848 if(now > rc->delay_up_to_timestamp) {
2849 rc->delay_up_current = rc->delay_up_duration;
2850 rc->delay_down_current = rc->delay_down_duration;
2852 rc->delay_up_to_timestamp = 0;
2855 rc->delay_up_current = (int)(rc->delay_up_current * rc->delay_multiplier);
2856 if(rc->delay_up_current > rc->delay_max_duration) rc->delay_up_current = rc->delay_max_duration;
2858 rc->delay_down_current = (int)(rc->delay_down_current * rc->delay_multiplier);
2859 if(rc->delay_down_current > rc->delay_max_duration) rc->delay_down_current = rc->delay_max_duration;
2862 if(status > rc->status)
2863 delay = rc->delay_up_current;
2865 delay = rc->delay_down_current;
2867 // COMMENTED: because we do need to send raising alarms
2868 // if(now + delay < rc->delay_up_to_timestamp)
2869 // delay = (int)(rc->delay_up_to_timestamp - now);
2871 rc->delay_last = delay;
2872 rc->delay_up_to_timestamp = now + delay;
2873 health_alarm_log(&localhost, rc->id, rc->next_event_id++, now, rc->name, rc->rrdset->id, rc->rrdset->family, rc->exec, rc->recipient, now - rc->last_status_change, rc->old_value, rc->value, rc->status, status, rc->source, rc->units, rc->info, rc->delay_last);
2874 rc->last_status_change = now;
2875 rc->status = status;
2878 rc->last_updated = now;
2879 rc->next_update = now + rc->update_every;
2881 if (next_run > rc->next_update)
2882 next_run = rc->next_update;
2885 rrdhost_unlock(&localhost);
2888 if (unlikely(pthread_setcancelstate(oldstate, NULL) != 0))
2889 error("Cannot set pthread cancel state to RESTORE (%d).", oldstate);
2891 if(unlikely(netdata_exit))
2894 // execute notifications
2896 health_alarm_log_process(&localhost);
2898 if(unlikely(netdata_exit))
2902 if(now < next_run) {
2903 debug(D_HEALTH, "Health monitoring iteration no %u done. Next iteration in %d secs",
2904 loop, (int) (next_run - now));
2905 sleep_usec(1000000 * (unsigned long long) (next_run - now));
2908 debug(D_HEALTH, "Health monitoring iteration no %u done. Next iteration now", loop);
2914 info("HEALTH thread exiting");