unified streaming logging

[netdata.git] / src / health.c

diff --git a/src/health.c b/src/health.c
index 83390e95a6e7eb01203f920aa7f77f5833577623..d8bc9db59ace4306c08d83e24fc7c118016969fc 100644 (file)
--- a/src/health.c
+++ b/src/health.c
@@ -15,7 +15,7 @@ inline char *health_config_dir(void) {
 void health_init(void) {
     debug(D_HEALTH, "Health configuration initializing");
 
-    if(!central_netdata_to_push_data) {
+    if(!rrdpush_exclusive) {
         if(!(default_health_enabled = config_get_boolean("health", "enabled", 1))) {
             debug(D_HEALTH, "Health is disabled.");
             return;
@@ -26,11 +26,6 @@ void health_init(void) {
         config_set_boolean("health", "enabled", 0);
         default_health_enabled = 0;
     }
-
-    char pathname[FILENAME_MAX + 1];
-    snprintfz(pathname, FILENAME_MAX, "%s/health", netdata_configured_varlib_dir);
-    if(mkdir(pathname, 0770) == -1 && errno != EEXIST)
-        fatal("Cannot create directory '%s'.", pathname);
 }
 
 // ----------------------------------------------------------------------------
@@ -345,9 +340,14 @@ void *health_main(void *ptr) {
 
         int oldstate, runnable = 0;
         time_t now = now_realtime_sec();
+        time_t now_boot = now_boottime_sec();
         time_t next_run = now + min_run_every;
         RRDCALC *rc;
 
+        time_t last_now = now;
+        time_t last_now_boot = now_boot;
+        time_t hibernation_delay = config_get_number("health", "postpone alarms during hibernation for seconds", 60);
+
         if(unlikely(pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, &oldstate) != 0))
             error("Cannot set pthread cancel state to DISABLE.");
 
@@ -355,7 +355,15 @@ void *health_main(void *ptr) {
 
         RRDHOST *host;
         rrdhost_foreach_read(host) {
-            if(unlikely(!host->health_enabled)) continue;
+            if(now - last_now > 2 * (now_boot - last_now_boot)) {
+                info("Postponing alarm checks for %ld seconds, on host '%s', due to boottime discrepancy (realtime dt: %ld, boottime dt: %ld).",
+                     hibernation_delay, host->hostname, (long)(now - last_now), (long)(now_boot - last_now_boot));
+                host->health_delay_up_to = now + hibernation_delay;
+            }
+            last_now = now;
+            last_now_boot = now_boot;
+
+            if(unlikely(!host->health_enabled || now < host->health_delay_up_to)) continue;
 
             rrdhost_rdlock(host);
 
@@ -616,6 +624,8 @@ void *health_main(void *ptr) {
         }
         else
             debug(D_HEALTH, "Health monitoring iteration no %u done. Next iteration now", loop);
+
+        now_boot = now_boottime_sec();
     }
 
     buffer_free(wb);