src/health.c

   1 #include "common.h"
   2
   3 #define RRDVAR_MAX_LENGTH 1024
   4
   5 static const char *health_default_exec = PLUGINS_DIR "/alarm-email.sh";
   6 static const char *health_default_recipient = "root";
   7 int health_enabled = 1;
   8
   9 // ----------------------------------------------------------------------------
  10 // RRDVAR management
  11
  12 static inline int rrdvar_fix_name(char *variable) {
  13     int fixed = 0;
  14     while(*variable) {
  15         if (!isalnum(*variable) && *variable != '.' && *variable != '_') {
  16             *variable++ = '_';
  17             fixed++;
  18         }
  19         else
  20             variable++;
  21     }
  22
  23     return fixed;
  24 }
  25
  26 int rrdvar_compare(void* a, void* b) {
  27     if(((RRDVAR *)a)->hash < ((RRDVAR *)b)->hash) return -1;
  28     else if(((RRDVAR *)a)->hash > ((RRDVAR *)b)->hash) return 1;
  29     else return strcmp(((RRDVAR *)a)->name, ((RRDVAR *)b)->name);
  30 }
  31
  32 static inline RRDVAR *rrdvar_index_add(avl_tree_lock *tree, RRDVAR *rv) {
  33     RRDVAR *ret = (RRDVAR *)avl_insert_lock(tree, (avl *)(rv));
  34     if(ret != rv)
  35         debug(D_VARIABLES, "Request to insert RRDVAR '%s' into index failed. Already exists.", rv->name);
  36
  37     return ret;
  38 }
  39
  40 static inline RRDVAR *rrdvar_index_del(avl_tree_lock *tree, RRDVAR *rv) {
  41     RRDVAR *ret = (RRDVAR *)avl_remove_lock(tree, (avl *)(rv));
  42     if(!ret)
  43         error("Request to remove RRDVAR '%s' from index failed. Not Found.", rv->name);
  44
  45     return ret;
  46 }
  47
  48 static inline RRDVAR *rrdvar_index_find(avl_tree_lock *tree, const char *name, uint32_t hash) {
  49     RRDVAR tmp;
  50     tmp.name = (char *)name;
  51     tmp.hash = (hash)?hash:simple_hash(tmp.name);
  52
  53     return (RRDVAR *)avl_search_lock(tree, (avl *)&tmp);
  54 }
  55
  56 static inline void rrdvar_free(RRDHOST *host, avl_tree_lock *tree, RRDVAR *rv) {
  57     (void)host;
  58
  59     if(!rv) return;
  60
  61     if(tree)
  62         rrdvar_index_del(tree, rv);
  63
  64     freez(rv->name);
  65     freez(rv);
  66 }
  67
  68 static inline RRDVAR *rrdvar_create_and_index(const char *scope, avl_tree_lock *tree, const char *name, int type, calculated_number *value) {
  69     char *variable = strdupz(name);
  70     rrdvar_fix_name(variable);
  71     uint32_t hash = simple_hash(variable);
  72
  73     RRDVAR *rv = rrdvar_index_find(tree, variable, hash);
  74     if(unlikely(!rv)) {
  75         debug(D_VARIABLES, "Variable '%s' not found in scope '%s'. Creating a new one.", variable, scope);
  76
  77         rv = callocz(1, sizeof(RRDVAR));
  78         rv->name = variable;
  79         rv->hash = hash;
  80         rv->type = type;
  81         rv->value = value;
  82
  83         RRDVAR *ret = rrdvar_index_add(tree, rv);
  84         if(unlikely(ret != rv)) {
  85             debug(D_VARIABLES, "Variable '%s' in scope '%s' already exists", variable, scope);
  86             rrdvar_free(NULL, NULL, rv);
  87             rv = NULL;
  88         }
  89         else
  90             debug(D_VARIABLES, "Variable '%s' created in scope '%s'", variable, scope);
  91     }
  92     else {
  93         // already exists
  94         freez(variable);
  95         rv = NULL;
  96     }
  97
  98     return rv;
  99 }
 100
 101 // ----------------------------------------------------------------------------
 102 // RRDVAR lookup
 103
 104 calculated_number rrdvar2number(RRDVAR *rv) {
 105     switch(rv->type) {
 106         case RRDVAR_TYPE_CALCULATED: {
 107             calculated_number *n = (calculated_number *)rv->value;
 108             return *n;
 109         }
 110
 111         case RRDVAR_TYPE_TIME_T: {
 112             time_t *n = (time_t *)rv->value;
 113             return *n;
 114         }
 115
 116         case RRDVAR_TYPE_COLLECTED: {
 117             collected_number *n = (collected_number *)rv->value;
 118             return *n;
 119         }
 120
 121         case RRDVAR_TYPE_TOTAL: {
 122             total_number *n = (total_number *)rv->value;
 123             return *n;
 124         }
 125
 126         case RRDVAR_TYPE_INT: {
 127             int *n = (int *)rv->value;
 128             return *n;
 129         }
 130
 131         default:
 132             error("I don't know how to convert RRDVAR type %d to calculated_number", rv->type);
 133             return NAN;
 134     }
 135 }
 136
 137 void dump_variable(void *data) {
 138     RRDVAR *rv = (RRDVAR *)data;
 139     debug(D_HEALTH, "%50s : %20.5Lf", rv->name, rrdvar2number(rv));
 140 }
 141
 142 int health_variable_lookup(const char *variable, uint32_t hash, RRDCALC *rc, calculated_number *result) {
 143     RRDSET *st = rc->rrdset;
 144     RRDVAR *rv;
 145
 146     if(!st) return 0;
 147
 148     rv = rrdvar_index_find(&st->variables_root_index, variable, hash);
 149     if(rv) {
 150         *result = rrdvar2number(rv);
 151         return 1;
 152     }
 153
 154     rv = rrdvar_index_find(&st->rrdfamily->variables_root_index, variable, hash);
 155     if(rv) {
 156         *result = rrdvar2number(rv);
 157         return 1;
 158     }
 159
 160     rv = rrdvar_index_find(&st->rrdhost->variables_root_index, variable, hash);
 161     if(rv) {
 162         *result = rrdvar2number(rv);
 163         return 1;
 164     }
 165
 166     debug(D_HEALTH, "Available local chart '%s' variables:", st->id);
 167     avl_traverse_lock(&st->variables_root_index, dump_variable);
 168
 169     debug(D_HEALTH, "Available family '%s' variables:", st->rrdfamily->family);
 170     avl_traverse_lock(&st->rrdfamily->variables_root_index, dump_variable);
 171
 172     debug(D_HEALTH, "Available host '%s' variables:", st->rrdhost->hostname);
 173     avl_traverse_lock(&st->rrdhost->variables_root_index, dump_variable);
 174
 175     return 0;
 176 }
 177
 178 // ----------------------------------------------------------------------------
 179 // RRDSETVAR management
 180
 181 RRDSETVAR *rrdsetvar_create(RRDSET *st, const char *variable, int type, void *value, uint32_t options) {
 182     debug(D_VARIABLES, "RRDVARSET create for chart id '%s' name '%s' with variable name '%s'", st->id, st->name, variable);
 183     RRDSETVAR *rs = (RRDSETVAR *)callocz(1, sizeof(RRDSETVAR));
 184
 185     char buffer[RRDVAR_MAX_LENGTH + 1];
 186     snprintfz(buffer, RRDVAR_MAX_LENGTH, "%s.%s", st->id, variable);
 187     rs->fullid = strdupz(buffer);
 188
 189     snprintfz(buffer, RRDVAR_MAX_LENGTH, "%s.%s", st->name, variable);
 190     rs->fullname = strdupz(buffer);
 191
 192     rs->variable = strdupz(variable);
 193
 194     rs->type = type;
 195     rs->value = value;
 196     rs->options = options;
 197     rs->rrdset = st;
 198
 199     rs->local       = rrdvar_create_and_index("local",  &st->variables_root_index, rs->variable, rs->type, rs->value);
 200     rs->family      = rrdvar_create_and_index("family", &st->rrdfamily->variables_root_index, rs->fullid, rs->type, rs->value);
 201     rs->host        = rrdvar_create_and_index("host",   &st->rrdhost->variables_root_index, rs->fullid, rs->type, rs->value);
 202     rs->family_name = rrdvar_create_and_index("family", &st->rrdfamily->variables_root_index, rs->fullname, rs->type, rs->value);
 203     rs->host_name   = rrdvar_create_and_index("host",   &st->rrdhost->variables_root_index, rs->fullname, rs->type, rs->value);
 204
 205     rs->next = st->variables;
 206     st->variables = rs;
 207
 208     return rs;
 209 }
 210
 211 void rrdsetvar_rename_all(RRDSET *st) {
 212     debug(D_VARIABLES, "RRDSETVAR rename for chart id '%s' name '%s'", st->id, st->name);
 213
 214     // only these 2 can change name
 215     // rs->family_name
 216     // rs->host_name
 217
 218     char buffer[RRDVAR_MAX_LENGTH + 1];
 219     RRDSETVAR *rs, *next = st->variables;
 220     while((rs = next)) {
 221         next = rs->next;
 222
 223         snprintfz(buffer, RRDVAR_MAX_LENGTH, "%s.%s", st->name, rs->variable);
 224
 225         if (strcmp(buffer, rs->fullname)) {
 226             // name changed
 227             rrdvar_free(st->rrdhost, &st->rrdfamily->variables_root_index, rs->family_name);
 228             rrdvar_free(st->rrdhost, &st->rrdhost->variables_root_index, rs->host_name);
 229
 230             freez(rs->fullname);
 231             rs->fullname = strdupz(st->name);
 232             rs->family_name = rrdvar_create_and_index("family", &st->rrdfamily->variables_root_index, rs->fullname, rs->type, rs->value);
 233             rs->host_name   = rrdvar_create_and_index("host",   &st->rrdhost->variables_root_index, rs->fullname, rs->type, rs->value);
 234         }
 235     }
 236
 237     rrdsetcalc_link_matching(st);
 238 }
 239
 240 void rrdsetvar_free(RRDSETVAR *rs) {
 241     RRDSET *st = rs->rrdset;
 242     debug(D_VARIABLES, "RRDSETVAR free for chart id '%s' name '%s', variable '%s'", st->id, st->name, rs->variable);
 243
 244     rrdvar_free(st->rrdhost, &st->variables_root_index, rs->local);
 245     rrdvar_free(st->rrdhost, &st->rrdfamily->variables_root_index, rs->family);
 246     rrdvar_free(st->rrdhost, &st->rrdhost->variables_root_index, rs->host);
 247     rrdvar_free(st->rrdhost, &st->rrdfamily->variables_root_index, rs->family_name);
 248     rrdvar_free(st->rrdhost, &st->rrdhost->variables_root_index, rs->host_name);
 249
 250     if(st->variables == rs) {
 251         st->variables = rs->next;
 252     }
 253     else {
 254         RRDSETVAR *t;
 255         for (t = st->variables; t && t->next != rs; t = t->next);
 256         if(!t) error("RRDSETVAR '%s' not found in chart '%s' variables linked list", rs->fullname, st->id);
 257         else t->next = rs->next;
 258     }
 259
 260     freez(rs->fullid);
 261     freez(rs->fullname);
 262     freez(rs->variable);
 263     freez(rs);
 264 }
 265
 266 // ----------------------------------------------------------------------------
 267 // RRDDIMVAR management
 268
 269 #define RRDDIMVAR_ID_MAX 1024
 270
 271 RRDDIMVAR *rrddimvar_create(RRDDIM *rd, int type, const char *prefix, const char *suffix, void *value, uint32_t options) {
 272     RRDSET *st = rd->rrdset;
 273
 274     debug(D_VARIABLES, "RRDDIMSET create for chart id '%s' name '%s', dimension id '%s', name '%s%s%s'", st->id, st->name, rd->id, (prefix)?prefix:"", rd->name, (suffix)?suffix:"");
 275
 276     if(!prefix) prefix = "";
 277     if(!suffix) suffix = "";
 278
 279     char buffer[RRDDIMVAR_ID_MAX + 1];
 280     RRDDIMVAR *rs = (RRDDIMVAR *)callocz(1, sizeof(RRDDIMVAR));
 281
 282     rs->prefix = strdupz(prefix);
 283     rs->suffix = strdupz(suffix);
 284
 285     snprintfz(buffer, RRDDIMVAR_ID_MAX, "%s%s%s", rs->prefix, rd->id, rs->suffix);
 286     rs->id = strdupz(buffer);
 287
 288     snprintfz(buffer, RRDDIMVAR_ID_MAX, "%s%s%s", rs->prefix, rd->name, rs->suffix);
 289     rs->name = strdupz(buffer);
 290
 291     snprintfz(buffer, RRDDIMVAR_ID_MAX, "%s.%s", rd->rrdset->id, rs->id);
 292     rs->fullidid = strdupz(buffer);
 293
 294     snprintfz(buffer, RRDDIMVAR_ID_MAX, "%s.%s", rd->rrdset->id, rs->name);
 295     rs->fullidname = strdupz(buffer);
 296
 297     snprintfz(buffer, RRDDIMVAR_ID_MAX, "%s.%s", rd->rrdset->name, rs->id);
 298     rs->fullnameid = strdupz(buffer);
 299
 300     snprintfz(buffer, RRDDIMVAR_ID_MAX, "%s.%s", rd->rrdset->name, rs->name);
 301     rs->fullnamename = strdupz(buffer);
 302
 303     rs->type = type;
 304     rs->value = value;
 305     rs->options = options;
 306     rs->rrddim = rd;
 307
 308     rs->local_id     = rrdvar_create_and_index("local", &st->variables_root_index, rs->id, rs->type, rs->value);
 309     rs->local_name   = rrdvar_create_and_index("local", &st->variables_root_index, rs->name, rs->type, rs->value);
 310
 311     rs->family_id    = rrdvar_create_and_index("family", &st->rrdfamily->variables_root_index, rs->id, rs->type, rs->value);
 312     rs->family_name  = rrdvar_create_and_index("family", &st->rrdfamily->variables_root_index, rs->name, rs->type, rs->value);
 313
 314     rs->host_fullidid     = rrdvar_create_and_index("host", &st->rrdhost->variables_root_index, rs->fullidid, rs->type, rs->value);
 315     rs->host_fullidname   = rrdvar_create_and_index("host", &st->rrdhost->variables_root_index, rs->fullidname, rs->type, rs->value);
 316     rs->host_fullnameid   = rrdvar_create_and_index("host", &st->rrdhost->variables_root_index, rs->fullnameid, rs->type, rs->value);
 317     rs->host_fullnamename = rrdvar_create_and_index("host", &st->rrdhost->variables_root_index, rs->fullnamename, rs->type, rs->value);
 318
 319     rs->next = rd->variables;
 320     rd->variables = rs;
 321
 322     return rs;
 323 }
 324
 325 void rrddimvar_rename_all(RRDDIM *rd) {
 326     RRDSET *st = rd->rrdset;
 327     debug(D_VARIABLES, "RRDDIMSET rename for chart id '%s' name '%s', dimension id '%s', name '%s'", st->id, st->name, rd->id, rd->name);
 328
 329     RRDDIMVAR *rs, *next = rd->variables;
 330     while((rs = next)) {
 331         next = rs->next;
 332
 333         if (strcmp(rd->name, rs->name)) {
 334             char buffer[RRDDIMVAR_ID_MAX + 1];
 335             // name changed
 336
 337             // name
 338             rrdvar_free(st->rrdhost, &st->variables_root_index, rs->local_name);
 339             freez(rs->name);
 340             snprintfz(buffer, RRDDIMVAR_ID_MAX, "%s%s%s", rs->prefix, rd->name, rs->suffix);
 341             rs->name = strdupz(buffer);
 342             rs->local_name = rrdvar_create_and_index("local", &st->variables_root_index, rs->name, rs->type, rs->value);
 343
 344             rrdvar_free(st->rrdhost, &st->rrdhost->variables_root_index, rs->host_fullidname);
 345             freez(rs->fullidname);
 346             snprintfz(buffer, RRDDIMVAR_ID_MAX, "%s.%s", st->id, rs->name);
 347             rs->fullidname = strdupz(buffer);
 348             rs->host_fullidname = rrdvar_create_and_index("host", &st->rrdhost->variables_root_index,
 349                                                              rs->fullidname, rs->type, rs->value);
 350
 351             // fullnameid
 352             rrdvar_free(st->rrdhost, &st->rrdhost->variables_root_index, rs->host_fullnameid);
 353             freez(rs->fullnameid);
 354             snprintfz(buffer, RRDDIMVAR_ID_MAX, "%s.%s", st->name, rs->id);
 355             rs->fullnameid = strdupz(buffer);
 356             rs->host_fullnameid = rrdvar_create_and_index("host", &st->rrdhost->variables_root_index,
 357                                                           rs->fullnameid, rs->type, rs->value);
 358
 359             // fullnamename
 360             rrdvar_free(st->rrdhost, &st->rrdhost->variables_root_index, rs->host_fullnamename);
 361             freez(rs->fullnamename);
 362             snprintfz(buffer, RRDDIMVAR_ID_MAX, "%s.%s", st->name, rs->name);
 363             rs->fullnamename = strdupz(buffer);
 364             rs->host_fullnamename = rrdvar_create_and_index("host", &st->rrdhost->variables_root_index,
 365                                                           rs->fullnamename, rs->type, rs->value);
 366         }
 367     }
 368 }
 369
 370 void rrddimvar_free(RRDDIMVAR *rs) {
 371     RRDDIM *rd = rs->rrddim;
 372     RRDSET *st = rd->rrdset;
 373     debug(D_VARIABLES, "RRDDIMSET free for chart id '%s' name '%s', dimension id '%s', name '%s', prefix='%s', suffix='%s'", st->id, st->name, rd->id, rd->name, rs->prefix, rs->suffix);
 374
 375     rrdvar_free(st->rrdhost, &st->variables_root_index, rs->local_id);
 376     rrdvar_free(st->rrdhost, &st->variables_root_index, rs->local_name);
 377
 378     rrdvar_free(st->rrdhost, &st->rrdfamily->variables_root_index, rs->family_id);
 379     rrdvar_free(st->rrdhost, &st->rrdfamily->variables_root_index, rs->family_name);
 380
 381     rrdvar_free(st->rrdhost, &st->rrdhost->variables_root_index, rs->host_fullidid);
 382     rrdvar_free(st->rrdhost, &st->rrdhost->variables_root_index, rs->host_fullidname);
 383     rrdvar_free(st->rrdhost, &st->rrdhost->variables_root_index, rs->host_fullnameid);
 384     rrdvar_free(st->rrdhost, &st->rrdhost->variables_root_index, rs->host_fullnamename);
 385
 386     if(rd->variables == rs) {
 387         debug(D_VARIABLES, "RRDDIMSET removing first entry for chart id '%s' name '%s', dimension id '%s', name '%s'", st->id, st->name, rd->id, rd->name);
 388         rd->variables = rs->next;
 389     }
 390     else {
 391         debug(D_VARIABLES, "RRDDIMSET removing non-first entry for chart id '%s' name '%s', dimension id '%s', name '%s'", st->id, st->name, rd->id, rd->name);
 392         RRDDIMVAR *t;
 393         for (t = rd->variables; t && t->next != rs; t = t->next) ;
 394         if(!t) error("RRDDIMVAR '%s' not found in dimension '%s/%s' variables linked list", rs->name, st->id, rd->id);
 395         else t->next = rs->next;
 396     }
 397
 398     freez(rs->prefix);
 399     freez(rs->suffix);
 400     freez(rs->id);
 401     freez(rs->name);
 402     freez(rs->fullidid);
 403     freez(rs->fullidname);
 404     freez(rs->fullnameid);
 405     freez(rs->fullnamename);
 406     freez(rs);
 407 }
 408
 409 // ----------------------------------------------------------------------------
 410 // RRDCALC management
 411
 412 static inline const char *rrdcalc_status2string(int status) {
 413     switch(status) {
 414         case RRDCALC_STATUS_UNINITIALIZED:
 415             return "UNINITIALIZED";
 416
 417         case RRDCALC_STATUS_UNDEFINED:
 418             return "UNDEFINED";
 419
 420         case RRDCALC_STATUS_CLEAR:
 421             return "CLEAR";
 422
 423         case RRDCALC_STATUS_RAISED:
 424             return "RAISED";
 425
 426         case RRDCALC_STATUS_WARNING:
 427             return "WARNING";
 428
 429         case RRDCALC_STATUS_CRITICAL:
 430             return "CRITICAL";
 431
 432         default:
 433             return "UNKNOWN";
 434     }
 435 }
 436
 437 static void rrdsetcalc_link(RRDSET *st, RRDCALC *rc) {
 438     debug(D_HEALTH, "Health linking alarm '%s.%s' to chart '%s' of host '%s'", rc->chart?rc->chart:"NOCHART", rc->name, st->id, st->rrdhost->hostname);
 439
 440     rc->last_status_change = time(NULL);
 441     rc->rrdset = st;
 442
 443     rc->rrdset_next = st->alarms;
 444     rc->rrdset_prev = NULL;
 445     st->alarms = rc;
 446
 447     if(rc->update_every < rc->rrdset->update_every) {
 448         error("Health alarm '%s.%s' has update every %d, less than chart update every %d. Setting alarm update frequency to %d.", rc->rrdset->id, rc->name, rc->update_every, rc->rrdset->update_every, rc->rrdset->update_every);
 449         rc->update_every = rc->rrdset->update_every;
 450     }
 451
 452     if(!isnan(rc->green) && isnan(st->green)) {
 453         debug(D_HEALTH, "Health alarm '%s.%s' green threshold set from %Lf to %Lf.", rc->rrdset->id, rc->name, rc->rrdset->green, rc->green);
 454         st->green = rc->green;
 455     }
 456
 457     if(!isnan(rc->red) && isnan(st->red)) {
 458         debug(D_HEALTH, "Health alarm '%s.%s' red threshold set from %Lf to %Lf.", rc->rrdset->id, rc->name, rc->rrdset->red, rc->red);
 459         st->red = rc->red;
 460     }
 461
 462     rc->local  = rrdvar_create_and_index("local",  &st->variables_root_index, rc->name, RRDVAR_TYPE_CALCULATED, &rc->value);
 463     rc->family = rrdvar_create_and_index("family", &st->rrdfamily->variables_root_index, rc->name, RRDVAR_TYPE_CALCULATED, &rc->value);
 464
 465     char fullname[RRDVAR_MAX_LENGTH + 1];
 466     snprintfz(fullname, RRDVAR_MAX_LENGTH, "%s.%s", st->id, rc->name);
 467     rc->hostid   = rrdvar_create_and_index("host", &st->rrdhost->variables_root_index, fullname, RRDVAR_TYPE_CALCULATED, &rc->value);
 468
 469     snprintfz(fullname, RRDVAR_MAX_LENGTH, "%s.%s", st->name, rc->name);
 470     rc->hostname = rrdvar_create_and_index("host", &st->rrdhost->variables_root_index, fullname, RRDVAR_TYPE_CALCULATED, &rc->value);
 471
 472         if(!rc->units) rc->units = strdupz(st->units);
 473 }
 474
 475 static inline int rrdcalc_is_matching_this_rrdset(RRDCALC *rc, RRDSET *st) {
 476     if(     (rc->hash_chart == st->hash      && !strcmp(rc->chart, st->id)) ||
 477             (rc->hash_chart == st->hash_name && !strcmp(rc->chart, st->name)))
 478         return 1;
 479
 480     return 0;
 481 }
 482
 483 // this has to be called while the RRDHOST is locked
 484 inline void rrdsetcalc_link_matching(RRDSET *st) {
 485     // debug(D_HEALTH, "find matching alarms for chart '%s'", st->id);
 486
 487     RRDCALC *rc;
 488     for(rc = st->rrdhost->alarms; rc ; rc = rc->next) {
 489         if(unlikely(rc->rrdset))
 490             continue;
 491
 492         if(unlikely(rrdcalc_is_matching_this_rrdset(rc, st)))
 493             rrdsetcalc_link(st, rc);
 494     }
 495 }
 496
 497 // this has to be called while the RRDHOST is locked
 498 inline void rrdsetcalc_unlink(RRDCALC *rc) {
 499     RRDSET *st = rc->rrdset;
 500
 501     if(!st) {
 502         error("Requested to unlink RRDCALC '%s.%s' which is not linked to any RRDSET", rc->chart?rc->chart:"NOCHART", rc->name);
 503         return;
 504     }
 505
 506     RRDHOST *host = st->rrdhost;
 507
 508     debug(D_HEALTH, "Health unlinking alarm '%s.%s' from chart '%s' of host '%s'", rc->chart?rc->chart:"NOCHART", rc->name, st->id, host->hostname);
 509
 510     // unlink it
 511     if(rc->rrdset_prev)
 512         rc->rrdset_prev->rrdset_next = rc->rrdset_next;
 513
 514     if(rc->rrdset_next)
 515         rc->rrdset_next->rrdset_prev = rc->rrdset_prev;
 516
 517     if(st->alarms == rc)
 518         st->alarms = rc->rrdset_next;
 519
 520     rc->rrdset_prev = rc->rrdset_next = NULL;
 521
 522     rrdvar_free(st->rrdhost, &st->variables_root_index, rc->local);
 523     rc->local = NULL;
 524
 525     rrdvar_free(st->rrdhost, &st->rrdfamily->variables_root_index, rc->family);
 526     rc->family = NULL;
 527
 528     rrdvar_free(st->rrdhost, &st->rrdhost->variables_root_index, rc->hostid);
 529     rc->hostid = NULL;
 530
 531     rrdvar_free(st->rrdhost, &st->rrdhost->variables_root_index, rc->hostname);
 532     rc->hostname = NULL;
 533
 534     rc->rrdset = NULL;
 535
 536     // RRDCALC will remain in RRDHOST
 537     // so that if the matching chart is found in the future
 538     // it will be applied automatically
 539 }
 540
 541 RRDCALC *rrdcalc_find(RRDSET *st, const char *name) {
 542     RRDCALC *rc;
 543     uint32_t hash = simple_hash(name);
 544
 545     for( rc = st->alarms; rc ; rc = rc->rrdset_next ) {
 546         if(unlikely(rc->hash == hash && !strcmp(rc->name, name)))
 547             return rc;
 548     }
 549
 550     return NULL;
 551 }
 552
 553 static inline int rrdcalc_exists(RRDHOST *host, const char *name, uint32_t hash) {
 554     RRDCALC *rc;
 555
 556     // make sure it does not already exist
 557     for(rc = host->alarms; rc ; rc = rc->next) {
 558         if (unlikely(rc->hash == hash && !strcmp(name, rc->name))) {
 559             error("Health alarm '%s' already exists in host '%s'.", name, host->hostname);
 560             return 1;
 561         }
 562     }
 563
 564     return 0;
 565 }
 566
 567 static inline void rrdcalc_create_part2(RRDHOST *host, RRDCALC *rc) {
 568     rrdhost_check_rdlock(host);
 569
 570     if(rc->calculation) {
 571         rc->calculation->this = &rc->value;
 572         rc->calculation->after = &rc->db_after;
 573         rc->calculation->before = &rc->db_before;
 574         rc->calculation->rrdcalc = rc;
 575     }
 576
 577     if(rc->warning) {
 578         rc->warning->this = &rc->value;
 579         rc->warning->after = &rc->db_after;
 580         rc->warning->before = &rc->db_before;
 581         rc->warning->rrdcalc = rc;
 582     }
 583
 584     if(rc->critical) {
 585         rc->critical->this = &rc->value;
 586         rc->critical->after = &rc->db_after;
 587         rc->critical->before = &rc->db_before;
 588         rc->critical->rrdcalc = rc;
 589     }
 590
 591     // link it to the host
 592     if(likely(host->alarms)) {
 593         // append it
 594         RRDCALC *t;
 595         for(t = host->alarms; t && t->next ; t = t->next) ;
 596         t->next = rc;
 597     }
 598     else {
 599         host->alarms = rc;
 600     }
 601
 602     // link it to its chart
 603     RRDSET *st;
 604     for(st = host->rrdset_root; st ; st = st->next) {
 605         if(rrdcalc_is_matching_this_rrdset(rc, st)) {
 606             rrdsetcalc_link(st, rc);
 607             break;
 608         }
 609     }
 610 }
 611
 612 static inline uint32_t rrdcalc_fullname(char *fullname, size_t len, const char *chart, const char *name) {
 613     snprintfz(fullname, len - 1, "%s%s%s", chart?chart:"", chart?".":"", name);
 614     rrdvar_fix_name(fullname);
 615     return simple_hash(fullname);
 616 }
 617
 618 static inline RRDCALC *rrdcalc_create(RRDHOST *host, const char *name, const char *chart, const char *dimensions,
 619                         const char *units, const char *info,
 620                         int group_method, int after, int before, int update_every, uint32_t options,
 621                         calculated_number green, calculated_number red,
 622                         const char *exec, const char *recipient, const char *source,
 623                         const char *calc, const char *warn, const char *crit) {
 624
 625     char fullname[RRDVAR_MAX_LENGTH + 1];
 626     uint32_t hash = rrdcalc_fullname(fullname, RRDVAR_MAX_LENGTH + 1, chart, name);
 627
 628     if(rrdcalc_exists(host, fullname, hash))
 629         return NULL;
 630
 631     RRDCALC *rc = callocz(1, sizeof(RRDCALC));
 632     rc->id = host->health_log.next_alarm_id++;
 633     rc->next_event_id = 1;
 634     rc->name = strdupz(name);
 635     rc->hash = simple_hash(rc->name);
 636     rc->chart = strdupz(chart);
 637     rc->hash_chart = simple_hash(rc->chart);
 638
 639     if(dimensions) rc->dimensions = strdupz(dimensions);
 640
 641     rc->green = green;
 642     rc->red = red;
 643     rc->value = NAN;
 644     rc->old_value = NAN;
 645
 646     rc->group = group_method;
 647     rc->after = after;
 648     rc->before = before;
 649     rc->update_every = update_every;
 650     rc->options = options;
 651
 652     if(exec) rc->exec = strdupz(exec);
 653     if(recipient) rc->recipient = strdupz(recipient);
 654     if(source) rc->source = strdupz(source);
 655     if(units) rc->units = strdupz(units);
 656     if(info) rc->info = strdupz(info);
 657
 658     if(calc) {
 659         rc->calculation = expression_parse(calc, NULL, NULL);
 660         if(!rc->calculation)
 661             error("Health alarm '%s.%s': failed to parse calculation expression '%s'", chart, name, calc);
 662     }
 663     if(warn) {
 664         rc->warning = expression_parse(warn, NULL, NULL);
 665         if(!rc->warning)
 666             error("Health alarm '%s.%s': failed to re-parse warning expression '%s'", chart, name, warn);
 667     }
 668     if(crit) {
 669         rc->critical = expression_parse(crit, NULL, NULL);
 670         if(!rc->critical)
 671             error("Health alarm '%s.%s': failed to re-parse critical expression '%s'", chart, name, crit);
 672     }
 673
 674     debug(D_HEALTH, "Health runtime added alarm '%s.%s': exec '%s', recipient '%s', green %Lf, red %Lf, lookup: group %d, after %d, before %d, options %u, dimensions '%s', update every %d, calculation '%s', warning '%s', critical '%s', source '%s",
 675           (rc->chart)?rc->chart:"NOCHART",
 676           rc->name,
 677           (rc->exec)?rc->exec:"DEFAULT",
 678           (rc->recipient)?rc->recipient:"DEFAULT",
 679           rc->green,
 680           rc->red,
 681           rc->group,
 682           rc->after,
 683           rc->before,
 684           rc->options,
 685           (rc->dimensions)?rc->dimensions:"NONE",
 686           rc->update_every,
 687           (rc->calculation)?rc->calculation->parsed_as:"NONE",
 688           (rc->warning)?rc->warning->parsed_as:"NONE",
 689           (rc->critical)?rc->critical->parsed_as:"NONE",
 690           rc->source
 691     );
 692
 693     rrdcalc_create_part2(host, rc);
 694     return rc;
 695 }
 696
 697 void rrdcalc_free(RRDHOST *host, RRDCALC *rc) {
 698     if(!rc) return;
 699
 700     debug(D_HEALTH, "Health removing alarm '%s.%s' of host '%s'", rc->chart?rc->chart:"NOCHART", rc->name, host->hostname);
 701
 702     // unlink it from RRDSET
 703     if(rc->rrdset) rrdsetcalc_unlink(rc);
 704
 705     // unlink it from RRDHOST
 706     if(unlikely(rc == host->alarms))
 707         host->alarms = rc->next;
 708
 709     else if(likely(host->alarms)) {
 710         RRDCALC *t, *last = host->alarms;
 711
 712         for(t = last->next; t && t != rc; last = t, t = t->next) ;
 713         if(last && last->next == rc)
 714             last->next = rc->next;
 715         else
 716             error("Cannot unlink alarm '%s.%s' from host '%s': not found", rc->chart?rc->chart:"NOCHART", rc->name, host->hostname);
 717     }
 718     else
 719         error("Cannot unlink unlink '%s.%s' from host '%s': This host does not have any calculations", rc->chart?rc->chart:"NOCHART", rc->name, host->hostname);
 720
 721     expression_free(rc->calculation);
 722     expression_free(rc->warning);
 723     expression_free(rc->critical);
 724
 725     freez(rc->name);
 726     freez(rc->chart);
 727     freez(rc->family);
 728     freez(rc->dimensions);
 729     freez(rc->exec);
 730     freez(rc->recipient);
 731     freez(rc->source);
 732     freez(rc->units);
 733     freez(rc->info);
 734     freez(rc);
 735 }
 736
 737 // ----------------------------------------------------------------------------
 738 // RRDCALCTEMPLATE management
 739
 740 void rrdcalctemplate_link_matching(RRDSET *st) {
 741     RRDCALCTEMPLATE *rt;
 742
 743     for(rt = st->rrdhost->templates; rt ; rt = rt->next) {
 744         if(rt->hash_context == st->hash_context && !strcmp(rt->context, st->context)) {
 745             RRDCALC *rc = rrdcalc_create(st->rrdhost, rt->name, st->id,
 746                            rt->dimensions, rt->units, rt->info, rt->group, rt->after, rt->before, rt->update_every, rt->options,
 747                            rt->green, rt->red,
 748                            rt->exec, rt->recipient, rt->source,
 749                            (rt->calculation)?rt->calculation->source:NULL,
 750                            (rt->warning)?rt->warning->source:NULL,
 751                            (rt->critical)?rt->critical->source:NULL);
 752
 753             if(unlikely(!rc))
 754                 error("Health tried to create alarm from template '%s', but it failed", rt->name);
 755
 756 #ifdef NETDATA_INTERNAL_CHECKS
 757             else if(rc->rrdset != st)
 758                 error("Health alarm '%s.%s' should be linked to chart '%s', but it is not", rc->chart?rc->chart:"NOCHART", rc->name, st->id);
 759 #endif
 760         }
 761     }
 762 }
 763
 764 static inline void rrdcalctemplate_free(RRDHOST *host, RRDCALCTEMPLATE *rt) {
 765     debug(D_HEALTH, "Health removing template '%s' of host '%s'", rt->name, host->hostname);
 766
 767     if(host->templates) {
 768         if(host->templates == rt) {
 769             host->templates = rt->next;
 770         }
 771         else {
 772             RRDCALCTEMPLATE *t, *last = host->templates;
 773             for (t = last->next; t && t != rt; last = t, t = t->next ) ;
 774             if(last && last->next == rt) {
 775                 last->next = rt->next;
 776                 rt->next = NULL;
 777             }
 778             else
 779                 error("Cannot find RRDCALCTEMPLATE '%s' linked in host '%s'", rt->name, host->hostname);
 780         }
 781     }
 782
 783     expression_free(rt->calculation);
 784     expression_free(rt->warning);
 785     expression_free(rt->critical);
 786
 787     freez(rt->name);
 788     freez(rt->exec);
 789     freez(rt->recipient);
 790     freez(rt->context);
 791     freez(rt->source);
 792     freez(rt->units);
 793     freez(rt->info);
 794     freez(rt->dimensions);
 795     freez(rt);
 796 }
 797
 798 // ----------------------------------------------------------------------------
 799 // load health configuration
 800
 801 #define HEALTH_CONF_MAX_LINE 4096
 802
 803 #define HEALTH_ALARM_KEY "alarm"
 804 #define HEALTH_TEMPLATE_KEY "template"
 805 #define HEALTH_ON_KEY "on"
 806 #define HEALTH_LOOKUP_KEY "lookup"
 807 #define HEALTH_CALC_KEY "calc"
 808 #define HEALTH_EVERY_KEY "every"
 809 #define HEALTH_GREEN_KEY "green"
 810 #define HEALTH_RED_KEY "red"
 811 #define HEALTH_WARN_KEY "warn"
 812 #define HEALTH_CRIT_KEY "crit"
 813 #define HEALTH_EXEC_KEY "exec"
 814 #define HEALTH_RECIPIENT_KEY "to"
 815 #define HEALTH_UNITS_KEY "units"
 816 #define HEALTH_INFO_KEY "info"
 817
 818 static inline int rrdcalc_add_alarm_from_config(RRDHOST *host, RRDCALC *rc) {
 819     {
 820         char fullname[RRDVAR_MAX_LENGTH + 1];
 821         uint32_t hash = rrdcalc_fullname(fullname, RRDVAR_MAX_LENGTH + 1, rc->chart, rc->name);
 822
 823         if (rrdcalc_exists(host, fullname, hash))
 824             return 0;
 825     }
 826
 827     if(!rc->chart) {
 828         error("Health configuration for alarm '%s' does not have a chart", rc->name);
 829         return 0;
 830     }
 831
 832     if(!rc->update_every) {
 833         error("Health configuration for alarm '%s.%s' has no frequency (parameter 'every'). Ignoring it.", rc->chart?rc->chart:"NOCHART", rc->name);
 834         return 0;
 835     }
 836
 837     if(!RRDCALC_HAS_DB_LOOKUP(rc) && !rc->warning && !rc->critical) {
 838         error("Health configuration for alarm '%s.%s' is useless (no calculation, no warning and no critical evaluation)", rc->chart?rc->chart:"NOCHART", rc->name);
 839         return 0;
 840     }
 841
 842     debug(D_HEALTH, "Health configuration adding alarm '%s.%s': exec '%s', recipient '%s', green %Lf, red %Lf, lookup: group %d, after %d, before %d, options %u, dimensions '%s', update every %d, calculation '%s', warning '%s', critical '%s', source '%s",
 843           rc->chart?rc->chart:"NOCHART",
 844           rc->name,
 845           (rc->exec)?rc->exec:"DEFAULT",
 846           (rc->recipient)?rc->recipient:"DEFAULT",
 847           rc->green,
 848           rc->red,
 849           rc->group,
 850           rc->after,
 851           rc->before,
 852           rc->options,
 853           (rc->dimensions)?rc->dimensions:"NONE",
 854           rc->update_every,
 855           (rc->calculation)?rc->calculation->parsed_as:"NONE",
 856           (rc->warning)?rc->warning->parsed_as:"NONE",
 857           (rc->critical)?rc->critical->parsed_as:"NONE",
 858           rc->source
 859     );
 860
 861     rrdcalc_create_part2(host, rc);
 862     return 1;
 863 }
 864
 865 static inline int rrdcalctemplate_add_template_from_config(RRDHOST *host, RRDCALCTEMPLATE *rt) {
 866     if(unlikely(!rt->context)) {
 867         error("Health configuration for template '%s' does not have a context", rt->name);
 868         return 0;
 869     }
 870
 871     if(unlikely(!rt->update_every)) {
 872         error("Health configuration for template '%s' has no frequency (parameter 'every'). Ignoring it.", rt->name);
 873         return 0;
 874     }
 875
 876     if(unlikely(!RRDCALCTEMPLATE_HAS_CALCULATION(rt) && !rt->warning && !rt->critical)) {
 877         error("Health configuration for template '%s' is useless (no calculation, no warning and no critical evaluation)", rt->name);
 878         return 0;
 879     }
 880
 881     RRDCALCTEMPLATE *t, *last = NULL;
 882     for (t = host->templates; t ; last = t, t = t->next) {
 883         if(unlikely(t->hash_name == rt->hash_name && !strcmp(t->name, rt->name))) {
 884             error("Health configuration template '%s' already exists for host '%s'.", rt->name, host->hostname);
 885             return 0;
 886         }
 887     }
 888
 889     debug(D_HEALTH, "Health configuration adding template '%s': context '%s', exec '%s', recipient '%s', green %Lf, red %Lf, lookup: group %d, after %d, before %d, options %u, dimensions '%s', update every %d, calculation '%s', warning '%s', critical '%s', source '%s'",
 890           rt->name,
 891           (rt->context)?rt->context:"NONE",
 892           (rt->exec)?rt->exec:"DEFAULT",
 893           (rt->recipient)?rt->recipient:"DEFAULT",
 894           rt->green,
 895           rt->red,
 896           rt->group,
 897           rt->after,
 898           rt->before,
 899           rt->options,
 900           (rt->dimensions)?rt->dimensions:"NONE",
 901           rt->update_every,
 902           (rt->calculation)?rt->calculation->parsed_as:"NONE",
 903           (rt->warning)?rt->warning->parsed_as:"NONE",
 904           (rt->critical)?rt->critical->parsed_as:"NONE",
 905           rt->source
 906     );
 907
 908     if(likely(last)) {
 909         last->next = rt;
 910     }
 911     else {
 912         rt->next = host->templates;
 913         host->templates = rt;
 914     }
 915
 916     return 1;
 917 }
 918
 919 static inline int health_parse_duration(char *string, int *result) {
 920     // make sure it is a number
 921     if(!*string || !(isdigit(*string) || *string == '+' || *string == '-')) {
 922         *result = 0;
 923         return 0;
 924     }
 925
 926     char *e = NULL;
 927     calculated_number n = strtold(string, &e);
 928     if(e && *e) {
 929         switch (*e) {
 930             case 'Y':
 931                 *result = (int) (n * 86400 * 365);
 932                 break;
 933             case 'M':
 934                 *result = (int) (n * 86400 * 30);
 935                 break;
 936             case 'w':
 937                 *result = (int) (n * 86400 * 7);
 938                 break;
 939             case 'd':
 940                 *result = (int) (n * 86400);
 941                 break;
 942             case 'h':
 943                 *result = (int) (n * 3600);
 944                 break;
 945             case 'm':
 946                 *result = (int) (n * 60);
 947                 break;
 948
 949             default:
 950             case 's':
 951                 *result = (int) (n);
 952                 break;
 953         }
 954     }
 955     else
 956        *result = (int)(n);
 957
 958     return 1;
 959 }
 960
 961 static inline int health_parse_db_lookup(
 962         size_t line, const char *path, const char *file, char *string,
 963         int *group_method, int *after, int *before, int *every,
 964         uint32_t *options, char **dimensions
 965 ) {
 966     debug(D_HEALTH, "Health configuration parsing database lookup %zu@%s/%s: %s", line, path, file, string);
 967
 968     if(*dimensions) freez(*dimensions);
 969     *dimensions = NULL;
 970     *after = 0;
 971     *before = 0;
 972     *every = 0;
 973     *options = 0;
 974
 975     char *s = string, *key;
 976
 977     // first is the group method
 978     key = s;
 979     while(*s && !isspace(*s)) s++;
 980     while(*s && isspace(*s)) *s++ = '\0';
 981     if(!*s) {
 982         error("Health configuration invalid chart calculation at line %zu of file '%s/%s': expected group method followed by the 'after' time, but got '%s'",
 983               line, path, file, key);
 984         return 0;
 985     }
 986
 987     if((*group_method = web_client_api_request_v1_data_group(key, -1)) == -1) {
 988         error("Health configuration at line %zu of file '%s/%s': invalid group method '%s'",
 989               line, path, file, key);
 990         return 0;
 991     }
 992
 993     // then is the 'after' time
 994     key = s;
 995     while(*s && !isspace(*s)) s++;
 996     while(*s && isspace(*s)) *s++ = '\0';
 997
 998     if(!health_parse_duration(key, after)) {
 999         error("Health configuration at line %zu of file '%s/%s': invalid duration '%s' after group method",
1000               line, path, file, key);
1001         return 0;
1002     }
1003
1004     // sane defaults
1005     *every = abs(*after);
1006
1007     // now we may have optional parameters
1008     while(*s) {
1009         key = s;
1010         while(*s && !isspace(*s)) s++;
1011         while(*s && isspace(*s)) *s++ = '\0';
1012         if(!*key) break;
1013
1014         if(!strcasecmp(key, "at")) {
1015             char *value = s;
1016             while(*s && !isspace(*s)) s++;
1017             while(*s && isspace(*s)) *s++ = '\0';
1018
1019             if (!health_parse_duration(value, before)) {
1020                 error("Health configuration at line %zu of file '%s/%s': invalid duration '%s' for '%s' keyword",
1021                       line, path, file, value, key);
1022             }
1023         }
1024         else if(!strcasecmp(key, HEALTH_EVERY_KEY)) {
1025             char *value = s;
1026             while(*s && !isspace(*s)) s++;
1027             while(*s && isspace(*s)) *s++ = '\0';
1028
1029             if (!health_parse_duration(value, every)) {
1030                 error("Health configuration at line %zu of file '%s/%s': invalid duration '%s' for '%s' keyword",
1031                       line, path, file, value, key);
1032             }
1033         }
1034         else if(!strcasecmp(key, "absolute") || !strcasecmp(key, "abs") || !strcasecmp(key, "absolute_sum")) {
1035             *options |= RRDR_OPTION_ABSOLUTE;
1036         }
1037         else if(!strcasecmp(key, "min2max")) {
1038             *options |= RRDR_OPTION_MIN2MAX;
1039         }
1040         else if(!strcasecmp(key, "null2zero")) {
1041             *options |= RRDR_OPTION_NULL2ZERO;
1042         }
1043         else if(!strcasecmp(key, "percentage")) {
1044             *options |= RRDR_OPTION_PERCENTAGE;
1045         }
1046         else if(!strcasecmp(key, "unaligned")) {
1047             *options |= RRDR_OPTION_NOT_ALIGNED;
1048         }
1049         else if(!strcasecmp(key, "of")) {
1050             if(*s && strcasecmp(s, "all"))
1051                *dimensions = strdupz(s);
1052             break;
1053         }
1054         else {
1055             error("Health configuration at line %zu of file '%s/%s': unknown keyword '%s'",
1056                   line, path, file, key);
1057         }
1058     }
1059
1060     return 1;
1061 }
1062
1063 static inline char *health_source_file(size_t line, const char *path, const char *filename) {
1064     char buffer[FILENAME_MAX + 1];
1065     snprintfz(buffer, FILENAME_MAX, "%zu@%s/%s", line, path, filename);
1066     return strdupz(buffer);
1067 }
1068
1069 static inline void strip_quotes(char *s) {
1070     while(*s) {
1071         if(*s == '\'' || *s == '"') *s = ' ';
1072         s++;
1073     }
1074 }
1075
1076 int health_readfile(const char *path, const char *filename) {
1077     debug(D_HEALTH, "Health configuration reading file '%s/%s'", path, filename);
1078
1079     static uint32_t hash_alarm = 0, hash_template = 0, hash_on = 0, hash_calc = 0, hash_green = 0, hash_red = 0, hash_warn = 0, hash_crit = 0, hash_exec = 0, hash_every = 0, hash_lookup = 0, hash_units = 0, hash_info = 0, hash_recipient = 0;
1080     char buffer[HEALTH_CONF_MAX_LINE + 1];
1081
1082     if(unlikely(!hash_alarm)) {
1083         hash_alarm = simple_uhash(HEALTH_ALARM_KEY);
1084         hash_template = simple_uhash(HEALTH_TEMPLATE_KEY);
1085         hash_on = simple_uhash(HEALTH_ON_KEY);
1086         hash_calc = simple_uhash(HEALTH_CALC_KEY);
1087         hash_lookup = simple_uhash(HEALTH_LOOKUP_KEY);
1088         hash_green = simple_uhash(HEALTH_GREEN_KEY);
1089         hash_red = simple_uhash(HEALTH_RED_KEY);
1090         hash_warn = simple_uhash(HEALTH_WARN_KEY);
1091         hash_crit = simple_uhash(HEALTH_CRIT_KEY);
1092         hash_exec = simple_uhash(HEALTH_EXEC_KEY);
1093         hash_every = simple_uhash(HEALTH_EVERY_KEY);
1094         hash_units = simple_hash(HEALTH_UNITS_KEY);
1095         hash_info = simple_hash(HEALTH_INFO_KEY);
1096         hash_recipient = simple_hash(HEALTH_RECIPIENT_KEY);
1097     }
1098
1099     snprintfz(buffer, HEALTH_CONF_MAX_LINE, "%s/%s", path, filename);
1100     FILE *fp = fopen(buffer, "r");
1101     if(!fp) {
1102         error("Health configuration cannot read file '%s'.", buffer);
1103         return 0;
1104     }
1105
1106     RRDCALC *rc = NULL;
1107     RRDCALCTEMPLATE *rt = NULL;
1108
1109     size_t line = 0, append = 0;
1110     char *s;
1111     while((s = fgets(&buffer[append], (int)(HEALTH_CONF_MAX_LINE - append), fp)) || append) {
1112         int stop_appending = !s;
1113         line++;
1114         // info("Line %zu of file '%s/%s': '%s'", line, path, filename, s);
1115         s = trim(buffer);
1116         if(!s) continue;
1117         // info("Trimmed line %zu of file '%s/%s': '%s'", line, path, filename, s);
1118
1119         append = strlen(s);
1120         if(!stop_appending && s[append - 1] == '\\') {
1121             s[append - 1] = ' ';
1122             append = &s[append] - buffer;
1123             if(append < HEALTH_CONF_MAX_LINE)
1124                 continue;
1125             else {
1126                 error("Health configuration has too long muli-line at line %zu of file '%s/%s'.", line, path, filename);
1127             }
1128         }
1129         append = 0;
1130
1131         char *key = s;
1132         while(*s && *s != ':') s++;
1133         if(!*s) {
1134             error("Health configuration has invalid line %zu of file '%s/%s'. It does not contain a ':'. Ignoring it.", line, path, filename);
1135             continue;
1136         }
1137         *s = '\0';
1138         s++;
1139
1140         char *value = s;
1141         key = trim(key);
1142         value = trim(value);
1143
1144         if(!key) {
1145             error("Health configuration has invalid line %zu of file '%s/%s'. Keyword is empty. Ignoring it.", line, path, filename);
1146             continue;
1147         }
1148
1149         if(!value) {
1150             error("Health configuration has invalid line %zu of file '%s/%s'. value is empty. Ignoring it.", line, path, filename);
1151             continue;
1152         }
1153
1154         // info("Health file '%s/%s', key '%s', value '%s'", path, filename, key, value);
1155         uint32_t hash = simple_uhash(key);
1156
1157         if(hash == hash_alarm && !strcasecmp(key, HEALTH_ALARM_KEY)) {
1158             if(rc && !rrdcalc_add_alarm_from_config(&localhost, rc))
1159                 rrdcalc_free(&localhost, rc);
1160
1161             if(rt) {
1162                 if (!rrdcalctemplate_add_template_from_config(&localhost, rt))
1163                     rrdcalctemplate_free(&localhost, rt);
1164                 rt = NULL;
1165             }
1166
1167             rc = callocz(1, sizeof(RRDCALC));
1168             rc->id = localhost.health_log.next_alarm_id++;
1169             rc->next_event_id = 1;
1170             rc->name = strdupz(value);
1171             rc->hash = simple_hash(rc->name);
1172             rc->source = health_source_file(line, path, filename);
1173             rc->green = NAN;
1174             rc->red = NAN;
1175             rc->value = NAN;
1176             rc->old_value = NAN;
1177
1178             if(rrdvar_fix_name(rc->name))
1179                 error("Health configuration renamed alarm '%s' to '%s'", value, rc->name);
1180         }
1181         else if(hash == hash_template && !strcasecmp(key, HEALTH_TEMPLATE_KEY)) {
1182             if(rc) {
1183                 if(!rrdcalc_add_alarm_from_config(&localhost, rc))
1184                     rrdcalc_free(&localhost, rc);
1185                 rc = NULL;
1186             }
1187
1188             if(rt && !rrdcalctemplate_add_template_from_config(&localhost, rt))
1189                 rrdcalctemplate_free(&localhost, rt);
1190
1191             rt = callocz(1, sizeof(RRDCALCTEMPLATE));
1192             rt->name = strdupz(value);
1193             rt->hash_name = simple_hash(rt->name);
1194             rt->source = health_source_file(line, path, filename);
1195             rt->green = NAN;
1196             rt->red = NAN;
1197
1198             if(rrdvar_fix_name(rt->name))
1199                 error("Health configuration renamed template '%s' to '%s'", value, rt->name);
1200         }
1201         else if(rc) {
1202             if(hash == hash_on && !strcasecmp(key, HEALTH_ON_KEY)) {
1203                 if(rc->chart) {
1204                     if(strcmp(rc->chart, value))
1205                         info("Health configuration at line %zu of file '%s/%s' for alarm '%s' has key '%s' twice, once with value '%s' and later with value '%s'. Using ('%s').",
1206                              line, path, filename, rc->name, key, rc->chart, value, value);
1207
1208                     freez(rc->chart);
1209                 }
1210                 rc->chart = strdupz(value);
1211                 rc->hash_chart = simple_hash(rc->chart);
1212             }
1213             else if(hash == hash_lookup && !strcasecmp(key, HEALTH_LOOKUP_KEY)) {
1214                 health_parse_db_lookup(line, path, filename, value, &rc->group, &rc->after, &rc->before,
1215                                        &rc->update_every,
1216                                        &rc->options, &rc->dimensions);
1217             }
1218             else if(hash == hash_every && !strcasecmp(key, HEALTH_EVERY_KEY)) {
1219                 if(!health_parse_duration(value, &rc->update_every))
1220                     info("Health configuration at line %zu of file '%s/%s' for alarm '%s' at key '%s' cannot parse duration: '%s'.",
1221                          line, path, filename, rc->name, key, value);
1222             }
1223             else if(hash == hash_green && !strcasecmp(key, HEALTH_GREEN_KEY)) {
1224                 char *e;
1225                 rc->green = strtold(value, &e);
1226                 if(e && *e) {
1227                     info("Health configuration at line %zu of file '%s/%s' for alarm '%s' at key '%s' leaves this string unmatched: '%s'.",
1228                          line, path, filename, rc->name, key, e);
1229                 }
1230             }
1231             else if(hash == hash_red && !strcasecmp(key, HEALTH_RED_KEY)) {
1232                 char *e;
1233                 rc->red = strtold(value, &e);
1234                 if(e && *e) {
1235                     info("Health configuration at line %zu of file '%s/%s' for alarm '%s' at key '%s' leaves this string unmatched: '%s'.",
1236                          line, path, filename, rc->name, key, e);
1237                 }
1238             }
1239             else if(hash == hash_calc && !strcasecmp(key, HEALTH_CALC_KEY)) {
1240                 const char *failed_at = NULL;
1241                 int error = 0;
1242                 rc->calculation = expression_parse(value, &failed_at, &error);
1243                 if(!rc->calculation) {
1244                     error("Health configuration at line %zu of file '%s/%s' for alarm '%s' at key '%s' has unparse-able expression '%s': %s at '%s'",
1245                           line, path, filename, rc->name, key, value, expression_strerror(error), failed_at);
1246                 }
1247             }
1248             else if(hash == hash_warn && !strcasecmp(key, HEALTH_WARN_KEY)) {
1249                 const char *failed_at = NULL;
1250                 int error = 0;
1251                 rc->warning = expression_parse(value, &failed_at, &error);
1252                 if(!rc->warning) {
1253                     error("Health configuration at line %zu of file '%s/%s' for alarm '%s' at key '%s' has unparse-able expression '%s': %s at '%s'",
1254                           line, path, filename, rc->name, key, value, expression_strerror(error), failed_at);
1255                 }
1256             }
1257             else if(hash == hash_crit && !strcasecmp(key, HEALTH_CRIT_KEY)) {
1258                 const char *failed_at = NULL;
1259                 int error = 0;
1260                 rc->critical = expression_parse(value, &failed_at, &error);
1261                 if(!rc->critical) {
1262                     error("Health configuration at line %zu of file '%s/%s' for alarm '%s' at key '%s' has unparse-able expression '%s': %s at '%s'",
1263                           line, path, filename, rc->name, key, value, expression_strerror(error), failed_at);
1264                 }
1265             }
1266             else if(hash == hash_exec && !strcasecmp(key, HEALTH_EXEC_KEY)) {
1267                 if(rc->exec) {
1268                     if(strcmp(rc->exec, value))
1269                         info("Health configuration at line %zu of file '%s/%s' for alarm '%s' has key '%s' twice, once with value '%s' and later with value '%s'. Using ('%s').",
1270                              line, path, filename, rc->name, key, rc->exec, value, value);
1271
1272                     freez(rc->exec);
1273                 }
1274                 rc->exec = strdupz(value);
1275             }
1276             else if(hash == hash_recipient && !strcasecmp(key, HEALTH_RECIPIENT_KEY)) {
1277                 if(rc->recipient) {
1278                     if(strcmp(rc->recipient, value))
1279                         info("Health configuration at line %zu of file '%s/%s' for alarm '%s' has key '%s' twice, once with value '%s' and later with value '%s'. Using ('%s').",
1280                              line, path, filename, rc->name, key, rc->recipient, value, value);
1281
1282                     freez(rc->recipient);
1283                 }
1284                 rc->recipient = strdupz(value);
1285             }
1286             else if(hash == hash_units && !strcasecmp(key, HEALTH_UNITS_KEY)) {
1287                 if(rc->units) {
1288                     if(strcmp(rc->units, value))
1289                         info("Health configuration at line %zu of file '%s/%s' for alarm '%s' has key '%s' twice, once with value '%s' and later with value '%s'. Using ('%s').",
1290                              line, path, filename, rc->name, key, rc->units, value, value);
1291
1292                     freez(rc->units);
1293                 }
1294                 rc->units = strdupz(value);
1295                 strip_quotes(rc->units);
1296             }
1297             else if(hash == hash_info && !strcasecmp(key, HEALTH_INFO_KEY)) {
1298                 if(rc->info) {
1299                     if(strcmp(rc->info, value))
1300                         info("Health configuration at line %zu of file '%s/%s' for alarm '%s' has key '%s' twice, once with value '%s' and later with value '%s'. Using ('%s').",
1301                              line, path, filename, rc->name, key, rc->info, value, value);
1302
1303                     freez(rc->info);
1304                 }
1305                 rc->info = strdupz(value);
1306                 strip_quotes(rc->info);
1307             }
1308             else {
1309                 error("Health configuration at line %zu of file '%s/%s' for alarm '%s' has unknown key '%s'.",
1310                      line, path, filename, rc->name, key);
1311             }
1312         }
1313         else if(rt) {
1314             if(hash == hash_on && !strcasecmp(key, HEALTH_ON_KEY)) {
1315                 if(rt->context) {
1316                     if(strcmp(rt->context, value))
1317                         info("Health configuration at line %zu of file '%s/%s' for template '%s' has key '%s' twice, once with value '%s' and later with value '%s'. Using ('%s').",
1318                              line, path, filename, rt->name, key, rt->context, value, value);
1319
1320                     freez(rt->context);
1321                 }
1322                 rt->context = strdupz(value);
1323                 rt->hash_context = simple_hash(rt->context);
1324             }
1325             else if(hash == hash_lookup && !strcasecmp(key, HEALTH_LOOKUP_KEY)) {
1326                 health_parse_db_lookup(line, path, filename, value, &rt->group, &rt->after, &rt->before,
1327                                        &rt->update_every,
1328                                        &rt->options, &rt->dimensions);
1329             }
1330             else if(hash == hash_every && !strcasecmp(key, HEALTH_EVERY_KEY)) {
1331                 if(!health_parse_duration(value, &rt->update_every))
1332                     info("Health configuration at line %zu of file '%s/%s' for template '%s' at key '%s' cannot parse duration: '%s'.",
1333                          line, path, filename, rt->name, key, value);
1334             }
1335             else if(hash == hash_green && !strcasecmp(key, HEALTH_GREEN_KEY)) {
1336                 char *e;
1337                 rt->green = strtold(value, &e);
1338                 if(e && *e) {
1339                     info("Health configuration at line %zu of file '%s/%s' for template '%s' at key '%s' leaves this string unmatched: '%s'.",
1340                          line, path, filename, rt->name, key, e);
1341                 }
1342             }
1343             else if(hash == hash_red && !strcasecmp(key, HEALTH_RED_KEY)) {
1344                 char *e;
1345                 rt->red = strtold(value, &e);
1346                 if(e && *e) {
1347                     info("Health configuration at line %zu of file '%s/%s' for template '%s' at key '%s' leaves this string unmatched: '%s'.",
1348                          line, path, filename, rt->name, key, e);
1349                 }
1350             }
1351             else if(hash == hash_calc && !strcasecmp(key, HEALTH_CALC_KEY)) {
1352                 const char *failed_at = NULL;
1353                 int error = 0;
1354                 rt->calculation = expression_parse(value, &failed_at, &error);
1355                 if(!rt->calculation) {
1356                     error("Health configuration at line %zu of file '%s/%s' for template '%s' at key '%s' has unparse-able expression '%s': %s at '%s'",
1357                           line, path, filename, rt->name, key, value, expression_strerror(error), failed_at);
1358                 }
1359             }
1360             else if(hash == hash_warn && !strcasecmp(key, HEALTH_WARN_KEY)) {
1361                 const char *failed_at = NULL;
1362                 int error = 0;
1363                 rt->warning = expression_parse(value, &failed_at, &error);
1364                 if(!rt->warning) {
1365                     error("Health configuration at line %zu of file '%s/%s' for template '%s' at key '%s' has unparse-able expression '%s': %s at '%s'",
1366                           line, path, filename, rt->name, key, value, expression_strerror(error), failed_at);
1367                 }
1368             }
1369             else if(hash == hash_crit && !strcasecmp(key, HEALTH_CRIT_KEY)) {
1370                 const char *failed_at = NULL;
1371                 int error = 0;
1372                 rt->critical = expression_parse(value, &failed_at, &error);
1373                 if(!rt->critical) {
1374                     error("Health configuration at line %zu of file '%s/%s' for template '%s' at key '%s' has unparse-able expression '%s': %s at '%s'",
1375                           line, path, filename, rt->name, key, value, expression_strerror(error), failed_at);
1376                 }
1377             }
1378             else if(hash == hash_exec && !strcasecmp(key, HEALTH_EXEC_KEY)) {
1379                 if(rt->exec) {
1380                     if(strcmp(rt->exec, value))
1381                         info("Health configuration at line %zu of file '%s/%s' for template '%s' has key '%s' twice, once with value '%s' and later with value '%s'. Using ('%s').",
1382                              line, path, filename, rt->name, key, rt->exec, value, value);
1383
1384                     freez(rt->exec);
1385                 }
1386                 rt->exec = strdupz(value);
1387             }
1388             else if(hash == hash_recipient && !strcasecmp(key, HEALTH_RECIPIENT_KEY)) {
1389                 if(rt->recipient) {
1390                     if(strcmp(rt->recipient, value))
1391                         info("Health configuration at line %zu of file '%s/%s' for template '%s' has key '%s' twice, once with value '%s' and later with value '%s'. Using ('%s').",
1392                              line, path, filename, rt->name, key, rt->recipient, value, value);
1393
1394                     freez(rt->recipient);
1395                 }
1396                 rt->recipient = strdupz(value);
1397             }
1398             else if(hash == hash_units && !strcasecmp(key, HEALTH_UNITS_KEY)) {
1399                 if(rt->units) {
1400                     if(strcmp(rt->units, value))
1401                         info("Health configuration at line %zu of file '%s/%s' for template '%s' has key '%s' twice, once with value '%s' and later with value '%s'. Using ('%s').",
1402                              line, path, filename, rt->name, key, rt->units, value, value);
1403
1404                     freez(rt->units);
1405                 }
1406                 rt->units = strdupz(value);
1407                 strip_quotes(rt->units);
1408             }
1409             else if(hash == hash_info && !strcasecmp(key, HEALTH_INFO_KEY)) {
1410                 if(rt->info) {
1411                     if(strcmp(rt->info, value))
1412                         info("Health configuration at line %zu of file '%s/%s' for template '%s' has key '%s' twice, once with value '%s' and later with value '%s'. Using ('%s').",
1413                              line, path, filename, rt->name, key, rt->info, value, value);
1414
1415                     freez(rt->info);
1416                 }
1417                 rt->info = strdupz(value);
1418                 strip_quotes(rt->info);
1419             }
1420             else {
1421                 error("Health configuration at line %zu of file '%s/%s' for template '%s' has unknown key '%s'.",
1422                       line, path, filename, rt->name, key);
1423             }
1424         }
1425         else {
1426             error("Health configuration at line %zu of file '%s/%s' has unknown key '%s'. Expected either '" HEALTH_ALARM_KEY "' or '" HEALTH_TEMPLATE_KEY "'.",
1427                   line, path, filename, key);
1428         }
1429     }
1430
1431     if(rc && !rrdcalc_add_alarm_from_config(&localhost, rc))
1432         rrdcalc_free(&localhost, rc);
1433
1434     if(rt && !rrdcalctemplate_add_template_from_config(&localhost, rt))
1435         rrdcalctemplate_free(&localhost, rt);
1436
1437     fclose(fp);
1438     return 1;
1439 }
1440
1441 void health_readdir(const char *path) {
1442     size_t pathlen = strlen(path);
1443
1444     debug(D_HEALTH, "Health configuration reading directory '%s'", path);
1445
1446     DIR *dir = opendir(path);
1447     if (!dir) {
1448         error("Health configuration cannot open directory '%s'.", path);
1449         return;
1450     }
1451
1452     struct dirent *de = NULL;
1453     while ((de = readdir(dir))) {
1454         size_t len = strlen(de->d_name);
1455
1456         if(de->d_type == DT_DIR
1457            && (
1458                    (de->d_name[0] == '.' && de->d_name[1] == '\0')
1459                    || (de->d_name[0] == '.' && de->d_name[1] == '.' && de->d_name[2] == '\0')
1460            ))
1461             continue;
1462
1463         else if(de->d_type == DT_DIR) {
1464             char *s = mallocz(pathlen + strlen(de->d_name) + 2);
1465             strcpy(s, path);
1466             strcat(s, "/");
1467             strcat(s, de->d_name);
1468             health_readdir(s);
1469             freez(s);
1470             continue;
1471         }
1472
1473         else if((de->d_type == DT_LNK || de->d_type == DT_REG) &&
1474                 len > 5 && !strcmp(&de->d_name[len - 5], ".conf")) {
1475             health_readfile(path, de->d_name);
1476         }
1477     }
1478
1479     closedir(dir);
1480 }
1481
1482 static inline char *health_config_dir(void) {
1483     char buffer[FILENAME_MAX + 1];
1484     snprintfz(buffer, FILENAME_MAX, "%s/health.d", config_get("global", "config directory", CONFIG_DIR));
1485     return config_get("health", "health configuration directory", buffer);
1486 }
1487
1488 void health_init(void) {
1489     debug(D_HEALTH, "Health configuration initializing");
1490
1491     if(!(health_enabled = config_get_boolean("health", "enabled", 1))) {
1492         debug(D_HEALTH, "Health is disabled.");
1493         return;
1494     }
1495
1496     char *path = health_config_dir();
1497
1498     {
1499         char buffer[FILENAME_MAX + 1];
1500         snprintfz(buffer, FILENAME_MAX, "%s/alarm-email.sh", config_get("global", "plugins directory", PLUGINS_DIR));
1501         health_default_exec = config_get("health", "script to execute on alarm", buffer);
1502     }
1503
1504     long n = config_get_number("health", "in memory max health log entries", (long)localhost.health_log.max);
1505     if(n < 2) {
1506         error("Health configuration has invalid max log entries %ld. Using default %u", n, localhost.health_log.max);
1507         config_set_number("health", "in memory max health log entries", (long)localhost.health_log.max);
1508     }
1509     else localhost.health_log.max = (unsigned int)n;
1510
1511     rrdhost_rwlock(&localhost);
1512     health_readdir(path);
1513     rrdhost_unlock(&localhost);
1514 }
1515
1516 // ----------------------------------------------------------------------------
1517 // JSON generation
1518
1519 static inline void health_string2json(BUFFER *wb, const char *prefix, const char *label, const char *value, const char *suffix) {
1520     if(value && *value)
1521         buffer_sprintf(wb, "%s\"%s\":\"%s\"%s", prefix, label, value, suffix);
1522     else
1523         buffer_sprintf(wb, "%s\"%s\":null%s", prefix, label, suffix);
1524 }
1525
1526 static inline void health_alarm_entry2json_nolock(BUFFER *wb, ALARM_ENTRY *ae) {
1527     buffer_sprintf(wb, "\n\t{\n"
1528                            "\t\t\"unique_id\":%u,\n"
1529                            "\t\t\"alarm_id\":%u,\n"
1530                            "\t\t\"alarm_event_id\":%u,\n"
1531                            "\t\t\"name\":\"%s\",\n"
1532                            "\t\t\"chart\":\"%s\",\n"
1533                            "\t\t\"family\":\"%s\",\n"
1534                            "\t\t\"processed\":%s,\n"
1535                            "\t\t\"updated\":%s,\n"
1536                            "\t\t\"exec_run\":%s,\n"
1537                            "\t\t\"exec_failed\":%s,\n"
1538                            "\t\t\"exec\":\"%s\",\n"
1539                            "\t\t\"recipient\":\"%s\",\n"
1540                            "\t\t\"exec_code\":%d,\n"
1541                            "\t\t\"source\":\"%s\",\n"
1542                            "\t\t\"units\":\"%s\",\n"
1543                            "\t\t\"info\":\"%s\",\n"
1544                            "\t\t\"when\":%lu,\n"
1545                            "\t\t\"duration\":%lu,\n"
1546                            "\t\t\"non_clear_duration\":%lu,\n"
1547                            "\t\t\"status\":\"%s\",\n"
1548                            "\t\t\"old_status\":\"%s\",\n",
1549                    ae->unique_id,
1550                    ae->alarm_id,
1551                    ae->alarm_event_id,
1552                    ae->name,
1553                    ae->chart,
1554                    ae->family,
1555                    (ae->notifications & HEALTH_ENTRY_NOTIFICATIONS_PROCESSED)?"true":"false",
1556                    (ae->notifications & HEALTH_ENTRY_NOTIFICATIONS_UPDATED)?"true":"false",
1557                    (ae->notifications & HEALTH_ENTRY_NOTIFICATIONS_EXEC_RUN)?"true":"false",
1558                    (ae->notifications & HEALTH_ENTRY_NOTIFICATIONS_EXEC_FAILED)?"true":"false",
1559                    ae->exec?ae->exec:health_default_exec,
1560                    ae->recipient?ae->recipient:health_default_recipient,
1561                    ae->exec_code,
1562                    ae->source,
1563                    ae->units?ae->units:"",
1564                    ae->info?ae->info:"",
1565                    (unsigned long)ae->when,
1566                    (unsigned long)ae->duration,
1567                    (unsigned long)ae->non_clear_duration,
1568                    rrdcalc_status2string(ae->new_status),
1569                    rrdcalc_status2string(ae->old_status)
1570     );
1571
1572     buffer_strcat(wb, "\t\t\"value\":");
1573     buffer_rrd_value(wb, ae->new_value);
1574     buffer_strcat(wb, ",\n");
1575
1576     buffer_strcat(wb, "\t\t\"old_value\":");
1577     buffer_rrd_value(wb, ae->old_value);
1578     buffer_strcat(wb, "\n");
1579
1580     buffer_strcat(wb, "\t}");
1581 }
1582
1583 void health_alarm_log2json(RRDHOST *host, BUFFER *wb) {
1584     pthread_rwlock_rdlock(&host->health_log.alarm_log_rwlock);
1585
1586     buffer_strcat(wb, "[");
1587
1588     unsigned int max = host->health_log.max;
1589     unsigned int count = 0;
1590     ALARM_ENTRY *ae;
1591     for(ae = host->health_log.alarms; ae && count < max ; count++, ae = ae->next) {
1592         if(likely(count)) buffer_strcat(wb, ",");
1593         health_alarm_entry2json_nolock(wb, ae);
1594     }
1595
1596     buffer_strcat(wb, "\n]\n");
1597
1598     pthread_rwlock_unlock(&host->health_log.alarm_log_rwlock);
1599 }
1600
1601 static inline void health_rrdcalc2json_nolock(BUFFER *wb, RRDCALC *rc) {
1602     buffer_sprintf(wb,
1603            "\t\t\"%s.%s\": {\n"
1604                    "\t\t\t\"name\": \"%s\",\n"
1605                    "\t\t\t\"chart\": \"%s\",\n"
1606                    "\t\t\t\"family\": \"%s\",\n"
1607                    "\t\t\t\"active\": %s,\n"
1608                    "\t\t\t\"exec\": \"%s\",\n"
1609                    "\t\t\t\"recipient\": \"%s\",\n"
1610                    "\t\t\t\"source\": \"%s\",\n"
1611                    "\t\t\t\"units\": \"%s\",\n"
1612                    "\t\t\t\"info\": \"%s\",\n"
1613                                    "\t\t\t\"status\": \"%s\",\n"
1614                    "\t\t\t\"last_status_change\": %lu,\n"
1615                    "\t\t\t\"last_updated\": %lu,\n"
1616                    "\t\t\t\"next_update\": %lu,\n"
1617                    "\t\t\t\"update_every\": %d,\n"
1618             , rc->chart, rc->name
1619             , rc->name
1620             , rc->chart
1621             , (rc->rrdset && rc->rrdset->family)?rc->rrdset->family:""
1622             , (rc->rrdset)?"true":"false"
1623             , rc->exec?rc->exec:health_default_exec
1624             , rc->recipient?rc->recipient:health_default_recipient
1625             , rc->source
1626             , rc->units?rc->units:""
1627             , rc->info?rc->info:""
1628             , rrdcalc_status2string(rc->status)
1629             , (unsigned long)rc->last_status_change
1630             , (unsigned long)rc->last_updated
1631             , (unsigned long)rc->next_update
1632             , rc->update_every
1633     );
1634
1635     if(RRDCALC_HAS_DB_LOOKUP(rc)) {
1636         if(rc->dimensions && *rc->dimensions)
1637             health_string2json(wb, "\t\t\t", "lookup_dimensions", rc->dimensions, ",\n");
1638
1639         buffer_sprintf(wb,
1640                        "\t\t\t\"db_after\": %lu,\n"
1641                        "\t\t\t\"db_before\": %lu,\n"
1642                        "\t\t\t\"lookup_method\": \"%s\",\n"
1643                        "\t\t\t\"lookup_after\": %d,\n"
1644                        "\t\t\t\"lookup_before\": %d,\n"
1645                        "\t\t\t\"lookup_options\": \"",
1646                        (unsigned long) rc->db_after,
1647                        (unsigned long) rc->db_before,
1648                        group_method2string(rc->group),
1649                        rc->after,
1650                        rc->before
1651         );
1652         buffer_data_options2string(wb, rc->options);
1653         buffer_strcat(wb, "\",\n");
1654     }
1655
1656     if(rc->calculation) {
1657         health_string2json(wb, "\t\t\t", "calc", rc->calculation->source, ",\n");
1658         health_string2json(wb, "\t\t\t", "calc_parsed", rc->calculation->parsed_as, ",\n");
1659     }
1660
1661     if(rc->warning) {
1662         health_string2json(wb, "\t\t\t", "warn", rc->warning->source, ",\n");
1663         health_string2json(wb, "\t\t\t", "warn_parsed", rc->warning->parsed_as, ",\n");
1664     }
1665
1666     if(rc->critical) {
1667         health_string2json(wb, "\t\t\t", "crit", rc->critical->source, ",\n");
1668         health_string2json(wb, "\t\t\t", "crit_parsed", rc->critical->parsed_as, ",\n");
1669     }
1670
1671     buffer_strcat(wb, "\t\t\t\"green\":");
1672     buffer_rrd_value(wb, rc->green);
1673     buffer_strcat(wb, ",\n");
1674
1675     buffer_strcat(wb, "\t\t\t\"red\":");
1676     buffer_rrd_value(wb, rc->red);
1677     buffer_strcat(wb, ",\n");
1678
1679     buffer_strcat(wb, "\t\t\t\"value\":");
1680     buffer_rrd_value(wb, rc->value);
1681     buffer_strcat(wb, "\n");
1682
1683     buffer_strcat(wb, "\t\t}");
1684 }
1685
1686 //void health_rrdcalctemplate2json_nolock(BUFFER *wb, RRDCALCTEMPLATE *rt) {
1687 //
1688 //}
1689
1690 void health_alarms2json(RRDHOST *host, BUFFER *wb, int all) {
1691     int i;
1692     rrdhost_rdlock(&localhost);
1693
1694     buffer_strcat(wb, "{\n\t\"alarms\": {\n");
1695     RRDCALC *rc;
1696     for(i = 0, rc = host->alarms; rc ; rc = rc->next) {
1697         if(unlikely(!rc->rrdset || !rc->rrdset->last_collected_time.tv_sec))
1698             continue;
1699
1700         if(likely(!all && !(rc->status == RRDCALC_STATUS_WARNING || rc->status == RRDCALC_STATUS_CRITICAL)))
1701             continue;
1702
1703         if(likely(i)) buffer_strcat(wb, ",\n");
1704         health_rrdcalc2json_nolock(wb, rc);
1705         i++;
1706     }
1707
1708 //    buffer_strcat(wb, "\n\t},\n\t\"templates\": {");
1709
1710 //    RRDCALCTEMPLATE *rt;
1711 //    for(rt = host->templates; rt ; rt = rt->next)
1712 //        health_rrdcalctemplate2json_nolock(wb, rt);
1713
1714     buffer_sprintf(wb, "\n\t},\n\t\"now\": %lu\n}\n", (unsigned long)time(NULL));
1715     rrdhost_unlock(&localhost);
1716 }
1717
1718
1719 // ----------------------------------------------------------------------------
1720 // re-load health configuration
1721
1722 static inline void health_free_all_nolock(RRDHOST *host) {
1723     while(host->templates)
1724         rrdcalctemplate_free(host, host->templates);
1725
1726     while(host->alarms)
1727         rrdcalc_free(host, host->alarms);
1728 }
1729
1730 void health_reload(void) {
1731     if(!health_enabled) {
1732         error("Health reload is requested, but health is not enabled.");
1733         return;
1734     }
1735
1736     char *path = health_config_dir();
1737
1738     rrdhost_rwlock(&localhost);
1739     health_free_all_nolock(&localhost);
1740     rrdhost_unlock(&localhost);
1741
1742     RRDSET *st;
1743     for(st = localhost.rrdset_root; st ; st = st->next) {
1744         st->green = NAN;
1745         st->red = NAN;
1746     }
1747
1748     rrdhost_rwlock(&localhost);
1749     health_readdir(path);
1750     rrdhost_unlock(&localhost);
1751
1752     for(st = localhost.rrdset_root; st ; st = st->next) {
1753         rrdhost_rwlock(&localhost);
1754
1755         rrdsetcalc_link_matching(st);
1756         rrdcalctemplate_link_matching(st);
1757
1758         rrdhost_unlock(&localhost);
1759     }
1760 }
1761
1762
1763 // ----------------------------------------------------------------------------
1764 // health main thread and friends
1765
1766 static inline int rrdcalc_value2status(calculated_number n) {
1767     if(isnan(n)) return RRDCALC_STATUS_UNDEFINED;
1768     if(n) return RRDCALC_STATUS_RAISED;
1769     return RRDCALC_STATUS_CLEAR;
1770 }
1771
1772 static inline void health_alarm_execute(RRDHOST *host, ALARM_ENTRY *ae) {
1773     if(ae->old_status == RRDCALC_STATUS_UNINITIALIZED && ae->new_status == RRDCALC_STATUS_CLEAR)
1774         return;
1775
1776     char buffer[FILENAME_MAX + 1];
1777     pid_t command_pid;
1778
1779     const char *exec = ae->exec;
1780     if(!exec) exec = health_default_exec;
1781
1782     const char *recipient = ae->recipient;
1783     if(!recipient) recipient = health_default_recipient;
1784
1785     snprintfz(buffer, FILENAME_MAX, "exec %s '%s' '%s' '%u' '%u' '%u' '%zu' '%s' '%s' '%s' '%s' '%s' '%0.0Lf' '%0.0Lf' '%s' '%u' '%u' '%s' '%s'",
1786               exec,
1787               recipient,
1788               host->hostname,
1789               ae->unique_id,
1790               ae->alarm_id,
1791               ae->alarm_event_id,
1792               ae->when,
1793               ae->name,
1794               ae->chart?ae->chart:"NOCAHRT",
1795               ae->family?ae->family:"NOFAMILY",
1796               rrdcalc_status2string(ae->new_status),
1797               rrdcalc_status2string(ae->old_status),
1798               ae->new_value,
1799               ae->old_value,
1800               ae->source?ae->source:"UNKNOWN",
1801               (uint32_t)ae->duration,
1802               (uint32_t)ae->non_clear_duration,
1803               ae->units?ae->units:"",
1804               ae->info?ae->info:""
1805     );
1806
1807     ae->notifications |= HEALTH_ENTRY_NOTIFICATIONS_EXEC_RUN;
1808
1809     debug(D_HEALTH, "executing command '%s'", buffer);
1810     FILE *fp = mypopen(buffer, &command_pid);
1811     if(!fp) {
1812         error("HEALTH: Cannot popen(\"%s\", \"r\").", buffer);
1813         return;
1814     }
1815     debug(D_HEALTH, "HEALTH reading from command");
1816     char *s = fgets(buffer, FILENAME_MAX, fp);
1817     (void)s;
1818     debug(D_HEALTH, "HEALTH closing command");
1819     ae->exec_code = mypclose(fp, command_pid);
1820     debug(D_HEALTH, "done executing command - returned with code %d", ae->exec_code);
1821
1822     if(ae->exec_code != 0)
1823         ae->notifications |= HEALTH_ENTRY_NOTIFICATIONS_EXEC_FAILED;
1824 }
1825
1826 static inline void health_process_notifications(RRDHOST *host, ALARM_ENTRY *ae) {
1827     info("Health alarm '%s.%s' = %0.2Lf - changed status from %s to %s",
1828          ae->chart?ae->chart:"NOCHART", ae->name,
1829          ae->new_value,
1830          rrdcalc_status2string(ae->old_status),
1831          rrdcalc_status2string(ae->new_status)
1832     );
1833
1834     health_alarm_execute(host, ae);
1835 }
1836
1837 static inline void health_alarm_log(RRDHOST *host,
1838                 uint32_t alarm_id, uint32_t alarm_event_id,
1839                 time_t when,
1840                 const char *name, const char *chart, const char *family,
1841                 const char *exec, const char *recipient, time_t duration,
1842                 calculated_number old_value, calculated_number new_value,
1843                 int old_status, int new_status,
1844                 const char *source,
1845                 const char *units,
1846                 const char *info
1847 ) {
1848     ALARM_ENTRY *ae = callocz(1, sizeof(ALARM_ENTRY));
1849     ae->name = strdupz(name);
1850     ae->hash_name = simple_hash(ae->name);
1851
1852     if(chart) {
1853         ae->chart = strdupz(chart);
1854         ae->hash_chart = simple_hash(ae->chart);
1855     }
1856
1857     if(family)
1858         ae->family = strdupz(family);
1859
1860     if(exec) ae->exec = strdupz(exec);
1861     if(recipient) ae->recipient = strdupz(recipient);
1862     if(source) ae->source = strdupz(source);
1863     if(units) ae->units = strdupz(units);
1864     if(info) ae->info = strdupz(info);
1865
1866     ae->unique_id = host->health_log.next_log_id++;
1867     ae->alarm_id = alarm_id;
1868     ae->alarm_event_id = alarm_event_id;
1869     ae->when = when;
1870     ae->old_value = old_value;
1871     ae->new_value = new_value;
1872     ae->old_status = old_status;
1873     ae->new_status = new_status;
1874     ae->duration = duration;
1875
1876     if(ae->old_status == RRDCALC_STATUS_WARNING || ae->old_status == RRDCALC_STATUS_CRITICAL)
1877         ae->non_clear_duration += ae->duration;
1878
1879     // link it
1880     pthread_rwlock_wrlock(&host->health_log.alarm_log_rwlock);
1881     ae->next = host->health_log.alarms;
1882     host->health_log.alarms = ae;
1883     host->health_log.count++;
1884     pthread_rwlock_unlock(&host->health_log.alarm_log_rwlock);
1885
1886     // match previous alarms
1887     pthread_rwlock_rdlock(&host->health_log.alarm_log_rwlock);
1888     ALARM_ENTRY *t;
1889     for(t = host->health_log.alarms ; t ; t = t->next) {
1890         if(t != ae &&
1891                 t->hash_name == ae->hash_name &&
1892                 t->hash_chart == ae->hash_chart &&
1893                 !strcmp(t->name, ae->name) &&
1894                 t->chart && ae->chart && !strcmp(t->chart, ae->chart)) {
1895
1896             if(!(t->notifications & HEALTH_ENTRY_NOTIFICATIONS_UPDATED) && !t->updated_by) {
1897                 t->notifications |= HEALTH_ENTRY_NOTIFICATIONS_UPDATED;
1898                 t->updated_by = ae;
1899
1900                 if((t->new_status == RRDCALC_STATUS_WARNING || t->new_status == RRDCALC_STATUS_CRITICAL) &&
1901                    (t->old_status == RRDCALC_STATUS_WARNING || t->old_status == RRDCALC_STATUS_CRITICAL))
1902                     ae->non_clear_duration += t->non_clear_duration;
1903             }
1904             else {
1905                 // no need to continue
1906                 break;
1907             }
1908         }
1909     }
1910     pthread_rwlock_unlock(&host->health_log.alarm_log_rwlock);
1911 }
1912
1913 static inline void health_alarm_log_process(RRDHOST *host) {
1914     static uint32_t last_processed = 0;
1915     ALARM_ENTRY *ae;
1916
1917     pthread_rwlock_rdlock(&host->health_log.alarm_log_rwlock);
1918
1919     for(ae = host->health_log.alarms; ae ;ae = ae->next) {
1920         if(last_processed >= ae->unique_id) break;
1921
1922         if(!(ae->notifications & HEALTH_ENTRY_NOTIFICATIONS_PROCESSED) &&
1923                 !(ae->notifications & HEALTH_ENTRY_NOTIFICATIONS_UPDATED)) {
1924             ae->notifications |= HEALTH_ENTRY_NOTIFICATIONS_PROCESSED;
1925             health_process_notifications(host, ae);
1926         }
1927     }
1928
1929     if(host->health_log.alarms)
1930         last_processed = host->health_log.alarms->unique_id;
1931
1932     pthread_rwlock_unlock(&host->health_log.alarm_log_rwlock);
1933
1934     if(host->health_log.count <= host->health_log.max)
1935         return;
1936
1937     // cleanup excess entries in the log
1938     pthread_rwlock_wrlock(&host->health_log.alarm_log_rwlock);
1939
1940     ALARM_ENTRY *last = NULL;
1941     unsigned int count = host->health_log.max;
1942     for(ae = host->health_log.alarms; ae && count ; count--, last = ae, ae = ae->next) ;
1943
1944     if(ae && last && last->next == ae)
1945         last->next = NULL;
1946     else
1947         ae = NULL;
1948
1949     while(ae) {
1950         ALARM_ENTRY *t = ae->next;
1951
1952         freez(ae->name);
1953         freez(ae->chart);
1954         freez(ae->family);
1955         freez(ae->exec);
1956         freez(ae->recipient);
1957         freez(ae->source);
1958         freez(ae->units);
1959         freez(ae->info);
1960         freez(ae);
1961
1962         ae = t;
1963     }
1964
1965     pthread_rwlock_unlock(&host->health_log.alarm_log_rwlock);
1966 }
1967
1968 static inline int rrdcalc_isrunnable(RRDCALC *rc, time_t now, time_t *next_run) {
1969     if (unlikely(!rc->rrdset)) {
1970         debug(D_HEALTH, "Health not running alarm '%s.%s'. It is not linked to a chart.", rc->chart?rc->chart:"NOCHART", rc->name);
1971         return 0;
1972     }
1973
1974     if (unlikely(!rc->rrdset->last_collected_time.tv_sec)) {
1975         debug(D_HEALTH, "Health not running alarm '%s.%s'. Chart is not yet collected.", rc->chart?rc->chart:"NOCHART", rc->name);
1976         return 0;
1977     }
1978
1979     if (unlikely(!rc->update_every)) {
1980         debug(D_HEALTH, "Health not running alarm '%s.%s'. It does not have an update frequency", rc->chart?rc->chart:"NOCHART", rc->name);
1981         return 0;
1982     }
1983
1984     if (unlikely(rc->next_update > now)) {
1985         if (unlikely(*next_run > rc->next_update))
1986             *next_run = rc->next_update;
1987
1988         debug(D_HEALTH, "Health not examining alarm '%s.%s' yet (will do in %d secs).", rc->chart?rc->chart:"NOCHART", rc->name, (int) (rc->next_update - now));
1989         return 0;
1990     }
1991
1992     // FIXME
1993     // we should check that the DB lookup is possible
1994     // i.e.
1995     // - the duration of the chart includes the required timeframe
1996     // we SHOULD NOT check the dimensions - there might be alarms that refer non-existing dimensions (e.g. cpu steal)
1997
1998     return 1;
1999 }
2000
2001 void *health_main(void *ptr) {
2002     (void)ptr;
2003
2004     info("HEALTH thread created with task id %d", gettid());
2005
2006     if(pthread_setcanceltype(PTHREAD_CANCEL_DEFERRED, NULL) != 0)
2007         error("Cannot set pthread cancel type to DEFERRED.");
2008
2009     if(pthread_setcancelstate(PTHREAD_CANCEL_ENABLE, NULL) != 0)
2010         error("Cannot set pthread cancel state to ENABLE.");
2011
2012     int min_run_every = (int)config_get_number("health", "run at least every seconds", 10);
2013     if(min_run_every < 1) min_run_every = 1;
2014
2015     BUFFER *wb = buffer_create(100);
2016
2017     unsigned int loop = 0;
2018     while(health_enabled) {
2019         loop++;
2020         debug(D_HEALTH, "Health monitoring iteration no %u started", loop);
2021
2022         int oldstate, runnable = 0;
2023         time_t now = time(NULL);
2024         time_t next_run = now + min_run_every;
2025         RRDCALC *rc;
2026
2027         if (unlikely(pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, &oldstate) != 0))
2028             error("Cannot set pthread cancel state to DISABLE.");
2029
2030         rrdhost_rdlock(&localhost);
2031
2032         // the first loop is to lookup values from the db
2033         for (rc = localhost.alarms; rc; rc = rc->next) {
2034             if (unlikely(!rrdcalc_isrunnable(rc, now, &next_run)))
2035                 continue;
2036
2037             runnable++;
2038             rc->old_value = rc->value;
2039
2040             // 1. if there is database lookup, do it
2041             // 2. if there is calculation expression, run it
2042
2043             if (unlikely(RRDCALC_HAS_DB_LOOKUP(rc))) {
2044                 time_t old_db_timestamp = rc->db_before;
2045                 int value_is_null = 0;
2046
2047                 int ret = rrd2value(rc->rrdset, wb, &rc->value,
2048                                     rc->dimensions, 1, rc->after, rc->before, rc->group,
2049                                     rc->options, &rc->db_after, &rc->db_before, &value_is_null);
2050
2051                 if (unlikely(ret != 200)) {
2052                     // database lookup failed
2053                     rc->value = NAN;
2054
2055                     debug(D_HEALTH, "Health alarm '%s.%s': database lookup returned error %d", rc->chart?rc->chart:"NOCHART", rc->name, ret);
2056
2057                     if (unlikely(!(rc->rrdcalc_flags & RRDCALC_FLAG_DB_ERROR))) {
2058                         rc->rrdcalc_flags |= RRDCALC_FLAG_DB_ERROR;
2059                         error("Health alarm '%s.%s': database lookup returned error %d", rc->chart?rc->chart:"NOCHART", rc->name, ret);
2060                     }
2061                 }
2062                 else if (unlikely(rc->rrdcalc_flags & RRDCALC_FLAG_DB_ERROR))
2063                     rc->rrdcalc_flags &= ~RRDCALC_FLAG_DB_ERROR;
2064
2065                 if (unlikely(old_db_timestamp == rc->db_before)) {
2066                     // database is stale
2067
2068                     debug(D_HEALTH, "Health alarm '%s.%s': database is stale", rc->chart?rc->chart:"NOCHART", rc->name);
2069
2070                     if (unlikely(!(rc->rrdcalc_flags & RRDCALC_FLAG_DB_STALE))) {
2071                         rc->rrdcalc_flags |= RRDCALC_FLAG_DB_STALE;
2072                         error("Health alarm '%s.%s': database is stale", rc->chart?rc->chart:"NOCHART", rc->name);
2073                     }
2074                 }
2075                 else if (unlikely(rc->rrdcalc_flags & RRDCALC_FLAG_DB_STALE))
2076                     rc->rrdcalc_flags &= ~RRDCALC_FLAG_DB_STALE;
2077
2078                 if (unlikely(value_is_null)) {
2079                     // collected value is null
2080
2081                     rc->value = NAN;
2082
2083                     debug(D_HEALTH, "Health alarm '%s.%s': database lookup returned empty value (possibly value is not collected yet)",
2084                           rc->chart?rc->chart:"NOCHART", rc->name);
2085
2086                     if (unlikely(!(rc->rrdcalc_flags & RRDCALC_FLAG_DB_NAN))) {
2087                         rc->rrdcalc_flags |= RRDCALC_FLAG_DB_NAN;
2088                         error("Health alarm '%s.%s': database lookup returned empty value (possibly value is not collected yet)",
2089                               rc->chart?rc->chart:"NOCHART", rc->name);
2090                     }
2091                 }
2092                 else if (unlikely(rc->rrdcalc_flags & RRDCALC_FLAG_DB_NAN))
2093                     rc->rrdcalc_flags &= ~RRDCALC_FLAG_DB_NAN;
2094
2095                 debug(D_HEALTH, "Health alarm '%s.%s': database lookup gave value "
2096                         CALCULATED_NUMBER_FORMAT, rc->chart?rc->chart:"NOCHART", rc->name, rc->value);
2097             }
2098
2099             if(unlikely(rc->calculation)) {
2100                 if (unlikely(!expression_evaluate(rc->calculation))) {
2101                     // calculation failed
2102
2103                     rc->value = NAN;
2104
2105                     debug(D_HEALTH, "Health alarm '%s.%s': failed to evaluate calculation with error: %s",
2106                           rc->chart?rc->chart:"NOCHART", rc->name, buffer_tostring(rc->calculation->error_msg));
2107
2108                     if (unlikely(!(rc->rrdcalc_flags & RRDCALC_FLAG_CALC_ERROR))) {
2109                         rc->rrdcalc_flags |= RRDCALC_FLAG_CALC_ERROR;
2110                         error("Health alarm '%s.%s': failed to evaluate calculation with error: %s",
2111                               rc->chart?rc->chart:"NOCHART", rc->name, buffer_tostring(rc->calculation->error_msg));
2112                     }
2113                 }
2114                 else {
2115                     if (unlikely(rc->rrdcalc_flags & RRDCALC_FLAG_CALC_ERROR))
2116                         rc->rrdcalc_flags &= ~RRDCALC_FLAG_CALC_ERROR;
2117
2118                     debug(D_HEALTH, "Health alarm '%s.%s': calculation expression gave value "
2119                             CALCULATED_NUMBER_FORMAT
2120                             ": %s (source: %s)",
2121                           rc->chart?rc->chart:"NOCHART", rc->name,
2122                           rc->calculation->result,
2123                           buffer_tostring(rc->calculation->error_msg),
2124                           rc->source
2125                     );
2126
2127                     rc->value = rc->calculation->result;
2128                 }
2129             }
2130         }
2131         rrdhost_unlock(&localhost);
2132
2133         if (runnable) {
2134             rrdhost_rdlock(&localhost);
2135
2136             for (rc = localhost.alarms; rc; rc = rc->next) {
2137                 if (unlikely(!rrdcalc_isrunnable(rc, now, &next_run)))
2138                     continue;
2139
2140                 int warning_status  = RRDCALC_STATUS_UNDEFINED;
2141                 int critical_status = RRDCALC_STATUS_UNDEFINED;
2142
2143                 if(likely(rc->warning)) {
2144                     if(unlikely(!expression_evaluate(rc->warning))) {
2145                         // calculation failed
2146
2147                         debug(D_HEALTH, "Health alarm '%s.%s': warning expression failed with error: %s",
2148                               rc->chart?rc->chart:"NOCHART", rc->name, buffer_tostring(rc->warning->error_msg));
2149
2150                         if (unlikely(!(rc->rrdcalc_flags & RRDCALC_FLAG_WARN_ERROR))) {
2151                             rc->rrdcalc_flags |= RRDCALC_FLAG_WARN_ERROR;
2152                             error("Health alarm '%s.%s': warning expression failed with error: %s",
2153                                   rc->chart?rc->chart:"NOCHART", rc->name, buffer_tostring(rc->warning->error_msg));
2154                         }
2155                     }
2156                     else {
2157                         if(unlikely(rc->rrdcalc_flags & RRDCALC_FLAG_WARN_ERROR))
2158                             rc->rrdcalc_flags &= ~RRDCALC_FLAG_WARN_ERROR;
2159
2160                         debug(D_HEALTH, "Health alarm '%s.%s': warning expression gave value "
2161                                 CALCULATED_NUMBER_FORMAT
2162                                 ": %s (source: %s)",
2163                               rc->chart?rc->chart:"NOCHART", rc->name,
2164                               rc->warning->result,
2165                               buffer_tostring(rc->warning->error_msg),
2166                               rc->source
2167                         );
2168
2169                         warning_status = rrdcalc_value2status(rc->warning->result);
2170                     }
2171                 }
2172
2173                 if(likely(rc->critical)) {
2174                     if(unlikely(!expression_evaluate(rc->critical))) {
2175                         // calculation failed
2176
2177                         debug(D_HEALTH, "Health alarm '%s.%s': critical expression failed with error: %s",
2178                               rc->chart?rc->chart:"NOCHART", rc->name, buffer_tostring(rc->critical->error_msg));
2179
2180                         if (unlikely(!(rc->rrdcalc_flags & RRDCALC_FLAG_CRIT_ERROR))) {
2181                             rc->rrdcalc_flags |= RRDCALC_FLAG_CRIT_ERROR;
2182                             error("Health alarm '%s.%s': critical expression failed with error: %s",
2183                                   rc->chart?rc->chart:"NOCHART", rc->name, buffer_tostring(rc->critical->error_msg));
2184                         }
2185                     }
2186                     else {
2187                         if(unlikely(rc->rrdcalc_flags & RRDCALC_FLAG_CRIT_ERROR))
2188                             rc->rrdcalc_flags &= ~RRDCALC_FLAG_CRIT_ERROR;
2189
2190                         debug(D_HEALTH, "Health alarm '%s.%s': critical expression gave value "
2191                                 CALCULATED_NUMBER_FORMAT
2192                                 ": %s (source: %s)",
2193                               rc->chart?rc->chart:"NOCHART", rc->name,
2194                               rc->critical->result,
2195                               buffer_tostring(rc->critical->error_msg),
2196                               rc->source
2197                         );
2198
2199                         critical_status = rrdcalc_value2status(rc->critical->result);
2200                     }
2201                 }
2202
2203                 int status = RRDCALC_STATUS_UNDEFINED;
2204
2205                 switch(warning_status) {
2206                     case RRDCALC_STATUS_CLEAR:
2207                         status = RRDCALC_STATUS_CLEAR;
2208                         break;
2209
2210                     case RRDCALC_STATUS_RAISED:
2211                         status = RRDCALC_STATUS_WARNING;
2212                         break;
2213
2214                     default:
2215                         break;
2216                 }
2217
2218                 switch(critical_status) {
2219                     case RRDCALC_STATUS_CLEAR:
2220                         if(status == RRDCALC_STATUS_UNDEFINED)
2221                             status = RRDCALC_STATUS_CLEAR;
2222                         break;
2223
2224                     case RRDCALC_STATUS_RAISED:
2225                         status = RRDCALC_STATUS_CRITICAL;
2226                         break;
2227
2228                     default:
2229                         break;
2230                 }
2231
2232                 if(status != rc->status) {
2233                     health_alarm_log(&localhost, rc->id, rc->next_event_id++, time(NULL), rc->name, rc->rrdset->id, rc->rrdset->family, rc->exec, rc->recipient, now - rc->last_status_change, rc->old_value, rc->value, rc->status, status, rc->source, rc->units, rc->info);
2234                     rc->last_status_change = now;
2235                     rc->status = status;
2236                 }
2237
2238                 rc->last_updated = now;
2239                 rc->next_update = now + rc->update_every;
2240
2241                 if (next_run > rc->next_update)
2242                     next_run = rc->next_update;
2243             }
2244
2245             rrdhost_unlock(&localhost);
2246         }
2247
2248         if (unlikely(pthread_setcancelstate(oldstate, NULL) != 0))
2249             error("Cannot set pthread cancel state to RESTORE (%d).", oldstate);
2250
2251         // execute notifications
2252         // and cleanup
2253         health_alarm_log_process(&localhost);
2254
2255         now = time(NULL);
2256         if(now < next_run) {
2257             debug(D_HEALTH, "Health monitoring iteration no %u done. Next iteration in %d secs",
2258                   loop, (int) (next_run - now));
2259             sleep_usec(1000000 * (unsigned long long) (next_run - now));
2260         }
2261         else {
2262             debug(D_HEALTH, "Health monitoring iteration no %u done. Next iteration now", loop);
2263         }
2264     }
2265
2266     buffer_free(wb);
2267
2268     info("HEALTH thread exiting");
2269     pthread_exit(NULL);
2270     return NULL;
2271 }