src/health.c

   1 #include "common.h"
   2
   3 #define RRDVAR_MAX_LENGTH 1024
   4
   5 static const char *health_default_exec = PLUGINS_DIR "/alarm-email.sh";
   6 static const char *health_default_recipient = "root";
   7 int health_enabled = 1;
   8
   9 // ----------------------------------------------------------------------------
  10 // RRDVAR management
  11
  12 static inline int rrdvar_fix_name(char *variable) {
  13     int fixed = 0;
  14     while(*variable) {
  15         if (!isalnum(*variable) && *variable != '.' && *variable != '_') {
  16             *variable++ = '_';
  17             fixed++;
  18         }
  19         else
  20             variable++;
  21     }
  22
  23     return fixed;
  24 }
  25
  26 int rrdvar_compare(void* a, void* b) {
  27     if(((RRDVAR *)a)->hash < ((RRDVAR *)b)->hash) return -1;
  28     else if(((RRDVAR *)a)->hash > ((RRDVAR *)b)->hash) return 1;
  29     else return strcmp(((RRDVAR *)a)->name, ((RRDVAR *)b)->name);
  30 }
  31
  32 static inline RRDVAR *rrdvar_index_add(avl_tree_lock *tree, RRDVAR *rv) {
  33     RRDVAR *ret = (RRDVAR *)avl_insert_lock(tree, (avl *)(rv));
  34     if(ret != rv)
  35         debug(D_VARIABLES, "Request to insert RRDVAR '%s' into index failed. Already exists.", rv->name);
  36
  37     return ret;
  38 }
  39
  40 static inline RRDVAR *rrdvar_index_del(avl_tree_lock *tree, RRDVAR *rv) {
  41     RRDVAR *ret = (RRDVAR *)avl_remove_lock(tree, (avl *)(rv));
  42     if(!ret)
  43         error("Request to remove RRDVAR '%s' from index failed. Not Found.", rv->name);
  44
  45     return ret;
  46 }
  47
  48 static inline RRDVAR *rrdvar_index_find(avl_tree_lock *tree, const char *name, uint32_t hash) {
  49     RRDVAR tmp;
  50     tmp.name = (char *)name;
  51     tmp.hash = (hash)?hash:simple_hash(tmp.name);
  52
  53     return (RRDVAR *)avl_search_lock(tree, (avl *)&tmp);
  54 }
  55
  56 static inline void rrdvar_free(RRDHOST *host, avl_tree_lock *tree, RRDVAR *rv) {
  57     (void)host;
  58
  59     if(!rv) return;
  60
  61     if(tree)
  62         rrdvar_index_del(tree, rv);
  63
  64     freez(rv->name);
  65     freez(rv);
  66 }
  67
  68 static inline RRDVAR *rrdvar_create_and_index(const char *scope, avl_tree_lock *tree, const char *name, int type, calculated_number *value) {
  69     char *variable = strdupz(name);
  70     rrdvar_fix_name(variable);
  71     uint32_t hash = simple_hash(variable);
  72
  73     RRDVAR *rv = rrdvar_index_find(tree, variable, hash);
  74     if(unlikely(!rv)) {
  75         debug(D_VARIABLES, "Variable '%s' not found in scope '%s'. Creating a new one.", variable, scope);
  76
  77         rv = callocz(1, sizeof(RRDVAR));
  78         rv->name = variable;
  79         rv->hash = hash;
  80         rv->type = type;
  81         rv->value = value;
  82
  83         RRDVAR *ret = rrdvar_index_add(tree, rv);
  84         if(unlikely(ret != rv)) {
  85             debug(D_VARIABLES, "Variable '%s' in scope '%s' already exists", variable, scope);
  86             rrdvar_free(NULL, NULL, rv);
  87             rv = NULL;
  88         }
  89         else
  90             debug(D_VARIABLES, "Variable '%s' created in scope '%s'", variable, scope);
  91     }
  92     else {
  93         // already exists
  94         freez(variable);
  95         rv = NULL;
  96     }
  97
  98     return rv;
  99 }
 100
 101 // ----------------------------------------------------------------------------
 102 // RRDVAR lookup
 103
 104 calculated_number rrdvar2number(RRDVAR *rv) {
 105     switch(rv->type) {
 106         case RRDVAR_TYPE_CALCULATED: {
 107             calculated_number *n = (calculated_number *)rv->value;
 108             return *n;
 109         }
 110
 111         case RRDVAR_TYPE_TIME_T: {
 112             time_t *n = (time_t *)rv->value;
 113             return *n;
 114         }
 115
 116         case RRDVAR_TYPE_COLLECTED: {
 117             collected_number *n = (collected_number *)rv->value;
 118             return *n;
 119         }
 120
 121         case RRDVAR_TYPE_TOTAL: {
 122             total_number *n = (total_number *)rv->value;
 123             return *n;
 124         }
 125
 126         case RRDVAR_TYPE_INT: {
 127             int *n = (int *)rv->value;
 128             return *n;
 129         }
 130
 131         default:
 132             error("I don't know how to convert RRDVAR type %d to calculated_number", rv->type);
 133             return NAN;
 134     }
 135 }
 136
 137 void dump_variable(void *data) {
 138     RRDVAR *rv = (RRDVAR *)data;
 139     debug(D_HEALTH, "%50s : %20.5Lf", rv->name, rrdvar2number(rv));
 140 }
 141
 142 int health_variable_lookup(const char *variable, uint32_t hash, RRDCALC *rc, calculated_number *result) {
 143     RRDSET *st = rc->rrdset;
 144     RRDVAR *rv;
 145
 146     if(!st) return 0;
 147
 148     rv = rrdvar_index_find(&st->variables_root_index, variable, hash);
 149     if(rv) {
 150         *result = rrdvar2number(rv);
 151         return 1;
 152     }
 153
 154     rv = rrdvar_index_find(&st->rrdfamily->variables_root_index, variable, hash);
 155     if(rv) {
 156         *result = rrdvar2number(rv);
 157         return 1;
 158     }
 159
 160     rv = rrdvar_index_find(&st->rrdhost->variables_root_index, variable, hash);
 161     if(rv) {
 162         *result = rrdvar2number(rv);
 163         return 1;
 164     }
 165
 166     debug(D_HEALTH, "Available local chart '%s' variables:", st->id);
 167     avl_traverse_lock(&st->variables_root_index, dump_variable);
 168
 169     debug(D_HEALTH, "Available family '%s' variables:", st->rrdfamily->family);
 170     avl_traverse_lock(&st->rrdfamily->variables_root_index, dump_variable);
 171
 172     debug(D_HEALTH, "Available host '%s' variables:", st->rrdhost->hostname);
 173     avl_traverse_lock(&st->rrdhost->variables_root_index, dump_variable);
 174
 175     return 0;
 176 }
 177
 178 // ----------------------------------------------------------------------------
 179 // RRDSETVAR management
 180
 181 RRDSETVAR *rrdsetvar_create(RRDSET *st, const char *variable, int type, void *value, uint32_t options) {
 182     debug(D_VARIABLES, "RRDVARSET create for chart id '%s' name '%s' with variable name '%s'", st->id, st->name, variable);
 183     RRDSETVAR *rs = (RRDSETVAR *)callocz(1, sizeof(RRDSETVAR));
 184
 185     char buffer[RRDVAR_MAX_LENGTH + 1];
 186     snprintfz(buffer, RRDVAR_MAX_LENGTH, "%s.%s", st->id, variable);
 187     rs->fullid = strdupz(buffer);
 188
 189     snprintfz(buffer, RRDVAR_MAX_LENGTH, "%s.%s", st->name, variable);
 190     rs->fullname = strdupz(buffer);
 191
 192     rs->variable = strdupz(variable);
 193
 194     rs->type = type;
 195     rs->value = value;
 196     rs->options = options;
 197     rs->rrdset = st;
 198
 199     rs->local       = rrdvar_create_and_index("local",  &st->variables_root_index, rs->variable, rs->type, rs->value);
 200     rs->family      = rrdvar_create_and_index("family", &st->rrdfamily->variables_root_index, rs->fullid, rs->type, rs->value);
 201     rs->host        = rrdvar_create_and_index("host",   &st->rrdhost->variables_root_index, rs->fullid, rs->type, rs->value);
 202     rs->family_name = rrdvar_create_and_index("family", &st->rrdfamily->variables_root_index, rs->fullname, rs->type, rs->value);
 203     rs->host_name   = rrdvar_create_and_index("host",   &st->rrdhost->variables_root_index, rs->fullname, rs->type, rs->value);
 204
 205     rs->next = st->variables;
 206     st->variables = rs;
 207
 208     return rs;
 209 }
 210
 211 void rrdsetvar_rename_all(RRDSET *st) {
 212     debug(D_VARIABLES, "RRDSETVAR rename for chart id '%s' name '%s'", st->id, st->name);
 213
 214     // only these 2 can change name
 215     // rs->family_name
 216     // rs->host_name
 217
 218     char buffer[RRDVAR_MAX_LENGTH + 1];
 219     RRDSETVAR *rs, *next = st->variables;
 220     while((rs = next)) {
 221         next = rs->next;
 222
 223         snprintfz(buffer, RRDVAR_MAX_LENGTH, "%s.%s", st->name, rs->variable);
 224
 225         if (strcmp(buffer, rs->fullname)) {
 226             // name changed
 227             rrdvar_free(st->rrdhost, &st->rrdfamily->variables_root_index, rs->family_name);
 228             rrdvar_free(st->rrdhost, &st->rrdhost->variables_root_index, rs->host_name);
 229
 230             freez(rs->fullname);
 231             rs->fullname = strdupz(st->name);
 232             rs->family_name = rrdvar_create_and_index("family", &st->rrdfamily->variables_root_index, rs->fullname, rs->type, rs->value);
 233             rs->host_name   = rrdvar_create_and_index("host",   &st->rrdhost->variables_root_index, rs->fullname, rs->type, rs->value);
 234         }
 235     }
 236
 237     rrdsetcalc_link_matching(st);
 238 }
 239
 240 void rrdsetvar_free(RRDSETVAR *rs) {
 241     RRDSET *st = rs->rrdset;
 242     debug(D_VARIABLES, "RRDSETVAR free for chart id '%s' name '%s', variable '%s'", st->id, st->name, rs->variable);
 243
 244     rrdvar_free(st->rrdhost, &st->variables_root_index, rs->local);
 245     rrdvar_free(st->rrdhost, &st->rrdfamily->variables_root_index, rs->family);
 246     rrdvar_free(st->rrdhost, &st->rrdhost->variables_root_index, rs->host);
 247     rrdvar_free(st->rrdhost, &st->rrdfamily->variables_root_index, rs->family_name);
 248     rrdvar_free(st->rrdhost, &st->rrdhost->variables_root_index, rs->host_name);
 249
 250     if(st->variables == rs) {
 251         st->variables = rs->next;
 252     }
 253     else {
 254         RRDSETVAR *t;
 255         for (t = st->variables; t && t->next != rs; t = t->next);
 256         if(!t) error("RRDSETVAR '%s' not found in chart '%s' variables linked list", rs->fullname, st->id);
 257         else t->next = rs->next;
 258     }
 259
 260     freez(rs->fullid);
 261     freez(rs->fullname);
 262     freez(rs->variable);
 263     freez(rs);
 264 }
 265
 266 // ----------------------------------------------------------------------------
 267 // RRDDIMVAR management
 268
 269 #define RRDDIMVAR_ID_MAX 1024
 270
 271 RRDDIMVAR *rrddimvar_create(RRDDIM *rd, int type, const char *prefix, const char *suffix, void *value, uint32_t options) {
 272     RRDSET *st = rd->rrdset;
 273
 274     debug(D_VARIABLES, "RRDDIMSET create for chart id '%s' name '%s', dimension id '%s', name '%s%s%s'", st->id, st->name, rd->id, (prefix)?prefix:"", rd->name, (suffix)?suffix:"");
 275
 276     if(!prefix) prefix = "";
 277     if(!suffix) suffix = "";
 278
 279     char buffer[RRDDIMVAR_ID_MAX + 1];
 280     RRDDIMVAR *rs = (RRDDIMVAR *)callocz(1, sizeof(RRDDIMVAR));
 281
 282     rs->prefix = strdupz(prefix);
 283     rs->suffix = strdupz(suffix);
 284
 285     snprintfz(buffer, RRDDIMVAR_ID_MAX, "%s%s%s", rs->prefix, rd->id, rs->suffix);
 286     rs->id = strdupz(buffer);
 287
 288     snprintfz(buffer, RRDDIMVAR_ID_MAX, "%s%s%s", rs->prefix, rd->name, rs->suffix);
 289     rs->name = strdupz(buffer);
 290
 291     snprintfz(buffer, RRDDIMVAR_ID_MAX, "%s.%s", rd->rrdset->id, rs->id);
 292     rs->fullidid = strdupz(buffer);
 293
 294     snprintfz(buffer, RRDDIMVAR_ID_MAX, "%s.%s", rd->rrdset->id, rs->name);
 295     rs->fullidname = strdupz(buffer);
 296
 297     snprintfz(buffer, RRDDIMVAR_ID_MAX, "%s.%s", rd->rrdset->name, rs->id);
 298     rs->fullnameid = strdupz(buffer);
 299
 300     snprintfz(buffer, RRDDIMVAR_ID_MAX, "%s.%s", rd->rrdset->name, rs->name);
 301     rs->fullnamename = strdupz(buffer);
 302
 303     rs->type = type;
 304     rs->value = value;
 305     rs->options = options;
 306     rs->rrddim = rd;
 307
 308     rs->local_id     = rrdvar_create_and_index("local", &st->variables_root_index, rs->id, rs->type, rs->value);
 309     rs->local_name   = rrdvar_create_and_index("local", &st->variables_root_index, rs->name, rs->type, rs->value);
 310
 311     rs->family_id    = rrdvar_create_and_index("family", &st->rrdfamily->variables_root_index, rs->id, rs->type, rs->value);
 312     rs->family_name  = rrdvar_create_and_index("family", &st->rrdfamily->variables_root_index, rs->name, rs->type, rs->value);
 313
 314     rs->host_fullidid     = rrdvar_create_and_index("host", &st->rrdhost->variables_root_index, rs->fullidid, rs->type, rs->value);
 315     rs->host_fullidname   = rrdvar_create_and_index("host", &st->rrdhost->variables_root_index, rs->fullidname, rs->type, rs->value);
 316     rs->host_fullnameid   = rrdvar_create_and_index("host", &st->rrdhost->variables_root_index, rs->fullnameid, rs->type, rs->value);
 317     rs->host_fullnamename = rrdvar_create_and_index("host", &st->rrdhost->variables_root_index, rs->fullnamename, rs->type, rs->value);
 318
 319     rs->next = rd->variables;
 320     rd->variables = rs;
 321
 322     return rs;
 323 }
 324
 325 void rrddimvar_rename_all(RRDDIM *rd) {
 326     RRDSET *st = rd->rrdset;
 327     debug(D_VARIABLES, "RRDDIMSET rename for chart id '%s' name '%s', dimension id '%s', name '%s'", st->id, st->name, rd->id, rd->name);
 328
 329     RRDDIMVAR *rs, *next = rd->variables;
 330     while((rs = next)) {
 331         next = rs->next;
 332
 333         if (strcmp(rd->name, rs->name)) {
 334             char buffer[RRDDIMVAR_ID_MAX + 1];
 335             // name changed
 336
 337             // name
 338             rrdvar_free(st->rrdhost, &st->variables_root_index, rs->local_name);
 339             freez(rs->name);
 340             snprintfz(buffer, RRDDIMVAR_ID_MAX, "%s%s%s", rs->prefix, rd->name, rs->suffix);
 341             rs->name = strdupz(buffer);
 342             rs->local_name = rrdvar_create_and_index("local", &st->variables_root_index, rs->name, rs->type, rs->value);
 343
 344             rrdvar_free(st->rrdhost, &st->rrdhost->variables_root_index, rs->host_fullidname);
 345             freez(rs->fullidname);
 346             snprintfz(buffer, RRDDIMVAR_ID_MAX, "%s.%s", st->id, rs->name);
 347             rs->fullidname = strdupz(buffer);
 348             rs->host_fullidname = rrdvar_create_and_index("host", &st->rrdhost->variables_root_index,
 349                                                              rs->fullidname, rs->type, rs->value);
 350
 351             // fullnameid
 352             rrdvar_free(st->rrdhost, &st->rrdhost->variables_root_index, rs->host_fullnameid);
 353             freez(rs->fullnameid);
 354             snprintfz(buffer, RRDDIMVAR_ID_MAX, "%s.%s", st->name, rs->id);
 355             rs->fullnameid = strdupz(buffer);
 356             rs->host_fullnameid = rrdvar_create_and_index("host", &st->rrdhost->variables_root_index,
 357                                                           rs->fullnameid, rs->type, rs->value);
 358
 359             // fullnamename
 360             rrdvar_free(st->rrdhost, &st->rrdhost->variables_root_index, rs->host_fullnamename);
 361             freez(rs->fullnamename);
 362             snprintfz(buffer, RRDDIMVAR_ID_MAX, "%s.%s", st->name, rs->name);
 363             rs->fullnamename = strdupz(buffer);
 364             rs->host_fullnamename = rrdvar_create_and_index("host", &st->rrdhost->variables_root_index,
 365                                                           rs->fullnamename, rs->type, rs->value);
 366         }
 367     }
 368 }
 369
 370 void rrddimvar_free(RRDDIMVAR *rs) {
 371     RRDDIM *rd = rs->rrddim;
 372     RRDSET *st = rd->rrdset;
 373     debug(D_VARIABLES, "RRDDIMSET free for chart id '%s' name '%s', dimension id '%s', name '%s', prefix='%s', suffix='%s'", st->id, st->name, rd->id, rd->name, rs->prefix, rs->suffix);
 374
 375     rrdvar_free(st->rrdhost, &st->variables_root_index, rs->local_id);
 376     rrdvar_free(st->rrdhost, &st->variables_root_index, rs->local_name);
 377
 378     rrdvar_free(st->rrdhost, &st->rrdfamily->variables_root_index, rs->family_id);
 379     rrdvar_free(st->rrdhost, &st->rrdfamily->variables_root_index, rs->family_name);
 380
 381     rrdvar_free(st->rrdhost, &st->rrdhost->variables_root_index, rs->host_fullidid);
 382     rrdvar_free(st->rrdhost, &st->rrdhost->variables_root_index, rs->host_fullidname);
 383     rrdvar_free(st->rrdhost, &st->rrdhost->variables_root_index, rs->host_fullnameid);
 384     rrdvar_free(st->rrdhost, &st->rrdhost->variables_root_index, rs->host_fullnamename);
 385
 386     if(rd->variables == rs) {
 387         debug(D_VARIABLES, "RRDDIMSET removing first entry for chart id '%s' name '%s', dimension id '%s', name '%s'", st->id, st->name, rd->id, rd->name);
 388         rd->variables = rs->next;
 389     }
 390     else {
 391         debug(D_VARIABLES, "RRDDIMSET removing non-first entry for chart id '%s' name '%s', dimension id '%s', name '%s'", st->id, st->name, rd->id, rd->name);
 392         RRDDIMVAR *t;
 393         for (t = rd->variables; t && t->next != rs; t = t->next) ;
 394         if(!t) error("RRDDIMVAR '%s' not found in dimension '%s/%s' variables linked list", rs->name, st->id, rd->id);
 395         else t->next = rs->next;
 396     }
 397
 398     freez(rs->prefix);
 399     freez(rs->suffix);
 400     freez(rs->id);
 401     freez(rs->name);
 402     freez(rs->fullidid);
 403     freez(rs->fullidname);
 404     freez(rs->fullnameid);
 405     freez(rs->fullnamename);
 406     freez(rs);
 407 }
 408
 409 // ----------------------------------------------------------------------------
 410 // RRDCALC management
 411
 412 static inline const char *rrdcalc_status2string(int status) {
 413     switch(status) {
 414         case RRDCALC_STATUS_UNINITIALIZED:
 415             return "UNINITIALIZED";
 416
 417         case RRDCALC_STATUS_UNDEFINED:
 418             return "UNDEFINED";
 419
 420         case RRDCALC_STATUS_CLEAR:
 421             return "CLEAR";
 422
 423         case RRDCALC_STATUS_RAISED:
 424             return "RAISED";
 425
 426         case RRDCALC_STATUS_WARNING:
 427             return "WARNING";
 428
 429         case RRDCALC_STATUS_CRITICAL:
 430             return "CRITICAL";
 431
 432         default:
 433             return "UNKNOWN";
 434     }
 435 }
 436
 437 static void rrdsetcalc_link(RRDSET *st, RRDCALC *rc) {
 438     debug(D_HEALTH, "Health linking alarm '%s.%s' to chart '%s' of host '%s'", rc->chart?rc->chart:"NOCHART", rc->name, st->id, st->rrdhost->hostname);
 439
 440     rc->last_status_change = time(NULL);
 441     rc->rrdset = st;
 442
 443     rc->rrdset_next = st->alarms;
 444     rc->rrdset_prev = NULL;
 445
 446     if(rc->rrdset_next)
 447         rc->rrdset_next->rrdset_prev = rc;
 448
 449     st->alarms = rc;
 450
 451     if(rc->update_every < rc->rrdset->update_every) {
 452         error("Health alarm '%s.%s' has update every %d, less than chart update every %d. Setting alarm update frequency to %d.", rc->rrdset->id, rc->name, rc->update_every, rc->rrdset->update_every, rc->rrdset->update_every);
 453         rc->update_every = rc->rrdset->update_every;
 454     }
 455
 456     if(!isnan(rc->green) && isnan(st->green)) {
 457         debug(D_HEALTH, "Health alarm '%s.%s' green threshold set from %Lf to %Lf.", rc->rrdset->id, rc->name, rc->rrdset->green, rc->green);
 458         st->green = rc->green;
 459     }
 460
 461     if(!isnan(rc->red) && isnan(st->red)) {
 462         debug(D_HEALTH, "Health alarm '%s.%s' red threshold set from %Lf to %Lf.", rc->rrdset->id, rc->name, rc->rrdset->red, rc->red);
 463         st->red = rc->red;
 464     }
 465
 466     rc->local  = rrdvar_create_and_index("local",  &st->variables_root_index, rc->name, RRDVAR_TYPE_CALCULATED, &rc->value);
 467     rc->family = rrdvar_create_and_index("family", &st->rrdfamily->variables_root_index, rc->name, RRDVAR_TYPE_CALCULATED, &rc->value);
 468
 469     char fullname[RRDVAR_MAX_LENGTH + 1];
 470     snprintfz(fullname, RRDVAR_MAX_LENGTH, "%s.%s", st->id, rc->name);
 471     rc->hostid   = rrdvar_create_and_index("host", &st->rrdhost->variables_root_index, fullname, RRDVAR_TYPE_CALCULATED, &rc->value);
 472
 473     snprintfz(fullname, RRDVAR_MAX_LENGTH, "%s.%s", st->name, rc->name);
 474     rc->hostname = rrdvar_create_and_index("host", &st->rrdhost->variables_root_index, fullname, RRDVAR_TYPE_CALCULATED, &rc->value);
 475
 476         if(!rc->units) rc->units = strdupz(st->units);
 477 }
 478
 479 static inline int rrdcalc_is_matching_this_rrdset(RRDCALC *rc, RRDSET *st) {
 480     if(     (rc->hash_chart == st->hash      && !strcmp(rc->chart, st->id)) ||
 481             (rc->hash_chart == st->hash_name && !strcmp(rc->chart, st->name)))
 482         return 1;
 483
 484     return 0;
 485 }
 486
 487 // this has to be called while the RRDHOST is locked
 488 inline void rrdsetcalc_link_matching(RRDSET *st) {
 489     // debug(D_HEALTH, "find matching alarms for chart '%s'", st->id);
 490
 491     RRDCALC *rc;
 492     for(rc = st->rrdhost->alarms; rc ; rc = rc->next) {
 493         if(unlikely(rc->rrdset))
 494             continue;
 495
 496         if(unlikely(rrdcalc_is_matching_this_rrdset(rc, st)))
 497             rrdsetcalc_link(st, rc);
 498     }
 499 }
 500
 501 // this has to be called while the RRDHOST is locked
 502 inline void rrdsetcalc_unlink(RRDCALC *rc) {
 503     RRDSET *st = rc->rrdset;
 504
 505     if(!st) {
 506         error("Requested to unlink RRDCALC '%s.%s' which is not linked to any RRDSET", rc->chart?rc->chart:"NOCHART", rc->name);
 507         return;
 508     }
 509
 510     RRDHOST *host = st->rrdhost;
 511
 512     debug(D_HEALTH, "Health unlinking alarm '%s.%s' from chart '%s' of host '%s'", rc->chart?rc->chart:"NOCHART", rc->name, st->id, host->hostname);
 513
 514     // unlink it
 515     if(rc->rrdset_prev)
 516         rc->rrdset_prev->rrdset_next = rc->rrdset_next;
 517
 518     if(rc->rrdset_next)
 519         rc->rrdset_next->rrdset_prev = rc->rrdset_prev;
 520
 521     if(st->alarms == rc)
 522         st->alarms = rc->rrdset_next;
 523
 524     rc->rrdset_prev = rc->rrdset_next = NULL;
 525
 526     rrdvar_free(st->rrdhost, &st->variables_root_index, rc->local);
 527     rc->local = NULL;
 528
 529     rrdvar_free(st->rrdhost, &st->rrdfamily->variables_root_index, rc->family);
 530     rc->family = NULL;
 531
 532     rrdvar_free(st->rrdhost, &st->rrdhost->variables_root_index, rc->hostid);
 533     rc->hostid = NULL;
 534
 535     rrdvar_free(st->rrdhost, &st->rrdhost->variables_root_index, rc->hostname);
 536     rc->hostname = NULL;
 537
 538     rc->rrdset = NULL;
 539
 540     // RRDCALC will remain in RRDHOST
 541     // so that if the matching chart is found in the future
 542     // it will be applied automatically
 543 }
 544
 545 RRDCALC *rrdcalc_find(RRDSET *st, const char *name) {
 546     RRDCALC *rc;
 547     uint32_t hash = simple_hash(name);
 548
 549     for( rc = st->alarms; rc ; rc = rc->rrdset_next ) {
 550         if(unlikely(rc->hash == hash && !strcmp(rc->name, name)))
 551             return rc;
 552     }
 553
 554     return NULL;
 555 }
 556
 557 static inline int rrdcalc_exists(RRDHOST *host, const char *name, uint32_t hash) {
 558     RRDCALC *rc;
 559
 560     // make sure it does not already exist
 561     for(rc = host->alarms; rc ; rc = rc->next) {
 562         if (unlikely(rc->hash == hash && !strcmp(name, rc->name))) {
 563             error("Health alarm '%s' already exists in host '%s'.", name, host->hostname);
 564             return 1;
 565         }
 566     }
 567
 568     return 0;
 569 }
 570
 571 static inline void rrdcalc_create_part2(RRDHOST *host, RRDCALC *rc) {
 572     rrdhost_check_rdlock(host);
 573
 574     if(rc->calculation) {
 575         rc->calculation->this = &rc->value;
 576         rc->calculation->after = &rc->db_after;
 577         rc->calculation->before = &rc->db_before;
 578         rc->calculation->rrdcalc = rc;
 579     }
 580
 581     if(rc->warning) {
 582         rc->warning->this = &rc->value;
 583         rc->warning->after = &rc->db_after;
 584         rc->warning->before = &rc->db_before;
 585         rc->warning->rrdcalc = rc;
 586     }
 587
 588     if(rc->critical) {
 589         rc->critical->this = &rc->value;
 590         rc->critical->after = &rc->db_after;
 591         rc->critical->before = &rc->db_before;
 592         rc->critical->rrdcalc = rc;
 593     }
 594
 595     // link it to the host
 596     if(likely(host->alarms)) {
 597         // append it
 598         RRDCALC *t;
 599         for(t = host->alarms; t && t->next ; t = t->next) ;
 600         t->next = rc;
 601     }
 602     else {
 603         host->alarms = rc;
 604     }
 605
 606     // link it to its chart
 607     RRDSET *st;
 608     for(st = host->rrdset_root; st ; st = st->next) {
 609         if(rrdcalc_is_matching_this_rrdset(rc, st)) {
 610             rrdsetcalc_link(st, rc);
 611             break;
 612         }
 613     }
 614 }
 615
 616 static inline uint32_t rrdcalc_fullname(char *fullname, size_t len, const char *chart, const char *name) {
 617     snprintfz(fullname, len - 1, "%s%s%s", chart?chart:"", chart?".":"", name);
 618     rrdvar_fix_name(fullname);
 619     return simple_hash(fullname);
 620 }
 621
 622 static inline RRDCALC *rrdcalc_create(RRDHOST *host, const char *name, const char *chart, const char *dimensions,
 623                         const char *units, const char *info,
 624                         int group_method, int after, int before, int update_every, uint32_t options,
 625                         calculated_number green, calculated_number red,
 626                         const char *exec, const char *recipient, const char *source,
 627                         const char *calc, const char *warn, const char *crit) {
 628
 629     char fullname[RRDVAR_MAX_LENGTH + 1];
 630     uint32_t hash = rrdcalc_fullname(fullname, RRDVAR_MAX_LENGTH + 1, chart, name);
 631
 632     if(rrdcalc_exists(host, fullname, hash))
 633         return NULL;
 634
 635     RRDCALC *rc = callocz(1, sizeof(RRDCALC));
 636     rc->id = host->health_log.next_alarm_id++;
 637     rc->next_event_id = 1;
 638     rc->name = strdupz(name);
 639     rc->hash = simple_hash(rc->name);
 640     rc->chart = strdupz(chart);
 641     rc->hash_chart = simple_hash(rc->chart);
 642
 643     if(dimensions) rc->dimensions = strdupz(dimensions);
 644
 645     rc->green = green;
 646     rc->red = red;
 647     rc->value = NAN;
 648     rc->old_value = NAN;
 649
 650     rc->group = group_method;
 651     rc->after = after;
 652     rc->before = before;
 653     rc->update_every = update_every;
 654     rc->options = options;
 655
 656     if(exec) rc->exec = strdupz(exec);
 657     if(recipient) rc->recipient = strdupz(recipient);
 658     if(source) rc->source = strdupz(source);
 659     if(units) rc->units = strdupz(units);
 660     if(info) rc->info = strdupz(info);
 661
 662     if(calc) {
 663         rc->calculation = expression_parse(calc, NULL, NULL);
 664         if(!rc->calculation)
 665             error("Health alarm '%s.%s': failed to parse calculation expression '%s'", chart, name, calc);
 666     }
 667     if(warn) {
 668         rc->warning = expression_parse(warn, NULL, NULL);
 669         if(!rc->warning)
 670             error("Health alarm '%s.%s': failed to re-parse warning expression '%s'", chart, name, warn);
 671     }
 672     if(crit) {
 673         rc->critical = expression_parse(crit, NULL, NULL);
 674         if(!rc->critical)
 675             error("Health alarm '%s.%s': failed to re-parse critical expression '%s'", chart, name, crit);
 676     }
 677
 678     debug(D_HEALTH, "Health runtime added alarm '%s.%s': exec '%s', recipient '%s', green %Lf, red %Lf, lookup: group %d, after %d, before %d, options %u, dimensions '%s', update every %d, calculation '%s', warning '%s', critical '%s', source '%s",
 679           (rc->chart)?rc->chart:"NOCHART",
 680           rc->name,
 681           (rc->exec)?rc->exec:"DEFAULT",
 682           (rc->recipient)?rc->recipient:"DEFAULT",
 683           rc->green,
 684           rc->red,
 685           rc->group,
 686           rc->after,
 687           rc->before,
 688           rc->options,
 689           (rc->dimensions)?rc->dimensions:"NONE",
 690           rc->update_every,
 691           (rc->calculation)?rc->calculation->parsed_as:"NONE",
 692           (rc->warning)?rc->warning->parsed_as:"NONE",
 693           (rc->critical)?rc->critical->parsed_as:"NONE",
 694           rc->source
 695     );
 696
 697     rrdcalc_create_part2(host, rc);
 698     return rc;
 699 }
 700
 701 void rrdcalc_free(RRDHOST *host, RRDCALC *rc) {
 702     if(!rc) return;
 703
 704     debug(D_HEALTH, "Health removing alarm '%s.%s' of host '%s'", rc->chart?rc->chart:"NOCHART", rc->name, host->hostname);
 705
 706     // unlink it from RRDSET
 707     if(rc->rrdset) rrdsetcalc_unlink(rc);
 708
 709     // unlink it from RRDHOST
 710     if(unlikely(rc == host->alarms))
 711         host->alarms = rc->next;
 712
 713     else if(likely(host->alarms)) {
 714         RRDCALC *t, *last = host->alarms;
 715         for(t = last->next; t && t != rc; last = t, t = t->next) ;
 716         if(last && last->next == rc)
 717             last->next = rc->next;
 718         else
 719             error("Cannot unlink alarm '%s.%s' from host '%s': not found", rc->chart?rc->chart:"NOCHART", rc->name, host->hostname);
 720     }
 721     else
 722         error("Cannot unlink unlink '%s.%s' from host '%s': This host does not have any calculations", rc->chart?rc->chart:"NOCHART", rc->name, host->hostname);
 723
 724     expression_free(rc->calculation);
 725     expression_free(rc->warning);
 726     expression_free(rc->critical);
 727
 728     freez(rc->name);
 729     freez(rc->chart);
 730     freez(rc->family);
 731     freez(rc->dimensions);
 732     freez(rc->exec);
 733     freez(rc->recipient);
 734     freez(rc->source);
 735     freez(rc->units);
 736     freez(rc->info);
 737     freez(rc);
 738 }
 739
 740 // ----------------------------------------------------------------------------
 741 // RRDCALCTEMPLATE management
 742
 743 void rrdcalctemplate_link_matching(RRDSET *st) {
 744     RRDCALCTEMPLATE *rt;
 745
 746     for(rt = st->rrdhost->templates; rt ; rt = rt->next) {
 747         if(rt->hash_context == st->hash_context && !strcmp(rt->context, st->context)) {
 748             RRDCALC *rc = rrdcalc_create(st->rrdhost, rt->name, st->id,
 749                            rt->dimensions, rt->units, rt->info, rt->group, rt->after, rt->before, rt->update_every, rt->options,
 750                            rt->green, rt->red,
 751                            rt->exec, rt->recipient, rt->source,
 752                            (rt->calculation)?rt->calculation->source:NULL,
 753                            (rt->warning)?rt->warning->source:NULL,
 754                            (rt->critical)?rt->critical->source:NULL);
 755
 756             if(unlikely(!rc))
 757                 error("Health tried to create alarm from template '%s', but it failed", rt->name);
 758
 759 #ifdef NETDATA_INTERNAL_CHECKS
 760             else if(rc->rrdset != st)
 761                 error("Health alarm '%s.%s' should be linked to chart '%s', but it is not", rc->chart?rc->chart:"NOCHART", rc->name, st->id);
 762 #endif
 763         }
 764     }
 765 }
 766
 767 static inline void rrdcalctemplate_free(RRDHOST *host, RRDCALCTEMPLATE *rt) {
 768     debug(D_HEALTH, "Health removing template '%s' of host '%s'", rt->name, host->hostname);
 769
 770     if(host->templates) {
 771         if(host->templates == rt) {
 772             host->templates = rt->next;
 773         }
 774         else {
 775             RRDCALCTEMPLATE *t, *last = host->templates;
 776             for (t = last->next; t && t != rt; last = t, t = t->next ) ;
 777             if(last && last->next == rt) {
 778                 last->next = rt->next;
 779                 rt->next = NULL;
 780             }
 781             else
 782                 error("Cannot find RRDCALCTEMPLATE '%s' linked in host '%s'", rt->name, host->hostname);
 783         }
 784     }
 785
 786     expression_free(rt->calculation);
 787     expression_free(rt->warning);
 788     expression_free(rt->critical);
 789
 790     freez(rt->name);
 791     freez(rt->exec);
 792     freez(rt->recipient);
 793     freez(rt->context);
 794     freez(rt->source);
 795     freez(rt->units);
 796     freez(rt->info);
 797     freez(rt->dimensions);
 798     freez(rt);
 799 }
 800
 801 // ----------------------------------------------------------------------------
 802 // load health configuration
 803
 804 #define HEALTH_CONF_MAX_LINE 4096
 805
 806 #define HEALTH_ALARM_KEY "alarm"
 807 #define HEALTH_TEMPLATE_KEY "template"
 808 #define HEALTH_ON_KEY "on"
 809 #define HEALTH_LOOKUP_KEY "lookup"
 810 #define HEALTH_CALC_KEY "calc"
 811 #define HEALTH_EVERY_KEY "every"
 812 #define HEALTH_GREEN_KEY "green"
 813 #define HEALTH_RED_KEY "red"
 814 #define HEALTH_WARN_KEY "warn"
 815 #define HEALTH_CRIT_KEY "crit"
 816 #define HEALTH_EXEC_KEY "exec"
 817 #define HEALTH_RECIPIENT_KEY "to"
 818 #define HEALTH_UNITS_KEY "units"
 819 #define HEALTH_INFO_KEY "info"
 820
 821 static inline int rrdcalc_add_alarm_from_config(RRDHOST *host, RRDCALC *rc) {
 822     {
 823         char fullname[RRDVAR_MAX_LENGTH + 1];
 824         uint32_t hash = rrdcalc_fullname(fullname, RRDVAR_MAX_LENGTH + 1, rc->chart, rc->name);
 825
 826         if (rrdcalc_exists(host, fullname, hash))
 827             return 0;
 828     }
 829
 830     if(!rc->chart) {
 831         error("Health configuration for alarm '%s' does not have a chart", rc->name);
 832         return 0;
 833     }
 834
 835     if(!rc->update_every) {
 836         error("Health configuration for alarm '%s.%s' has no frequency (parameter 'every'). Ignoring it.", rc->chart?rc->chart:"NOCHART", rc->name);
 837         return 0;
 838     }
 839
 840     if(!RRDCALC_HAS_DB_LOOKUP(rc) && !rc->warning && !rc->critical) {
 841         error("Health configuration for alarm '%s.%s' is useless (no calculation, no warning and no critical evaluation)", rc->chart?rc->chart:"NOCHART", rc->name);
 842         return 0;
 843     }
 844
 845     debug(D_HEALTH, "Health configuration adding alarm '%s.%s': exec '%s', recipient '%s', green %Lf, red %Lf, lookup: group %d, after %d, before %d, options %u, dimensions '%s', update every %d, calculation '%s', warning '%s', critical '%s', source '%s",
 846           rc->chart?rc->chart:"NOCHART",
 847           rc->name,
 848           (rc->exec)?rc->exec:"DEFAULT",
 849           (rc->recipient)?rc->recipient:"DEFAULT",
 850           rc->green,
 851           rc->red,
 852           rc->group,
 853           rc->after,
 854           rc->before,
 855           rc->options,
 856           (rc->dimensions)?rc->dimensions:"NONE",
 857           rc->update_every,
 858           (rc->calculation)?rc->calculation->parsed_as:"NONE",
 859           (rc->warning)?rc->warning->parsed_as:"NONE",
 860           (rc->critical)?rc->critical->parsed_as:"NONE",
 861           rc->source
 862     );
 863
 864     rrdcalc_create_part2(host, rc);
 865     return 1;
 866 }
 867
 868 static inline int rrdcalctemplate_add_template_from_config(RRDHOST *host, RRDCALCTEMPLATE *rt) {
 869     if(unlikely(!rt->context)) {
 870         error("Health configuration for template '%s' does not have a context", rt->name);
 871         return 0;
 872     }
 873
 874     if(unlikely(!rt->update_every)) {
 875         error("Health configuration for template '%s' has no frequency (parameter 'every'). Ignoring it.", rt->name);
 876         return 0;
 877     }
 878
 879     if(unlikely(!RRDCALCTEMPLATE_HAS_CALCULATION(rt) && !rt->warning && !rt->critical)) {
 880         error("Health configuration for template '%s' is useless (no calculation, no warning and no critical evaluation)", rt->name);
 881         return 0;
 882     }
 883
 884     RRDCALCTEMPLATE *t, *last = NULL;
 885     for (t = host->templates; t ; last = t, t = t->next) {
 886         if(unlikely(t->hash_name == rt->hash_name && !strcmp(t->name, rt->name))) {
 887             error("Health configuration template '%s' already exists for host '%s'.", rt->name, host->hostname);
 888             return 0;
 889         }
 890     }
 891
 892     debug(D_HEALTH, "Health configuration adding template '%s': context '%s', exec '%s', recipient '%s', green %Lf, red %Lf, lookup: group %d, after %d, before %d, options %u, dimensions '%s', update every %d, calculation '%s', warning '%s', critical '%s', source '%s'",
 893           rt->name,
 894           (rt->context)?rt->context:"NONE",
 895           (rt->exec)?rt->exec:"DEFAULT",
 896           (rt->recipient)?rt->recipient:"DEFAULT",
 897           rt->green,
 898           rt->red,
 899           rt->group,
 900           rt->after,
 901           rt->before,
 902           rt->options,
 903           (rt->dimensions)?rt->dimensions:"NONE",
 904           rt->update_every,
 905           (rt->calculation)?rt->calculation->parsed_as:"NONE",
 906           (rt->warning)?rt->warning->parsed_as:"NONE",
 907           (rt->critical)?rt->critical->parsed_as:"NONE",
 908           rt->source
 909     );
 910
 911     if(likely(last)) {
 912         last->next = rt;
 913     }
 914     else {
 915         rt->next = host->templates;
 916         host->templates = rt;
 917     }
 918
 919     return 1;
 920 }
 921
 922 static inline int health_parse_duration(char *string, int *result) {
 923     // make sure it is a number
 924     if(!*string || !(isdigit(*string) || *string == '+' || *string == '-')) {
 925         *result = 0;
 926         return 0;
 927     }
 928
 929     char *e = NULL;
 930     calculated_number n = strtold(string, &e);
 931     if(e && *e) {
 932         switch (*e) {
 933             case 'Y':
 934                 *result = (int) (n * 86400 * 365);
 935                 break;
 936             case 'M':
 937                 *result = (int) (n * 86400 * 30);
 938                 break;
 939             case 'w':
 940                 *result = (int) (n * 86400 * 7);
 941                 break;
 942             case 'd':
 943                 *result = (int) (n * 86400);
 944                 break;
 945             case 'h':
 946                 *result = (int) (n * 3600);
 947                 break;
 948             case 'm':
 949                 *result = (int) (n * 60);
 950                 break;
 951
 952             default:
 953             case 's':
 954                 *result = (int) (n);
 955                 break;
 956         }
 957     }
 958     else
 959        *result = (int)(n);
 960
 961     return 1;
 962 }
 963
 964 static inline int health_parse_db_lookup(
 965         size_t line, const char *path, const char *file, char *string,
 966         int *group_method, int *after, int *before, int *every,
 967         uint32_t *options, char **dimensions
 968 ) {
 969     debug(D_HEALTH, "Health configuration parsing database lookup %zu@%s/%s: %s", line, path, file, string);
 970
 971     if(*dimensions) freez(*dimensions);
 972     *dimensions = NULL;
 973     *after = 0;
 974     *before = 0;
 975     *every = 0;
 976     *options = 0;
 977
 978     char *s = string, *key;
 979
 980     // first is the group method
 981     key = s;
 982     while(*s && !isspace(*s)) s++;
 983     while(*s && isspace(*s)) *s++ = '\0';
 984     if(!*s) {
 985         error("Health configuration invalid chart calculation at line %zu of file '%s/%s': expected group method followed by the 'after' time, but got '%s'",
 986               line, path, file, key);
 987         return 0;
 988     }
 989
 990     if((*group_method = web_client_api_request_v1_data_group(key, -1)) == -1) {
 991         error("Health configuration at line %zu of file '%s/%s': invalid group method '%s'",
 992               line, path, file, key);
 993         return 0;
 994     }
 995
 996     // then is the 'after' time
 997     key = s;
 998     while(*s && !isspace(*s)) s++;
 999     while(*s && isspace(*s)) *s++ = '\0';
1000
1001     if(!health_parse_duration(key, after)) {
1002         error("Health configuration at line %zu of file '%s/%s': invalid duration '%s' after group method",
1003               line, path, file, key);
1004         return 0;
1005     }
1006
1007     // sane defaults
1008     *every = abs(*after);
1009
1010     // now we may have optional parameters
1011     while(*s) {
1012         key = s;
1013         while(*s && !isspace(*s)) s++;
1014         while(*s && isspace(*s)) *s++ = '\0';
1015         if(!*key) break;
1016
1017         if(!strcasecmp(key, "at")) {
1018             char *value = s;
1019             while(*s && !isspace(*s)) s++;
1020             while(*s && isspace(*s)) *s++ = '\0';
1021
1022             if (!health_parse_duration(value, before)) {
1023                 error("Health configuration at line %zu of file '%s/%s': invalid duration '%s' for '%s' keyword",
1024                       line, path, file, value, key);
1025             }
1026         }
1027         else if(!strcasecmp(key, HEALTH_EVERY_KEY)) {
1028             char *value = s;
1029             while(*s && !isspace(*s)) s++;
1030             while(*s && isspace(*s)) *s++ = '\0';
1031
1032             if (!health_parse_duration(value, every)) {
1033                 error("Health configuration at line %zu of file '%s/%s': invalid duration '%s' for '%s' keyword",
1034                       line, path, file, value, key);
1035             }
1036         }
1037         else if(!strcasecmp(key, "absolute") || !strcasecmp(key, "abs") || !strcasecmp(key, "absolute_sum")) {
1038             *options |= RRDR_OPTION_ABSOLUTE;
1039         }
1040         else if(!strcasecmp(key, "min2max")) {
1041             *options |= RRDR_OPTION_MIN2MAX;
1042         }
1043         else if(!strcasecmp(key, "null2zero")) {
1044             *options |= RRDR_OPTION_NULL2ZERO;
1045         }
1046         else if(!strcasecmp(key, "percentage")) {
1047             *options |= RRDR_OPTION_PERCENTAGE;
1048         }
1049         else if(!strcasecmp(key, "unaligned")) {
1050             *options |= RRDR_OPTION_NOT_ALIGNED;
1051         }
1052         else if(!strcasecmp(key, "of")) {
1053             if(*s && strcasecmp(s, "all"))
1054                *dimensions = strdupz(s);
1055             break;
1056         }
1057         else {
1058             error("Health configuration at line %zu of file '%s/%s': unknown keyword '%s'",
1059                   line, path, file, key);
1060         }
1061     }
1062
1063     return 1;
1064 }
1065
1066 static inline char *health_source_file(size_t line, const char *path, const char *filename) {
1067     char buffer[FILENAME_MAX + 1];
1068     snprintfz(buffer, FILENAME_MAX, "%zu@%s/%s", line, path, filename);
1069     return strdupz(buffer);
1070 }
1071
1072 static inline void strip_quotes(char *s) {
1073     while(*s) {
1074         if(*s == '\'' || *s == '"') *s = ' ';
1075         s++;
1076     }
1077 }
1078
1079 int health_readfile(const char *path, const char *filename) {
1080     debug(D_HEALTH, "Health configuration reading file '%s/%s'", path, filename);
1081
1082     static uint32_t hash_alarm = 0, hash_template = 0, hash_on = 0, hash_calc = 0, hash_green = 0, hash_red = 0, hash_warn = 0, hash_crit = 0, hash_exec = 0, hash_every = 0, hash_lookup = 0, hash_units = 0, hash_info = 0, hash_recipient = 0;
1083     char buffer[HEALTH_CONF_MAX_LINE + 1];
1084
1085     if(unlikely(!hash_alarm)) {
1086         hash_alarm = simple_uhash(HEALTH_ALARM_KEY);
1087         hash_template = simple_uhash(HEALTH_TEMPLATE_KEY);
1088         hash_on = simple_uhash(HEALTH_ON_KEY);
1089         hash_calc = simple_uhash(HEALTH_CALC_KEY);
1090         hash_lookup = simple_uhash(HEALTH_LOOKUP_KEY);
1091         hash_green = simple_uhash(HEALTH_GREEN_KEY);
1092         hash_red = simple_uhash(HEALTH_RED_KEY);
1093         hash_warn = simple_uhash(HEALTH_WARN_KEY);
1094         hash_crit = simple_uhash(HEALTH_CRIT_KEY);
1095         hash_exec = simple_uhash(HEALTH_EXEC_KEY);
1096         hash_every = simple_uhash(HEALTH_EVERY_KEY);
1097         hash_units = simple_hash(HEALTH_UNITS_KEY);
1098         hash_info = simple_hash(HEALTH_INFO_KEY);
1099         hash_recipient = simple_hash(HEALTH_RECIPIENT_KEY);
1100     }
1101
1102     snprintfz(buffer, HEALTH_CONF_MAX_LINE, "%s/%s", path, filename);
1103     FILE *fp = fopen(buffer, "r");
1104     if(!fp) {
1105         error("Health configuration cannot read file '%s'.", buffer);
1106         return 0;
1107     }
1108
1109     RRDCALC *rc = NULL;
1110     RRDCALCTEMPLATE *rt = NULL;
1111
1112     size_t line = 0, append = 0;
1113     char *s;
1114     while((s = fgets(&buffer[append], (int)(HEALTH_CONF_MAX_LINE - append), fp)) || append) {
1115         int stop_appending = !s;
1116         line++;
1117         // info("Line %zu of file '%s/%s': '%s'", line, path, filename, s);
1118         s = trim(buffer);
1119         if(!s) continue;
1120         // info("Trimmed line %zu of file '%s/%s': '%s'", line, path, filename, s);
1121
1122         append = strlen(s);
1123         if(!stop_appending && s[append - 1] == '\\') {
1124             s[append - 1] = ' ';
1125             append = &s[append] - buffer;
1126             if(append < HEALTH_CONF_MAX_LINE)
1127                 continue;
1128             else {
1129                 error("Health configuration has too long muli-line at line %zu of file '%s/%s'.", line, path, filename);
1130             }
1131         }
1132         append = 0;
1133
1134         char *key = s;
1135         while(*s && *s != ':') s++;
1136         if(!*s) {
1137             error("Health configuration has invalid line %zu of file '%s/%s'. It does not contain a ':'. Ignoring it.", line, path, filename);
1138             continue;
1139         }
1140         *s = '\0';
1141         s++;
1142
1143         char *value = s;
1144         key = trim(key);
1145         value = trim(value);
1146
1147         if(!key) {
1148             error("Health configuration has invalid line %zu of file '%s/%s'. Keyword is empty. Ignoring it.", line, path, filename);
1149             continue;
1150         }
1151
1152         if(!value) {
1153             error("Health configuration has invalid line %zu of file '%s/%s'. value is empty. Ignoring it.", line, path, filename);
1154             continue;
1155         }
1156
1157         // info("Health file '%s/%s', key '%s', value '%s'", path, filename, key, value);
1158         uint32_t hash = simple_uhash(key);
1159
1160         if(hash == hash_alarm && !strcasecmp(key, HEALTH_ALARM_KEY)) {
1161             if(rc && !rrdcalc_add_alarm_from_config(&localhost, rc))
1162                 rrdcalc_free(&localhost, rc);
1163
1164             if(rt) {
1165                 if (!rrdcalctemplate_add_template_from_config(&localhost, rt))
1166                     rrdcalctemplate_free(&localhost, rt);
1167                 rt = NULL;
1168             }
1169
1170             rc = callocz(1, sizeof(RRDCALC));
1171             rc->id = localhost.health_log.next_alarm_id++;
1172             rc->next_event_id = 1;
1173             rc->name = strdupz(value);
1174             rc->hash = simple_hash(rc->name);
1175             rc->source = health_source_file(line, path, filename);
1176             rc->green = NAN;
1177             rc->red = NAN;
1178             rc->value = NAN;
1179             rc->old_value = NAN;
1180
1181             if(rrdvar_fix_name(rc->name))
1182                 error("Health configuration renamed alarm '%s' to '%s'", value, rc->name);
1183         }
1184         else if(hash == hash_template && !strcasecmp(key, HEALTH_TEMPLATE_KEY)) {
1185             if(rc) {
1186                 if(!rrdcalc_add_alarm_from_config(&localhost, rc))
1187                     rrdcalc_free(&localhost, rc);
1188                 rc = NULL;
1189             }
1190
1191             if(rt && !rrdcalctemplate_add_template_from_config(&localhost, rt))
1192                 rrdcalctemplate_free(&localhost, rt);
1193
1194             rt = callocz(1, sizeof(RRDCALCTEMPLATE));
1195             rt->name = strdupz(value);
1196             rt->hash_name = simple_hash(rt->name);
1197             rt->source = health_source_file(line, path, filename);
1198             rt->green = NAN;
1199             rt->red = NAN;
1200
1201             if(rrdvar_fix_name(rt->name))
1202                 error("Health configuration renamed template '%s' to '%s'", value, rt->name);
1203         }
1204         else if(rc) {
1205             if(hash == hash_on && !strcasecmp(key, HEALTH_ON_KEY)) {
1206                 if(rc->chart) {
1207                     if(strcmp(rc->chart, value))
1208                         info("Health configuration at line %zu of file '%s/%s' for alarm '%s' has key '%s' twice, once with value '%s' and later with value '%s'. Using ('%s').",
1209                              line, path, filename, rc->name, key, rc->chart, value, value);
1210
1211                     freez(rc->chart);
1212                 }
1213                 rc->chart = strdupz(value);
1214                 rc->hash_chart = simple_hash(rc->chart);
1215             }
1216             else if(hash == hash_lookup && !strcasecmp(key, HEALTH_LOOKUP_KEY)) {
1217                 health_parse_db_lookup(line, path, filename, value, &rc->group, &rc->after, &rc->before,
1218                                        &rc->update_every,
1219                                        &rc->options, &rc->dimensions);
1220             }
1221             else if(hash == hash_every && !strcasecmp(key, HEALTH_EVERY_KEY)) {
1222                 if(!health_parse_duration(value, &rc->update_every))
1223                     info("Health configuration at line %zu of file '%s/%s' for alarm '%s' at key '%s' cannot parse duration: '%s'.",
1224                          line, path, filename, rc->name, key, value);
1225             }
1226             else if(hash == hash_green && !strcasecmp(key, HEALTH_GREEN_KEY)) {
1227                 char *e;
1228                 rc->green = strtold(value, &e);
1229                 if(e && *e) {
1230                     info("Health configuration at line %zu of file '%s/%s' for alarm '%s' at key '%s' leaves this string unmatched: '%s'.",
1231                          line, path, filename, rc->name, key, e);
1232                 }
1233             }
1234             else if(hash == hash_red && !strcasecmp(key, HEALTH_RED_KEY)) {
1235                 char *e;
1236                 rc->red = strtold(value, &e);
1237                 if(e && *e) {
1238                     info("Health configuration at line %zu of file '%s/%s' for alarm '%s' at key '%s' leaves this string unmatched: '%s'.",
1239                          line, path, filename, rc->name, key, e);
1240                 }
1241             }
1242             else if(hash == hash_calc && !strcasecmp(key, HEALTH_CALC_KEY)) {
1243                 const char *failed_at = NULL;
1244                 int error = 0;
1245                 rc->calculation = expression_parse(value, &failed_at, &error);
1246                 if(!rc->calculation) {
1247                     error("Health configuration at line %zu of file '%s/%s' for alarm '%s' at key '%s' has unparse-able expression '%s': %s at '%s'",
1248                           line, path, filename, rc->name, key, value, expression_strerror(error), failed_at);
1249                 }
1250             }
1251             else if(hash == hash_warn && !strcasecmp(key, HEALTH_WARN_KEY)) {
1252                 const char *failed_at = NULL;
1253                 int error = 0;
1254                 rc->warning = expression_parse(value, &failed_at, &error);
1255                 if(!rc->warning) {
1256                     error("Health configuration at line %zu of file '%s/%s' for alarm '%s' at key '%s' has unparse-able expression '%s': %s at '%s'",
1257                           line, path, filename, rc->name, key, value, expression_strerror(error), failed_at);
1258                 }
1259             }
1260             else if(hash == hash_crit && !strcasecmp(key, HEALTH_CRIT_KEY)) {
1261                 const char *failed_at = NULL;
1262                 int error = 0;
1263                 rc->critical = expression_parse(value, &failed_at, &error);
1264                 if(!rc->critical) {
1265                     error("Health configuration at line %zu of file '%s/%s' for alarm '%s' at key '%s' has unparse-able expression '%s': %s at '%s'",
1266                           line, path, filename, rc->name, key, value, expression_strerror(error), failed_at);
1267                 }
1268             }
1269             else if(hash == hash_exec && !strcasecmp(key, HEALTH_EXEC_KEY)) {
1270                 if(rc->exec) {
1271                     if(strcmp(rc->exec, value))
1272                         info("Health configuration at line %zu of file '%s/%s' for alarm '%s' has key '%s' twice, once with value '%s' and later with value '%s'. Using ('%s').",
1273                              line, path, filename, rc->name, key, rc->exec, value, value);
1274
1275                     freez(rc->exec);
1276                 }
1277                 rc->exec = strdupz(value);
1278             }
1279             else if(hash == hash_recipient && !strcasecmp(key, HEALTH_RECIPIENT_KEY)) {
1280                 if(rc->recipient) {
1281                     if(strcmp(rc->recipient, value))
1282                         info("Health configuration at line %zu of file '%s/%s' for alarm '%s' has key '%s' twice, once with value '%s' and later with value '%s'. Using ('%s').",
1283                              line, path, filename, rc->name, key, rc->recipient, value, value);
1284
1285                     freez(rc->recipient);
1286                 }
1287                 rc->recipient = strdupz(value);
1288             }
1289             else if(hash == hash_units && !strcasecmp(key, HEALTH_UNITS_KEY)) {
1290                 if(rc->units) {
1291                     if(strcmp(rc->units, value))
1292                         info("Health configuration at line %zu of file '%s/%s' for alarm '%s' has key '%s' twice, once with value '%s' and later with value '%s'. Using ('%s').",
1293                              line, path, filename, rc->name, key, rc->units, value, value);
1294
1295                     freez(rc->units);
1296                 }
1297                 rc->units = strdupz(value);
1298                 strip_quotes(rc->units);
1299             }
1300             else if(hash == hash_info && !strcasecmp(key, HEALTH_INFO_KEY)) {
1301                 if(rc->info) {
1302                     if(strcmp(rc->info, value))
1303                         info("Health configuration at line %zu of file '%s/%s' for alarm '%s' has key '%s' twice, once with value '%s' and later with value '%s'. Using ('%s').",
1304                              line, path, filename, rc->name, key, rc->info, value, value);
1305
1306                     freez(rc->info);
1307                 }
1308                 rc->info = strdupz(value);
1309                 strip_quotes(rc->info);
1310             }
1311             else {
1312                 error("Health configuration at line %zu of file '%s/%s' for alarm '%s' has unknown key '%s'.",
1313                      line, path, filename, rc->name, key);
1314             }
1315         }
1316         else if(rt) {
1317             if(hash == hash_on && !strcasecmp(key, HEALTH_ON_KEY)) {
1318                 if(rt->context) {
1319                     if(strcmp(rt->context, value))
1320                         info("Health configuration at line %zu of file '%s/%s' for template '%s' has key '%s' twice, once with value '%s' and later with value '%s'. Using ('%s').",
1321                              line, path, filename, rt->name, key, rt->context, value, value);
1322
1323                     freez(rt->context);
1324                 }
1325                 rt->context = strdupz(value);
1326                 rt->hash_context = simple_hash(rt->context);
1327             }
1328             else if(hash == hash_lookup && !strcasecmp(key, HEALTH_LOOKUP_KEY)) {
1329                 health_parse_db_lookup(line, path, filename, value, &rt->group, &rt->after, &rt->before,
1330                                        &rt->update_every,
1331                                        &rt->options, &rt->dimensions);
1332             }
1333             else if(hash == hash_every && !strcasecmp(key, HEALTH_EVERY_KEY)) {
1334                 if(!health_parse_duration(value, &rt->update_every))
1335                     info("Health configuration at line %zu of file '%s/%s' for template '%s' at key '%s' cannot parse duration: '%s'.",
1336                          line, path, filename, rt->name, key, value);
1337             }
1338             else if(hash == hash_green && !strcasecmp(key, HEALTH_GREEN_KEY)) {
1339                 char *e;
1340                 rt->green = strtold(value, &e);
1341                 if(e && *e) {
1342                     info("Health configuration at line %zu of file '%s/%s' for template '%s' at key '%s' leaves this string unmatched: '%s'.",
1343                          line, path, filename, rt->name, key, e);
1344                 }
1345             }
1346             else if(hash == hash_red && !strcasecmp(key, HEALTH_RED_KEY)) {
1347                 char *e;
1348                 rt->red = strtold(value, &e);
1349                 if(e && *e) {
1350                     info("Health configuration at line %zu of file '%s/%s' for template '%s' at key '%s' leaves this string unmatched: '%s'.",
1351                          line, path, filename, rt->name, key, e);
1352                 }
1353             }
1354             else if(hash == hash_calc && !strcasecmp(key, HEALTH_CALC_KEY)) {
1355                 const char *failed_at = NULL;
1356                 int error = 0;
1357                 rt->calculation = expression_parse(value, &failed_at, &error);
1358                 if(!rt->calculation) {
1359                     error("Health configuration at line %zu of file '%s/%s' for template '%s' at key '%s' has unparse-able expression '%s': %s at '%s'",
1360                           line, path, filename, rt->name, key, value, expression_strerror(error), failed_at);
1361                 }
1362             }
1363             else if(hash == hash_warn && !strcasecmp(key, HEALTH_WARN_KEY)) {
1364                 const char *failed_at = NULL;
1365                 int error = 0;
1366                 rt->warning = expression_parse(value, &failed_at, &error);
1367                 if(!rt->warning) {
1368                     error("Health configuration at line %zu of file '%s/%s' for template '%s' at key '%s' has unparse-able expression '%s': %s at '%s'",
1369                           line, path, filename, rt->name, key, value, expression_strerror(error), failed_at);
1370                 }
1371             }
1372             else if(hash == hash_crit && !strcasecmp(key, HEALTH_CRIT_KEY)) {
1373                 const char *failed_at = NULL;
1374                 int error = 0;
1375                 rt->critical = expression_parse(value, &failed_at, &error);
1376                 if(!rt->critical) {
1377                     error("Health configuration at line %zu of file '%s/%s' for template '%s' at key '%s' has unparse-able expression '%s': %s at '%s'",
1378                           line, path, filename, rt->name, key, value, expression_strerror(error), failed_at);
1379                 }
1380             }
1381             else if(hash == hash_exec && !strcasecmp(key, HEALTH_EXEC_KEY)) {
1382                 if(rt->exec) {
1383                     if(strcmp(rt->exec, value))
1384                         info("Health configuration at line %zu of file '%s/%s' for template '%s' has key '%s' twice, once with value '%s' and later with value '%s'. Using ('%s').",
1385                              line, path, filename, rt->name, key, rt->exec, value, value);
1386
1387                     freez(rt->exec);
1388                 }
1389                 rt->exec = strdupz(value);
1390             }
1391             else if(hash == hash_recipient && !strcasecmp(key, HEALTH_RECIPIENT_KEY)) {
1392                 if(rt->recipient) {
1393                     if(strcmp(rt->recipient, value))
1394                         info("Health configuration at line %zu of file '%s/%s' for template '%s' has key '%s' twice, once with value '%s' and later with value '%s'. Using ('%s').",
1395                              line, path, filename, rt->name, key, rt->recipient, value, value);
1396
1397                     freez(rt->recipient);
1398                 }
1399                 rt->recipient = strdupz(value);
1400             }
1401             else if(hash == hash_units && !strcasecmp(key, HEALTH_UNITS_KEY)) {
1402                 if(rt->units) {
1403                     if(strcmp(rt->units, value))
1404                         info("Health configuration at line %zu of file '%s/%s' for template '%s' has key '%s' twice, once with value '%s' and later with value '%s'. Using ('%s').",
1405                              line, path, filename, rt->name, key, rt->units, value, value);
1406
1407                     freez(rt->units);
1408                 }
1409                 rt->units = strdupz(value);
1410                 strip_quotes(rt->units);
1411             }
1412             else if(hash == hash_info && !strcasecmp(key, HEALTH_INFO_KEY)) {
1413                 if(rt->info) {
1414                     if(strcmp(rt->info, value))
1415                         info("Health configuration at line %zu of file '%s/%s' for template '%s' has key '%s' twice, once with value '%s' and later with value '%s'. Using ('%s').",
1416                              line, path, filename, rt->name, key, rt->info, value, value);
1417
1418                     freez(rt->info);
1419                 }
1420                 rt->info = strdupz(value);
1421                 strip_quotes(rt->info);
1422             }
1423             else {
1424                 error("Health configuration at line %zu of file '%s/%s' for template '%s' has unknown key '%s'.",
1425                       line, path, filename, rt->name, key);
1426             }
1427         }
1428         else {
1429             error("Health configuration at line %zu of file '%s/%s' has unknown key '%s'. Expected either '" HEALTH_ALARM_KEY "' or '" HEALTH_TEMPLATE_KEY "'.",
1430                   line, path, filename, key);
1431         }
1432     }
1433
1434     if(rc && !rrdcalc_add_alarm_from_config(&localhost, rc))
1435         rrdcalc_free(&localhost, rc);
1436
1437     if(rt && !rrdcalctemplate_add_template_from_config(&localhost, rt))
1438         rrdcalctemplate_free(&localhost, rt);
1439
1440     fclose(fp);
1441     return 1;
1442 }
1443
1444 void health_readdir(const char *path) {
1445     size_t pathlen = strlen(path);
1446
1447     debug(D_HEALTH, "Health configuration reading directory '%s'", path);
1448
1449     DIR *dir = opendir(path);
1450     if (!dir) {
1451         error("Health configuration cannot open directory '%s'.", path);
1452         return;
1453     }
1454
1455     struct dirent *de = NULL;
1456     while ((de = readdir(dir))) {
1457         size_t len = strlen(de->d_name);
1458
1459         if(de->d_type == DT_DIR
1460            && (
1461                    (de->d_name[0] == '.' && de->d_name[1] == '\0')
1462                    || (de->d_name[0] == '.' && de->d_name[1] == '.' && de->d_name[2] == '\0')
1463            ))
1464             continue;
1465
1466         else if(de->d_type == DT_DIR) {
1467             char *s = mallocz(pathlen + strlen(de->d_name) + 2);
1468             strcpy(s, path);
1469             strcat(s, "/");
1470             strcat(s, de->d_name);
1471             health_readdir(s);
1472             freez(s);
1473             continue;
1474         }
1475
1476         else if((de->d_type == DT_LNK || de->d_type == DT_REG) &&
1477                 len > 5 && !strcmp(&de->d_name[len - 5], ".conf")) {
1478             health_readfile(path, de->d_name);
1479         }
1480     }
1481
1482     closedir(dir);
1483 }
1484
1485 static inline char *health_config_dir(void) {
1486     char buffer[FILENAME_MAX + 1];
1487     snprintfz(buffer, FILENAME_MAX, "%s/health.d", config_get("global", "config directory", CONFIG_DIR));
1488     return config_get("health", "health configuration directory", buffer);
1489 }
1490
1491 void health_init(void) {
1492     debug(D_HEALTH, "Health configuration initializing");
1493
1494     if(!(health_enabled = config_get_boolean("health", "enabled", 1))) {
1495         debug(D_HEALTH, "Health is disabled.");
1496         return;
1497     }
1498
1499     char *path = health_config_dir();
1500
1501     {
1502         char buffer[FILENAME_MAX + 1];
1503         snprintfz(buffer, FILENAME_MAX, "%s/alarm-email.sh", config_get("global", "plugins directory", PLUGINS_DIR));
1504         health_default_exec = config_get("health", "script to execute on alarm", buffer);
1505     }
1506
1507     long n = config_get_number("health", "in memory max health log entries", (long)localhost.health_log.max);
1508     if(n < 2) {
1509         error("Health configuration has invalid max log entries %ld. Using default %u", n, localhost.health_log.max);
1510         config_set_number("health", "in memory max health log entries", (long)localhost.health_log.max);
1511     }
1512     else localhost.health_log.max = (unsigned int)n;
1513
1514     rrdhost_rwlock(&localhost);
1515     health_readdir(path);
1516     rrdhost_unlock(&localhost);
1517 }
1518
1519 // ----------------------------------------------------------------------------
1520 // JSON generation
1521
1522 static inline void health_string2json(BUFFER *wb, const char *prefix, const char *label, const char *value, const char *suffix) {
1523     if(value && *value)
1524         buffer_sprintf(wb, "%s\"%s\":\"%s\"%s", prefix, label, value, suffix);
1525     else
1526         buffer_sprintf(wb, "%s\"%s\":null%s", prefix, label, suffix);
1527 }
1528
1529 static inline void health_alarm_entry2json_nolock(BUFFER *wb, ALARM_ENTRY *ae, RRDHOST *host) {
1530     buffer_sprintf(wb, "\n\t{\n"
1531                            "\t\t\"hostname\":\"%s\",\n"
1532                            "\t\t\"unique_id\":%u,\n"
1533                            "\t\t\"alarm_id\":%u,\n"
1534                            "\t\t\"alarm_event_id\":%u,\n"
1535                            "\t\t\"name\":\"%s\",\n"
1536                            "\t\t\"chart\":\"%s\",\n"
1537                            "\t\t\"family\":\"%s\",\n"
1538                            "\t\t\"processed\":%s,\n"
1539                            "\t\t\"updated\":%s,\n"
1540                            "\t\t\"exec_run\":%s,\n"
1541                            "\t\t\"exec_failed\":%s,\n"
1542                            "\t\t\"exec\":\"%s\",\n"
1543                            "\t\t\"recipient\":\"%s\",\n"
1544                            "\t\t\"exec_code\":%d,\n"
1545                            "\t\t\"source\":\"%s\",\n"
1546                            "\t\t\"units\":\"%s\",\n"
1547                            "\t\t\"info\":\"%s\",\n"
1548                            "\t\t\"when\":%lu,\n"
1549                            "\t\t\"duration\":%lu,\n"
1550                            "\t\t\"non_clear_duration\":%lu,\n"
1551                            "\t\t\"status\":\"%s\",\n"
1552                            "\t\t\"old_status\":\"%s\",\n",
1553                    host->hostname,
1554                    ae->unique_id,
1555                    ae->alarm_id,
1556                    ae->alarm_event_id,
1557                    ae->name,
1558                    ae->chart,
1559                    ae->family,
1560                    (ae->notifications & HEALTH_ENTRY_NOTIFICATIONS_PROCESSED)?"true":"false",
1561                    (ae->notifications & HEALTH_ENTRY_NOTIFICATIONS_UPDATED)?"true":"false",
1562                    (ae->notifications & HEALTH_ENTRY_NOTIFICATIONS_EXEC_RUN)?"true":"false",
1563                    (ae->notifications & HEALTH_ENTRY_NOTIFICATIONS_EXEC_FAILED)?"true":"false",
1564                    ae->exec?ae->exec:health_default_exec,
1565                    ae->recipient?ae->recipient:health_default_recipient,
1566                    ae->exec_code,
1567                    ae->source,
1568                    ae->units?ae->units:"",
1569                    ae->info?ae->info:"",
1570                    (unsigned long)ae->when,
1571                    (unsigned long)ae->duration,
1572                    (unsigned long)ae->non_clear_duration,
1573                    rrdcalc_status2string(ae->new_status),
1574                    rrdcalc_status2string(ae->old_status)
1575     );
1576
1577     buffer_strcat(wb, "\t\t\"value\":");
1578     buffer_rrd_value(wb, ae->new_value);
1579     buffer_strcat(wb, ",\n");
1580
1581     buffer_strcat(wb, "\t\t\"old_value\":");
1582     buffer_rrd_value(wb, ae->old_value);
1583     buffer_strcat(wb, "\n");
1584
1585     buffer_strcat(wb, "\t}");
1586 }
1587
1588 void health_alarm_log2json(RRDHOST *host, BUFFER *wb, uint32_t after) {
1589     pthread_rwlock_rdlock(&host->health_log.alarm_log_rwlock);
1590
1591     buffer_strcat(wb, "[");
1592
1593     unsigned int max = host->health_log.max;
1594     unsigned int count = 0;
1595     ALARM_ENTRY *ae;
1596     for(ae = host->health_log.alarms; ae && count < max ; count++, ae = ae->next) {
1597         if(ae->unique_id > after) {
1598             if(likely(count)) buffer_strcat(wb, ",");
1599             health_alarm_entry2json_nolock(wb, ae, host);
1600         }
1601     }
1602
1603     buffer_strcat(wb, "\n]\n");
1604
1605     pthread_rwlock_unlock(&host->health_log.alarm_log_rwlock);
1606 }
1607
1608 static inline void health_rrdcalc2json_nolock(BUFFER *wb, RRDCALC *rc) {
1609     buffer_sprintf(wb,
1610            "\t\t\"%s.%s\": {\n"
1611                    "\t\t\t\"name\": \"%s\",\n"
1612                    "\t\t\t\"chart\": \"%s\",\n"
1613                    "\t\t\t\"family\": \"%s\",\n"
1614                    "\t\t\t\"active\": %s,\n"
1615                    "\t\t\t\"exec\": \"%s\",\n"
1616                    "\t\t\t\"recipient\": \"%s\",\n"
1617                    "\t\t\t\"source\": \"%s\",\n"
1618                    "\t\t\t\"units\": \"%s\",\n"
1619                    "\t\t\t\"info\": \"%s\",\n"
1620                                    "\t\t\t\"status\": \"%s\",\n"
1621                    "\t\t\t\"last_status_change\": %lu,\n"
1622                    "\t\t\t\"last_updated\": %lu,\n"
1623                    "\t\t\t\"next_update\": %lu,\n"
1624                    "\t\t\t\"update_every\": %d,\n"
1625             , rc->chart, rc->name
1626             , rc->name
1627             , rc->chart
1628             , (rc->rrdset && rc->rrdset->family)?rc->rrdset->family:""
1629             , (rc->rrdset)?"true":"false"
1630             , rc->exec?rc->exec:health_default_exec
1631             , rc->recipient?rc->recipient:health_default_recipient
1632             , rc->source
1633             , rc->units?rc->units:""
1634             , rc->info?rc->info:""
1635             , rrdcalc_status2string(rc->status)
1636             , (unsigned long)rc->last_status_change
1637             , (unsigned long)rc->last_updated
1638             , (unsigned long)rc->next_update
1639             , rc->update_every
1640     );
1641
1642     if(RRDCALC_HAS_DB_LOOKUP(rc)) {
1643         if(rc->dimensions && *rc->dimensions)
1644             health_string2json(wb, "\t\t\t", "lookup_dimensions", rc->dimensions, ",\n");
1645
1646         buffer_sprintf(wb,
1647                        "\t\t\t\"db_after\": %lu,\n"
1648                        "\t\t\t\"db_before\": %lu,\n"
1649                        "\t\t\t\"lookup_method\": \"%s\",\n"
1650                        "\t\t\t\"lookup_after\": %d,\n"
1651                        "\t\t\t\"lookup_before\": %d,\n"
1652                        "\t\t\t\"lookup_options\": \"",
1653                        (unsigned long) rc->db_after,
1654                        (unsigned long) rc->db_before,
1655                        group_method2string(rc->group),
1656                        rc->after,
1657                        rc->before
1658         );
1659         buffer_data_options2string(wb, rc->options);
1660         buffer_strcat(wb, "\",\n");
1661     }
1662
1663     if(rc->calculation) {
1664         health_string2json(wb, "\t\t\t", "calc", rc->calculation->source, ",\n");
1665         health_string2json(wb, "\t\t\t", "calc_parsed", rc->calculation->parsed_as, ",\n");
1666     }
1667
1668     if(rc->warning) {
1669         health_string2json(wb, "\t\t\t", "warn", rc->warning->source, ",\n");
1670         health_string2json(wb, "\t\t\t", "warn_parsed", rc->warning->parsed_as, ",\n");
1671     }
1672
1673     if(rc->critical) {
1674         health_string2json(wb, "\t\t\t", "crit", rc->critical->source, ",\n");
1675         health_string2json(wb, "\t\t\t", "crit_parsed", rc->critical->parsed_as, ",\n");
1676     }
1677
1678     buffer_strcat(wb, "\t\t\t\"green\":");
1679     buffer_rrd_value(wb, rc->green);
1680     buffer_strcat(wb, ",\n");
1681
1682     buffer_strcat(wb, "\t\t\t\"red\":");
1683     buffer_rrd_value(wb, rc->red);
1684     buffer_strcat(wb, ",\n");
1685
1686     buffer_strcat(wb, "\t\t\t\"value\":");
1687     buffer_rrd_value(wb, rc->value);
1688     buffer_strcat(wb, "\n");
1689
1690     buffer_strcat(wb, "\t\t}");
1691 }
1692
1693 //void health_rrdcalctemplate2json_nolock(BUFFER *wb, RRDCALCTEMPLATE *rt) {
1694 //
1695 //}
1696
1697 void health_alarms2json(RRDHOST *host, BUFFER *wb, int all) {
1698     int i;
1699     rrdhost_rdlock(&localhost);
1700
1701     buffer_strcat(wb, "{\n\t\"alarms\": {\n");
1702     RRDCALC *rc;
1703     for(i = 0, rc = host->alarms; rc ; rc = rc->next) {
1704         if(unlikely(!rc->rrdset || !rc->rrdset->last_collected_time.tv_sec))
1705             continue;
1706
1707         if(likely(!all && !(rc->status == RRDCALC_STATUS_WARNING || rc->status == RRDCALC_STATUS_CRITICAL)))
1708             continue;
1709
1710         if(likely(i)) buffer_strcat(wb, ",\n");
1711         health_rrdcalc2json_nolock(wb, rc);
1712         i++;
1713     }
1714
1715 //    buffer_strcat(wb, "\n\t},\n\t\"templates\": {");
1716
1717 //    RRDCALCTEMPLATE *rt;
1718 //    for(rt = host->templates; rt ; rt = rt->next)
1719 //        health_rrdcalctemplate2json_nolock(wb, rt);
1720
1721     buffer_sprintf(wb, "\n\t},\n\t\"now\": %lu\n}\n", (unsigned long)time(NULL));
1722     rrdhost_unlock(&localhost);
1723 }
1724
1725
1726 // ----------------------------------------------------------------------------
1727 // re-load health configuration
1728
1729 static inline void health_free_all_nolock(RRDHOST *host) {
1730     while(host->templates)
1731         rrdcalctemplate_free(host, host->templates);
1732
1733     while(host->alarms)
1734         rrdcalc_free(host, host->alarms);
1735 }
1736
1737 void health_reload(void) {
1738     if(!health_enabled) {
1739         error("Health reload is requested, but health is not enabled.");
1740         return;
1741     }
1742
1743     char *path = health_config_dir();
1744
1745     // free all running alarms
1746     rrdhost_rwlock(&localhost);
1747     health_free_all_nolock(&localhost);
1748     rrdhost_unlock(&localhost);
1749
1750     // invalidate all previous entries in the alarm log
1751     ALARM_ENTRY *t;
1752     for(t = localhost.health_log.alarms ; t ; t = t->next) {
1753         t->notifications |= HEALTH_ENTRY_NOTIFICATIONS_UPDATED;
1754     }
1755
1756     // reset all thresholds to all charts
1757     RRDSET *st;
1758     for(st = localhost.rrdset_root; st ; st = st->next) {
1759         st->green = NAN;
1760         st->red = NAN;
1761     }
1762
1763     // load the new alarms
1764     rrdhost_rwlock(&localhost);
1765     health_readdir(path);
1766     rrdhost_unlock(&localhost);
1767
1768     // link the loaded alarms to their charts
1769     for(st = localhost.rrdset_root; st ; st = st->next) {
1770         rrdhost_rwlock(&localhost);
1771
1772         rrdsetcalc_link_matching(st);
1773         rrdcalctemplate_link_matching(st);
1774
1775         rrdhost_unlock(&localhost);
1776     }
1777 }
1778
1779
1780 // ----------------------------------------------------------------------------
1781 // health main thread and friends
1782
1783 static inline int rrdcalc_value2status(calculated_number n) {
1784     if(isnan(n)) return RRDCALC_STATUS_UNDEFINED;
1785     if(n) return RRDCALC_STATUS_RAISED;
1786     return RRDCALC_STATUS_CLEAR;
1787 }
1788
1789 static inline void health_alarm_execute(RRDHOST *host, ALARM_ENTRY *ae) {
1790     if(ae->old_status == RRDCALC_STATUS_UNINITIALIZED && ae->new_status == RRDCALC_STATUS_CLEAR)
1791         return;
1792
1793     char buffer[FILENAME_MAX + 1];
1794     pid_t command_pid;
1795
1796     const char *exec = ae->exec;
1797     if(!exec) exec = health_default_exec;
1798
1799     const char *recipient = ae->recipient;
1800     if(!recipient) recipient = health_default_recipient;
1801
1802     snprintfz(buffer, FILENAME_MAX, "exec %s '%s' '%s' '%u' '%u' '%u' '%zu' '%s' '%s' '%s' '%s' '%s' '%0.0Lf' '%0.0Lf' '%s' '%u' '%u' '%s' '%s'",
1803               exec,
1804               recipient,
1805               host->hostname,
1806               ae->unique_id,
1807               ae->alarm_id,
1808               ae->alarm_event_id,
1809               ae->when,
1810               ae->name,
1811               ae->chart?ae->chart:"NOCAHRT",
1812               ae->family?ae->family:"NOFAMILY",
1813               rrdcalc_status2string(ae->new_status),
1814               rrdcalc_status2string(ae->old_status),
1815               ae->new_value,
1816               ae->old_value,
1817               ae->source?ae->source:"UNKNOWN",
1818               (uint32_t)ae->duration,
1819               (uint32_t)ae->non_clear_duration,
1820               ae->units?ae->units:"",
1821               ae->info?ae->info:""
1822     );
1823
1824     ae->notifications |= HEALTH_ENTRY_NOTIFICATIONS_EXEC_RUN;
1825
1826     debug(D_HEALTH, "executing command '%s'", buffer);
1827     FILE *fp = mypopen(buffer, &command_pid);
1828     if(!fp) {
1829         error("HEALTH: Cannot popen(\"%s\", \"r\").", buffer);
1830         return;
1831     }
1832     debug(D_HEALTH, "HEALTH reading from command");
1833     char *s = fgets(buffer, FILENAME_MAX, fp);
1834     (void)s;
1835     debug(D_HEALTH, "HEALTH closing command");
1836     ae->exec_code = mypclose(fp, command_pid);
1837     debug(D_HEALTH, "done executing command - returned with code %d", ae->exec_code);
1838
1839     if(ae->exec_code != 0)
1840         ae->notifications |= HEALTH_ENTRY_NOTIFICATIONS_EXEC_FAILED;
1841 }
1842
1843 static inline void health_process_notifications(RRDHOST *host, ALARM_ENTRY *ae) {
1844     info("Health alarm '%s.%s' = %0.2Lf - changed status from %s to %s",
1845          ae->chart?ae->chart:"NOCHART", ae->name,
1846          ae->new_value,
1847          rrdcalc_status2string(ae->old_status),
1848          rrdcalc_status2string(ae->new_status)
1849     );
1850
1851     health_alarm_execute(host, ae);
1852 }
1853
1854 static inline void health_alarm_log(RRDHOST *host,
1855                 uint32_t alarm_id, uint32_t alarm_event_id,
1856                 time_t when,
1857                 const char *name, const char *chart, const char *family,
1858                 const char *exec, const char *recipient, time_t duration,
1859                 calculated_number old_value, calculated_number new_value,
1860                 int old_status, int new_status,
1861                 const char *source,
1862                 const char *units,
1863                 const char *info
1864 ) {
1865     ALARM_ENTRY *ae = callocz(1, sizeof(ALARM_ENTRY));
1866     ae->name = strdupz(name);
1867     ae->hash_name = simple_hash(ae->name);
1868
1869     if(chart) {
1870         ae->chart = strdupz(chart);
1871         ae->hash_chart = simple_hash(ae->chart);
1872     }
1873
1874     if(family)
1875         ae->family = strdupz(family);
1876
1877     if(exec) ae->exec = strdupz(exec);
1878     if(recipient) ae->recipient = strdupz(recipient);
1879     if(source) ae->source = strdupz(source);
1880     if(units) ae->units = strdupz(units);
1881     if(info) ae->info = strdupz(info);
1882
1883     ae->unique_id = host->health_log.next_log_id++;
1884     ae->alarm_id = alarm_id;
1885     ae->alarm_event_id = alarm_event_id;
1886     ae->when = when;
1887     ae->old_value = old_value;
1888     ae->new_value = new_value;
1889     ae->old_status = old_status;
1890     ae->new_status = new_status;
1891     ae->duration = duration;
1892
1893     if(ae->old_status == RRDCALC_STATUS_WARNING || ae->old_status == RRDCALC_STATUS_CRITICAL)
1894         ae->non_clear_duration += ae->duration;
1895
1896     // link it
1897     pthread_rwlock_wrlock(&host->health_log.alarm_log_rwlock);
1898     ae->next = host->health_log.alarms;
1899     host->health_log.alarms = ae;
1900     host->health_log.count++;
1901     pthread_rwlock_unlock(&host->health_log.alarm_log_rwlock);
1902
1903     // match previous alarms
1904     pthread_rwlock_rdlock(&host->health_log.alarm_log_rwlock);
1905     ALARM_ENTRY *t;
1906     for(t = host->health_log.alarms ; t ; t = t->next) {
1907         if(t != ae &&
1908                 t->hash_name == ae->hash_name &&
1909                 t->hash_chart == ae->hash_chart &&
1910                 !strcmp(t->name, ae->name) &&
1911                 t->chart && ae->chart && !strcmp(t->chart, ae->chart)) {
1912
1913             if(!(t->notifications & HEALTH_ENTRY_NOTIFICATIONS_UPDATED) && !t->updated_by) {
1914                 t->notifications |= HEALTH_ENTRY_NOTIFICATIONS_UPDATED;
1915                 t->updated_by = ae;
1916
1917                 if((t->new_status == RRDCALC_STATUS_WARNING || t->new_status == RRDCALC_STATUS_CRITICAL) &&
1918                    (t->old_status == RRDCALC_STATUS_WARNING || t->old_status == RRDCALC_STATUS_CRITICAL))
1919                     ae->non_clear_duration += t->non_clear_duration;
1920             }
1921             else {
1922                 // no need to continue
1923                 break;
1924             }
1925         }
1926     }
1927     pthread_rwlock_unlock(&host->health_log.alarm_log_rwlock);
1928 }
1929
1930 static inline void health_alarm_log_process(RRDHOST *host) {
1931     static uint32_t last_processed = 0;
1932     ALARM_ENTRY *ae;
1933
1934     pthread_rwlock_rdlock(&host->health_log.alarm_log_rwlock);
1935
1936     for(ae = host->health_log.alarms; ae ;ae = ae->next) {
1937         if(last_processed >= ae->unique_id) break;
1938
1939         if(!(ae->notifications & HEALTH_ENTRY_NOTIFICATIONS_PROCESSED) &&
1940                 !(ae->notifications & HEALTH_ENTRY_NOTIFICATIONS_UPDATED)) {
1941             ae->notifications |= HEALTH_ENTRY_NOTIFICATIONS_PROCESSED;
1942             health_process_notifications(host, ae);
1943         }
1944     }
1945
1946     if(host->health_log.alarms)
1947         last_processed = host->health_log.alarms->unique_id;
1948
1949     pthread_rwlock_unlock(&host->health_log.alarm_log_rwlock);
1950
1951     if(host->health_log.count <= host->health_log.max)
1952         return;
1953
1954     // cleanup excess entries in the log
1955     pthread_rwlock_wrlock(&host->health_log.alarm_log_rwlock);
1956
1957     ALARM_ENTRY *last = NULL;
1958     unsigned int count = host->health_log.max;
1959     for(ae = host->health_log.alarms; ae && count ; count--, last = ae, ae = ae->next) ;
1960
1961     if(ae && last && last->next == ae)
1962         last->next = NULL;
1963     else
1964         ae = NULL;
1965
1966     while(ae) {
1967         ALARM_ENTRY *t = ae->next;
1968
1969         freez(ae->name);
1970         freez(ae->chart);
1971         freez(ae->family);
1972         freez(ae->exec);
1973         freez(ae->recipient);
1974         freez(ae->source);
1975         freez(ae->units);
1976         freez(ae->info);
1977         freez(ae);
1978
1979         ae = t;
1980     }
1981
1982     pthread_rwlock_unlock(&host->health_log.alarm_log_rwlock);
1983 }
1984
1985 static inline int rrdcalc_isrunnable(RRDCALC *rc, time_t now, time_t *next_run) {
1986     if (unlikely(!rc->rrdset)) {
1987         debug(D_HEALTH, "Health not running alarm '%s.%s'. It is not linked to a chart.", rc->chart?rc->chart:"NOCHART", rc->name);
1988         return 0;
1989     }
1990
1991     if (unlikely(!rc->rrdset->last_collected_time.tv_sec)) {
1992         debug(D_HEALTH, "Health not running alarm '%s.%s'. Chart is not yet collected.", rc->chart?rc->chart:"NOCHART", rc->name);
1993         return 0;
1994     }
1995
1996     if (unlikely(!rc->update_every)) {
1997         debug(D_HEALTH, "Health not running alarm '%s.%s'. It does not have an update frequency", rc->chart?rc->chart:"NOCHART", rc->name);
1998         return 0;
1999     }
2000
2001     if (unlikely(rc->next_update > now)) {
2002         if (unlikely(*next_run > rc->next_update))
2003             *next_run = rc->next_update;
2004
2005         debug(D_HEALTH, "Health not examining alarm '%s.%s' yet (will do in %d secs).", rc->chart?rc->chart:"NOCHART", rc->name, (int) (rc->next_update - now));
2006         return 0;
2007     }
2008
2009     // FIXME
2010     // we should check that the DB lookup is possible
2011     // i.e.
2012     // - the duration of the chart includes the required timeframe
2013     // we SHOULD NOT check the dimensions - there might be alarms that refer non-existing dimensions (e.g. cpu steal)
2014
2015     return 1;
2016 }
2017
2018 void *health_main(void *ptr) {
2019     (void)ptr;
2020
2021     info("HEALTH thread created with task id %d", gettid());
2022
2023     if(pthread_setcanceltype(PTHREAD_CANCEL_DEFERRED, NULL) != 0)
2024         error("Cannot set pthread cancel type to DEFERRED.");
2025
2026     if(pthread_setcancelstate(PTHREAD_CANCEL_ENABLE, NULL) != 0)
2027         error("Cannot set pthread cancel state to ENABLE.");
2028
2029     int min_run_every = (int)config_get_number("health", "run at least every seconds", 10);
2030     if(min_run_every < 1) min_run_every = 1;
2031
2032     BUFFER *wb = buffer_create(100);
2033
2034     unsigned int loop = 0;
2035     while(health_enabled) {
2036         loop++;
2037         debug(D_HEALTH, "Health monitoring iteration no %u started", loop);
2038
2039         int oldstate, runnable = 0;
2040         time_t now = time(NULL);
2041         time_t next_run = now + min_run_every;
2042         RRDCALC *rc;
2043
2044         if (unlikely(pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, &oldstate) != 0))
2045             error("Cannot set pthread cancel state to DISABLE.");
2046
2047         rrdhost_rdlock(&localhost);
2048
2049         // the first loop is to lookup values from the db
2050         for (rc = localhost.alarms; rc; rc = rc->next) {
2051             if (unlikely(!rrdcalc_isrunnable(rc, now, &next_run)))
2052                 continue;
2053
2054             runnable++;
2055             rc->old_value = rc->value;
2056
2057             // 1. if there is database lookup, do it
2058             // 2. if there is calculation expression, run it
2059
2060             if (unlikely(RRDCALC_HAS_DB_LOOKUP(rc))) {
2061                 time_t old_db_timestamp = rc->db_before;
2062                 int value_is_null = 0;
2063
2064                 int ret = rrd2value(rc->rrdset, wb, &rc->value,
2065                                     rc->dimensions, 1, rc->after, rc->before, rc->group,
2066                                     rc->options, &rc->db_after, &rc->db_before, &value_is_null);
2067
2068                 if (unlikely(ret != 200)) {
2069                     // database lookup failed
2070                     rc->value = NAN;
2071
2072                     debug(D_HEALTH, "Health alarm '%s.%s': database lookup returned error %d", rc->chart?rc->chart:"NOCHART", rc->name, ret);
2073
2074                     if (unlikely(!(rc->rrdcalc_flags & RRDCALC_FLAG_DB_ERROR))) {
2075                         rc->rrdcalc_flags |= RRDCALC_FLAG_DB_ERROR;
2076                         error("Health alarm '%s.%s': database lookup returned error %d", rc->chart?rc->chart:"NOCHART", rc->name, ret);
2077                     }
2078                 }
2079                 else if (unlikely(rc->rrdcalc_flags & RRDCALC_FLAG_DB_ERROR))
2080                     rc->rrdcalc_flags &= ~RRDCALC_FLAG_DB_ERROR;
2081
2082                 if (unlikely(old_db_timestamp == rc->db_before)) {
2083                     // database is stale
2084
2085                     debug(D_HEALTH, "Health alarm '%s.%s': database is stale", rc->chart?rc->chart:"NOCHART", rc->name);
2086
2087                     if (unlikely(!(rc->rrdcalc_flags & RRDCALC_FLAG_DB_STALE))) {
2088                         rc->rrdcalc_flags |= RRDCALC_FLAG_DB_STALE;
2089                         error("Health alarm '%s.%s': database is stale", rc->chart?rc->chart:"NOCHART", rc->name);
2090                     }
2091                 }
2092                 else if (unlikely(rc->rrdcalc_flags & RRDCALC_FLAG_DB_STALE))
2093                     rc->rrdcalc_flags &= ~RRDCALC_FLAG_DB_STALE;
2094
2095                 if (unlikely(value_is_null)) {
2096                     // collected value is null
2097
2098                     rc->value = NAN;
2099
2100                     debug(D_HEALTH, "Health alarm '%s.%s': database lookup returned empty value (possibly value is not collected yet)",
2101                           rc->chart?rc->chart:"NOCHART", rc->name);
2102
2103                     if (unlikely(!(rc->rrdcalc_flags & RRDCALC_FLAG_DB_NAN))) {
2104                         rc->rrdcalc_flags |= RRDCALC_FLAG_DB_NAN;
2105                         error("Health alarm '%s.%s': database lookup returned empty value (possibly value is not collected yet)",
2106                               rc->chart?rc->chart:"NOCHART", rc->name);
2107                     }
2108                 }
2109                 else if (unlikely(rc->rrdcalc_flags & RRDCALC_FLAG_DB_NAN))
2110                     rc->rrdcalc_flags &= ~RRDCALC_FLAG_DB_NAN;
2111
2112                 debug(D_HEALTH, "Health alarm '%s.%s': database lookup gave value "
2113                         CALCULATED_NUMBER_FORMAT, rc->chart?rc->chart:"NOCHART", rc->name, rc->value);
2114             }
2115
2116             if(unlikely(rc->calculation)) {
2117                 if (unlikely(!expression_evaluate(rc->calculation))) {
2118                     // calculation failed
2119
2120                     rc->value = NAN;
2121
2122                     debug(D_HEALTH, "Health alarm '%s.%s': failed to evaluate calculation with error: %s",
2123                           rc->chart?rc->chart:"NOCHART", rc->name, buffer_tostring(rc->calculation->error_msg));
2124
2125                     if (unlikely(!(rc->rrdcalc_flags & RRDCALC_FLAG_CALC_ERROR))) {
2126                         rc->rrdcalc_flags |= RRDCALC_FLAG_CALC_ERROR;
2127                         error("Health alarm '%s.%s': failed to evaluate calculation with error: %s",
2128                               rc->chart?rc->chart:"NOCHART", rc->name, buffer_tostring(rc->calculation->error_msg));
2129                     }
2130                 }
2131                 else {
2132                     if (unlikely(rc->rrdcalc_flags & RRDCALC_FLAG_CALC_ERROR))
2133                         rc->rrdcalc_flags &= ~RRDCALC_FLAG_CALC_ERROR;
2134
2135                     debug(D_HEALTH, "Health alarm '%s.%s': calculation expression gave value "
2136                             CALCULATED_NUMBER_FORMAT
2137                             ": %s (source: %s)",
2138                           rc->chart?rc->chart:"NOCHART", rc->name,
2139                           rc->calculation->result,
2140                           buffer_tostring(rc->calculation->error_msg),
2141                           rc->source
2142                     );
2143
2144                     rc->value = rc->calculation->result;
2145                 }
2146             }
2147         }
2148         rrdhost_unlock(&localhost);
2149
2150         if (runnable) {
2151             rrdhost_rdlock(&localhost);
2152
2153             for (rc = localhost.alarms; rc; rc = rc->next) {
2154                 if (unlikely(!rrdcalc_isrunnable(rc, now, &next_run)))
2155                     continue;
2156
2157                 int warning_status  = RRDCALC_STATUS_UNDEFINED;
2158                 int critical_status = RRDCALC_STATUS_UNDEFINED;
2159
2160                 if(likely(rc->warning)) {
2161                     if(unlikely(!expression_evaluate(rc->warning))) {
2162                         // calculation failed
2163
2164                         debug(D_HEALTH, "Health alarm '%s.%s': warning expression failed with error: %s",
2165                               rc->chart?rc->chart:"NOCHART", rc->name, buffer_tostring(rc->warning->error_msg));
2166
2167                         if (unlikely(!(rc->rrdcalc_flags & RRDCALC_FLAG_WARN_ERROR))) {
2168                             rc->rrdcalc_flags |= RRDCALC_FLAG_WARN_ERROR;
2169                             error("Health alarm '%s.%s': warning expression failed with error: %s",
2170                                   rc->chart?rc->chart:"NOCHART", rc->name, buffer_tostring(rc->warning->error_msg));
2171                         }
2172                     }
2173                     else {
2174                         if(unlikely(rc->rrdcalc_flags & RRDCALC_FLAG_WARN_ERROR))
2175                             rc->rrdcalc_flags &= ~RRDCALC_FLAG_WARN_ERROR;
2176
2177                         debug(D_HEALTH, "Health alarm '%s.%s': warning expression gave value "
2178                                 CALCULATED_NUMBER_FORMAT
2179                                 ": %s (source: %s)",
2180                               rc->chart?rc->chart:"NOCHART", rc->name,
2181                               rc->warning->result,
2182                               buffer_tostring(rc->warning->error_msg),
2183                               rc->source
2184                         );
2185
2186                         warning_status = rrdcalc_value2status(rc->warning->result);
2187                     }
2188                 }
2189
2190                 if(likely(rc->critical)) {
2191                     if(unlikely(!expression_evaluate(rc->critical))) {
2192                         // calculation failed
2193
2194                         debug(D_HEALTH, "Health alarm '%s.%s': critical expression failed with error: %s",
2195                               rc->chart?rc->chart:"NOCHART", rc->name, buffer_tostring(rc->critical->error_msg));
2196
2197                         if (unlikely(!(rc->rrdcalc_flags & RRDCALC_FLAG_CRIT_ERROR))) {
2198                             rc->rrdcalc_flags |= RRDCALC_FLAG_CRIT_ERROR;
2199                             error("Health alarm '%s.%s': critical expression failed with error: %s",
2200                                   rc->chart?rc->chart:"NOCHART", rc->name, buffer_tostring(rc->critical->error_msg));
2201                         }
2202                     }
2203                     else {
2204                         if(unlikely(rc->rrdcalc_flags & RRDCALC_FLAG_CRIT_ERROR))
2205                             rc->rrdcalc_flags &= ~RRDCALC_FLAG_CRIT_ERROR;
2206
2207                         debug(D_HEALTH, "Health alarm '%s.%s': critical expression gave value "
2208                                 CALCULATED_NUMBER_FORMAT
2209                                 ": %s (source: %s)",
2210                               rc->chart?rc->chart:"NOCHART", rc->name,
2211                               rc->critical->result,
2212                               buffer_tostring(rc->critical->error_msg),
2213                               rc->source
2214                         );
2215
2216                         critical_status = rrdcalc_value2status(rc->critical->result);
2217                     }
2218                 }
2219
2220                 int status = RRDCALC_STATUS_UNDEFINED;
2221
2222                 switch(warning_status) {
2223                     case RRDCALC_STATUS_CLEAR:
2224                         status = RRDCALC_STATUS_CLEAR;
2225                         break;
2226
2227                     case RRDCALC_STATUS_RAISED:
2228                         status = RRDCALC_STATUS_WARNING;
2229                         break;
2230
2231                     default:
2232                         break;
2233                 }
2234
2235                 switch(critical_status) {
2236                     case RRDCALC_STATUS_CLEAR:
2237                         if(status == RRDCALC_STATUS_UNDEFINED)
2238                             status = RRDCALC_STATUS_CLEAR;
2239                         break;
2240
2241                     case RRDCALC_STATUS_RAISED:
2242                         status = RRDCALC_STATUS_CRITICAL;
2243                         break;
2244
2245                     default:
2246                         break;
2247                 }
2248
2249                 if(status != rc->status) {
2250                     health_alarm_log(&localhost, rc->id, rc->next_event_id++, time(NULL), rc->name, rc->rrdset->id, rc->rrdset->family, rc->exec, rc->recipient, now - rc->last_status_change, rc->old_value, rc->value, rc->status, status, rc->source, rc->units, rc->info);
2251                     rc->last_status_change = now;
2252                     rc->status = status;
2253                 }
2254
2255                 rc->last_updated = now;
2256                 rc->next_update = now + rc->update_every;
2257
2258                 if (next_run > rc->next_update)
2259                     next_run = rc->next_update;
2260             }
2261
2262             rrdhost_unlock(&localhost);
2263         }
2264
2265         if (unlikely(pthread_setcancelstate(oldstate, NULL) != 0))
2266             error("Cannot set pthread cancel state to RESTORE (%d).", oldstate);
2267
2268         // execute notifications
2269         // and cleanup
2270         health_alarm_log_process(&localhost);
2271
2272         now = time(NULL);
2273         if(now < next_run) {
2274             debug(D_HEALTH, "Health monitoring iteration no %u done. Next iteration in %d secs",
2275                   loop, (int) (next_run - now));
2276             sleep_usec(1000000 * (unsigned long long) (next_run - now));
2277         }
2278         else {
2279             debug(D_HEALTH, "Health monitoring iteration no %u done. Next iteration now", loop);
2280         }
2281     }
2282
2283     buffer_free(wb);
2284
2285     info("HEALTH thread exiting");
2286     pthread_exit(NULL);
2287     return NULL;
2288 }