src/health.c

   1 #include "common.h"
   2
   3 #define RRDVAR_MAX_LENGTH 1024
   4
   5 struct health_options {
   6     const char *health_default_exec;
   7     const char *health_default_recipient;
   8     const char *log_filename;
   9     size_t log_entries_written;
  10     FILE *log_fp;
  11 };
  12
  13 static struct health_options health = {
  14     .health_default_exec = PLUGINS_DIR "/alarm-notify.sh",
  15     .health_default_recipient = "root",
  16     .log_filename = VARLIB_DIR "/health/alarm_log.db",
  17     .log_entries_written = 0,
  18     .log_fp = NULL
  19 };
  20
  21 int health_enabled = 1;
  22
  23 // ----------------------------------------------------------------------------
  24 // health alarm log load/save
  25 // no need for locking - only one thread is reading / writing the alarms log
  26
  27 static inline int health_alarm_log_open(void) {
  28     if(health.log_fp)
  29         fclose(health.log_fp);
  30
  31     health.log_fp = fopen(health.log_filename, "a");
  32
  33     if(health.log_fp) {
  34         if (setvbuf(health.log_fp, NULL, _IOLBF, 0) != 0)
  35             error("Health: cannot set line buffering on health log file.");
  36         return 0;
  37     }
  38
  39     error("Health: cannot open health log file '%s'. Health data will be lost in case of netdata or server crash.", health.log_filename);
  40     return -1;
  41 }
  42
  43 static inline void health_alarm_log_close(void) {
  44     if(health.log_fp) {
  45         fclose(health.log_fp);
  46         health.log_fp = NULL;
  47     }
  48 }
  49
  50 static inline void health_log_rotate(void) {
  51     static size_t rotate_every = 0;
  52
  53     if(unlikely(rotate_every == 0)) {
  54         rotate_every = (size_t)config_get_number("health", "rotate log every lines", 2000);
  55         if(rotate_every < 100) rotate_every = 100;
  56     }
  57
  58     if(unlikely(health.log_entries_written > rotate_every)) {
  59         health_alarm_log_close();
  60
  61         char old_filename[FILENAME_MAX + 1];
  62         snprintfz(old_filename, FILENAME_MAX, "%s.old", health.log_filename);
  63
  64         if(unlink(old_filename) == -1 && errno != ENOENT)
  65             error("Health: cannot remove old alarms log file '%s'", old_filename);
  66
  67         if(link(health.log_filename, old_filename) == -1 && errno != ENOENT)
  68             error("Health: cannot move file '%s' to '%s'.", health.log_filename, old_filename);
  69
  70         if(unlink(health.log_filename) == -1 && errno != ENOENT)
  71             error("Health: cannot remove old alarms log file '%s'", health.log_filename);
  72
  73         // open it with truncate
  74         health.log_fp = fopen(health.log_filename, "w");
  75
  76         if(health.log_fp)
  77             fclose(health.log_fp);
  78         else
  79             error("Health: cannot truncate health log '%s'", health.log_filename);
  80
  81         health.log_fp = NULL;
  82
  83         health.log_entries_written = 0;
  84         health_alarm_log_open();
  85     }
  86 }
  87
  88 static inline void health_alarm_log_save(RRDHOST *host, ALARM_ENTRY *ae) {
  89     health_log_rotate();
  90
  91     if(likely(health.log_fp)) {
  92         if(unlikely(fprintf(health.log_fp
  93                 , "%c\t%s"
  94                   "\t%08x\t%08x\t%08x\t%08x\t%08x"
  95                   "\t%08x\t%08x\t%08x"
  96                   "\t%08x\t%08x\t%08x"
  97                   "\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s"
  98                   "\t%d\t%d\t%d\t%d"
  99                   "\t%Lf\t%Lf"
 100                   "\n"
 101                 , (ae->flags & HEALTH_ENTRY_FLAG_SAVED)?'U':'A'
 102                 , host->hostname
 103
 104                 , ae->unique_id
 105                 , ae->alarm_id
 106                 , ae->alarm_event_id
 107                 , ae->updated_by_id
 108                 , ae->updates_id
 109
 110                 , (uint32_t)ae->when
 111                 , (uint32_t)ae->duration
 112                 , (uint32_t)ae->non_clear_duration
 113                 , (uint32_t)ae->flags
 114                 , (uint32_t)ae->exec_run_timestamp
 115                 , (uint32_t)ae->delay_up_to_timestamp
 116
 117                 , (ae->name)?ae->name:""
 118                 , (ae->chart)?ae->chart:""
 119                 , (ae->family)?ae->family:""
 120                 , (ae->exec)?ae->exec:""
 121                 , (ae->recipient)?ae->recipient:""
 122                 , (ae->source)?ae->source:""
 123                 , (ae->units)?ae->units:""
 124                 , (ae->info)?ae->info:""
 125
 126                 , ae->exec_code
 127                 , ae->new_status
 128                 , ae->old_status
 129                 , ae->delay
 130
 131                 , (long double)ae->new_value
 132                 , (long double)ae->old_value
 133         ) < 0))
 134             error("Health: failed to save alarm log entry. Health data may be lost in case of abnormal restart.");
 135         else {
 136             ae->flags |= HEALTH_ENTRY_FLAG_SAVED;
 137             health.log_entries_written++;
 138         }
 139     }
 140 }
 141
 142 static inline ssize_t health_alarm_log_read(RRDHOST *host, FILE *fp, const char *filename) {
 143     static uint32_t max_unique_id = 0, max_alarm_id = 0;
 144     ssize_t loaded = -1, updated = -1, errored = -1, duplicate = -1;
 145
 146     errno = 0;
 147
 148     char *s, *buf = mallocz(65536 + 1);
 149     size_t line = 0, len = 0;
 150     loaded = updated = errored = duplicate = 0;
 151
 152     pthread_rwlock_rdlock(&host->health_log.alarm_log_rwlock);
 153
 154     while((s = fgets_trim_len(buf, 65536, fp, &len))) {
 155         health.log_entries_written++;
 156         line++;
 157
 158         int max_entries = 30, entries = 0;
 159         char *pointers[max_entries];
 160
 161         pointers[entries++] = s++;
 162         while(*s) {
 163             if(unlikely(*s == '\t')) {
 164                 *s = '\0';
 165                 pointers[entries++] = ++s;
 166                 if(entries >= max_entries) {
 167                     error("Health: line %zu of file '%s' has more than %d entries. Ignoring excessive entries.", line, filename, max_entries);
 168                     break;
 169                 }
 170             }
 171             else s++;
 172         }
 173
 174         if(likely(*pointers[0] == 'U' || *pointers[0] == 'A')) {
 175             ALARM_ENTRY *ae = NULL;
 176
 177             if(entries < 26) {
 178                 error("Health: line %zu of file '%s' should have at least 26 entries, but it has %d. Ignoring line.", line, filename, entries);
 179                 errored++;
 180                 continue;
 181             }
 182
 183             // check that we have valid ids
 184             uint32_t unique_id = (uint32_t)strtoul(pointers[2], NULL, 16);
 185             if(!unique_id) {
 186                 error("Health: line %zu of file '%s' states alarm entry with unique id %u (%s). Ignoring line.", line, filename, unique_id, pointers[2]);
 187                 errored++;
 188                 continue;
 189             }
 190
 191             uint32_t alarm_id = (uint32_t)strtoul(pointers[3], NULL, 16);
 192             if(!alarm_id) {
 193                 error("Health: line %zu of file '%s' states alarm entry for alarm id %u (%s). Ignoring line.", line, filename, alarm_id, pointers[3]);
 194                 errored++;
 195                 continue;
 196             }
 197
 198             // find a possible overwrite
 199             for(ae = host->health_log.alarms; ae; ae = ae->next) {
 200                 if(unlikely(ae->unique_id == unique_id)) {
 201                     if(unlikely(*pointers[0] == 'A')) {
 202                         error("Health: line %zu of file '%s' adds duplicate alarm log entry with unique id %u."
 203                               , line, filename, unique_id);
 204                         *pointers[0] = 'U';
 205                         duplicate++;
 206                     }
 207                     break;
 208                 }
 209             }
 210
 211             // if not found, create a new one
 212             if(likely(!ae)) {
 213
 214                 // if it is an update, but we haven't found it, make it an addition
 215                 if(unlikely(*pointers[0] == 'U')) {
 216                     *pointers[0] = 'A';
 217                     error("Health: line %zu of file '%s' updates alarm log entry with unique id %u, but it is not found.", line, filename, unique_id);
 218                 }
 219
 220                 // alarms should be added in the right order
 221                 if(unlikely(unique_id < max_unique_id)) {
 222                     error("Health: line %zu of file '%s' has alarm log entry with %u in wrong order.", line, filename, unique_id);
 223                 }
 224
 225                 ae = callocz(1, sizeof(ALARM_ENTRY));
 226             }
 227
 228             // check for a possible host missmatch
 229             if(strcmp(pointers[1], host->hostname))
 230                 error("Health: line %zu of file '%s' provides an alarm for host '%s' but this is named '%s'.", line, filename, pointers[1], host->hostname);
 231
 232             ae->unique_id               = unique_id;
 233             ae->alarm_id                = alarm_id;
 234             ae->alarm_event_id          = (uint32_t)strtoul(pointers[4], NULL, 16);
 235             ae->updated_by_id           = (uint32_t)strtoul(pointers[5], NULL, 16);
 236             ae->updates_id              = (uint32_t)strtoul(pointers[6], NULL, 16);
 237
 238             ae->when                    = (uint32_t)strtoul(pointers[7], NULL, 16);
 239             ae->duration                = (uint32_t)strtoul(pointers[8], NULL, 16);
 240             ae->non_clear_duration      = (uint32_t)strtoul(pointers[9], NULL, 16);
 241
 242             ae->flags                   = (uint32_t)strtoul(pointers[10], NULL, 16);
 243             ae->flags |= HEALTH_ENTRY_FLAG_SAVED;
 244
 245             ae->exec_run_timestamp      = (uint32_t)strtoul(pointers[11], NULL, 16);
 246             ae->delay_up_to_timestamp   = (uint32_t)strtoul(pointers[12], NULL, 16);
 247
 248             if(unlikely(ae->name)) freez(ae->name);
 249             ae->name = strdupz(pointers[13]);
 250             ae->hash_name = simple_hash(ae->name);
 251
 252             if(unlikely(ae->chart)) freez(ae->chart);
 253             ae->chart = strdupz(pointers[14]);
 254             ae->hash_chart = simple_hash(ae->chart);
 255
 256             if(unlikely(ae->family)) freez(ae->family);
 257             ae->family = strdupz(pointers[15]);
 258
 259             if(unlikely(ae->exec)) freez(ae->exec);
 260             ae->exec = strdupz(pointers[16]);
 261             if(!*ae->exec) { freez(ae->exec); ae->exec = NULL; }
 262
 263             if(unlikely(ae->recipient)) freez(ae->recipient);
 264             ae->recipient = strdupz(pointers[17]);
 265             if(!*ae->recipient) { freez(ae->recipient); ae->recipient = NULL; }
 266
 267             if(unlikely(ae->source)) freez(ae->source);
 268             ae->source = strdupz(pointers[18]);
 269             if(!*ae->source) { freez(ae->source); ae->source = NULL; }
 270
 271             if(unlikely(ae->units)) freez(ae->units);
 272             ae->units = strdupz(pointers[19]);
 273             if(!*ae->units) { freez(ae->units); ae->units = NULL; }
 274
 275             if(unlikely(ae->info)) freez(ae->info);
 276             ae->info = strdupz(pointers[20]);
 277             if(!*ae->info) { freez(ae->info); ae->info = NULL; }
 278
 279             ae->exec_code   = atoi(pointers[21]);
 280             ae->new_status  = atoi(pointers[22]);
 281             ae->old_status  = atoi(pointers[23]);
 282             ae->delay       = atoi(pointers[24]);
 283
 284             ae->new_value   = strtold(pointers[25], NULL);
 285             ae->old_value   = strtold(pointers[26], NULL);
 286
 287             // add it to host if not already there
 288             if(unlikely(*pointers[0] == 'A')) {
 289                 ae->next = host->health_log.alarms;
 290                 host->health_log.alarms = ae;
 291                 loaded++;
 292             }
 293             else updated++;
 294
 295             if(unlikely(ae->unique_id > max_unique_id))
 296                 max_unique_id = ae->unique_id;
 297
 298             if(unlikely(ae->alarm_id >= max_alarm_id))
 299                 max_alarm_id = ae->alarm_id;
 300         }
 301         else {
 302             error("Health: line %zu of file '%s' is invalid (unrecognized entry type '%s').", line, filename, pointers[0]);
 303             errored++;
 304         }
 305     }
 306
 307     pthread_rwlock_unlock(&host->health_log.alarm_log_rwlock);
 308
 309     freez(buf);
 310
 311     if(!max_unique_id) max_unique_id = (uint32_t)time(NULL);
 312     if(!max_alarm_id)  max_alarm_id  = (uint32_t)time(NULL);
 313
 314     host->health_log.next_log_id = max_unique_id + 1;
 315     host->health_log.next_alarm_id = max_alarm_id + 1;
 316
 317     info("Health: loaded file '%s' with %zd new alarm entries, updated %zd alarms, errors %zd entries, duplicate %zd", filename, loaded, updated, errored, duplicate);
 318     return loaded;
 319 }
 320
 321 static inline void health_alarm_log_load(RRDHOST *host) {
 322     health_alarm_log_close();
 323
 324     char buffer[FILENAME_MAX + 1];
 325     snprintfz(buffer, FILENAME_MAX, "%s.old", health.log_filename);
 326     FILE *fp = fopen(buffer, "r");
 327     if(!fp)
 328         error("Health: cannot open health file: %s", health.log_filename);
 329     else {
 330         health_alarm_log_read(host, fp, health.log_filename);
 331         fclose(fp);
 332     }
 333
 334     health.log_entries_written = 0;
 335     fp = fopen(health.log_filename, "r");
 336     if(!fp)
 337         error("Health: cannot open health file: %s", health.log_filename);
 338     else {
 339         health_alarm_log_read(host, fp, health.log_filename);
 340         fclose(fp);
 341     }
 342
 343     health_alarm_log_open();
 344 }
 345
 346
 347 // ----------------------------------------------------------------------------
 348 // health alarm log management
 349
 350 static inline void health_alarm_log(RRDHOST *host,
 351                 uint32_t alarm_id, uint32_t alarm_event_id,
 352                 time_t when,
 353                 const char *name, const char *chart, const char *family,
 354                 const char *exec, const char *recipient, time_t duration,
 355                 calculated_number old_value, calculated_number new_value,
 356                 int old_status, int new_status,
 357                 const char *source,
 358                 const char *units,
 359                 const char *info,
 360                 int delay
 361 ) {
 362     debug(D_HEALTH, "Health adding alarm log entry with id: %u", host->health_log.next_log_id);
 363
 364     ALARM_ENTRY *ae = callocz(1, sizeof(ALARM_ENTRY));
 365     ae->name = strdupz(name);
 366     ae->hash_name = simple_hash(ae->name);
 367
 368     if(chart) {
 369         ae->chart = strdupz(chart);
 370         ae->hash_chart = simple_hash(ae->chart);
 371     }
 372
 373     if(family)
 374         ae->family = strdupz(family);
 375
 376     if(exec) ae->exec = strdupz(exec);
 377     if(recipient) ae->recipient = strdupz(recipient);
 378     if(source) ae->source = strdupz(source);
 379     if(units) ae->units = strdupz(units);
 380     if(info) ae->info = strdupz(info);
 381
 382     ae->unique_id = host->health_log.next_log_id++;
 383     ae->alarm_id = alarm_id;
 384     ae->alarm_event_id = alarm_event_id;
 385     ae->when = when;
 386     ae->old_value = old_value;
 387     ae->new_value = new_value;
 388     ae->old_status = old_status;
 389     ae->new_status = new_status;
 390     ae->duration = duration;
 391     ae->delay = delay;
 392     ae->delay_up_to_timestamp = when + delay;
 393
 394     if(ae->old_status == RRDCALC_STATUS_WARNING || ae->old_status == RRDCALC_STATUS_CRITICAL)
 395         ae->non_clear_duration += ae->duration;
 396
 397     // link it
 398     pthread_rwlock_wrlock(&host->health_log.alarm_log_rwlock);
 399     ae->next = host->health_log.alarms;
 400     host->health_log.alarms = ae;
 401     host->health_log.count++;
 402     pthread_rwlock_unlock(&host->health_log.alarm_log_rwlock);
 403
 404     // match previous alarms
 405     pthread_rwlock_rdlock(&host->health_log.alarm_log_rwlock);
 406     ALARM_ENTRY *t;
 407     for(t = host->health_log.alarms ; t ; t = t->next) {
 408         if(t != ae && t->alarm_id == ae->alarm_id) {
 409             if(!(t->flags & HEALTH_ENTRY_FLAG_UPDATED) && !t->updated_by_id) {
 410                 t->flags |= HEALTH_ENTRY_FLAG_UPDATED;
 411                 t->updated_by_id = ae->unique_id;
 412                 ae->updates_id = t->unique_id;
 413
 414                 if((t->new_status == RRDCALC_STATUS_WARNING || t->new_status == RRDCALC_STATUS_CRITICAL) &&
 415                    (t->old_status == RRDCALC_STATUS_WARNING || t->old_status == RRDCALC_STATUS_CRITICAL))
 416                     ae->non_clear_duration += t->non_clear_duration;
 417
 418                 health_alarm_log_save(host, t);
 419             }
 420
 421             // no need to continue
 422             break;
 423         }
 424     }
 425     pthread_rwlock_unlock(&host->health_log.alarm_log_rwlock);
 426
 427     health_alarm_log_save(host, ae);
 428 }
 429
 430 // ----------------------------------------------------------------------------
 431 // RRDVAR management
 432
 433 static inline int rrdvar_fix_name(char *variable) {
 434     int fixed = 0;
 435     while(*variable) {
 436         if (!isalnum(*variable) && *variable != '.' && *variable != '_') {
 437             *variable++ = '_';
 438             fixed++;
 439         }
 440         else
 441             variable++;
 442     }
 443
 444     return fixed;
 445 }
 446
 447 int rrdvar_compare(void* a, void* b) {
 448     if(((RRDVAR *)a)->hash < ((RRDVAR *)b)->hash) return -1;
 449     else if(((RRDVAR *)a)->hash > ((RRDVAR *)b)->hash) return 1;
 450     else return strcmp(((RRDVAR *)a)->name, ((RRDVAR *)b)->name);
 451 }
 452
 453 static inline RRDVAR *rrdvar_index_add(avl_tree_lock *tree, RRDVAR *rv) {
 454     RRDVAR *ret = (RRDVAR *)avl_insert_lock(tree, (avl *)(rv));
 455     if(ret != rv)
 456         debug(D_VARIABLES, "Request to insert RRDVAR '%s' into index failed. Already exists.", rv->name);
 457
 458     return ret;
 459 }
 460
 461 static inline RRDVAR *rrdvar_index_del(avl_tree_lock *tree, RRDVAR *rv) {
 462     RRDVAR *ret = (RRDVAR *)avl_remove_lock(tree, (avl *)(rv));
 463     if(!ret)
 464         error("Request to remove RRDVAR '%s' from index failed. Not Found.", rv->name);
 465
 466     return ret;
 467 }
 468
 469 static inline RRDVAR *rrdvar_index_find(avl_tree_lock *tree, const char *name, uint32_t hash) {
 470     RRDVAR tmp;
 471     tmp.name = (char *)name;
 472     tmp.hash = (hash)?hash:simple_hash(tmp.name);
 473
 474     return (RRDVAR *)avl_search_lock(tree, (avl *)&tmp);
 475 }
 476
 477 static inline void rrdvar_free(RRDHOST *host, avl_tree_lock *tree, RRDVAR *rv) {
 478     (void)host;
 479
 480     if(!rv) return;
 481
 482     if(tree)
 483         rrdvar_index_del(tree, rv);
 484
 485     freez(rv->name);
 486     freez(rv);
 487 }
 488
 489 static inline RRDVAR *rrdvar_create_and_index(const char *scope, avl_tree_lock *tree, const char *name, int type, calculated_number *value) {
 490     char *variable = strdupz(name);
 491     rrdvar_fix_name(variable);
 492     uint32_t hash = simple_hash(variable);
 493
 494     RRDVAR *rv = rrdvar_index_find(tree, variable, hash);
 495     if(unlikely(!rv)) {
 496         debug(D_VARIABLES, "Variable '%s' not found in scope '%s'. Creating a new one.", variable, scope);
 497
 498         rv = callocz(1, sizeof(RRDVAR));
 499         rv->name = variable;
 500         rv->hash = hash;
 501         rv->type = type;
 502         rv->value = value;
 503
 504         RRDVAR *ret = rrdvar_index_add(tree, rv);
 505         if(unlikely(ret != rv)) {
 506             debug(D_VARIABLES, "Variable '%s' in scope '%s' already exists", variable, scope);
 507             rrdvar_free(NULL, NULL, rv);
 508             rv = NULL;
 509         }
 510         else
 511             debug(D_VARIABLES, "Variable '%s' created in scope '%s'", variable, scope);
 512     }
 513     else {
 514         // already exists
 515         freez(variable);
 516         rv = NULL;
 517     }
 518
 519     return rv;
 520 }
 521
 522 // ----------------------------------------------------------------------------
 523 // RRDVAR lookup
 524
 525 calculated_number rrdvar2number(RRDVAR *rv) {
 526     switch(rv->type) {
 527         case RRDVAR_TYPE_CALCULATED: {
 528             calculated_number *n = (calculated_number *)rv->value;
 529             return *n;
 530         }
 531
 532         case RRDVAR_TYPE_TIME_T: {
 533             time_t *n = (time_t *)rv->value;
 534             return *n;
 535         }
 536
 537         case RRDVAR_TYPE_COLLECTED: {
 538             collected_number *n = (collected_number *)rv->value;
 539             return *n;
 540         }
 541
 542         case RRDVAR_TYPE_TOTAL: {
 543             total_number *n = (total_number *)rv->value;
 544             return *n;
 545         }
 546
 547         case RRDVAR_TYPE_INT: {
 548             int *n = (int *)rv->value;
 549             return *n;
 550         }
 551
 552         default:
 553             error("I don't know how to convert RRDVAR type %d to calculated_number", rv->type);
 554             return NAN;
 555     }
 556 }
 557
 558 void dump_variable(void *data) {
 559     RRDVAR *rv = (RRDVAR *)data;
 560     debug(D_HEALTH, "%50s : %20.5Lf", rv->name, rrdvar2number(rv));
 561 }
 562
 563 int health_variable_lookup(const char *variable, uint32_t hash, RRDCALC *rc, calculated_number *result) {
 564     RRDSET *st = rc->rrdset;
 565     RRDVAR *rv;
 566
 567     if(!st) return 0;
 568
 569     rv = rrdvar_index_find(&st->variables_root_index, variable, hash);
 570     if(rv) {
 571         *result = rrdvar2number(rv);
 572         return 1;
 573     }
 574
 575     rv = rrdvar_index_find(&st->rrdfamily->variables_root_index, variable, hash);
 576     if(rv) {
 577         *result = rrdvar2number(rv);
 578         return 1;
 579     }
 580
 581     rv = rrdvar_index_find(&st->rrdhost->variables_root_index, variable, hash);
 582     if(rv) {
 583         *result = rrdvar2number(rv);
 584         return 1;
 585     }
 586
 587     debug(D_HEALTH, "Available local chart '%s' variables:", st->id);
 588     avl_traverse_lock(&st->variables_root_index, dump_variable);
 589
 590     debug(D_HEALTH, "Available family '%s' variables:", st->rrdfamily->family);
 591     avl_traverse_lock(&st->rrdfamily->variables_root_index, dump_variable);
 592
 593     debug(D_HEALTH, "Available host '%s' variables:", st->rrdhost->hostname);
 594     avl_traverse_lock(&st->rrdhost->variables_root_index, dump_variable);
 595
 596     return 0;
 597 }
 598
 599 // ----------------------------------------------------------------------------
 600 // RRDSETVAR management
 601
 602 RRDSETVAR *rrdsetvar_create(RRDSET *st, const char *variable, int type, void *value, uint32_t options) {
 603     debug(D_VARIABLES, "RRDVARSET create for chart id '%s' name '%s' with variable name '%s'", st->id, st->name, variable);
 604     RRDSETVAR *rs = (RRDSETVAR *)callocz(1, sizeof(RRDSETVAR));
 605
 606     char buffer[RRDVAR_MAX_LENGTH + 1];
 607     snprintfz(buffer, RRDVAR_MAX_LENGTH, "%s.%s", st->id, variable);
 608     rs->fullid = strdupz(buffer);
 609
 610     snprintfz(buffer, RRDVAR_MAX_LENGTH, "%s.%s", st->name, variable);
 611     rs->fullname = strdupz(buffer);
 612
 613     rs->variable = strdupz(variable);
 614
 615     rs->type = type;
 616     rs->value = value;
 617     rs->options = options;
 618     rs->rrdset = st;
 619
 620     rs->local       = rrdvar_create_and_index("local",  &st->variables_root_index, rs->variable, rs->type, rs->value);
 621     rs->family      = rrdvar_create_and_index("family", &st->rrdfamily->variables_root_index, rs->fullid, rs->type, rs->value);
 622     rs->host        = rrdvar_create_and_index("host",   &st->rrdhost->variables_root_index, rs->fullid, rs->type, rs->value);
 623     rs->family_name = rrdvar_create_and_index("family", &st->rrdfamily->variables_root_index, rs->fullname, rs->type, rs->value);
 624     rs->host_name   = rrdvar_create_and_index("host",   &st->rrdhost->variables_root_index, rs->fullname, rs->type, rs->value);
 625
 626     rs->next = st->variables;
 627     st->variables = rs;
 628
 629     return rs;
 630 }
 631
 632 void rrdsetvar_rename_all(RRDSET *st) {
 633     debug(D_VARIABLES, "RRDSETVAR rename for chart id '%s' name '%s'", st->id, st->name);
 634
 635     // only these 2 can change name
 636     // rs->family_name
 637     // rs->host_name
 638
 639     char buffer[RRDVAR_MAX_LENGTH + 1];
 640     RRDSETVAR *rs, *next = st->variables;
 641     while((rs = next)) {
 642         next = rs->next;
 643
 644         snprintfz(buffer, RRDVAR_MAX_LENGTH, "%s.%s", st->name, rs->variable);
 645
 646         if (strcmp(buffer, rs->fullname)) {
 647             // name changed
 648             rrdvar_free(st->rrdhost, &st->rrdfamily->variables_root_index, rs->family_name);
 649             rrdvar_free(st->rrdhost, &st->rrdhost->variables_root_index, rs->host_name);
 650
 651             freez(rs->fullname);
 652             rs->fullname = strdupz(st->name);
 653             rs->family_name = rrdvar_create_and_index("family", &st->rrdfamily->variables_root_index, rs->fullname, rs->type, rs->value);
 654             rs->host_name   = rrdvar_create_and_index("host",   &st->rrdhost->variables_root_index, rs->fullname, rs->type, rs->value);
 655         }
 656     }
 657
 658     rrdsetcalc_link_matching(st);
 659 }
 660
 661 void rrdsetvar_free(RRDSETVAR *rs) {
 662     RRDSET *st = rs->rrdset;
 663     debug(D_VARIABLES, "RRDSETVAR free for chart id '%s' name '%s', variable '%s'", st->id, st->name, rs->variable);
 664
 665     rrdvar_free(st->rrdhost, &st->variables_root_index, rs->local);
 666     rrdvar_free(st->rrdhost, &st->rrdfamily->variables_root_index, rs->family);
 667     rrdvar_free(st->rrdhost, &st->rrdhost->variables_root_index, rs->host);
 668     rrdvar_free(st->rrdhost, &st->rrdfamily->variables_root_index, rs->family_name);
 669     rrdvar_free(st->rrdhost, &st->rrdhost->variables_root_index, rs->host_name);
 670
 671     if(st->variables == rs) {
 672         st->variables = rs->next;
 673     }
 674     else {
 675         RRDSETVAR *t;
 676         for (t = st->variables; t && t->next != rs; t = t->next);
 677         if(!t) error("RRDSETVAR '%s' not found in chart '%s' variables linked list", rs->fullname, st->id);
 678         else t->next = rs->next;
 679     }
 680
 681     freez(rs->fullid);
 682     freez(rs->fullname);
 683     freez(rs->variable);
 684     freez(rs);
 685 }
 686
 687 // ----------------------------------------------------------------------------
 688 // RRDDIMVAR management
 689
 690 #define RRDDIMVAR_ID_MAX 1024
 691
 692 RRDDIMVAR *rrddimvar_create(RRDDIM *rd, int type, const char *prefix, const char *suffix, void *value, uint32_t options) {
 693     RRDSET *st = rd->rrdset;
 694
 695     debug(D_VARIABLES, "RRDDIMSET create for chart id '%s' name '%s', dimension id '%s', name '%s%s%s'", st->id, st->name, rd->id, (prefix)?prefix:"", rd->name, (suffix)?suffix:"");
 696
 697     if(!prefix) prefix = "";
 698     if(!suffix) suffix = "";
 699
 700     char buffer[RRDDIMVAR_ID_MAX + 1];
 701     RRDDIMVAR *rs = (RRDDIMVAR *)callocz(1, sizeof(RRDDIMVAR));
 702
 703     rs->prefix = strdupz(prefix);
 704     rs->suffix = strdupz(suffix);
 705
 706     snprintfz(buffer, RRDDIMVAR_ID_MAX, "%s%s%s", rs->prefix, rd->id, rs->suffix);
 707     rs->id = strdupz(buffer);
 708
 709     snprintfz(buffer, RRDDIMVAR_ID_MAX, "%s%s%s", rs->prefix, rd->name, rs->suffix);
 710     rs->name = strdupz(buffer);
 711
 712     snprintfz(buffer, RRDDIMVAR_ID_MAX, "%s.%s", rd->rrdset->id, rs->id);
 713     rs->fullidid = strdupz(buffer);
 714
 715     snprintfz(buffer, RRDDIMVAR_ID_MAX, "%s.%s", rd->rrdset->id, rs->name);
 716     rs->fullidname = strdupz(buffer);
 717
 718     snprintfz(buffer, RRDDIMVAR_ID_MAX, "%s.%s", rd->rrdset->name, rs->id);
 719     rs->fullnameid = strdupz(buffer);
 720
 721     snprintfz(buffer, RRDDIMVAR_ID_MAX, "%s.%s", rd->rrdset->name, rs->name);
 722     rs->fullnamename = strdupz(buffer);
 723
 724     rs->type = type;
 725     rs->value = value;
 726     rs->options = options;
 727     rs->rrddim = rd;
 728
 729     rs->local_id     = rrdvar_create_and_index("local", &st->variables_root_index, rs->id, rs->type, rs->value);
 730     rs->local_name   = rrdvar_create_and_index("local", &st->variables_root_index, rs->name, rs->type, rs->value);
 731
 732     rs->family_id    = rrdvar_create_and_index("family", &st->rrdfamily->variables_root_index, rs->id, rs->type, rs->value);
 733     rs->family_name  = rrdvar_create_and_index("family", &st->rrdfamily->variables_root_index, rs->name, rs->type, rs->value);
 734
 735     rs->host_fullidid     = rrdvar_create_and_index("host", &st->rrdhost->variables_root_index, rs->fullidid, rs->type, rs->value);
 736     rs->host_fullidname   = rrdvar_create_and_index("host", &st->rrdhost->variables_root_index, rs->fullidname, rs->type, rs->value);
 737     rs->host_fullnameid   = rrdvar_create_and_index("host", &st->rrdhost->variables_root_index, rs->fullnameid, rs->type, rs->value);
 738     rs->host_fullnamename = rrdvar_create_and_index("host", &st->rrdhost->variables_root_index, rs->fullnamename, rs->type, rs->value);
 739
 740     rs->next = rd->variables;
 741     rd->variables = rs;
 742
 743     return rs;
 744 }
 745
 746 void rrddimvar_rename_all(RRDDIM *rd) {
 747     RRDSET *st = rd->rrdset;
 748     debug(D_VARIABLES, "RRDDIMSET rename for chart id '%s' name '%s', dimension id '%s', name '%s'", st->id, st->name, rd->id, rd->name);
 749
 750     RRDDIMVAR *rs, *next = rd->variables;
 751     while((rs = next)) {
 752         next = rs->next;
 753
 754         if (strcmp(rd->name, rs->name)) {
 755             char buffer[RRDDIMVAR_ID_MAX + 1];
 756             // name changed
 757
 758             // name
 759             rrdvar_free(st->rrdhost, &st->variables_root_index, rs->local_name);
 760             freez(rs->name);
 761             snprintfz(buffer, RRDDIMVAR_ID_MAX, "%s%s%s", rs->prefix, rd->name, rs->suffix);
 762             rs->name = strdupz(buffer);
 763             rs->local_name = rrdvar_create_and_index("local", &st->variables_root_index, rs->name, rs->type, rs->value);
 764
 765             rrdvar_free(st->rrdhost, &st->rrdhost->variables_root_index, rs->host_fullidname);
 766             freez(rs->fullidname);
 767             snprintfz(buffer, RRDDIMVAR_ID_MAX, "%s.%s", st->id, rs->name);
 768             rs->fullidname = strdupz(buffer);
 769             rs->host_fullidname = rrdvar_create_and_index("host", &st->rrdhost->variables_root_index,
 770                                                              rs->fullidname, rs->type, rs->value);
 771
 772             // fullnameid
 773             rrdvar_free(st->rrdhost, &st->rrdhost->variables_root_index, rs->host_fullnameid);
 774             freez(rs->fullnameid);
 775             snprintfz(buffer, RRDDIMVAR_ID_MAX, "%s.%s", st->name, rs->id);
 776             rs->fullnameid = strdupz(buffer);
 777             rs->host_fullnameid = rrdvar_create_and_index("host", &st->rrdhost->variables_root_index,
 778                                                           rs->fullnameid, rs->type, rs->value);
 779
 780             // fullnamename
 781             rrdvar_free(st->rrdhost, &st->rrdhost->variables_root_index, rs->host_fullnamename);
 782             freez(rs->fullnamename);
 783             snprintfz(buffer, RRDDIMVAR_ID_MAX, "%s.%s", st->name, rs->name);
 784             rs->fullnamename = strdupz(buffer);
 785             rs->host_fullnamename = rrdvar_create_and_index("host", &st->rrdhost->variables_root_index,
 786                                                           rs->fullnamename, rs->type, rs->value);
 787         }
 788     }
 789 }
 790
 791 void rrddimvar_free(RRDDIMVAR *rs) {
 792     RRDDIM *rd = rs->rrddim;
 793     RRDSET *st = rd->rrdset;
 794     debug(D_VARIABLES, "RRDDIMSET free for chart id '%s' name '%s', dimension id '%s', name '%s', prefix='%s', suffix='%s'", st->id, st->name, rd->id, rd->name, rs->prefix, rs->suffix);
 795
 796     rrdvar_free(st->rrdhost, &st->variables_root_index, rs->local_id);
 797     rrdvar_free(st->rrdhost, &st->variables_root_index, rs->local_name);
 798
 799     rrdvar_free(st->rrdhost, &st->rrdfamily->variables_root_index, rs->family_id);
 800     rrdvar_free(st->rrdhost, &st->rrdfamily->variables_root_index, rs->family_name);
 801
 802     rrdvar_free(st->rrdhost, &st->rrdhost->variables_root_index, rs->host_fullidid);
 803     rrdvar_free(st->rrdhost, &st->rrdhost->variables_root_index, rs->host_fullidname);
 804     rrdvar_free(st->rrdhost, &st->rrdhost->variables_root_index, rs->host_fullnameid);
 805     rrdvar_free(st->rrdhost, &st->rrdhost->variables_root_index, rs->host_fullnamename);
 806
 807     if(rd->variables == rs) {
 808         debug(D_VARIABLES, "RRDDIMSET removing first entry for chart id '%s' name '%s', dimension id '%s', name '%s'", st->id, st->name, rd->id, rd->name);
 809         rd->variables = rs->next;
 810     }
 811     else {
 812         debug(D_VARIABLES, "RRDDIMSET removing non-first entry for chart id '%s' name '%s', dimension id '%s', name '%s'", st->id, st->name, rd->id, rd->name);
 813         RRDDIMVAR *t;
 814         for (t = rd->variables; t && t->next != rs; t = t->next) ;
 815         if(!t) error("RRDDIMVAR '%s' not found in dimension '%s/%s' variables linked list", rs->name, st->id, rd->id);
 816         else t->next = rs->next;
 817     }
 818
 819     freez(rs->prefix);
 820     freez(rs->suffix);
 821     freez(rs->id);
 822     freez(rs->name);
 823     freez(rs->fullidid);
 824     freez(rs->fullidname);
 825     freez(rs->fullnameid);
 826     freez(rs->fullnamename);
 827     freez(rs);
 828 }
 829
 830 // ----------------------------------------------------------------------------
 831 // RRDCALC management
 832
 833 static inline const char *rrdcalc_status2string(int status) {
 834     switch(status) {
 835         case RRDCALC_STATUS_REMOVED:
 836             return "REMOVED";
 837
 838         case RRDCALC_STATUS_UNDEFINED:
 839             return "UNDEFINED";
 840
 841         case RRDCALC_STATUS_UNINITIALIZED:
 842             return "UNINITIALIZED";
 843
 844         case RRDCALC_STATUS_CLEAR:
 845             return "CLEAR";
 846
 847         case RRDCALC_STATUS_RAISED:
 848             return "RAISED";
 849
 850         case RRDCALC_STATUS_WARNING:
 851             return "WARNING";
 852
 853         case RRDCALC_STATUS_CRITICAL:
 854             return "CRITICAL";
 855
 856         default:
 857             error("Unknown alarm status %d", status);
 858             return "UNKNOWN";
 859     }
 860 }
 861
 862 static void rrdsetcalc_link(RRDSET *st, RRDCALC *rc) {
 863     debug(D_HEALTH, "Health linking alarm '%s.%s' to chart '%s' of host '%s'", rc->chart?rc->chart:"NOCHART", rc->name, st->id, st->rrdhost->hostname);
 864
 865     rc->last_status_change = time(NULL);
 866     rc->rrdset = st;
 867
 868     rc->rrdset_next = st->alarms;
 869     rc->rrdset_prev = NULL;
 870
 871     if(rc->rrdset_next)
 872         rc->rrdset_next->rrdset_prev = rc;
 873
 874     st->alarms = rc;
 875
 876     if(rc->update_every < rc->rrdset->update_every) {
 877         error("Health alarm '%s.%s' has update every %d, less than chart update every %d. Setting alarm update frequency to %d.", rc->rrdset->id, rc->name, rc->update_every, rc->rrdset->update_every, rc->rrdset->update_every);
 878         rc->update_every = rc->rrdset->update_every;
 879     }
 880
 881     if(!isnan(rc->green) && isnan(st->green)) {
 882         debug(D_HEALTH, "Health alarm '%s.%s' green threshold set from %Lf to %Lf.", rc->rrdset->id, rc->name, rc->rrdset->green, rc->green);
 883         st->green = rc->green;
 884     }
 885
 886     if(!isnan(rc->red) && isnan(st->red)) {
 887         debug(D_HEALTH, "Health alarm '%s.%s' red threshold set from %Lf to %Lf.", rc->rrdset->id, rc->name, rc->rrdset->red, rc->red);
 888         st->red = rc->red;
 889     }
 890
 891     rc->local  = rrdvar_create_and_index("local",  &st->variables_root_index, rc->name, RRDVAR_TYPE_CALCULATED, &rc->value);
 892     rc->family = rrdvar_create_and_index("family", &st->rrdfamily->variables_root_index, rc->name, RRDVAR_TYPE_CALCULATED, &rc->value);
 893
 894     char fullname[RRDVAR_MAX_LENGTH + 1];
 895     snprintfz(fullname, RRDVAR_MAX_LENGTH, "%s.%s", st->id, rc->name);
 896     rc->hostid   = rrdvar_create_and_index("host", &st->rrdhost->variables_root_index, fullname, RRDVAR_TYPE_CALCULATED, &rc->value);
 897
 898     snprintfz(fullname, RRDVAR_MAX_LENGTH, "%s.%s", st->name, rc->name);
 899     rc->hostname = rrdvar_create_and_index("host", &st->rrdhost->variables_root_index, fullname, RRDVAR_TYPE_CALCULATED, &rc->value);
 900
 901         if(!rc->units) rc->units = strdupz(st->units);
 902
 903     {
 904         time_t now = time(NULL);
 905         health_alarm_log(st->rrdhost, rc->id, rc->next_event_id++, now, rc->name, rc->rrdset->id, rc->rrdset->family, rc->exec, rc->recipient, now - rc->last_status_change, rc->old_value, rc->value, rc->status, RRDCALC_STATUS_UNINITIALIZED, rc->source, rc->units, rc->info, 0);
 906     }
 907 }
 908
 909 static inline int rrdcalc_is_matching_this_rrdset(RRDCALC *rc, RRDSET *st) {
 910     if(     (rc->hash_chart == st->hash      && !strcmp(rc->chart, st->id)) ||
 911             (rc->hash_chart == st->hash_name && !strcmp(rc->chart, st->name)))
 912         return 1;
 913
 914     return 0;
 915 }
 916
 917 // this has to be called while the RRDHOST is locked
 918 inline void rrdsetcalc_link_matching(RRDSET *st) {
 919     // debug(D_HEALTH, "find matching alarms for chart '%s'", st->id);
 920
 921     RRDCALC *rc;
 922     for(rc = st->rrdhost->alarms; rc ; rc = rc->next) {
 923         if(unlikely(rc->rrdset))
 924             continue;
 925
 926         if(unlikely(rrdcalc_is_matching_this_rrdset(rc, st)))
 927             rrdsetcalc_link(st, rc);
 928     }
 929 }
 930
 931 // this has to be called while the RRDHOST is locked
 932 inline void rrdsetcalc_unlink(RRDCALC *rc) {
 933     RRDSET *st = rc->rrdset;
 934
 935     if(!st) {
 936         debug(D_HEALTH, "Requested to unlink RRDCALC '%s.%s' which is not linked to any RRDSET", rc->chart?rc->chart:"NOCHART", rc->name);
 937         error("Requested to unlink RRDCALC '%s.%s' which is not linked to any RRDSET", rc->chart?rc->chart:"NOCHART", rc->name);
 938         return;
 939     }
 940
 941     {
 942         time_t now = time(NULL);
 943         health_alarm_log(st->rrdhost, rc->id, rc->next_event_id++, now, rc->name, rc->rrdset->id, rc->rrdset->family, rc->exec, rc->recipient, now - rc->last_status_change, rc->old_value, rc->value, rc->status, RRDCALC_STATUS_REMOVED, rc->source, rc->units, rc->info, 0);
 944     }
 945
 946     RRDHOST *host = st->rrdhost;
 947
 948     debug(D_HEALTH, "Health unlinking alarm '%s.%s' from chart '%s' of host '%s'", rc->chart?rc->chart:"NOCHART", rc->name, st->id, host->hostname);
 949
 950     // unlink it
 951     if(rc->rrdset_prev)
 952         rc->rrdset_prev->rrdset_next = rc->rrdset_next;
 953
 954     if(rc->rrdset_next)
 955         rc->rrdset_next->rrdset_prev = rc->rrdset_prev;
 956
 957     if(st->alarms == rc)
 958         st->alarms = rc->rrdset_next;
 959
 960     rc->rrdset_prev = rc->rrdset_next = NULL;
 961
 962     rrdvar_free(st->rrdhost, &st->variables_root_index, rc->local);
 963     rc->local = NULL;
 964
 965     rrdvar_free(st->rrdhost, &st->rrdfamily->variables_root_index, rc->family);
 966     rc->family = NULL;
 967
 968     rrdvar_free(st->rrdhost, &st->rrdhost->variables_root_index, rc->hostid);
 969     rc->hostid = NULL;
 970
 971     rrdvar_free(st->rrdhost, &st->rrdhost->variables_root_index, rc->hostname);
 972     rc->hostname = NULL;
 973
 974     rc->rrdset = NULL;
 975
 976     // RRDCALC will remain in RRDHOST
 977     // so that if the matching chart is found in the future
 978     // it will be applied automatically
 979 }
 980
 981 RRDCALC *rrdcalc_find(RRDSET *st, const char *name) {
 982     RRDCALC *rc;
 983     uint32_t hash = simple_hash(name);
 984
 985     for( rc = st->alarms; rc ; rc = rc->rrdset_next ) {
 986         if(unlikely(rc->hash == hash && !strcmp(rc->name, name)))
 987             return rc;
 988     }
 989
 990     return NULL;
 991 }
 992
 993 static inline int rrdcalc_exists(RRDHOST *host, const char *chart, const char *name, uint32_t hash_chart, uint32_t hash_name) {
 994     RRDCALC *rc;
 995
 996     if(unlikely(!chart)) {
 997         error("attempt to find RRDCALC '%s' without giving a chart name", name);
 998         return 1;
 999     }
1000
1001     if(unlikely(!hash_chart)) hash_chart = simple_hash(chart);
1002     if(unlikely(!hash_name))  hash_name  = simple_hash(name);
1003
1004     // make sure it does not already exist
1005     for(rc = host->alarms; rc ; rc = rc->next) {
1006         if (unlikely(rc->chart && rc->hash == hash_name && rc->hash_chart == hash_chart && !strcmp(name, rc->name) && !strcmp(chart, rc->chart))) {
1007             debug(D_HEALTH, "Health alarm '%s.%s' already exists in host '%s'.", chart, name, host->hostname);
1008             error("Health alarm '%s.%s' already exists in host '%s'.", chart, name, host->hostname);
1009             return 1;
1010         }
1011     }
1012
1013     return 0;
1014 }
1015
1016 static inline uint32_t rrdcalc_get_unique_id(RRDHOST *host, const char *chart, const char *name, uint32_t *next_event_id) {
1017     if(chart && name) {
1018         uint32_t hash_chart = simple_hash(chart);
1019         uint32_t hash_name = simple_hash(name);
1020
1021         // re-use old IDs, by looking them up in the alarm log
1022         ALARM_ENTRY *ae;
1023         for(ae = host->health_log.alarms; ae ;ae = ae->next) {
1024             if(unlikely(ae->hash_name == hash_name && ae->hash_chart == hash_chart && !strcmp(name, ae->name) && !strcmp(chart, ae->chart))) {
1025                 if(next_event_id) *next_event_id = ae->alarm_event_id + 1;
1026                 return ae->alarm_id;
1027             }
1028         }
1029     }
1030
1031     return host->health_log.next_alarm_id++;
1032 }
1033
1034 static inline void rrdcalc_create_part2(RRDHOST *host, RRDCALC *rc) {
1035     rrdhost_check_rdlock(host);
1036
1037     if(rc->calculation) {
1038         rc->calculation->status = &rc->status;
1039         rc->calculation->this = &rc->value;
1040         rc->calculation->after = &rc->db_after;
1041         rc->calculation->before = &rc->db_before;
1042         rc->calculation->rrdcalc = rc;
1043     }
1044
1045     if(rc->warning) {
1046         rc->warning->status = &rc->status;
1047         rc->warning->this = &rc->value;
1048         rc->warning->after = &rc->db_after;
1049         rc->warning->before = &rc->db_before;
1050         rc->warning->rrdcalc = rc;
1051     }
1052
1053     if(rc->critical) {
1054         rc->critical->status = &rc->status;
1055         rc->critical->this = &rc->value;
1056         rc->critical->after = &rc->db_after;
1057         rc->critical->before = &rc->db_before;
1058         rc->critical->rrdcalc = rc;
1059     }
1060
1061     // link it to the host
1062     if(likely(host->alarms)) {
1063         // append it
1064         RRDCALC *t;
1065         for(t = host->alarms; t && t->next ; t = t->next) ;
1066         t->next = rc;
1067     }
1068     else {
1069         host->alarms = rc;
1070     }
1071
1072     // link it to its chart
1073     RRDSET *st;
1074     for(st = host->rrdset_root; st ; st = st->next) {
1075         if(rrdcalc_is_matching_this_rrdset(rc, st)) {
1076             rrdsetcalc_link(st, rc);
1077             break;
1078         }
1079     }
1080 }
1081
1082 static inline RRDCALC *rrdcalc_create(RRDHOST *host, RRDCALCTEMPLATE *rt, const char *chart) {
1083
1084     debug(D_HEALTH, "Health creating dynamic alarm (from template) '%s.%s'", chart, rt->name);
1085
1086     if(rrdcalc_exists(host, chart, rt->name, 0, 0))
1087         return NULL;
1088
1089     RRDCALC *rc = callocz(1, sizeof(RRDCALC));
1090     rc->next_event_id = 1;
1091     rc->id = rrdcalc_get_unique_id(host, chart, rt->name, &rc->next_event_id);
1092     rc->name = strdupz(rt->name);
1093     rc->hash = simple_hash(rc->name);
1094     rc->chart = strdupz(chart);
1095     rc->hash_chart = simple_hash(rc->chart);
1096
1097     if(rt->dimensions) rc->dimensions = strdupz(rt->dimensions);
1098
1099     rc->green = rt->green;
1100     rc->red = rt->red;
1101     rc->value = NAN;
1102     rc->old_value = NAN;
1103
1104     rc->delay_up_duration = rt->delay_up_duration;
1105     rc->delay_down_duration = rt->delay_down_duration;
1106     rc->delay_max_duration = rt->delay_max_duration;
1107     rc->delay_multiplier = rt->delay_multiplier;
1108
1109     rc->group = rt->group;
1110     rc->after = rt->after;
1111     rc->before = rt->before;
1112     rc->update_every = rt->update_every;
1113     rc->options = rt->options;
1114
1115     if(rt->exec) rc->exec = strdupz(rt->exec);
1116     if(rt->recipient) rc->recipient = strdupz(rt->recipient);
1117     if(rt->source) rc->source = strdupz(rt->source);
1118     if(rt->units) rc->units = strdupz(rt->units);
1119     if(rt->info) rc->info = strdupz(rt->info);
1120
1121     if(rt->calculation) {
1122         rc->calculation = expression_parse(rt->calculation->source, NULL, NULL);
1123         if(!rc->calculation)
1124             error("Health alarm '%s.%s': failed to parse calculation expression '%s'", chart, rt->name, rt->calculation->source);
1125     }
1126     if(rt->warning) {
1127         rc->warning = expression_parse(rt->warning->source, NULL, NULL);
1128         if(!rc->warning)
1129             error("Health alarm '%s.%s': failed to re-parse warning expression '%s'", chart, rt->name, rt->warning->source);
1130     }
1131     if(rt->critical) {
1132         rc->critical = expression_parse(rt->critical->source, NULL, NULL);
1133         if(!rc->critical)
1134             error("Health alarm '%s.%s': failed to re-parse critical expression '%s'", chart, rt->name, rt->critical->source);
1135     }
1136
1137     debug(D_HEALTH, "Health runtime added alarm '%s.%s': exec '%s', recipient '%s', green %Lf, red %Lf, lookup: group %d, after %d, before %d, options %u, dimensions '%s', update every %d, calculation '%s', warning '%s', critical '%s', source '%s', delay up %d, delay down %d, delay max %d, delay_multiplier %f",
1138           (rc->chart)?rc->chart:"NOCHART",
1139           rc->name,
1140           (rc->exec)?rc->exec:"DEFAULT",
1141           (rc->recipient)?rc->recipient:"DEFAULT",
1142           rc->green,
1143           rc->red,
1144           rc->group,
1145           rc->after,
1146           rc->before,
1147           rc->options,
1148           (rc->dimensions)?rc->dimensions:"NONE",
1149           rc->update_every,
1150           (rc->calculation)?rc->calculation->parsed_as:"NONE",
1151           (rc->warning)?rc->warning->parsed_as:"NONE",
1152           (rc->critical)?rc->critical->parsed_as:"NONE",
1153           rc->source,
1154           rc->delay_up_duration,
1155           rc->delay_down_duration,
1156           rc->delay_max_duration,
1157           rc->delay_multiplier
1158     );
1159
1160     rrdcalc_create_part2(host, rc);
1161     return rc;
1162 }
1163
1164 void rrdcalc_free(RRDHOST *host, RRDCALC *rc) {
1165     if(!rc) return;
1166
1167     debug(D_HEALTH, "Health removing alarm '%s.%s' of host '%s'", rc->chart?rc->chart:"NOCHART", rc->name, host->hostname);
1168
1169     // unlink it from RRDSET
1170     if(rc->rrdset) rrdsetcalc_unlink(rc);
1171
1172     // unlink it from RRDHOST
1173     if(unlikely(rc == host->alarms))
1174         host->alarms = rc->next;
1175
1176     else if(likely(host->alarms)) {
1177         RRDCALC *t, *last = host->alarms;
1178         for(t = last->next; t && t != rc; last = t, t = t->next) ;
1179         if(last->next == rc)
1180             last->next = rc->next;
1181         else
1182             error("Cannot unlink alarm '%s.%s' from host '%s': not found", rc->chart?rc->chart:"NOCHART", rc->name, host->hostname);
1183     }
1184     else
1185         error("Cannot unlink unlink '%s.%s' from host '%s': This host does not have any calculations", rc->chart?rc->chart:"NOCHART", rc->name, host->hostname);
1186
1187     expression_free(rc->calculation);
1188     expression_free(rc->warning);
1189     expression_free(rc->critical);
1190
1191     freez(rc->name);
1192     freez(rc->chart);
1193     freez(rc->family);
1194     freez(rc->dimensions);
1195     freez(rc->exec);
1196     freez(rc->recipient);
1197     freez(rc->source);
1198     freez(rc->units);
1199     freez(rc->info);
1200     freez(rc);
1201 }
1202
1203 // ----------------------------------------------------------------------------
1204 // RRDCALCTEMPLATE management
1205
1206 void rrdcalctemplate_link_matching(RRDSET *st) {
1207     RRDCALCTEMPLATE *rt;
1208
1209     for(rt = st->rrdhost->templates; rt ; rt = rt->next) {
1210         if(rt->hash_context == st->hash_context && !strcmp(rt->context, st->context)) {
1211             RRDCALC *rc = rrdcalc_create(st->rrdhost, rt, st->id);
1212             if(unlikely(!rc))
1213                 error("Health tried to create alarm from template '%s', but it failed", rt->name);
1214
1215 #ifdef NETDATA_INTERNAL_CHECKS
1216             else if(rc->rrdset != st)
1217                 error("Health alarm '%s.%s' should be linked to chart '%s', but it is not", rc->chart?rc->chart:"NOCHART", rc->name, st->id);
1218 #endif
1219         }
1220     }
1221 }
1222
1223 static inline void rrdcalctemplate_free(RRDHOST *host, RRDCALCTEMPLATE *rt) {
1224     debug(D_HEALTH, "Health removing template '%s' of host '%s'", rt->name, host->hostname);
1225
1226     if(host->templates) {
1227         if(host->templates == rt) {
1228             host->templates = rt->next;
1229         }
1230         else {
1231             RRDCALCTEMPLATE *t, *last = host->templates;
1232             for (t = last->next; t && t != rt; last = t, t = t->next ) ;
1233             if(last && last->next == rt) {
1234                 last->next = rt->next;
1235                 rt->next = NULL;
1236             }
1237             else
1238                 error("Cannot find RRDCALCTEMPLATE '%s' linked in host '%s'", rt->name, host->hostname);
1239         }
1240     }
1241
1242     expression_free(rt->calculation);
1243     expression_free(rt->warning);
1244     expression_free(rt->critical);
1245
1246     freez(rt->name);
1247     freez(rt->exec);
1248     freez(rt->recipient);
1249     freez(rt->context);
1250     freez(rt->source);
1251     freez(rt->units);
1252     freez(rt->info);
1253     freez(rt->dimensions);
1254     freez(rt);
1255 }
1256
1257 // ----------------------------------------------------------------------------
1258 // load health configuration
1259
1260 #define HEALTH_CONF_MAX_LINE 4096
1261
1262 #define HEALTH_ALARM_KEY "alarm"
1263 #define HEALTH_TEMPLATE_KEY "template"
1264 #define HEALTH_ON_KEY "on"
1265 #define HEALTH_LOOKUP_KEY "lookup"
1266 #define HEALTH_CALC_KEY "calc"
1267 #define HEALTH_EVERY_KEY "every"
1268 #define HEALTH_GREEN_KEY "green"
1269 #define HEALTH_RED_KEY "red"
1270 #define HEALTH_WARN_KEY "warn"
1271 #define HEALTH_CRIT_KEY "crit"
1272 #define HEALTH_EXEC_KEY "exec"
1273 #define HEALTH_RECIPIENT_KEY "to"
1274 #define HEALTH_UNITS_KEY "units"
1275 #define HEALTH_INFO_KEY "info"
1276 #define HEALTH_DELAY_KEY "delay"
1277
1278 static inline int rrdcalc_add_alarm_from_config(RRDHOST *host, RRDCALC *rc) {
1279     if(!rc->chart) {
1280         error("Health configuration for alarm '%s' does not have a chart", rc->name);
1281         return 0;
1282     }
1283
1284     if(!rc->update_every) {
1285         error("Health configuration for alarm '%s.%s' has no frequency (parameter 'every'). Ignoring it.", rc->chart?rc->chart:"NOCHART", rc->name);
1286         return 0;
1287     }
1288
1289     if(!RRDCALC_HAS_DB_LOOKUP(rc) && !rc->warning && !rc->critical) {
1290         error("Health configuration for alarm '%s.%s' is useless (no calculation, no warning and no critical evaluation)", rc->chart?rc->chart:"NOCHART", rc->name);
1291         return 0;
1292     }
1293
1294     if (rrdcalc_exists(host, rc->chart, rc->name, rc->hash_chart, rc->hash))
1295         return 0;
1296
1297     rc->id = rrdcalc_get_unique_id(&localhost, rc->chart, rc->name, &rc->next_event_id);
1298
1299     debug(D_HEALTH, "Health configuration adding alarm '%s.%s' (%u): exec '%s', recipient '%s', green %Lf, red %Lf, lookup: group %d, after %d, before %d, options %u, dimensions '%s', update every %d, calculation '%s', warning '%s', critical '%s', source '%s', delay up %d, delay down %d, delay max %d, delay_multiplier %f",
1300           rc->chart?rc->chart:"NOCHART",
1301           rc->name,
1302           rc->id,
1303           (rc->exec)?rc->exec:"DEFAULT",
1304           (rc->recipient)?rc->recipient:"DEFAULT",
1305           rc->green,
1306           rc->red,
1307           rc->group,
1308           rc->after,
1309           rc->before,
1310           rc->options,
1311           (rc->dimensions)?rc->dimensions:"NONE",
1312           rc->update_every,
1313           (rc->calculation)?rc->calculation->parsed_as:"NONE",
1314           (rc->warning)?rc->warning->parsed_as:"NONE",
1315           (rc->critical)?rc->critical->parsed_as:"NONE",
1316           rc->source,
1317           rc->delay_up_duration,
1318           rc->delay_down_duration,
1319           rc->delay_max_duration,
1320           rc->delay_multiplier
1321     );
1322
1323     rrdcalc_create_part2(host, rc);
1324     return 1;
1325 }
1326
1327 static inline int rrdcalctemplate_add_template_from_config(RRDHOST *host, RRDCALCTEMPLATE *rt) {
1328     if(unlikely(!rt->context)) {
1329         error("Health configuration for template '%s' does not have a context", rt->name);
1330         return 0;
1331     }
1332
1333     if(unlikely(!rt->update_every)) {
1334         error("Health configuration for template '%s' has no frequency (parameter 'every'). Ignoring it.", rt->name);
1335         return 0;
1336     }
1337
1338     if(unlikely(!RRDCALCTEMPLATE_HAS_CALCULATION(rt) && !rt->warning && !rt->critical)) {
1339         error("Health configuration for template '%s' is useless (no calculation, no warning and no critical evaluation)", rt->name);
1340         return 0;
1341     }
1342
1343     RRDCALCTEMPLATE *t, *last = NULL;
1344     for (t = host->templates; t ; last = t, t = t->next) {
1345         if(unlikely(t->hash_name == rt->hash_name && !strcmp(t->name, rt->name))) {
1346             error("Health configuration template '%s' already exists for host '%s'.", rt->name, host->hostname);
1347             return 0;
1348         }
1349     }
1350
1351     debug(D_HEALTH, "Health configuration adding template '%s': context '%s', exec '%s', recipient '%s', green %Lf, red %Lf, lookup: group %d, after %d, before %d, options %u, dimensions '%s', update every %d, calculation '%s', warning '%s', critical '%s', source '%s', delay up %d, delay down %d, delay max %d, delay_multiplier %f",
1352           rt->name,
1353           (rt->context)?rt->context:"NONE",
1354           (rt->exec)?rt->exec:"DEFAULT",
1355           (rt->recipient)?rt->recipient:"DEFAULT",
1356           rt->green,
1357           rt->red,
1358           rt->group,
1359           rt->after,
1360           rt->before,
1361           rt->options,
1362           (rt->dimensions)?rt->dimensions:"NONE",
1363           rt->update_every,
1364           (rt->calculation)?rt->calculation->parsed_as:"NONE",
1365           (rt->warning)?rt->warning->parsed_as:"NONE",
1366           (rt->critical)?rt->critical->parsed_as:"NONE",
1367           rt->source,
1368           rt->delay_up_duration,
1369           rt->delay_down_duration,
1370           rt->delay_max_duration,
1371           rt->delay_multiplier
1372     );
1373
1374     if(likely(last)) {
1375         last->next = rt;
1376     }
1377     else {
1378         rt->next = host->templates;
1379         host->templates = rt;
1380     }
1381
1382     return 1;
1383 }
1384
1385 static inline int health_parse_duration(char *string, int *result) {
1386     // make sure it is a number
1387     if(!*string || !(isdigit(*string) || *string == '+' || *string == '-')) {
1388         *result = 0;
1389         return 0;
1390     }
1391
1392     char *e = NULL;
1393     calculated_number n = strtold(string, &e);
1394     if(e && *e) {
1395         switch (*e) {
1396             case 'Y':
1397                 *result = (int) (n * 86400 * 365);
1398                 break;
1399             case 'M':
1400                 *result = (int) (n * 86400 * 30);
1401                 break;
1402             case 'w':
1403                 *result = (int) (n * 86400 * 7);
1404                 break;
1405             case 'd':
1406                 *result = (int) (n * 86400);
1407                 break;
1408             case 'h':
1409                 *result = (int) (n * 3600);
1410                 break;
1411             case 'm':
1412                 *result = (int) (n * 60);
1413                 break;
1414
1415             default:
1416             case 's':
1417                 *result = (int) (n);
1418                 break;
1419         }
1420     }
1421     else
1422        *result = (int)(n);
1423
1424     return 1;
1425 }
1426
1427 static inline int health_parse_delay(
1428         size_t line, const char *path, const char *file, char *string,
1429         int *delay_up_duration,
1430         int *delay_down_duration,
1431         int *delay_max_duration,
1432         float *delay_multiplier) {
1433
1434     char given_up = 0;
1435     char given_down = 0;
1436     char given_max = 0;
1437     char given_multiplier = 0;
1438
1439     char *s = string;
1440     while(*s) {
1441         char *key = s;
1442
1443         while(*s && !isspace(*s)) s++;
1444         while(*s && isspace(*s)) *s++ = '\0';
1445
1446         if(!*key) break;
1447
1448         char *value = s;
1449         while(*s && !isspace(*s)) s++;
1450         while(*s && isspace(*s)) *s++ = '\0';
1451
1452         if(!strcasecmp(key, "up")) {
1453             if (!health_parse_duration(value, delay_up_duration)) {
1454                 error("Health configuration at line %zu of file '%s/%s': invalid value '%s' for '%s' keyword",
1455                       line, path, file, value, key);
1456             }
1457             else given_up = 1;
1458         }
1459         else if(!strcasecmp(key, "down")) {
1460             if (!health_parse_duration(value, delay_down_duration)) {
1461                 error("Health configuration at line %zu of file '%s/%s': invalid value '%s' for '%s' keyword",
1462                       line, path, file, value, key);
1463             }
1464             else given_down = 1;
1465         }
1466         else if(!strcasecmp(key, "multiplier")) {
1467             *delay_multiplier = strtof(value, NULL);
1468             if(isnan(*delay_multiplier) || isinf(*delay_multiplier) || islessequal(*delay_multiplier, 0)) {
1469                 error("Health configuration at line %zu of file '%s/%s': invalid value '%s' for '%s' keyword",
1470                       line, path, file, value, key);
1471             }
1472             else given_multiplier = 1;
1473         }
1474         else if(!strcasecmp(key, "max")) {
1475             if (!health_parse_duration(value, delay_max_duration)) {
1476                 error("Health configuration at line %zu of file '%s/%s': invalid value '%s' for '%s' keyword",
1477                       line, path, file, value, key);
1478             }
1479             else given_max = 1;
1480         }
1481         else {
1482             error("Health configuration at line %zu of file '%s/%s': unknown keyword '%s'",
1483                   line, path, file, key);
1484         }
1485     }
1486
1487     if(!given_up)
1488         *delay_up_duration = 0;
1489
1490     if(!given_down)
1491         *delay_down_duration = 0;
1492
1493     if(!given_multiplier)
1494         *delay_multiplier = 1.0;
1495
1496     if(!given_max) {
1497         if((*delay_max_duration) < (*delay_up_duration) * (*delay_multiplier))
1498             *delay_max_duration = (*delay_up_duration) * (*delay_multiplier);
1499
1500         if((*delay_max_duration) < (*delay_down_duration) * (*delay_multiplier))
1501             *delay_max_duration = (*delay_down_duration) * (*delay_multiplier);
1502     }
1503
1504     return 1;
1505 }
1506
1507 static inline int health_parse_db_lookup(
1508         size_t line, const char *path, const char *file, char *string,
1509         int *group_method, int *after, int *before, int *every,
1510         uint32_t *options, char **dimensions
1511 ) {
1512     debug(D_HEALTH, "Health configuration parsing database lookup %zu@%s/%s: %s", line, path, file, string);
1513
1514     if(*dimensions) freez(*dimensions);
1515     *dimensions = NULL;
1516     *after = 0;
1517     *before = 0;
1518     *every = 0;
1519     *options = 0;
1520
1521     char *s = string, *key;
1522
1523     // first is the group method
1524     key = s;
1525     while(*s && !isspace(*s)) s++;
1526     while(*s && isspace(*s)) *s++ = '\0';
1527     if(!*s) {
1528         error("Health configuration invalid chart calculation at line %zu of file '%s/%s': expected group method followed by the 'after' time, but got '%s'",
1529               line, path, file, key);
1530         return 0;
1531     }
1532
1533     if((*group_method = web_client_api_request_v1_data_group(key, -1)) == -1) {
1534         error("Health configuration at line %zu of file '%s/%s': invalid group method '%s'",
1535               line, path, file, key);
1536         return 0;
1537     }
1538
1539     // then is the 'after' time
1540     key = s;
1541     while(*s && !isspace(*s)) s++;
1542     while(*s && isspace(*s)) *s++ = '\0';
1543
1544     if(!health_parse_duration(key, after)) {
1545         error("Health configuration at line %zu of file '%s/%s': invalid duration '%s' after group method",
1546               line, path, file, key);
1547         return 0;
1548     }
1549
1550     // sane defaults
1551     *every = abs(*after);
1552
1553     // now we may have optional parameters
1554     while(*s) {
1555         key = s;
1556         while(*s && !isspace(*s)) s++;
1557         while(*s && isspace(*s)) *s++ = '\0';
1558         if(!*key) break;
1559
1560         if(!strcasecmp(key, "at")) {
1561             char *value = s;
1562             while(*s && !isspace(*s)) s++;
1563             while(*s && isspace(*s)) *s++ = '\0';
1564
1565             if (!health_parse_duration(value, before)) {
1566                 error("Health configuration at line %zu of file '%s/%s': invalid duration '%s' for '%s' keyword",
1567                       line, path, file, value, key);
1568             }
1569         }
1570         else if(!strcasecmp(key, HEALTH_EVERY_KEY)) {
1571             char *value = s;
1572             while(*s && !isspace(*s)) s++;
1573             while(*s && isspace(*s)) *s++ = '\0';
1574
1575             if (!health_parse_duration(value, every)) {
1576                 error("Health configuration at line %zu of file '%s/%s': invalid duration '%s' for '%s' keyword",
1577                       line, path, file, value, key);
1578             }
1579         }
1580         else if(!strcasecmp(key, "absolute") || !strcasecmp(key, "abs") || !strcasecmp(key, "absolute_sum")) {
1581             *options |= RRDR_OPTION_ABSOLUTE;
1582         }
1583         else if(!strcasecmp(key, "min2max")) {
1584             *options |= RRDR_OPTION_MIN2MAX;
1585         }
1586         else if(!strcasecmp(key, "null2zero")) {
1587             *options |= RRDR_OPTION_NULL2ZERO;
1588         }
1589         else if(!strcasecmp(key, "percentage")) {
1590             *options |= RRDR_OPTION_PERCENTAGE;
1591         }
1592         else if(!strcasecmp(key, "unaligned")) {
1593             *options |= RRDR_OPTION_NOT_ALIGNED;
1594         }
1595         else if(!strcasecmp(key, "of")) {
1596             if(*s && strcasecmp(s, "all"))
1597                *dimensions = strdupz(s);
1598             break;
1599         }
1600         else {
1601             error("Health configuration at line %zu of file '%s/%s': unknown keyword '%s'",
1602                   line, path, file, key);
1603         }
1604     }
1605
1606     return 1;
1607 }
1608
1609 static inline char *tabs2spaces(char *s) {
1610     char *t = s;
1611     while(*t) {
1612         if(unlikely(*t == '\t')) *t = ' ';
1613         t++;
1614     }
1615
1616     return s;
1617 }
1618
1619 static inline char *health_source_file(size_t line, const char *path, const char *filename) {
1620     char buffer[FILENAME_MAX + 1];
1621     snprintfz(buffer, FILENAME_MAX, "%zu@%s/%s", line, path, filename);
1622     return strdupz(buffer);
1623 }
1624
1625 static inline void strip_quotes(char *s) {
1626     while(*s) {
1627         if(*s == '\'' || *s == '"') *s = ' ';
1628         s++;
1629     }
1630 }
1631
1632 int health_readfile(const char *path, const char *filename) {
1633     debug(D_HEALTH, "Health configuration reading file '%s/%s'", path, filename);
1634
1635     static uint32_t hash_alarm = 0, hash_template = 0, hash_on = 0, hash_calc = 0, hash_green = 0, hash_red = 0, hash_warn = 0, hash_crit = 0, hash_exec = 0, hash_every = 0, hash_lookup = 0, hash_units = 0, hash_info = 0, hash_recipient = 0, hash_delay = 0;
1636     char buffer[HEALTH_CONF_MAX_LINE + 1];
1637
1638     if(unlikely(!hash_alarm)) {
1639         hash_alarm = simple_uhash(HEALTH_ALARM_KEY);
1640         hash_template = simple_uhash(HEALTH_TEMPLATE_KEY);
1641         hash_on = simple_uhash(HEALTH_ON_KEY);
1642         hash_calc = simple_uhash(HEALTH_CALC_KEY);
1643         hash_lookup = simple_uhash(HEALTH_LOOKUP_KEY);
1644         hash_green = simple_uhash(HEALTH_GREEN_KEY);
1645         hash_red = simple_uhash(HEALTH_RED_KEY);
1646         hash_warn = simple_uhash(HEALTH_WARN_KEY);
1647         hash_crit = simple_uhash(HEALTH_CRIT_KEY);
1648         hash_exec = simple_uhash(HEALTH_EXEC_KEY);
1649         hash_every = simple_uhash(HEALTH_EVERY_KEY);
1650         hash_units = simple_hash(HEALTH_UNITS_KEY);
1651         hash_info = simple_hash(HEALTH_INFO_KEY);
1652         hash_recipient = simple_hash(HEALTH_RECIPIENT_KEY);
1653         hash_delay = simple_uhash(HEALTH_DELAY_KEY);
1654     }
1655
1656     snprintfz(buffer, HEALTH_CONF_MAX_LINE, "%s/%s", path, filename);
1657     FILE *fp = fopen(buffer, "r");
1658     if(!fp) {
1659         error("Health configuration cannot read file '%s'.", buffer);
1660         return 0;
1661     }
1662
1663     RRDCALC *rc = NULL;
1664     RRDCALCTEMPLATE *rt = NULL;
1665
1666     size_t line = 0, append = 0;
1667     char *s;
1668     while((s = fgets(&buffer[append], (int)(HEALTH_CONF_MAX_LINE - append), fp)) || append) {
1669         int stop_appending = !s;
1670         line++;
1671         // info("Line %zu of file '%s/%s': '%s'", line, path, filename, s);
1672         s = trim(buffer);
1673         if(!s) continue;
1674         // info("Trimmed line %zu of file '%s/%s': '%s'", line, path, filename, s);
1675
1676         append = strlen(s);
1677         if(!stop_appending && s[append - 1] == '\\') {
1678             s[append - 1] = ' ';
1679             append = &s[append] - buffer;
1680             if(append < HEALTH_CONF_MAX_LINE)
1681                 continue;
1682             else {
1683                 error("Health configuration has too long muli-line at line %zu of file '%s/%s'.", line, path, filename);
1684             }
1685         }
1686         append = 0;
1687
1688         char *key = s;
1689         while(*s && *s != ':') s++;
1690         if(!*s) {
1691             error("Health configuration has invalid line %zu of file '%s/%s'. It does not contain a ':'. Ignoring it.", line, path, filename);
1692             continue;
1693         }
1694         *s = '\0';
1695         s++;
1696
1697         char *value = s;
1698         key = trim(key);
1699         value = trim(value);
1700
1701         if(!key) {
1702             error("Health configuration has invalid line %zu of file '%s/%s'. Keyword is empty. Ignoring it.", line, path, filename);
1703             continue;
1704         }
1705
1706         if(!value) {
1707             error("Health configuration has invalid line %zu of file '%s/%s'. value is empty. Ignoring it.", line, path, filename);
1708             continue;
1709         }
1710
1711         // info("Health file '%s/%s', key '%s', value '%s'", path, filename, key, value);
1712         uint32_t hash = simple_uhash(key);
1713
1714         if(hash == hash_alarm && !strcasecmp(key, HEALTH_ALARM_KEY)) {
1715             if(rc && !rrdcalc_add_alarm_from_config(&localhost, rc))
1716                 rrdcalc_free(&localhost, rc);
1717
1718             if(rt) {
1719                 if (!rrdcalctemplate_add_template_from_config(&localhost, rt))
1720                     rrdcalctemplate_free(&localhost, rt);
1721                 rt = NULL;
1722             }
1723
1724             rc = callocz(1, sizeof(RRDCALC));
1725             rc->next_event_id = 1;
1726             rc->name = tabs2spaces(strdupz(value));
1727             rc->hash = simple_hash(rc->name);
1728             rc->source = health_source_file(line, path, filename);
1729             rc->green = NAN;
1730             rc->red = NAN;
1731             rc->value = NAN;
1732             rc->old_value = NAN;
1733             rc->delay_multiplier = 1.0;
1734
1735             if(rrdvar_fix_name(rc->name))
1736                 error("Health configuration renamed alarm '%s' to '%s'", value, rc->name);
1737         }
1738         else if(hash == hash_template && !strcasecmp(key, HEALTH_TEMPLATE_KEY)) {
1739             if(rc) {
1740                 if(!rrdcalc_add_alarm_from_config(&localhost, rc))
1741                     rrdcalc_free(&localhost, rc);
1742                 rc = NULL;
1743             }
1744
1745             if(rt && !rrdcalctemplate_add_template_from_config(&localhost, rt))
1746                 rrdcalctemplate_free(&localhost, rt);
1747
1748             rt = callocz(1, sizeof(RRDCALCTEMPLATE));
1749             rt->name = tabs2spaces(strdupz(value));
1750             rt->hash_name = simple_hash(rt->name);
1751             rt->source = health_source_file(line, path, filename);
1752             rt->green = NAN;
1753             rt->red = NAN;
1754             rt->delay_multiplier = 1.0;
1755
1756             if(rrdvar_fix_name(rt->name))
1757                 error("Health configuration renamed template '%s' to '%s'", value, rt->name);
1758         }
1759         else if(rc) {
1760             if(hash == hash_on && !strcasecmp(key, HEALTH_ON_KEY)) {
1761                 if(rc->chart) {
1762                     if(strcmp(rc->chart, value))
1763                         info("Health configuration at line %zu of file '%s/%s' for alarm '%s' has key '%s' twice, once with value '%s' and later with value '%s'. Using ('%s').",
1764                              line, path, filename, rc->name, key, rc->chart, value, value);
1765
1766                     freez(rc->chart);
1767                 }
1768                 rc->chart = tabs2spaces(strdupz(value));
1769                 rc->hash_chart = simple_hash(rc->chart);
1770             }
1771             else if(hash == hash_lookup && !strcasecmp(key, HEALTH_LOOKUP_KEY)) {
1772                 health_parse_db_lookup(line, path, filename, value, &rc->group, &rc->after, &rc->before,
1773                                        &rc->update_every,
1774                                        &rc->options, &rc->dimensions);
1775             }
1776             else if(hash == hash_every && !strcasecmp(key, HEALTH_EVERY_KEY)) {
1777                 if(!health_parse_duration(value, &rc->update_every))
1778                     info("Health configuration at line %zu of file '%s/%s' for alarm '%s' at key '%s' cannot parse duration: '%s'.",
1779                          line, path, filename, rc->name, key, value);
1780             }
1781             else if(hash == hash_green && !strcasecmp(key, HEALTH_GREEN_KEY)) {
1782                 char *e;
1783                 rc->green = strtold(value, &e);
1784                 if(e && *e) {
1785                     info("Health configuration at line %zu of file '%s/%s' for alarm '%s' at key '%s' leaves this string unmatched: '%s'.",
1786                          line, path, filename, rc->name, key, e);
1787                 }
1788             }
1789             else if(hash == hash_red && !strcasecmp(key, HEALTH_RED_KEY)) {
1790                 char *e;
1791                 rc->red = strtold(value, &e);
1792                 if(e && *e) {
1793                     info("Health configuration at line %zu of file '%s/%s' for alarm '%s' at key '%s' leaves this string unmatched: '%s'.",
1794                          line, path, filename, rc->name, key, e);
1795                 }
1796             }
1797             else if(hash == hash_calc && !strcasecmp(key, HEALTH_CALC_KEY)) {
1798                 const char *failed_at = NULL;
1799                 int error = 0;
1800                 rc->calculation = expression_parse(value, &failed_at, &error);
1801                 if(!rc->calculation) {
1802                     error("Health configuration at line %zu of file '%s/%s' for alarm '%s' at key '%s' has unparse-able expression '%s': %s at '%s'",
1803                           line, path, filename, rc->name, key, value, expression_strerror(error), failed_at);
1804                 }
1805             }
1806             else if(hash == hash_warn && !strcasecmp(key, HEALTH_WARN_KEY)) {
1807                 const char *failed_at = NULL;
1808                 int error = 0;
1809                 rc->warning = expression_parse(value, &failed_at, &error);
1810                 if(!rc->warning) {
1811                     error("Health configuration at line %zu of file '%s/%s' for alarm '%s' at key '%s' has unparse-able expression '%s': %s at '%s'",
1812                           line, path, filename, rc->name, key, value, expression_strerror(error), failed_at);
1813                 }
1814             }
1815             else if(hash == hash_crit && !strcasecmp(key, HEALTH_CRIT_KEY)) {
1816                 const char *failed_at = NULL;
1817                 int error = 0;
1818                 rc->critical = expression_parse(value, &failed_at, &error);
1819                 if(!rc->critical) {
1820                     error("Health configuration at line %zu of file '%s/%s' for alarm '%s' at key '%s' has unparse-able expression '%s': %s at '%s'",
1821                           line, path, filename, rc->name, key, value, expression_strerror(error), failed_at);
1822                 }
1823             }
1824             else if(hash == hash_exec && !strcasecmp(key, HEALTH_EXEC_KEY)) {
1825                 if(rc->exec) {
1826                     if(strcmp(rc->exec, value))
1827                         info("Health configuration at line %zu of file '%s/%s' for alarm '%s' has key '%s' twice, once with value '%s' and later with value '%s'. Using ('%s').",
1828                              line, path, filename, rc->name, key, rc->exec, value, value);
1829
1830                     freez(rc->exec);
1831                 }
1832                 rc->exec = tabs2spaces(strdupz(value));
1833             }
1834             else if(hash == hash_recipient && !strcasecmp(key, HEALTH_RECIPIENT_KEY)) {
1835                 if(rc->recipient) {
1836                     if(strcmp(rc->recipient, value))
1837                         info("Health configuration at line %zu of file '%s/%s' for alarm '%s' has key '%s' twice, once with value '%s' and later with value '%s'. Using ('%s').",
1838                              line, path, filename, rc->name, key, rc->recipient, value, value);
1839
1840                     freez(rc->recipient);
1841                 }
1842                 rc->recipient = tabs2spaces(strdupz(value));
1843             }
1844             else if(hash == hash_units && !strcasecmp(key, HEALTH_UNITS_KEY)) {
1845                 if(rc->units) {
1846                     if(strcmp(rc->units, value))
1847                         info("Health configuration at line %zu of file '%s/%s' for alarm '%s' has key '%s' twice, once with value '%s' and later with value '%s'. Using ('%s').",
1848                              line, path, filename, rc->name, key, rc->units, value, value);
1849
1850                     freez(rc->units);
1851                 }
1852                 rc->units = tabs2spaces(strdupz(value));
1853                 strip_quotes(rc->units);
1854             }
1855             else if(hash == hash_info && !strcasecmp(key, HEALTH_INFO_KEY)) {
1856                 if(rc->info) {
1857                     if(strcmp(rc->info, value))
1858                         info("Health configuration at line %zu of file '%s/%s' for alarm '%s' has key '%s' twice, once with value '%s' and later with value '%s'. Using ('%s').",
1859                              line, path, filename, rc->name, key, rc->info, value, value);
1860
1861                     freez(rc->info);
1862                 }
1863                 rc->info = tabs2spaces(strdupz(value));
1864                 strip_quotes(rc->info);
1865             }
1866             else if(hash == hash_delay && !strcasecmp(key, HEALTH_DELAY_KEY)) {
1867                 health_parse_delay(line, path, filename, value, &rc->delay_up_duration, &rc->delay_down_duration, &rc->delay_max_duration, &rc->delay_multiplier);
1868             }
1869             else {
1870                 error("Health configuration at line %zu of file '%s/%s' for alarm '%s' has unknown key '%s'.",
1871                      line, path, filename, rc->name, key);
1872             }
1873         }
1874         else if(rt) {
1875             if(hash == hash_on && !strcasecmp(key, HEALTH_ON_KEY)) {
1876                 if(rt->context) {
1877                     if(strcmp(rt->context, value))
1878                         info("Health configuration at line %zu of file '%s/%s' for template '%s' has key '%s' twice, once with value '%s' and later with value '%s'. Using ('%s').",
1879                              line, path, filename, rt->name, key, rt->context, value, value);
1880
1881                     freez(rt->context);
1882                 }
1883                 rt->context = tabs2spaces(strdupz(value));
1884                 rt->hash_context = simple_hash(rt->context);
1885             }
1886             else if(hash == hash_lookup && !strcasecmp(key, HEALTH_LOOKUP_KEY)) {
1887                 health_parse_db_lookup(line, path, filename, value, &rt->group, &rt->after, &rt->before,
1888                                        &rt->update_every,
1889                                        &rt->options, &rt->dimensions);
1890             }
1891             else if(hash == hash_every && !strcasecmp(key, HEALTH_EVERY_KEY)) {
1892                 if(!health_parse_duration(value, &rt->update_every))
1893                     info("Health configuration at line %zu of file '%s/%s' for template '%s' at key '%s' cannot parse duration: '%s'.",
1894                          line, path, filename, rt->name, key, value);
1895             }
1896             else if(hash == hash_green && !strcasecmp(key, HEALTH_GREEN_KEY)) {
1897                 char *e;
1898                 rt->green = strtold(value, &e);
1899                 if(e && *e) {
1900                     info("Health configuration at line %zu of file '%s/%s' for template '%s' at key '%s' leaves this string unmatched: '%s'.",
1901                          line, path, filename, rt->name, key, e);
1902                 }
1903             }
1904             else if(hash == hash_red && !strcasecmp(key, HEALTH_RED_KEY)) {
1905                 char *e;
1906                 rt->red = strtold(value, &e);
1907                 if(e && *e) {
1908                     info("Health configuration at line %zu of file '%s/%s' for template '%s' at key '%s' leaves this string unmatched: '%s'.",
1909                          line, path, filename, rt->name, key, e);
1910                 }
1911             }
1912             else if(hash == hash_calc && !strcasecmp(key, HEALTH_CALC_KEY)) {
1913                 const char *failed_at = NULL;
1914                 int error = 0;
1915                 rt->calculation = expression_parse(value, &failed_at, &error);
1916                 if(!rt->calculation) {
1917                     error("Health configuration at line %zu of file '%s/%s' for template '%s' at key '%s' has unparse-able expression '%s': %s at '%s'",
1918                           line, path, filename, rt->name, key, value, expression_strerror(error), failed_at);
1919                 }
1920             }
1921             else if(hash == hash_warn && !strcasecmp(key, HEALTH_WARN_KEY)) {
1922                 const char *failed_at = NULL;
1923                 int error = 0;
1924                 rt->warning = expression_parse(value, &failed_at, &error);
1925                 if(!rt->warning) {
1926                     error("Health configuration at line %zu of file '%s/%s' for template '%s' at key '%s' has unparse-able expression '%s': %s at '%s'",
1927                           line, path, filename, rt->name, key, value, expression_strerror(error), failed_at);
1928                 }
1929             }
1930             else if(hash == hash_crit && !strcasecmp(key, HEALTH_CRIT_KEY)) {
1931                 const char *failed_at = NULL;
1932                 int error = 0;
1933                 rt->critical = expression_parse(value, &failed_at, &error);
1934                 if(!rt->critical) {
1935                     error("Health configuration at line %zu of file '%s/%s' for template '%s' at key '%s' has unparse-able expression '%s': %s at '%s'",
1936                           line, path, filename, rt->name, key, value, expression_strerror(error), failed_at);
1937                 }
1938             }
1939             else if(hash == hash_exec && !strcasecmp(key, HEALTH_EXEC_KEY)) {
1940                 if(rt->exec) {
1941                     if(strcmp(rt->exec, value))
1942                         info("Health configuration at line %zu of file '%s/%s' for template '%s' has key '%s' twice, once with value '%s' and later with value '%s'. Using ('%s').",
1943                              line, path, filename, rt->name, key, rt->exec, value, value);
1944
1945                     freez(rt->exec);
1946                 }
1947                 rt->exec = tabs2spaces(strdupz(value));
1948             }
1949             else if(hash == hash_recipient && !strcasecmp(key, HEALTH_RECIPIENT_KEY)) {
1950                 if(rt->recipient) {
1951                     if(strcmp(rt->recipient, value))
1952                         info("Health configuration at line %zu of file '%s/%s' for template '%s' has key '%s' twice, once with value '%s' and later with value '%s'. Using ('%s').",
1953                              line, path, filename, rt->name, key, rt->recipient, value, value);
1954
1955                     freez(rt->recipient);
1956                 }
1957                 rt->recipient = tabs2spaces(strdupz(value));
1958             }
1959             else if(hash == hash_units && !strcasecmp(key, HEALTH_UNITS_KEY)) {
1960                 if(rt->units) {
1961                     if(strcmp(rt->units, value))
1962                         info("Health configuration at line %zu of file '%s/%s' for template '%s' has key '%s' twice, once with value '%s' and later with value '%s'. Using ('%s').",
1963                              line, path, filename, rt->name, key, rt->units, value, value);
1964
1965                     freez(rt->units);
1966                 }
1967                 rt->units = tabs2spaces(strdupz(value));
1968                 strip_quotes(rt->units);
1969             }
1970             else if(hash == hash_info && !strcasecmp(key, HEALTH_INFO_KEY)) {
1971                 if(rt->info) {
1972                     if(strcmp(rt->info, value))
1973                         info("Health configuration at line %zu of file '%s/%s' for template '%s' has key '%s' twice, once with value '%s' and later with value '%s'. Using ('%s').",
1974                              line, path, filename, rt->name, key, rt->info, value, value);
1975
1976                     freez(rt->info);
1977                 }
1978                 rt->info = tabs2spaces(strdupz(value));
1979                 strip_quotes(rt->info);
1980             }
1981             else if(hash == hash_delay && !strcasecmp(key, HEALTH_DELAY_KEY)) {
1982                 health_parse_delay(line, path, filename, value, &rt->delay_up_duration, &rt->delay_down_duration, &rt->delay_max_duration, &rt->delay_multiplier);
1983             }
1984             else {
1985                 error("Health configuration at line %zu of file '%s/%s' for template '%s' has unknown key '%s'.",
1986                       line, path, filename, rt->name, key);
1987             }
1988         }
1989         else {
1990             error("Health configuration at line %zu of file '%s/%s' has unknown key '%s'. Expected either '" HEALTH_ALARM_KEY "' or '" HEALTH_TEMPLATE_KEY "'.",
1991                   line, path, filename, key);
1992         }
1993     }
1994
1995     if(rc && !rrdcalc_add_alarm_from_config(&localhost, rc))
1996         rrdcalc_free(&localhost, rc);
1997
1998     if(rt && !rrdcalctemplate_add_template_from_config(&localhost, rt))
1999         rrdcalctemplate_free(&localhost, rt);
2000
2001     fclose(fp);
2002     return 1;
2003 }
2004
2005 void health_readdir(const char *path) {
2006     size_t pathlen = strlen(path);
2007
2008     debug(D_HEALTH, "Health configuration reading directory '%s'", path);
2009
2010     DIR *dir = opendir(path);
2011     if (!dir) {
2012         error("Health configuration cannot open directory '%s'.", path);
2013         return;
2014     }
2015
2016     struct dirent *de = NULL;
2017     while ((de = readdir(dir))) {
2018         size_t len = strlen(de->d_name);
2019
2020         if(de->d_type == DT_DIR
2021            && (
2022                    (de->d_name[0] == '.' && de->d_name[1] == '\0')
2023                    || (de->d_name[0] == '.' && de->d_name[1] == '.' && de->d_name[2] == '\0')
2024            )) {
2025             debug(D_HEALTH, "Ignoring directory '%s'", de->d_name);
2026             continue;
2027         }
2028
2029         else if(de->d_type == DT_DIR) {
2030             char *s = mallocz(pathlen + strlen(de->d_name) + 2);
2031             strcpy(s, path);
2032             strcat(s, "/");
2033             strcat(s, de->d_name);
2034             health_readdir(s);
2035             freez(s);
2036             continue;
2037         }
2038
2039         else if((de->d_type == DT_LNK || de->d_type == DT_REG || de->d_type == DT_UNKNOWN) &&
2040                 len > 5 && !strcmp(&de->d_name[len - 5], ".conf")) {
2041             health_readfile(path, de->d_name);
2042         }
2043
2044         else debug(D_HEALTH, "Ignoring file '%s'", de->d_name);
2045     }
2046
2047     closedir(dir);
2048 }
2049
2050 static inline char *health_config_dir(void) {
2051     char buffer[FILENAME_MAX + 1];
2052     snprintfz(buffer, FILENAME_MAX, "%s/health.d", config_get("global", "config directory", CONFIG_DIR));
2053     return config_get("health", "health configuration directory", buffer);
2054 }
2055
2056 void health_init(void) {
2057     debug(D_HEALTH, "Health configuration initializing");
2058
2059     if(!(health_enabled = config_get_boolean("health", "enabled", 1))) {
2060         debug(D_HEALTH, "Health is disabled.");
2061         return;
2062     }
2063
2064     char *pathname = config_get("health", "health db directory", VARLIB_DIR "/health");
2065     if(mkdir(pathname, 0770) == -1 && errno != EEXIST)
2066         fatal("Cannot create directory '%s'.", pathname);
2067
2068     char filename[FILENAME_MAX + 1];
2069     snprintfz(filename, FILENAME_MAX, "%s/health-log.db", pathname);
2070     health.log_filename = config_get("health", "health db file", filename);
2071
2072     health_alarm_log_load(&localhost);
2073     health_alarm_log_open();
2074
2075     char *path = health_config_dir();
2076
2077     {
2078         char buffer[FILENAME_MAX + 1];
2079         snprintfz(buffer, FILENAME_MAX, "%s/alarm-notify.sh", config_get("global", "plugins directory", PLUGINS_DIR));
2080         health.health_default_exec = config_get("health", "script to execute on alarm", buffer);
2081     }
2082
2083     long n = config_get_number("health", "in memory max health log entries", (long)localhost.health_log.max);
2084     if(n < 10) {
2085         error("Health configuration has invalid max log entries %ld. Using default %u", n, localhost.health_log.max);
2086         config_set_number("health", "in memory max health log entries", (long)localhost.health_log.max);
2087     }
2088     else localhost.health_log.max = (unsigned int)n;
2089
2090     rrdhost_rwlock(&localhost);
2091     health_readdir(path);
2092     rrdhost_unlock(&localhost);
2093 }
2094
2095 // ----------------------------------------------------------------------------
2096 // JSON generation
2097
2098 static inline void health_string2json(BUFFER *wb, const char *prefix, const char *label, const char *value, const char *suffix) {
2099     if(value && *value)
2100         buffer_sprintf(wb, "%s\"%s\":\"%s\"%s", prefix, label, value, suffix);
2101     else
2102         buffer_sprintf(wb, "%s\"%s\":null%s", prefix, label, suffix);
2103 }
2104
2105 static inline void health_alarm_entry2json_nolock(BUFFER *wb, ALARM_ENTRY *ae, RRDHOST *host) {
2106     buffer_sprintf(wb, "\n\t{\n"
2107                            "\t\t\"hostname\": \"%s\",\n"
2108                            "\t\t\"unique_id\": %u,\n"
2109                            "\t\t\"alarm_id\": %u,\n"
2110                            "\t\t\"alarm_event_id\": %u,\n"
2111                            "\t\t\"name\": \"%s\",\n"
2112                            "\t\t\"chart\": \"%s\",\n"
2113                            "\t\t\"family\": \"%s\",\n"
2114                            "\t\t\"processed\": %s,\n"
2115                            "\t\t\"updated\": %s,\n"
2116                            "\t\t\"exec_run\": %lu,\n"
2117                            "\t\t\"exec_failed\": %s,\n"
2118                            "\t\t\"exec\": \"%s\",\n"
2119                            "\t\t\"recipient\": \"%s\",\n"
2120                            "\t\t\"exec_code\": %d,\n"
2121                            "\t\t\"source\": \"%s\",\n"
2122                            "\t\t\"units\": \"%s\",\n"
2123                            "\t\t\"info\": \"%s\",\n"
2124                            "\t\t\"when\": %lu,\n"
2125                            "\t\t\"duration\": %lu,\n"
2126                            "\t\t\"non_clear_duration\": %lu,\n"
2127                            "\t\t\"status\": \"%s\",\n"
2128                            "\t\t\"old_status\": \"%s\",\n"
2129                            "\t\t\"delay\": %d,\n"
2130                            "\t\t\"delay_up_to_timestamp\": %lu,\n"
2131                            "\t\t\"updated_by_id\": %u,\n"
2132                            "\t\t\"updates_id\": %u,\n",
2133                    host->hostname,
2134                    ae->unique_id,
2135                    ae->alarm_id,
2136                    ae->alarm_event_id,
2137                    ae->name,
2138                    ae->chart,
2139                    ae->family,
2140                    (ae->flags & HEALTH_ENTRY_FLAG_PROCESSED)?"true":"false",
2141                    (ae->flags & HEALTH_ENTRY_FLAG_UPDATED)?"true":"false",
2142                    (unsigned long)ae->exec_run_timestamp,
2143                    (ae->flags & HEALTH_ENTRY_FLAG_EXEC_FAILED)?"true":"false",
2144                    ae->exec?ae->exec:health.health_default_exec,
2145                    ae->recipient?ae->recipient:health.health_default_recipient,
2146                    ae->exec_code,
2147                    ae->source,
2148                    ae->units?ae->units:"",
2149                    ae->info?ae->info:"",
2150                    (unsigned long)ae->when,
2151                    (unsigned long)ae->duration,
2152                    (unsigned long)ae->non_clear_duration,
2153                    rrdcalc_status2string(ae->new_status),
2154                    rrdcalc_status2string(ae->old_status),
2155                    ae->delay,
2156                    (unsigned long)ae->delay_up_to_timestamp,
2157                    ae->updated_by_id,
2158                    ae->updates_id
2159     );
2160
2161     buffer_strcat(wb, "\t\t\"value\":");
2162     buffer_rrd_value(wb, ae->new_value);
2163     buffer_strcat(wb, ",\n");
2164
2165     buffer_strcat(wb, "\t\t\"old_value\":");
2166     buffer_rrd_value(wb, ae->old_value);
2167     buffer_strcat(wb, "\n");
2168
2169     buffer_strcat(wb, "\t}");
2170 }
2171
2172 void health_alarm_log2json(RRDHOST *host, BUFFER *wb, uint32_t after) {
2173     pthread_rwlock_rdlock(&host->health_log.alarm_log_rwlock);
2174
2175     buffer_strcat(wb, "[");
2176
2177     unsigned int max = host->health_log.max;
2178     unsigned int count = 0;
2179     ALARM_ENTRY *ae;
2180     for(ae = host->health_log.alarms; ae && count < max ; count++, ae = ae->next) {
2181         if(ae->unique_id > after) {
2182             if(likely(count)) buffer_strcat(wb, ",");
2183             health_alarm_entry2json_nolock(wb, ae, host);
2184         }
2185     }
2186
2187     buffer_strcat(wb, "\n]\n");
2188
2189     pthread_rwlock_unlock(&host->health_log.alarm_log_rwlock);
2190 }
2191
2192 static inline void health_rrdcalc2json_nolock(BUFFER *wb, RRDCALC *rc) {
2193     buffer_sprintf(wb,
2194            "\t\t\"%s.%s\": {\n"
2195                    "\t\t\t\"id\": %lu,\n"
2196                    "\t\t\t\"name\": \"%s\",\n"
2197                    "\t\t\t\"chart\": \"%s\",\n"
2198                    "\t\t\t\"family\": \"%s\",\n"
2199                    "\t\t\t\"active\": %s,\n"
2200                    "\t\t\t\"exec\": \"%s\",\n"
2201                    "\t\t\t\"recipient\": \"%s\",\n"
2202                    "\t\t\t\"source\": \"%s\",\n"
2203                    "\t\t\t\"units\": \"%s\",\n"
2204                    "\t\t\t\"info\": \"%s\",\n"
2205                                    "\t\t\t\"status\": \"%s\",\n"
2206                    "\t\t\t\"last_status_change\": %lu,\n"
2207                    "\t\t\t\"last_updated\": %lu,\n"
2208                    "\t\t\t\"next_update\": %lu,\n"
2209                    "\t\t\t\"update_every\": %d,\n"
2210                    "\t\t\t\"delay_up_duration\": %d,\n"
2211                    "\t\t\t\"delay_down_duration\": %d,\n"
2212                    "\t\t\t\"delay_max_duration\": %d,\n"
2213                    "\t\t\t\"delay_multiplier\": %f,\n"
2214                    "\t\t\t\"delay\": %d,\n"
2215                    "\t\t\t\"delay_up_to_timestamp\": %lu,\n"
2216             , rc->chart, rc->name
2217             , (unsigned long)rc->id
2218             , rc->name
2219             , rc->chart
2220             , (rc->rrdset && rc->rrdset->family)?rc->rrdset->family:""
2221             , (rc->rrdset)?"true":"false"
2222             , rc->exec?rc->exec:health.health_default_exec
2223             , rc->recipient?rc->recipient:health.health_default_recipient
2224             , rc->source
2225             , rc->units?rc->units:""
2226             , rc->info?rc->info:""
2227             , rrdcalc_status2string(rc->status)
2228             , (unsigned long)rc->last_status_change
2229             , (unsigned long)rc->last_updated
2230             , (unsigned long)rc->next_update
2231             , rc->update_every
2232             , rc->delay_up_duration
2233             , rc->delay_down_duration
2234             , rc->delay_max_duration
2235             , rc->delay_multiplier
2236             , rc->delay_last
2237             , (unsigned long)rc->delay_up_to_timestamp
2238     );
2239
2240     if(RRDCALC_HAS_DB_LOOKUP(rc)) {
2241         if(rc->dimensions && *rc->dimensions)
2242             health_string2json(wb, "\t\t\t", "lookup_dimensions", rc->dimensions, ",\n");
2243
2244         buffer_sprintf(wb,
2245                        "\t\t\t\"db_after\": %lu,\n"
2246                        "\t\t\t\"db_before\": %lu,\n"
2247                        "\t\t\t\"lookup_method\": \"%s\",\n"
2248                        "\t\t\t\"lookup_after\": %d,\n"
2249                        "\t\t\t\"lookup_before\": %d,\n"
2250                        "\t\t\t\"lookup_options\": \"",
2251                        (unsigned long) rc->db_after,
2252                        (unsigned long) rc->db_before,
2253                        group_method2string(rc->group),
2254                        rc->after,
2255                        rc->before
2256         );
2257         buffer_data_options2string(wb, rc->options);
2258         buffer_strcat(wb, "\",\n");
2259     }
2260
2261     if(rc->calculation) {
2262         health_string2json(wb, "\t\t\t", "calc", rc->calculation->source, ",\n");
2263         health_string2json(wb, "\t\t\t", "calc_parsed", rc->calculation->parsed_as, ",\n");
2264     }
2265
2266     if(rc->warning) {
2267         health_string2json(wb, "\t\t\t", "warn", rc->warning->source, ",\n");
2268         health_string2json(wb, "\t\t\t", "warn_parsed", rc->warning->parsed_as, ",\n");
2269     }
2270
2271     if(rc->critical) {
2272         health_string2json(wb, "\t\t\t", "crit", rc->critical->source, ",\n");
2273         health_string2json(wb, "\t\t\t", "crit_parsed", rc->critical->parsed_as, ",\n");
2274     }
2275
2276     buffer_strcat(wb, "\t\t\t\"green\":");
2277     buffer_rrd_value(wb, rc->green);
2278     buffer_strcat(wb, ",\n");
2279
2280     buffer_strcat(wb, "\t\t\t\"red\":");
2281     buffer_rrd_value(wb, rc->red);
2282     buffer_strcat(wb, ",\n");
2283
2284     buffer_strcat(wb, "\t\t\t\"value\":");
2285     buffer_rrd_value(wb, rc->value);
2286     buffer_strcat(wb, "\n");
2287
2288     buffer_strcat(wb, "\t\t}");
2289 }
2290
2291 //void health_rrdcalctemplate2json_nolock(BUFFER *wb, RRDCALCTEMPLATE *rt) {
2292 //
2293 //}
2294
2295 void health_alarms2json(RRDHOST *host, BUFFER *wb, int all) {
2296     int i;
2297
2298     rrdhost_rdlock(&localhost);
2299     buffer_sprintf(wb, "{\n\t\"hostname\": \"%s\","
2300                         "\n\t\"latest_alarm_log_unique_id\": %u,"
2301                         "\n\t\"status\": %s,"
2302                         "\n\t\"now\": %lu,"
2303                         "\n\t\"alarms\": {\n",
2304                         host->hostname,
2305                         (host->health_log.next_log_id > 0)?(host->health_log.next_log_id - 1):0,
2306                         health_enabled?"true":"false",
2307                         (unsigned long)time(NULL));
2308
2309     RRDCALC *rc;
2310     for(i = 0, rc = host->alarms; rc ; rc = rc->next) {
2311         if(unlikely(!rc->rrdset || !rc->rrdset->last_collected_time.tv_sec))
2312             continue;
2313
2314         if(likely(!all && !(rc->status == RRDCALC_STATUS_WARNING || rc->status == RRDCALC_STATUS_CRITICAL)))
2315             continue;
2316
2317         if(likely(i)) buffer_strcat(wb, ",\n");
2318         health_rrdcalc2json_nolock(wb, rc);
2319         i++;
2320     }
2321
2322 //    buffer_strcat(wb, "\n\t},\n\t\"templates\": {");
2323 //    RRDCALCTEMPLATE *rt;
2324 //    for(rt = host->templates; rt ; rt = rt->next)
2325 //        health_rrdcalctemplate2json_nolock(wb, rt);
2326
2327     buffer_strcat(wb, "\n\t}\n}\n");
2328     rrdhost_unlock(&localhost);
2329 }
2330
2331
2332 // ----------------------------------------------------------------------------
2333 // re-load health configuration
2334
2335 static inline void health_free_all_nolock(RRDHOST *host) {
2336     while(host->templates)
2337         rrdcalctemplate_free(host, host->templates);
2338
2339     while(host->alarms)
2340         rrdcalc_free(host, host->alarms);
2341 }
2342
2343 void health_reload(void) {
2344     if(!health_enabled) {
2345         error("Health reload is requested, but health is not enabled.");
2346         return;
2347     }
2348
2349     char *path = health_config_dir();
2350
2351     // free all running alarms
2352     rrdhost_rwlock(&localhost);
2353     health_free_all_nolock(&localhost);
2354     rrdhost_unlock(&localhost);
2355
2356     // invalidate all previous entries in the alarm log
2357     ALARM_ENTRY *t;
2358     for(t = localhost.health_log.alarms ; t ; t = t->next) {
2359         if(t->new_status != RRDCALC_STATUS_REMOVED)
2360             t->flags |= HEALTH_ENTRY_FLAG_UPDATED;
2361     }
2362
2363     // reset all thresholds to all charts
2364     RRDSET *st;
2365     for(st = localhost.rrdset_root; st ; st = st->next) {
2366         st->green = NAN;
2367         st->red = NAN;
2368     }
2369
2370     // load the new alarms
2371     rrdhost_rwlock(&localhost);
2372     health_readdir(path);
2373     rrdhost_unlock(&localhost);
2374
2375     // link the loaded alarms to their charts
2376     for(st = localhost.rrdset_root; st ; st = st->next) {
2377         rrdhost_rwlock(&localhost);
2378
2379         rrdsetcalc_link_matching(st);
2380         rrdcalctemplate_link_matching(st);
2381
2382         rrdhost_unlock(&localhost);
2383     }
2384 }
2385
2386 // ----------------------------------------------------------------------------
2387 // health main thread and friends
2388
2389 static inline int rrdcalc_value2status(calculated_number n) {
2390     if(isnan(n)) return RRDCALC_STATUS_UNDEFINED;
2391     if(n) return RRDCALC_STATUS_RAISED;
2392     return RRDCALC_STATUS_CLEAR;
2393 }
2394
2395 static inline void health_alarm_execute(RRDHOST *host, ALARM_ENTRY *ae) {
2396     ae->flags |= HEALTH_ENTRY_FLAG_PROCESSED;
2397
2398     if(unlikely(ae->new_status < RRDCALC_STATUS_CLEAR)) {
2399         // do not send notifications for internal statuses
2400         goto done;
2401     }
2402
2403     // find the previous notification for the same alarm
2404     // which we have run the exec script
2405     ALARM_ENTRY *t;
2406     for(t = ae->next; t ;t = t->next) {
2407         if(t->alarm_id == ae->alarm_id && t->flags & HEALTH_ENTRY_FLAG_EXEC_RUN)
2408             break;
2409     }
2410
2411     if(likely(t)) {
2412         // we have executed this alarm notification in the past
2413         if (t && t->new_status == ae->new_status) {
2414             // don't send the same notification again
2415             info("Health not sending again notification for alarm '%s.%s' status %s", ae->chart, ae->name,
2416                  rrdcalc_status2string(ae->new_status));
2417             goto done;
2418         }
2419     }
2420     else {
2421         // we have not executed this alarm notification in the past
2422         if(unlikely(ae->old_status == RRDCALC_STATUS_UNINITIALIZED && ae->new_status == RRDCALC_STATUS_CLEAR)) {
2423             info("Health not sending notification for first initialization of alarm '%s.%s' status %s", ae->chart, ae->name, rrdcalc_status2string(ae->new_status));
2424             goto done;
2425         }
2426     }
2427
2428     char buffer[FILENAME_MAX + 1];
2429     pid_t command_pid;
2430
2431     const char *exec = ae->exec;
2432     if(!exec) exec = health.health_default_exec;
2433
2434     const char *recipient = ae->recipient;
2435     if(!recipient) recipient = health.health_default_recipient;
2436
2437     snprintfz(buffer, FILENAME_MAX, "exec %s '%s' '%s' '%u' '%u' '%u' '%lu' '%s' '%s' '%s' '%s' '%s' '%0.0Lf' '%0.0Lf' '%s' '%u' '%u' '%s' '%s'",
2438               exec,
2439               recipient,
2440               host->hostname,
2441               ae->unique_id,
2442               ae->alarm_id,
2443               ae->alarm_event_id,
2444               (unsigned long)ae->when,
2445               ae->name,
2446               ae->chart?ae->chart:"NOCAHRT",
2447               ae->family?ae->family:"NOFAMILY",
2448               rrdcalc_status2string(ae->new_status),
2449               rrdcalc_status2string(ae->old_status),
2450               ae->new_value,
2451               ae->old_value,
2452               ae->source?ae->source:"UNKNOWN",
2453               (uint32_t)ae->duration,
2454               (uint32_t)ae->non_clear_duration,
2455               ae->units?ae->units:"",
2456               ae->info?ae->info:""
2457     );
2458
2459     ae->flags |= HEALTH_ENTRY_FLAG_EXEC_RUN;
2460     ae->exec_run_timestamp = time(NULL);
2461
2462     debug(D_HEALTH, "executing command '%s'", buffer);
2463     FILE *fp = mypopen(buffer, &command_pid);
2464     if(!fp) {
2465         error("HEALTH: Cannot popen(\"%s\", \"r\").", buffer);
2466         goto done;
2467     }
2468     debug(D_HEALTH, "HEALTH reading from command");
2469     char *s = fgets(buffer, FILENAME_MAX, fp);
2470     (void)s;
2471     ae->exec_code = mypclose(fp, command_pid);
2472     debug(D_HEALTH, "done executing command - returned with code %d", ae->exec_code);
2473
2474     if(ae->exec_code != 0)
2475         ae->flags |= HEALTH_ENTRY_FLAG_EXEC_FAILED;
2476
2477 done:
2478     health_alarm_log_save(host, ae);
2479     return;
2480 }
2481
2482 static inline void health_process_notifications(RRDHOST *host, ALARM_ENTRY *ae) {
2483     info("Health alarm '%s.%s' = %0.2Lf - changed status from %s to %s",
2484          ae->chart?ae->chart:"NOCHART", ae->name,
2485          ae->new_value,
2486          rrdcalc_status2string(ae->old_status),
2487          rrdcalc_status2string(ae->new_status)
2488     );
2489
2490     health_alarm_execute(host, ae);
2491 }
2492
2493 static inline void health_alarm_log_process(RRDHOST *host) {
2494     static uint32_t stop_at_id = 0;
2495     uint32_t first_waiting = (host->health_log.alarms)?host->health_log.alarms->unique_id:0;
2496     time_t now = time(NULL);
2497
2498     pthread_rwlock_rdlock(&host->health_log.alarm_log_rwlock);
2499
2500     ALARM_ENTRY *ae;
2501     for(ae = host->health_log.alarms; ae && ae->unique_id >= stop_at_id ; ae = ae->next) {
2502         if(unlikely(
2503             !(ae->flags & HEALTH_ENTRY_FLAG_PROCESSED) &&
2504             !(ae->flags & HEALTH_ENTRY_FLAG_UPDATED)
2505             )) {
2506
2507             if(unlikely(ae->unique_id < first_waiting))
2508                 first_waiting = ae->unique_id;
2509
2510             if(likely(now >= ae->delay_up_to_timestamp))
2511                 health_process_notifications(host, ae);
2512         }
2513     }
2514
2515     // remember this for the next iteration
2516     stop_at_id = first_waiting;
2517
2518     pthread_rwlock_unlock(&host->health_log.alarm_log_rwlock);
2519
2520     if(host->health_log.count <= host->health_log.max)
2521         return;
2522
2523     // cleanup excess entries in the log
2524     pthread_rwlock_wrlock(&host->health_log.alarm_log_rwlock);
2525
2526     ALARM_ENTRY *last = NULL;
2527     unsigned int count = host->health_log.max * 2 / 3;
2528     for(ae = host->health_log.alarms; ae && count ; count--, last = ae, ae = ae->next) ;
2529
2530     if(ae && last && last->next == ae)
2531         last->next = NULL;
2532     else
2533         ae = NULL;
2534
2535     while(ae) {
2536         debug(D_HEALTH, "Health removing alarm log entry with id: %u", ae->unique_id);
2537
2538         ALARM_ENTRY *t = ae->next;
2539
2540         freez(ae->name);
2541         freez(ae->chart);
2542         freez(ae->family);
2543         freez(ae->exec);
2544         freez(ae->recipient);
2545         freez(ae->source);
2546         freez(ae->units);
2547         freez(ae->info);
2548         freez(ae);
2549
2550         ae = t;
2551         host->health_log.count--;
2552     }
2553
2554     pthread_rwlock_unlock(&host->health_log.alarm_log_rwlock);
2555 }
2556
2557 static inline int rrdcalc_isrunnable(RRDCALC *rc, time_t now, time_t *next_run) {
2558     if (unlikely(!rc->rrdset)) {
2559         debug(D_HEALTH, "Health not running alarm '%s.%s'. It is not linked to a chart.", rc->chart?rc->chart:"NOCHART", rc->name);
2560         return 0;
2561     }
2562
2563     if (unlikely(!rc->rrdset->last_collected_time.tv_sec)) {
2564         debug(D_HEALTH, "Health not running alarm '%s.%s'. Chart is not yet collected.", rc->chart?rc->chart:"NOCHART", rc->name);
2565         return 0;
2566     }
2567
2568     if (unlikely(!rc->update_every)) {
2569         debug(D_HEALTH, "Health not running alarm '%s.%s'. It does not have an update frequency", rc->chart?rc->chart:"NOCHART", rc->name);
2570         return 0;
2571     }
2572
2573     if (unlikely(rc->next_update > now)) {
2574         if (unlikely(*next_run > rc->next_update))
2575             *next_run = rc->next_update;
2576
2577         debug(D_HEALTH, "Health not examining alarm '%s.%s' yet (will do in %d secs).", rc->chart?rc->chart:"NOCHART", rc->name, (int) (rc->next_update - now));
2578         return 0;
2579     }
2580
2581     // FIXME
2582     // we should check that the DB lookup is possible
2583     // i.e.
2584     // - the duration of the chart includes the required timeframe
2585     // we SHOULD NOT check the dimensions - there might be alarms that refer non-existing dimensions (e.g. cpu steal)
2586
2587     return 1;
2588 }
2589
2590 void *health_main(void *ptr) {
2591     (void)ptr;
2592
2593     info("HEALTH thread created with task id %d", gettid());
2594
2595     if(pthread_setcanceltype(PTHREAD_CANCEL_DEFERRED, NULL) != 0)
2596         error("Cannot set pthread cancel type to DEFERRED.");
2597
2598     if(pthread_setcancelstate(PTHREAD_CANCEL_ENABLE, NULL) != 0)
2599         error("Cannot set pthread cancel state to ENABLE.");
2600
2601     int min_run_every = (int)config_get_number("health", "run at least every seconds", 10);
2602     if(min_run_every < 1) min_run_every = 1;
2603
2604     BUFFER *wb = buffer_create(100);
2605
2606     unsigned int loop = 0;
2607     while(health_enabled && !netdata_exit) {
2608         loop++;
2609         debug(D_HEALTH, "Health monitoring iteration no %u started", loop);
2610
2611         int oldstate, runnable = 0;
2612         time_t now = time(NULL);
2613         time_t next_run = now + min_run_every;
2614         RRDCALC *rc;
2615
2616         if (unlikely(pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, &oldstate) != 0))
2617             error("Cannot set pthread cancel state to DISABLE.");
2618
2619         rrdhost_rdlock(&localhost);
2620
2621         // the first loop is to lookup values from the db
2622         for (rc = localhost.alarms; rc; rc = rc->next) {
2623             if (unlikely(!rrdcalc_isrunnable(rc, now, &next_run)))
2624                 continue;
2625
2626             runnable++;
2627             rc->old_value = rc->value;
2628
2629             // 1. if there is database lookup, do it
2630             // 2. if there is calculation expression, run it
2631
2632             if (unlikely(RRDCALC_HAS_DB_LOOKUP(rc))) {
2633                 time_t old_db_timestamp = rc->db_before;
2634                 int value_is_null = 0;
2635
2636                 int ret = rrd2value(rc->rrdset, wb, &rc->value,
2637                                     rc->dimensions, 1, rc->after, rc->before, rc->group,
2638                                     rc->options, &rc->db_after, &rc->db_before, &value_is_null);
2639
2640                 if (unlikely(ret != 200)) {
2641                     // database lookup failed
2642                     rc->value = NAN;
2643
2644                     debug(D_HEALTH, "Health alarm '%s.%s': database lookup returned error %d", rc->chart?rc->chart:"NOCHART", rc->name, ret);
2645
2646                     if (unlikely(!(rc->rrdcalc_flags & RRDCALC_FLAG_DB_ERROR))) {
2647                         rc->rrdcalc_flags |= RRDCALC_FLAG_DB_ERROR;
2648                         error("Health alarm '%s.%s': database lookup returned error %d", rc->chart?rc->chart:"NOCHART", rc->name, ret);
2649                     }
2650                 }
2651                 else if (unlikely(rc->rrdcalc_flags & RRDCALC_FLAG_DB_ERROR))
2652                     rc->rrdcalc_flags &= ~RRDCALC_FLAG_DB_ERROR;
2653
2654                 if (unlikely(old_db_timestamp == rc->db_before)) {
2655                     // database is stale
2656
2657                     debug(D_HEALTH, "Health alarm '%s.%s': database is stale", rc->chart?rc->chart:"NOCHART", rc->name);
2658
2659                     if (unlikely(!(rc->rrdcalc_flags & RRDCALC_FLAG_DB_STALE))) {
2660                         rc->rrdcalc_flags |= RRDCALC_FLAG_DB_STALE;
2661                         error("Health alarm '%s.%s': database is stale", rc->chart?rc->chart:"NOCHART", rc->name);
2662                     }
2663                 }
2664                 else if (unlikely(rc->rrdcalc_flags & RRDCALC_FLAG_DB_STALE))
2665                     rc->rrdcalc_flags &= ~RRDCALC_FLAG_DB_STALE;
2666
2667                 if (unlikely(value_is_null)) {
2668                     // collected value is null
2669
2670                     rc->value = NAN;
2671
2672                     debug(D_HEALTH, "Health alarm '%s.%s': database lookup returned empty value (possibly value is not collected yet)",
2673                           rc->chart?rc->chart:"NOCHART", rc->name);
2674
2675                     if (unlikely(!(rc->rrdcalc_flags & RRDCALC_FLAG_DB_NAN))) {
2676                         rc->rrdcalc_flags |= RRDCALC_FLAG_DB_NAN;
2677                         error("Health alarm '%s.%s': database lookup returned empty value (possibly value is not collected yet)",
2678                               rc->chart?rc->chart:"NOCHART", rc->name);
2679                     }
2680                 }
2681                 else if (unlikely(rc->rrdcalc_flags & RRDCALC_FLAG_DB_NAN))
2682                     rc->rrdcalc_flags &= ~RRDCALC_FLAG_DB_NAN;
2683
2684                 debug(D_HEALTH, "Health alarm '%s.%s': database lookup gave value "
2685                         CALCULATED_NUMBER_FORMAT, rc->chart?rc->chart:"NOCHART", rc->name, rc->value);
2686             }
2687
2688             if(unlikely(rc->calculation)) {
2689                 if (unlikely(!expression_evaluate(rc->calculation))) {
2690                     // calculation failed
2691
2692                     rc->value = NAN;
2693
2694                     debug(D_HEALTH, "Health alarm '%s.%s': failed to evaluate calculation with error: %s",
2695                           rc->chart?rc->chart:"NOCHART", rc->name, buffer_tostring(rc->calculation->error_msg));
2696
2697                     if (unlikely(!(rc->rrdcalc_flags & RRDCALC_FLAG_CALC_ERROR))) {
2698                         rc->rrdcalc_flags |= RRDCALC_FLAG_CALC_ERROR;
2699                         error("Health alarm '%s.%s': failed to evaluate calculation with error: %s",
2700                               rc->chart?rc->chart:"NOCHART", rc->name, buffer_tostring(rc->calculation->error_msg));
2701                     }
2702                 }
2703                 else {
2704                     if (unlikely(rc->rrdcalc_flags & RRDCALC_FLAG_CALC_ERROR))
2705                         rc->rrdcalc_flags &= ~RRDCALC_FLAG_CALC_ERROR;
2706
2707                     debug(D_HEALTH, "Health alarm '%s.%s': calculation expression gave value "
2708                             CALCULATED_NUMBER_FORMAT
2709                             ": %s (source: %s)",
2710                           rc->chart?rc->chart:"NOCHART", rc->name,
2711                           rc->calculation->result,
2712                           buffer_tostring(rc->calculation->error_msg),
2713                           rc->source
2714                     );
2715
2716                     rc->value = rc->calculation->result;
2717                 }
2718             }
2719         }
2720         rrdhost_unlock(&localhost);
2721
2722         if (unlikely(runnable && !netdata_exit)) {
2723             rrdhost_rdlock(&localhost);
2724
2725             for (rc = localhost.alarms; rc; rc = rc->next) {
2726                 if (unlikely(!rrdcalc_isrunnable(rc, now, &next_run)))
2727                     continue;
2728
2729                 int warning_status  = RRDCALC_STATUS_UNDEFINED;
2730                 int critical_status = RRDCALC_STATUS_UNDEFINED;
2731
2732                 if(likely(rc->warning)) {
2733                     if(unlikely(!expression_evaluate(rc->warning))) {
2734                         // calculation failed
2735
2736                         debug(D_HEALTH, "Health alarm '%s.%s': warning expression failed with error: %s",
2737                               rc->chart?rc->chart:"NOCHART", rc->name, buffer_tostring(rc->warning->error_msg));
2738
2739                         if (unlikely(!(rc->rrdcalc_flags & RRDCALC_FLAG_WARN_ERROR))) {
2740                             rc->rrdcalc_flags |= RRDCALC_FLAG_WARN_ERROR;
2741                             error("Health alarm '%s.%s': warning expression failed with error: %s",
2742                                   rc->chart?rc->chart:"NOCHART", rc->name, buffer_tostring(rc->warning->error_msg));
2743                         }
2744                     }
2745                     else {
2746                         if(unlikely(rc->rrdcalc_flags & RRDCALC_FLAG_WARN_ERROR))
2747                             rc->rrdcalc_flags &= ~RRDCALC_FLAG_WARN_ERROR;
2748
2749                         debug(D_HEALTH, "Health alarm '%s.%s': warning expression gave value "
2750                                 CALCULATED_NUMBER_FORMAT
2751                                 ": %s (source: %s)",
2752                               rc->chart?rc->chart:"NOCHART", rc->name,
2753                               rc->warning->result,
2754                               buffer_tostring(rc->warning->error_msg),
2755                               rc->source
2756                         );
2757
2758                         warning_status = rrdcalc_value2status(rc->warning->result);
2759                     }
2760                 }
2761
2762                 if(likely(rc->critical)) {
2763                     if(unlikely(!expression_evaluate(rc->critical))) {
2764                         // calculation failed
2765
2766                         debug(D_HEALTH, "Health alarm '%s.%s': critical expression failed with error: %s",
2767                               rc->chart?rc->chart:"NOCHART", rc->name, buffer_tostring(rc->critical->error_msg));
2768
2769                         if (unlikely(!(rc->rrdcalc_flags & RRDCALC_FLAG_CRIT_ERROR))) {
2770                             rc->rrdcalc_flags |= RRDCALC_FLAG_CRIT_ERROR;
2771                             error("Health alarm '%s.%s': critical expression failed with error: %s",
2772                                   rc->chart?rc->chart:"NOCHART", rc->name, buffer_tostring(rc->critical->error_msg));
2773                         }
2774                     }
2775                     else {
2776                         if(unlikely(rc->rrdcalc_flags & RRDCALC_FLAG_CRIT_ERROR))
2777                             rc->rrdcalc_flags &= ~RRDCALC_FLAG_CRIT_ERROR;
2778
2779                         debug(D_HEALTH, "Health alarm '%s.%s': critical expression gave value "
2780                                 CALCULATED_NUMBER_FORMAT
2781                                 ": %s (source: %s)",
2782                               rc->chart?rc->chart:"NOCHART", rc->name,
2783                               rc->critical->result,
2784                               buffer_tostring(rc->critical->error_msg),
2785                               rc->source
2786                         );
2787
2788                         critical_status = rrdcalc_value2status(rc->critical->result);
2789                     }
2790                 }
2791
2792                 int status = RRDCALC_STATUS_UNDEFINED;
2793
2794                 switch(warning_status) {
2795                     case RRDCALC_STATUS_CLEAR:
2796                         status = RRDCALC_STATUS_CLEAR;
2797                         break;
2798
2799                     case RRDCALC_STATUS_RAISED:
2800                         status = RRDCALC_STATUS_WARNING;
2801                         break;
2802
2803                     default:
2804                         break;
2805                 }
2806
2807                 switch(critical_status) {
2808                     case RRDCALC_STATUS_CLEAR:
2809                         if(status == RRDCALC_STATUS_UNDEFINED)
2810                             status = RRDCALC_STATUS_CLEAR;
2811                         break;
2812
2813                     case RRDCALC_STATUS_RAISED:
2814                         status = RRDCALC_STATUS_CRITICAL;
2815                         break;
2816
2817                     default:
2818                         break;
2819                 }
2820
2821                 if(status != rc->status) {
2822                     int delay = 0;
2823
2824                     if(now > rc->delay_up_to_timestamp) {
2825                         rc->delay_up_current = rc->delay_up_duration;
2826                         rc->delay_down_current = rc->delay_down_duration;
2827                         rc->delay_last = 0;
2828                         rc->delay_up_to_timestamp = 0;
2829                     }
2830                     else {
2831                         rc->delay_up_current = (int)(rc->delay_up_current * rc->delay_multiplier);
2832                         if(rc->delay_up_current > rc->delay_max_duration) rc->delay_up_current = rc->delay_max_duration;
2833
2834                         rc->delay_down_current = (int)(rc->delay_down_current * rc->delay_multiplier);
2835                         if(rc->delay_down_current > rc->delay_max_duration) rc->delay_down_current = rc->delay_max_duration;
2836                     }
2837
2838                     if(status > rc->status)
2839                         delay = rc->delay_up_current;
2840                     else
2841                         delay = rc->delay_down_current;
2842
2843                     // COMMENTED: because we do need to send raising alarms
2844                     // if(now + delay < rc->delay_up_to_timestamp)
2845                     //    delay = (int)(rc->delay_up_to_timestamp - now);
2846
2847                     rc->delay_last = delay;
2848                     rc->delay_up_to_timestamp = now + delay;
2849                     health_alarm_log(&localhost, rc->id, rc->next_event_id++, now, rc->name, rc->rrdset->id, rc->rrdset->family, rc->exec, rc->recipient, now - rc->last_status_change, rc->old_value, rc->value, rc->status, status, rc->source, rc->units, rc->info, rc->delay_last);
2850                     rc->last_status_change = now;
2851                     rc->status = status;
2852                 }
2853
2854                 rc->last_updated = now;
2855                 rc->next_update = now + rc->update_every;
2856
2857                 if (next_run > rc->next_update)
2858                     next_run = rc->next_update;
2859             }
2860
2861             rrdhost_unlock(&localhost);
2862         }
2863
2864         if (unlikely(pthread_setcancelstate(oldstate, NULL) != 0))
2865             error("Cannot set pthread cancel state to RESTORE (%d).", oldstate);
2866
2867         if(unlikely(netdata_exit))
2868             break;
2869
2870         // execute notifications
2871         // and cleanup
2872         health_alarm_log_process(&localhost);
2873
2874         if(unlikely(netdata_exit))
2875             break;
2876
2877         now = time(NULL);
2878         if(now < next_run) {
2879             debug(D_HEALTH, "Health monitoring iteration no %u done. Next iteration in %d secs",
2880                   loop, (int) (next_run - now));
2881             sleep_usec(1000000 * (unsigned long long) (next_run - now));
2882         }
2883         else {
2884             debug(D_HEALTH, "Health monitoring iteration no %u done. Next iteration now", loop);
2885         }
2886     }
2887
2888     buffer_free(wb);
2889
2890     info("HEALTH thread exiting");
2891     pthread_exit(NULL);
2892     return NULL;
2893 }