src/health_log.c

   1 #define NETDATA_HEALTH_INTERNALS
   2 #include "common.h"
   3
   4 // ----------------------------------------------------------------------------
   5 // health alarm log load/save
   6 // no need for locking - only one thread is reading / writing the alarms log
   7
   8 inline int health_alarm_log_open(RRDHOST *host) {
   9     if(host->health_log_fp)
  10         fclose(host->health_log_fp);
  11
  12     host->health_log_fp = fopen(host->health_log_filename, "a");
  13
  14     if(host->health_log_fp) {
  15         if (setvbuf(host->health_log_fp, NULL, _IOLBF, 0) != 0)
  16             error("HEALTH [%s]: cannot set line buffering on health log file '%s'.", host->hostname, host->health_log_filename);
  17         return 0;
  18     }
  19
  20     error("HEALTH [%s]: cannot open health log file '%s'. Health data will be lost in case of netdata or server crash.", host->hostname, host->health_log_filename);
  21     return -1;
  22 }
  23
  24 inline void health_alarm_log_close(RRDHOST *host) {
  25     if(host->health_log_fp) {
  26         fclose(host->health_log_fp);
  27         host->health_log_fp = NULL;
  28     }
  29 }
  30
  31 inline void health_log_rotate(RRDHOST *host) {
  32     static size_t rotate_every = 0;
  33
  34     if(unlikely(rotate_every == 0)) {
  35         rotate_every = (size_t)config_get_number(CONFIG_SECTION_HEALTH, "rotate log every lines", 2000);
  36         if(rotate_every < 100) rotate_every = 100;
  37     }
  38
  39     if(unlikely(host->health_log_entries_written > rotate_every)) {
  40         health_alarm_log_close(host);
  41
  42         char old_filename[FILENAME_MAX + 1];
  43         snprintfz(old_filename, FILENAME_MAX, "%s.old", host->health_log_filename);
  44
  45         if(unlink(old_filename) == -1 && errno != ENOENT)
  46             error("HEALTH [%s]: cannot remove old alarms log file '%s'", host->hostname, old_filename);
  47
  48         if(link(host->health_log_filename, old_filename) == -1 && errno != ENOENT)
  49             error("HEALTH [%s]: cannot move file '%s' to '%s'.", host->hostname, host->health_log_filename, old_filename);
  50
  51         if(unlink(host->health_log_filename) == -1 && errno != ENOENT)
  52             error("HEALTH [%s]: cannot remove old alarms log file '%s'", host->hostname, host->health_log_filename);
  53
  54         // open it with truncate
  55         host->health_log_fp = fopen(host->health_log_filename, "w");
  56
  57         if(host->health_log_fp)
  58             fclose(host->health_log_fp);
  59         else
  60             error("HEALTH [%s]: cannot truncate health log '%s'", host->hostname, host->health_log_filename);
  61
  62         host->health_log_fp = NULL;
  63
  64         host->health_log_entries_written = 0;
  65         health_alarm_log_open(host);
  66     }
  67 }
  68
  69 inline void health_alarm_log_save(RRDHOST *host, ALARM_ENTRY *ae) {
  70     health_log_rotate(host);
  71
  72     if(likely(host->health_log_fp)) {
  73         if(unlikely(fprintf(host->health_log_fp
  74                             , "%c\t%s"
  75                         "\t%08x\t%08x\t%08x\t%08x\t%08x"
  76                         "\t%08x\t%08x\t%08x"
  77                         "\t%08x\t%08x\t%08x"
  78                         "\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s"
  79                         "\t%d\t%d\t%d\t%d"
  80                         "\t%Lf\t%Lf"
  81                         "\n"
  82                             , (ae->flags & HEALTH_ENTRY_FLAG_SAVED)?'U':'A'
  83                             , host->hostname
  84
  85                             , ae->unique_id
  86                             , ae->alarm_id
  87                             , ae->alarm_event_id
  88                             , ae->updated_by_id
  89                             , ae->updates_id
  90
  91                             , (uint32_t)ae->when
  92                             , (uint32_t)ae->duration
  93                             , (uint32_t)ae->non_clear_duration
  94                             , (uint32_t)ae->flags
  95                             , (uint32_t)ae->exec_run_timestamp
  96                             , (uint32_t)ae->delay_up_to_timestamp
  97
  98                             , (ae->name)?ae->name:""
  99                             , (ae->chart)?ae->chart:""
 100                             , (ae->family)?ae->family:""
 101                             , (ae->exec)?ae->exec:""
 102                             , (ae->recipient)?ae->recipient:""
 103                             , (ae->source)?ae->source:""
 104                             , (ae->units)?ae->units:""
 105                             , (ae->info)?ae->info:""
 106
 107                             , ae->exec_code
 108                             , ae->new_status
 109                             , ae->old_status
 110                             , ae->delay
 111
 112                             , (long double)ae->new_value
 113                             , (long double)ae->old_value
 114         ) < 0))
 115             error("HEALTH [%s]: failed to save alarm log entry to '%s'. Health data may be lost in case of abnormal restart.", host->hostname, host->health_log_filename);
 116         else {
 117             ae->flags |= HEALTH_ENTRY_FLAG_SAVED;
 118             host->health_log_entries_written++;
 119         }
 120     }
 121 }
 122
 123 inline ssize_t health_alarm_log_read(RRDHOST *host, FILE *fp, const char *filename) {
 124     static uint32_t max_unique_id = 0, max_alarm_id = 0;
 125
 126     errno = 0;
 127
 128     char *s, *buf = mallocz(65536 + 1);
 129     size_t line = 0, len = 0;
 130     ssize_t loaded = 0, updated = 0, errored = 0, duplicate = 0;
 131
 132     pthread_rwlock_rdlock(&host->health_log.alarm_log_rwlock);
 133
 134     while((s = fgets_trim_len(buf, 65536, fp, &len))) {
 135         host->health_log_entries_written++;
 136         line++;
 137
 138         int max_entries = 30, entries = 0;
 139         char *pointers[max_entries];
 140
 141         pointers[entries++] = s++;
 142         while(*s) {
 143             if(unlikely(*s == '\t')) {
 144                 *s = '\0';
 145                 pointers[entries++] = ++s;
 146                 if(entries >= max_entries) {
 147                     error("HEALTH [%s]: line %zu of file '%s' has more than %d entries. Ignoring excessive entries.", host->hostname, line, filename, max_entries);
 148                     break;
 149                 }
 150             }
 151             else s++;
 152         }
 153
 154         if(likely(*pointers[0] == 'U' || *pointers[0] == 'A')) {
 155             ALARM_ENTRY *ae = NULL;
 156
 157             if(entries < 26) {
 158                 error("HEALTH [%s]: line %zu of file '%s' should have at least 26 entries, but it has %d. Ignoring it.", host->hostname, line, filename, entries);
 159                 errored++;
 160                 continue;
 161             }
 162
 163             // check that we have valid ids
 164             uint32_t unique_id = (uint32_t)strtoul(pointers[2], NULL, 16);
 165             if(!unique_id) {
 166                 error("HEALTH [%s]: line %zu of file '%s' states alarm entry with invalid unique id %u (%s). Ignoring it.", host->hostname, line, filename, unique_id, pointers[2]);
 167                 errored++;
 168                 continue;
 169             }
 170
 171             uint32_t alarm_id = (uint32_t)strtoul(pointers[3], NULL, 16);
 172             if(!alarm_id) {
 173                 error("HEALTH [%s]: line %zu of file '%s' states alarm entry for invalid alarm id %u (%s). Ignoring it.", host->hostname, line, filename, alarm_id, pointers[3]);
 174                 errored++;
 175                 continue;
 176             }
 177
 178             if(unlikely(*pointers[0] == 'A')) {
 179                 // make sure it is properly numbered
 180                 if(unlikely(host->health_log.alarms && unique_id < host->health_log.alarms->unique_id)) {
 181                     error("HEALTH [%s]: line %zu of file '%s' has alarm log entry %u in wrong order. Ignoring it.", host->hostname, line, filename, unique_id);
 182                     errored++;
 183                     continue;
 184                 }
 185
 186                 ae = callocz(1, sizeof(ALARM_ENTRY));
 187             }
 188             else if(unlikely(*pointers[0] == 'U')) {
 189                 // find the original
 190                 for(ae = host->health_log.alarms; ae; ae = ae->next) {
 191                     if(unlikely(unique_id == ae->unique_id)) {
 192                         if(unlikely(*pointers[0] == 'A')) {
 193                             error("HEALTH [%s]: line %zu of file '%s' adds duplicate alarm log entry %u. Using the later."
 194                                   , host->hostname, line, filename, unique_id);
 195                             *pointers[0] = 'U';
 196                             duplicate++;
 197                         }
 198                         break;
 199                     }
 200                     else if(unlikely(unique_id > ae->unique_id)) {
 201                         // no need to continue
 202                         // the linked list is sorted
 203                         ae = NULL;
 204                         break;
 205                     }
 206                 }
 207             }
 208
 209             // if not found, skip this line
 210             if(unlikely(!ae)) {
 211                 // error("HEALTH [%s]: line %zu of file '%s' updates alarm log entry with unique id %u, but it is not found.", host->hostname, line, filename, unique_id);
 212                 continue;
 213             }
 214
 215             // check for a possible host missmatch
 216             //if(strcmp(pointers[1], host->hostname))
 217             //    error("HEALTH [%s]: line %zu of file '%s' provides an alarm for host '%s' but this is named '%s'.", host->hostname, line, filename, pointers[1], host->hostname);
 218
 219             ae->unique_id               = unique_id;
 220             ae->alarm_id                = alarm_id;
 221             ae->alarm_event_id          = (uint32_t)strtoul(pointers[4], NULL, 16);
 222             ae->updated_by_id           = (uint32_t)strtoul(pointers[5], NULL, 16);
 223             ae->updates_id              = (uint32_t)strtoul(pointers[6], NULL, 16);
 224
 225             ae->when                    = (uint32_t)strtoul(pointers[7], NULL, 16);
 226             ae->duration                = (uint32_t)strtoul(pointers[8], NULL, 16);
 227             ae->non_clear_duration      = (uint32_t)strtoul(pointers[9], NULL, 16);
 228
 229             ae->flags                   = (uint32_t)strtoul(pointers[10], NULL, 16);
 230             ae->flags |= HEALTH_ENTRY_FLAG_SAVED;
 231
 232             ae->exec_run_timestamp      = (uint32_t)strtoul(pointers[11], NULL, 16);
 233             ae->delay_up_to_timestamp   = (uint32_t)strtoul(pointers[12], NULL, 16);
 234
 235             freez(ae->name);
 236             ae->name = strdupz(pointers[13]);
 237             ae->hash_name = simple_hash(ae->name);
 238
 239             freez(ae->chart);
 240             ae->chart = strdupz(pointers[14]);
 241             ae->hash_chart = simple_hash(ae->chart);
 242
 243             freez(ae->family);
 244             ae->family = strdupz(pointers[15]);
 245
 246             freez(ae->exec);
 247             ae->exec = strdupz(pointers[16]);
 248             if(!*ae->exec) { freez(ae->exec); ae->exec = NULL; }
 249
 250             freez(ae->recipient);
 251             ae->recipient = strdupz(pointers[17]);
 252             if(!*ae->recipient) { freez(ae->recipient); ae->recipient = NULL; }
 253
 254             freez(ae->source);
 255             ae->source = strdupz(pointers[18]);
 256             if(!*ae->source) { freez(ae->source); ae->source = NULL; }
 257
 258             freez(ae->units);
 259             ae->units = strdupz(pointers[19]);
 260             if(!*ae->units) { freez(ae->units); ae->units = NULL; }
 261
 262             freez(ae->info);
 263             ae->info = strdupz(pointers[20]);
 264             if(!*ae->info) { freez(ae->info); ae->info = NULL; }
 265
 266             ae->exec_code   = str2i(pointers[21]);
 267             ae->new_status  = str2i(pointers[22]);
 268             ae->old_status  = str2i(pointers[23]);
 269             ae->delay       = str2i(pointers[24]);
 270
 271             ae->new_value   = str2l(pointers[25]);
 272             ae->old_value   = str2l(pointers[26]);
 273
 274             static char value_string[100 + 1];
 275             freez(ae->old_value_string);
 276             freez(ae->new_value_string);
 277             ae->old_value_string = strdupz(format_value_and_unit(value_string, 100, ae->old_value, ae->units, -1));
 278             ae->new_value_string = strdupz(format_value_and_unit(value_string, 100, ae->new_value, ae->units, -1));
 279
 280             // add it to host if not already there
 281             if(unlikely(*pointers[0] == 'A')) {
 282                 ae->next = host->health_log.alarms;
 283                 host->health_log.alarms = ae;
 284                 loaded++;
 285             }
 286             else updated++;
 287
 288             if(unlikely(ae->unique_id > max_unique_id))
 289                 max_unique_id = ae->unique_id;
 290
 291             if(unlikely(ae->alarm_id >= max_alarm_id))
 292                 max_alarm_id = ae->alarm_id;
 293         }
 294         else {
 295             error("HEALTH [%s]: line %zu of file '%s' is invalid (unrecognized entry type '%s').", host->hostname, line, filename, pointers[0]);
 296             errored++;
 297         }
 298     }
 299
 300     pthread_rwlock_unlock(&host->health_log.alarm_log_rwlock);
 301
 302     freez(buf);
 303
 304     if(!max_unique_id) max_unique_id = (uint32_t)now_realtime_sec();
 305     if(!max_alarm_id)  max_alarm_id  = (uint32_t)now_realtime_sec();
 306
 307     host->health_log.next_log_id = max_unique_id + 1;
 308     host->health_log.next_alarm_id = max_alarm_id + 1;
 309
 310     debug(D_HEALTH, "HEALTH [%s]: loaded file '%s' with %zd new alarm entries, updated %zd alarms, errors %zd entries, duplicate %zd", host->hostname, filename, loaded, updated, errored, duplicate);
 311     return loaded;
 312 }
 313
 314 inline void health_alarm_log_load(RRDHOST *host) {
 315     health_alarm_log_close(host);
 316
 317     char filename[FILENAME_MAX + 1];
 318     snprintfz(filename, FILENAME_MAX, "%s.old", host->health_log_filename);
 319     FILE *fp = fopen(filename, "r");
 320     if(!fp)
 321         error("HEALTH [%s]: cannot open health file: %s", host->hostname, filename);
 322     else {
 323         health_alarm_log_read(host, fp, filename);
 324         fclose(fp);
 325     }
 326
 327     host->health_log_entries_written = 0;
 328     fp = fopen(host->health_log_filename, "r");
 329     if(!fp)
 330         error("HEALTH [%s]: cannot open health file: %s", host->hostname, host->health_log_filename);
 331     else {
 332         health_alarm_log_read(host, fp, host->health_log_filename);
 333         fclose(fp);
 334     }
 335
 336     health_alarm_log_open(host);
 337 }
 338
 339
 340 // ----------------------------------------------------------------------------
 341 // health alarm log management
 342
 343 inline void health_alarm_log(
 344         RRDHOST *host,
 345         uint32_t alarm_id,
 346         uint32_t alarm_event_id,
 347         time_t when,
 348         const char *name,
 349         const char *chart,
 350         const char *family,
 351         const char *exec,
 352         const char *recipient,
 353         time_t duration,
 354         calculated_number old_value,
 355         calculated_number new_value,
 356         int old_status,
 357         int new_status,
 358         const char *source,
 359         const char *units,
 360         const char *info,
 361         int delay,
 362         uint32_t flags
 363 ) {
 364     debug(D_HEALTH, "Health adding alarm log entry with id: %u", host->health_log.next_log_id);
 365
 366     ALARM_ENTRY *ae = callocz(1, sizeof(ALARM_ENTRY));
 367     ae->name = strdupz(name);
 368     ae->hash_name = simple_hash(ae->name);
 369
 370     if(chart) {
 371         ae->chart = strdupz(chart);
 372         ae->hash_chart = simple_hash(ae->chart);
 373     }
 374
 375     if(family)
 376         ae->family = strdupz(family);
 377
 378     if(exec) ae->exec = strdupz(exec);
 379     if(recipient) ae->recipient = strdupz(recipient);
 380     if(source) ae->source = strdupz(source);
 381     if(units) ae->units = strdupz(units);
 382     if(info) ae->info = strdupz(info);
 383
 384     ae->unique_id = host->health_log.next_log_id++;
 385     ae->alarm_id = alarm_id;
 386     ae->alarm_event_id = alarm_event_id;
 387     ae->when = when;
 388     ae->old_value = old_value;
 389     ae->new_value = new_value;
 390
 391     static char value_string[100 + 1];
 392     ae->old_value_string = strdupz(format_value_and_unit(value_string, 100, ae->old_value, ae->units, -1));
 393     ae->new_value_string = strdupz(format_value_and_unit(value_string, 100, ae->new_value, ae->units, -1));
 394
 395     ae->old_status = old_status;
 396     ae->new_status = new_status;
 397     ae->duration = duration;
 398     ae->delay = delay;
 399     ae->delay_up_to_timestamp = when + delay;
 400
 401     ae->flags |= flags;
 402
 403     if(ae->old_status == RRDCALC_STATUS_WARNING || ae->old_status == RRDCALC_STATUS_CRITICAL)
 404         ae->non_clear_duration += ae->duration;
 405
 406     // link it
 407     pthread_rwlock_wrlock(&host->health_log.alarm_log_rwlock);
 408     ae->next = host->health_log.alarms;
 409     host->health_log.alarms = ae;
 410     host->health_log.count++;
 411     pthread_rwlock_unlock(&host->health_log.alarm_log_rwlock);
 412
 413     // match previous alarms
 414     pthread_rwlock_rdlock(&host->health_log.alarm_log_rwlock);
 415     ALARM_ENTRY *t;
 416     for(t = host->health_log.alarms ; t ; t = t->next) {
 417         if(t != ae && t->alarm_id == ae->alarm_id) {
 418             if(!(t->flags & HEALTH_ENTRY_FLAG_UPDATED) && !t->updated_by_id) {
 419                 t->flags |= HEALTH_ENTRY_FLAG_UPDATED;
 420                 t->updated_by_id = ae->unique_id;
 421                 ae->updates_id = t->unique_id;
 422
 423                 if((t->new_status == RRDCALC_STATUS_WARNING || t->new_status == RRDCALC_STATUS_CRITICAL) &&
 424                    (t->old_status == RRDCALC_STATUS_WARNING || t->old_status == RRDCALC_STATUS_CRITICAL))
 425                     ae->non_clear_duration += t->non_clear_duration;
 426
 427                 health_alarm_log_save(host, t);
 428             }
 429
 430             // no need to continue
 431             break;
 432         }
 433     }
 434     pthread_rwlock_unlock(&host->health_log.alarm_log_rwlock);
 435
 436     health_alarm_log_save(host, ae);
 437 }
 438
 439 inline void health_alarm_log_free_one_nochecks_nounlink(ALARM_ENTRY *ae) {
 440     freez(ae->name);
 441     freez(ae->chart);
 442     freez(ae->family);
 443     freez(ae->exec);
 444     freez(ae->recipient);
 445     freez(ae->source);
 446     freez(ae->units);
 447     freez(ae->info);
 448     freez(ae->old_value_string);
 449     freez(ae->new_value_string);
 450     freez(ae);
 451 }
 452
 453 inline void health_alarm_log_free(RRDHOST *host) {
 454     rrdhost_check_wrlock(host);
 455
 456     pthread_rwlock_wrlock(&host->health_log.alarm_log_rwlock);
 457
 458     ALARM_ENTRY *ae;
 459     while((ae = host->health_log.alarms)) {
 460         host->health_log.alarms = ae->next;
 461         health_alarm_log_free_one_nochecks_nounlink(ae);
 462     }
 463
 464     pthread_rwlock_unlock(&host->health_log.alarm_log_rwlock);
 465 }