X-Git-Url: https://arthur.barton.de/gitweb/?a=blobdiff_plain;f=src%2Frrd.h;h=bc642d80bbe77c596e255fd7919070aa5704c6cc;hb=2065f19101bdfd8d83ec0335edf9b04a2b4c7d1f;hp=ad75c3d6a11fc34f4320832c8a91adedc1846d4b;hpb=35c1abbdc2833089667ffde39813d103e65dd54d;p=netdata.git diff --git a/src/rrd.h b/src/rrd.h index ad75c3d6..bc642d80 100644 --- a/src/rrd.h +++ b/src/rrd.h @@ -7,8 +7,8 @@ #define RRD_DEFAULT_HISTORY_ENTRIES 3600 #define RRD_HISTORY_ENTRIES_MAX (86400*10) -extern int default_localhost_rrd_update_every; -extern int default_localhost_rrd_history_entries; +extern int default_rrd_update_every; +extern int default_rrd_history_entries; #define RRD_ID_LENGTH_MAX 200 @@ -39,16 +39,18 @@ const char *rrdset_type_name(RRDSET_TYPE chart_type); // memory mode typedef enum rrd_memory_mode { - RRD_MEMORY_MODE_RAM = 0, - RRD_MEMORY_MODE_MAP = 1, - RRD_MEMORY_MODE_SAVE = 2 + RRD_MEMORY_MODE_NONE = 0, + RRD_MEMORY_MODE_RAM = 1, + RRD_MEMORY_MODE_MAP = 2, + RRD_MEMORY_MODE_SAVE = 3 } RRD_MEMORY_MODE; +#define RRD_MEMORY_MODE_NONE_NAME "none" #define RRD_MEMORY_MODE_RAM_NAME "ram" #define RRD_MEMORY_MODE_MAP_NAME "map" #define RRD_MEMORY_MODE_SAVE_NAME "save" -extern RRD_MEMORY_MODE default_localhost_rrd_memory_mode; +extern RRD_MEMORY_MODE default_rrd_memory_mode; extern const char *rrd_memory_mode_name(RRD_MEMORY_MODE id); extern RRD_MEMORY_MODE rrd_memory_mode_id(const char *name); @@ -72,20 +74,6 @@ typedef enum rrd_algorithm { extern RRD_ALGORITHM rrd_algorithm_id(const char *name); extern const char *rrd_algorithm_name(RRD_ALGORITHM algorithm); -// ---------------------------------------------------------------------------- -// flags - -typedef enum rrddim_flags { - RRDDIM_FLAG_HIDDEN = 1 << 0, // this dimension will not be offered to callers - RRDDIM_FLAG_DONT_DETECT_RESETS_OR_OVERFLOWS = 1 << 1, // do not offer RESET or OVERFLOW info to callers - RRDDIM_FLAG_UPDATED = 1 << 31 // the dimension has been updated since the last processing -} RRDDIM_FLAGS; - -#define rrddim_flag_check(rd, flag) ((rd)->flags & flag) -#define rrddim_flag_set(rd, flag) (rd)->flags |= flag -#define rrddim_flag_clear(rd, flag) (rd)->flags &= ~flag - - // ---------------------------------------------------------------------------- // RRD FAMILY @@ -102,6 +90,22 @@ struct rrdfamily { typedef struct rrdfamily RRDFAMILY; +// ---------------------------------------------------------------------------- +// flags +// use this for configuration flags, not for state control +// flags are set/unset in a manner that is not thread safe +// and may lead to missing information. + +typedef enum rrddim_flags { + RRDDIM_FLAG_HIDDEN = 1 << 0, // this dimension will not be offered to callers + RRDDIM_FLAG_DONT_DETECT_RESETS_OR_OVERFLOWS = 1 << 1 // do not offer RESET or OVERFLOW info to callers +} RRDDIM_FLAGS; + +#define rrddim_flag_check(rd, flag) ((rd)->flags & flag) +#define rrddim_flag_set(rd, flag) (rd)->flags |= flag +#define rrddim_flag_clear(rd, flag) (rd)->flags &= ~flag + + // ---------------------------------------------------------------------------- // RRD DIMENSION - this is a metric @@ -127,7 +131,7 @@ struct rrddim { collected_number multiplier; // the multiplier of the collected values collected_number divisor; // the divider of the collected values - uint32_t flags; // options and status options for the dimension + uint32_t flags; // configuration flags for the dimension // ------------------------------------------------------------------------ // members for temporary data we need for calculations @@ -140,7 +144,11 @@ struct rrddim { char *cache_filename; // the filename we load/save from/to this set - size_t counter; // the number of times we added values to this rrdim + size_t collections_counter; // the number of times we added values to this rrdim + size_t unused[10]; + + int updated:1; // 1 when the dimension has been updated since the last processing + int exposed:1; // 1 when set what have sent this dimension to the central netdata struct timeval last_collected_time; // when was this dimension last updated // this is actual date time we updated the last_collected_value @@ -184,15 +192,29 @@ struct rrddim { }; typedef struct rrddim RRDDIM; +// ---------------------------------------------------------------------------- +// these loop macros make sure the linked list is accessed with the right lock + +#define rrddim_foreach_read(rd, st) \ + for(rd = st->dimensions, rrdset_check_rdlock(st); rd ; rd = rd->next) + +#define rrddim_foreach_write(rd, st) \ + for(rd = st->dimensions, rrdset_check_wrlock(st); rd ; rd = rd->next) + // ---------------------------------------------------------------------------- // RRDSET - this is a chart +// use this for configuration flags, not for state control +// flags are set/unset in a manner that is not thread safe +// and may lead to missing information. + typedef enum rrdset_flags { - RRDSET_FLAG_ENABLED = 1 << 0, // enables or disables a chart - RRDSET_FLAG_DETAIL = 1 << 1, // if set, the data set should be considered as a detail of another - // (the master data set should be the one that has the same family and is not detail) - RRDSET_FLAG_DEBUG = 1 << 2 // enables or disables debugging for a chart + RRDSET_FLAG_ENABLED = 1 << 0, // enables or disables a chart + RRDSET_FLAG_DETAIL = 1 << 1, // if set, the data set should be considered as a detail of another + // (the master data set should be the one that has the same family and is not detail) + RRDSET_FLAG_DEBUG = 1 << 2, // enables or disables debugging for a chart + RRDSET_FLAG_OBSOLETE = 1 << 3 // this is marked by the collector/module as obsolete } RRDSET_FLAGS; #define rrdset_flag_check(st, flag) ((st)->flags & flag) @@ -235,7 +257,7 @@ struct rrdset { long current_entry; // the entry that is currently being updated // it goes around in a round-robin fashion - uint32_t flags; + uint32_t flags; // configuration flags int gap_when_lost_iterations_above; // after how many lost iterations a gap should be stored // netdata will interpolate values for gaps lower than this @@ -251,10 +273,13 @@ struct rrdset { char *cache_dir; // the directory to store dimensions char cache_filename[FILENAME_MAX+1]; // the filename to store this set - pthread_rwlock_t rrdset_rwlock; + pthread_rwlock_t rrdset_rwlock; // protects dimensions linked list - unsigned long counter; // the number of times we added values to this rrd - unsigned long counter_done; // the number of times we added values to this rrd + size_t counter; // the number of times we added values to this database + size_t counter_done; // the number of times rrdset_done() has been called + + time_t last_accessed_time; // the last time this RRDSET has been accessed + size_t unused[9]; uint32_t hash; // a simple hash on the id, to speed up searching // we first compare hashes, and only if the hashes are equal we do string comparisons @@ -300,9 +325,31 @@ struct rrdset { }; typedef struct rrdset RRDSET; -#define rrdset_rdlock(st) pthread_rwlock_rdlock(&((st)->rrdset_rwlock)) -#define rrdset_wrlock(st) pthread_rwlock_wrlock(&((st)->rrdset_rwlock)) -#define rrdset_unlock(st) pthread_rwlock_unlock(&((st)->rrdset_rwlock)) +// ---------------------------------------------------------------------------- +// these loop macros make sure the linked list is accessed with the right lock + +#define rrdset_foreach_read(st, host) \ + for(st = host->rrdset_root, rrdhost_check_rdlock(host); st ; st = st->next) + +#define rrdset_foreach_write(st, host) \ + for(st = host->rrdset_root, rrdhost_check_wrlock(host); st ; st = st->next) + + +// ---------------------------------------------------------------------------- +// RRDHOST flags +// use this for configuration flags, not for state control +// flags are set/unset in a manner that is not thread safe +// and may lead to missing information. + +typedef enum rrdhost_flags { + RRDHOST_ORPHAN = 1 << 0, // this host is orphan + RRDHOST_DELETE_OBSOLETE_FILES = 1 << 1, // delete files of obsolete charts + RRDHOST_DELETE_ORPHAN_FILES = 1 << 2 // delete the entire host when orphan +} RRDHOST_FLAGS; + +#define rrdhost_flag_check(host, flag) ((host)->flags & flag) +#define rrdhost_flag_set(host, flag) (host)->flags |= flag +#define rrdhost_flag_clear(host, flag) (host)->flags &= ~flag // ---------------------------------------------------------------------------- // RRD HOST @@ -310,30 +357,61 @@ typedef struct rrdset RRDSET; struct rrdhost { avl avl; // the index of hosts + // ------------------------------------------------------------------------ + // host information + char *hostname; // the hostname of this host uint32_t hash_hostname; // the hostname hash char machine_guid[GUID_LEN + 1]; // the unique ID of this host uint32_t hash_machine_guid; // the hash of the unique ID + char *os; // the O/S type of the host + + uint32_t flags; // flags about this RRDHOST + int rrd_update_every; // the update frequency of the host int rrd_history_entries; // the number of history entries for the host's charts - - int health_enabled; // 1 when this host has health enabled RRD_MEMORY_MODE rrd_memory_mode; // the memory more for the charts of this host - RRDSET *rrdset_root; // the host charts + char *cache_dir; // the directory to save RRD cache files + char *varlib_dir; // the directory to save health log - pthread_rwlock_t rrdhost_rwlock; // lock for this RRDHOST - avl_tree_lock rrdset_root_index; // the host's charts index (by id) - avl_tree_lock rrdset_root_index_name; // the host's charts index (by name) + // ------------------------------------------------------------------------ + // streaming of data to remote hosts - rrdpush - avl_tree_lock rrdfamily_root_index; // the host's chart families index - avl_tree_lock variables_root_index; // the host's chart variables index + int rrdpush_enabled:1; // 1 when this host sends metrics to another netdata + char *rrdpush_destination; // where to send metrics to + char *rrdpush_api_key; // the api key at the receiving netdata + volatile int rrdpush_connected:1; // 1 when the sender is ready to push metrics + volatile int rrdpush_spawn:1; // 1 when the sender thread has been spawn + volatile int rrdpush_error_shown:1; // 1 when we have logged a communication error + int rrdpush_socket; // the fd of the socket to the remote host, or -1 + pthread_t rrdpush_thread; // the sender thread + pthread_mutex_t rrdpush_mutex; // exclusive access to rrdpush_buffer + int rrdpush_pipe[2]; // collector to sender thread communication + BUFFER *rrdpush_buffer; // collector fills it, sender sends them - char *cache_dir; // the directory to save RRD cache files - char *varlib_dir; // the directory to save health log + + // ------------------------------------------------------------------------ + // streaming of data from remote hosts - rrdpush + + volatile size_t connected_senders; // when remote hosts are streaming to this + // host, this is the counter of connected clients + + time_t senders_disconnected_time; // the time the last sender was disconnected + + // ------------------------------------------------------------------------ + // health monitoring options + + int health_enabled:1; // 1 when this host has health enabled + time_t health_delay_up_to; // a timestamp to delay alarms processing up to + char *health_default_exec; // the full path of the alarms notifications program + char *health_default_recipient; // the default recipient for all alarms + char *health_log_filename; // the alarms event log filename + size_t health_log_entries_written; // the number of alarm events writtern to the alarms event log + FILE *health_log_fp; // the FILE pointer to the open alarms event log file // all RRDCALCs are primarily allocated and linked here // RRDCALCs may be linked to charts at any point @@ -347,38 +425,137 @@ struct rrdhost { // are created or renamed, that match them RRDCALCTEMPLATE *templates; - // health / alarm settings - char *health_default_exec; - char *health_default_recipient; - char *health_log_filename; - size_t health_log_entries_written; - FILE *health_log_fp; + + // ------------------------------------------------------------------------ + // the charts of the host + + RRDSET *rrdset_root; // the host charts + + + // ------------------------------------------------------------------------ + // locks + + pthread_rwlock_t rrdhost_rwlock; // lock for this RRDHOST (protects rrdset_root linked list) + + avl_tree_lock rrdset_root_index; // the host's charts index (by id) + avl_tree_lock rrdset_root_index_name; // the host's charts index (by name) + + avl_tree_lock rrdfamily_root_index; // the host's chart families index + avl_tree_lock variables_root_index; // the host's chart variables index struct rrdhost *next; }; typedef struct rrdhost RRDHOST; extern RRDHOST *localhost; -#define rrdhost_rdlock(h) pthread_rwlock_rdlock(&((h)->rrdhost_rwlock)) -#define rrdhost_wrlock(h) pthread_rwlock_wrlock(&((h)->rrdhost_rwlock)) -#define rrdhost_unlock(h) pthread_rwlock_unlock(&((h)->rrdhost_rwlock)) +static inline void rrdhost_rdlock(RRDHOST *host) { + if(unlikely(pthread_rwlock_rdlock(&host->rrdhost_rwlock) != 0)) + error("Cannot obtain read lock on host '%s'", host->hostname); +} + +static inline void rrdhost_wrlock(RRDHOST *host) { + if(unlikely(pthread_rwlock_wrlock(&host->rrdhost_rwlock) != 0)) + error("Cannot obtain write lock on host '%s'", host->hostname); +} + +static inline void rrdhost_unlock(RRDHOST *host) { + if(unlikely(pthread_rwlock_unlock(&host->rrdhost_rwlock) != 0)) + error("Cannot unlock host '%s'", host->hostname); +} + + +// ---------------------------------------------------------------------------- +// these loop macros make sure the linked list is accessed with the right lock + +#define rrdhost_foreach_read(var) \ + for(var = localhost, rrd_check_rdlock(); var ; var = var->next) + +#define rrdhost_foreach_write(var) \ + for(var = localhost, rrd_check_wrlock(); var ; var = var->next) + + +// ---------------------------------------------------------------------------- +// global lock for all RRDHOSTs + +extern pthread_rwlock_t rrd_rwlock; + +static inline void rrd_rdlock() { + if(unlikely(pthread_rwlock_rdlock(&rrd_rwlock) != 0)) + error("Cannot read lock the RRD database."); +} + +static inline void rrd_wrlock() { + if(unlikely(pthread_rwlock_wrlock(&rrd_rwlock) != 0)) + error("Cannot write lock the RRD database."); +} + +static inline void rrd_unlock() { + if(unlikely(pthread_rwlock_unlock(&rrd_rwlock) != 0)) + error("Cannot unlock the RRD database."); +} + +static inline void rrdset_rdlock(RRDSET *st) { + if(unlikely(pthread_rwlock_rdlock(&st->rrdset_rwlock) != 0)) + error("Cannot read lock RRDSET '%s' of host '%s'", st->id, st->rrdhost->hostname); +} + +static inline void rrdset_wrlock(RRDSET *st) { + if(unlikely(pthread_rwlock_wrlock(&st->rrdset_rwlock) != 0)) + error("Cannot write lock RRDSET '%s' of host '%s'", st->id, st->rrdhost->hostname); +} + +static inline void rrdset_unlock(RRDSET *st) { + if(unlikely(pthread_rwlock_unlock(&st->rrdset_rwlock) != 0)) + error("Cannot unlock RRDSET '%s' of host '%s'", st->id, st->rrdhost->hostname); +} + +// ---------------------------------------------------------------------------- + +extern size_t rrd_hosts_available; +extern time_t rrdhost_free_orphan_time; extern void rrd_init(char *hostname); -extern RRDHOST *rrdhost_find(const char *guid, uint32_t hash); -extern RRDHOST *rrdhost_find_or_create(const char *hostname, const char *guid); +extern RRDHOST *rrdhost_find_by_hostname(const char *hostname, uint32_t hash); +extern RRDHOST *rrdhost_find_by_guid(const char *guid, uint32_t hash); + +extern RRDHOST *rrdhost_find_or_create( + const char *hostname + , const char *guid + , const char *os + , int update_every + , int history + , RRD_MEMORY_MODE mode + , int health_enabled + , int rrdpush_enabled + , char *rrdpush_destination + , char *rrdpush_api_key +); #ifdef NETDATA_INTERNAL_CHECKS +extern void rrdhost_check_wrlock_int(RRDHOST *host, const char *file, const char *function, const unsigned long line); +extern void rrdhost_check_rdlock_int(RRDHOST *host, const char *file, const char *function, const unsigned long line); +extern void rrdset_check_rdlock_int(RRDSET *st, const char *file, const char *function, const unsigned long line); +extern void rrdset_check_wrlock_int(RRDSET *st, const char *file, const char *function, const unsigned long line); +extern void rrd_check_rdlock_int(const char *file, const char *function, const unsigned long line); +extern void rrd_check_wrlock_int(const char *file, const char *function, const unsigned long line); + #define rrdhost_check_rdlock(host) rrdhost_check_rdlock_int(host, __FILE__, __FUNCTION__, __LINE__) #define rrdhost_check_wrlock(host) rrdhost_check_wrlock_int(host, __FILE__, __FUNCTION__, __LINE__) +#define rrdset_check_rdlock(st) rrdset_check_rdlock_int(st, __FILE__, __FUNCTION__, __LINE__) +#define rrdset_check_wrlock(st) rrdset_check_wrlock_int(st, __FILE__, __FUNCTION__, __LINE__) +#define rrd_check_rdlock() rrd_check_rdlock_int(__FILE__, __FUNCTION__, __LINE__) +#define rrd_check_wrlock() rrd_check_wrlock_int(__FILE__, __FUNCTION__, __LINE__) + #else #define rrdhost_check_rdlock(host) (void)0 #define rrdhost_check_wrlock(host) (void)0 +#define rrdset_check_rdlock(host) (void)0 +#define rrdset_check_wrlock(host) (void)0 +#define rrd_check_rdlock() (void)0 +#define rrd_check_wrlock() (void)0 #endif -extern void rrdhost_check_wrlock_int(RRDHOST *host, const char *file, const char *function, const unsigned long line); -extern void rrdhost_check_rdlock_int(RRDHOST *host, const char *file, const char *function, const unsigned long line); - // ---------------------------------------------------------------------------- // RRDSET functions @@ -401,8 +578,10 @@ extern RRDSET *rrdset_create(RRDHOST *host extern void rrdhost_free_all(void); extern void rrdhost_save_all(void); +extern void rrdhost_cleanup_orphan(RRDHOST *protected); extern void rrdhost_free(RRDHOST *host); extern void rrdhost_save(RRDHOST *host); +extern void rrdhost_delete(RRDHOST *host); extern RRDSET *rrdset_find(RRDHOST *host, const char *id); #define rrdset_find_localhost(id) rrdset_find(localhost, id) @@ -417,7 +596,10 @@ extern void rrdset_next_usec_unfiltered(RRDSET *st, usec_t microseconds); extern void rrdset_next_usec(RRDSET *st, usec_t microseconds); #define rrdset_next(st) rrdset_next_usec(st, 0ULL) -extern usec_t rrdset_done(RRDSET *st); +extern void rrdset_done(RRDSET *st); + +// checks if the RRDSET should be offered to viewers +#define rrdset_is_available_for_viewers(st) (rrdset_flag_check(st, RRDSET_FLAG_ENABLED) && !rrdset_flag_check(st, RRDSET_FLAG_OBSOLETE) && (st)->dimensions) // get the total duration in seconds of the round robin database #define rrdset_duration(st) ((time_t)( (((st)->counter >= ((unsigned long)(st)->entries))?(unsigned long)(st)->entries:(st)->counter) * (st)->update_every )) @@ -466,20 +648,19 @@ extern int rrddim_unhide(RRDSET *st, const char *id); extern collected_number rrddim_set_by_pointer(RRDSET *st, RRDDIM *rd, collected_number value); extern collected_number rrddim_set(RRDSET *st, const char *id, collected_number value); +extern long align_entries_to_pagesize(RRD_MEMORY_MODE mode, long entries); + // ---------------------------------------------------------------------------- // RRD internal functions #ifdef NETDATA_RRD_INTERNALS -extern void rrdset_free(RRDSET *st); extern avl_tree_lock rrdhost_root_index; extern char *rrdset_strncpyz_name(char *to, const char *from, size_t length); extern char *rrdset_cache_dir(RRDHOST *host, const char *id, const char *config_section); -extern void rrdset_reset(RRDSET *st); - extern void rrddim_free(RRDSET *st, RRDDIM *rd); extern int rrddim_compare(void* a, void* b); @@ -494,6 +675,13 @@ extern void rrdfamily_free(RRDHOST *host, RRDFAMILY *rc); #define rrdset_index_del(host, st) (RRDSET *)avl_remove_lock(&((host)->rrdset_root_index), (avl *)(st)) extern RRDSET *rrdset_index_del_name(RRDHOST *host, RRDSET *st); +extern void rrdset_free(RRDSET *st); +extern void rrdset_reset(RRDSET *st); +extern void rrdset_save(RRDSET *st); +extern void rrdset_delete(RRDSET *st); + +extern void rrdhost_cleanup_obsolete(RRDHOST *host); + #endif /* NETDATA_RRD_INTERNALS */