delete obsolete charts and orphan hosts by default

diff --git a/src/rrd.h b/src/rrd.h
index 8c6259890da9807fdc23e38adb63acb65cd09ca0..2b872ed7dccacb6b312fd4cbbe17622303f44834 100644 (file)
--- a/src/rrd.h
+++ b/src/rrd.h
@@ -339,6 +339,22 @@ typedef struct rrdset RRDSET;
     for(st = host->rrdset_root, rrdhost_check_wrlock(host); st ; st = st->next)
 
 
+// ----------------------------------------------------------------------------
+// RRDHOST flags
+// use this for configuration flags, not for state control
+// flags are set/unset in a manner that is not thread safe
+// and may lead to missing information.
+
+typedef enum rrdhost_flags {
+    RRDHOST_ORPHAN                 = 1 << 0, // this host is orphan
+    RRDHOST_DELETE_OBSOLETE_FILES  = 1 << 1, // delete files of obsolete charts
+    RRDHOST_DELETE_ORPHAN_FILES    = 1 << 2  // delete the entire host when orphan
+} RRDHOST_FLAGS;
+
+#define rrdhost_flag_check(host, flag) ((host)->flags & flag)
+#define rrdhost_flag_set(host, flag)   (host)->flags |= flag
+#define rrdhost_flag_clear(host, flag) (host)->flags &= ~flag
+
 // ----------------------------------------------------------------------------
 // RRD HOST
 
@@ -355,6 +371,9 @@ struct rrdhost {
     uint32_t hash_machine_guid;                     // the hash of the unique ID
 
     char *os;                                       // the O/S type of the host
+
+    uint32_t flags;                                 // flags about this RRDHOST
+
     int rrd_update_every;                           // the update frequency of the host
     int rrd_history_entries;                        // the number of history entries for the host's charts
     RRD_MEMORY_MODE rrd_memory_mode;                // the memory more for the charts of this host
@@ -524,9 +543,10 @@ extern RRDSET *rrdset_create(RRDHOST *host
 extern void rrdhost_free_all(void);
 extern void rrdhost_save_all(void);
 
-extern void rrdhost_cleanup_remote_stale(RRDHOST *protected);
+extern void rrdhost_cleanup_orphan(RRDHOST *protected);
 extern void rrdhost_free(RRDHOST *host);
 extern void rrdhost_save(RRDHOST *host);
+extern void rrdhost_delete(RRDHOST *host);
 
 extern RRDSET *rrdset_find(RRDHOST *host, const char *id);
 #define rrdset_find_localhost(id) rrdset_find(localhost, id)
@@ -601,14 +621,11 @@ extern long align_entries_to_pagesize(RRD_MEMORY_MODE mode, long entries);
 
 #ifdef NETDATA_RRD_INTERNALS
 
-extern void rrdset_free(RRDSET *st);
 extern avl_tree_lock rrdhost_root_index;
 
 extern char *rrdset_strncpyz_name(char *to, const char *from, size_t length);
 extern char *rrdset_cache_dir(RRDHOST *host, const char *id, const char *config_section);
 
-extern void rrdset_reset(RRDSET *st);
-
 extern void rrddim_free(RRDSET *st, RRDDIM *rd);
 
 extern int rrddim_compare(void* a, void* b);
@@ -623,8 +640,12 @@ extern void rrdfamily_free(RRDHOST *host, RRDFAMILY *rc);
 #define rrdset_index_del(host, st) (RRDSET *)avl_remove_lock(&((host)->rrdset_root_index), (avl *)(st))
 extern RRDSET *rrdset_index_del_name(RRDHOST *host, RRDSET *st);
 
+extern void rrdset_free(RRDSET *st);
+extern void rrdset_reset(RRDSET *st);
 extern void rrdset_save(RRDSET *st);
-extern void rrdhost_cleanup(RRDHOST *host);
+extern void rrdset_delete(RRDSET *st);
+
+extern void rrdhost_cleanup_obsolete(RRDHOST *host);
 
 #endif /* NETDATA_RRD_INTERNALS */
 

diff --git a/src/rrdhost.c b/src/rrdhost.c
index c5bc6bd778964c3d565a482fcb6ce7d91033b96b..576de2b16f8b9d13d26f15ad7db45024e4d64d8d 100644 (file)
--- a/src/rrdhost.c
+++ b/src/rrdhost.c
@@ -120,6 +120,13 @@ RRDHOST *rrdhost_create(const char *hostname,
     avl_init_lock(&(host->rrdfamily_root_index),   rrdfamily_compare);
     avl_init_lock(&(host->variables_root_index),   rrdvar_compare);
 
+    if(config_get_boolean(CONFIG_SECTION_GLOBAL, "delete obsolete charts files", 1))
+        rrdhost_flag_set(host, RRDHOST_DELETE_OBSOLETE_FILES);
+
+    if(config_get_boolean(CONFIG_SECTION_GLOBAL, "delete orphan hosts files", 1) && !is_localhost)
+        rrdhost_flag_set(host, RRDHOST_DELETE_ORPHAN_FILES);
+
+
     // ------------------------------------------------------------------------
     // initialize health variables
 
@@ -310,27 +317,32 @@ RRDHOST *rrdhost_find_or_create(
             error("Host '%s' has memory mode '%s', but the wanted one is '%s'.", host->hostname, rrd_memory_mode_name(host->rrd_memory_mode), rrd_memory_mode_name(mode));
     }
 
-    rrdhost_cleanup_remote_stale(host);
+    rrdhost_cleanup_orphan(host);
 
     return host;
 }
 
-void rrdhost_cleanup_remote_stale(RRDHOST *protected) {
+void rrdhost_cleanup_orphan(RRDHOST *protected) {
     time_t now = now_realtime_sec();
 
     rrd_wrlock();
 
-    RRDHOST *h;
+    RRDHOST *host;
 
 restart_after_removal:
-    rrdhost_foreach_write(h) {
-        if(h != protected
-           && h != localhost
-           && !h->connected_senders
-           && h->senders_disconnected_time + rrdhost_free_orphan_time < now) {
-            info("Host '%s' with machine guid '%s' is obsolete - cleaning up.", h->hostname, h->machine_guid);
-            rrdhost_save(h);
-            rrdhost_free(h);
+    rrdhost_foreach_write(host) {
+        if(host != protected
+           && host != localhost
+           && !host->connected_senders
+           && host->senders_disconnected_time + rrdhost_free_orphan_time < now) {
+            info("Host '%s' with machine guid '%s' is obsolete - cleaning up.", host->hostname, host->machine_guid);
+
+            if(rrdset_flag_check(host, RRDHOST_ORPHAN))
+                rrdhost_delete(host);
+            else
+                rrdhost_save(host);
+
+            rrdhost_free(host);
             goto restart_after_removal;
         }
     }
@@ -495,6 +507,29 @@ void rrdhost_save(RRDHOST *host) {
     rrdhost_unlock(host);
 }
 
+// ----------------------------------------------------------------------------
+// RRDHOST - delete files
+
+void rrdhost_delete(RRDHOST *host) {
+    if(!host) return;
+
+    info("Deleting database of host '%s'...", host->hostname);
+
+    RRDSET *st;
+
+    // we get a write lock
+    // to ensure only one thread is saving the database
+    rrdhost_wrlock(host);
+
+    rrdset_foreach_write(st, host) {
+        rrdset_rdlock(st);
+        rrdset_delete(st);
+        rrdset_unlock(st);
+    }
+
+    rrdhost_unlock(host);
+}
+
 void rrdhost_save_all(void) {
     info("Saving database [%zu hosts(s)]...", rrd_hosts_available);
 
@@ -507,7 +542,7 @@ void rrdhost_save_all(void) {
     rrd_unlock();
 }
 
-void rrdhost_cleanup(RRDHOST *host) {
+void rrdhost_cleanup_obsolete(RRDHOST *host) {
     time_t now = now_realtime_sec();
 
     RRDSET *st;
@@ -521,7 +556,12 @@ restart_after_removal:
         )) {
 
             rrdset_rdlock(st);
-            rrdset_save(st);
+
+            if(rrdhost_flag_check(host, RRDHOST_DELETE_OBSOLETE_FILES))
+                rrdset_delete(st);
+            else
+                rrdset_save(st);
+
             rrdset_unlock(st);
 
             rrdset_free(st);

diff --git a/src/rrdpush.c b/src/rrdpush.c
index 069fae467eb8c8bec06b8bd73533749e1ac5d8da..f5f54d39a320e8158b2b7df2d4921510a0a6c1bc 100644 (file)
--- a/src/rrdpush.c
+++ b/src/rrdpush.c
@@ -219,6 +219,8 @@ static void rrdpush_sender_thread_cleanup_locked_all(RRDHOST *host) {
     host->rrdpush_buffer = NULL;
 
     host->rrdpush_spawn = 0;
+
+    rrdhost_flag_set(host, RRDHOST_ORPHAN);
 }
 
 void rrdpush_sender_thread_stop(RRDHOST *host) {
@@ -645,6 +647,7 @@ void rrdpush_sender_thread_spawn(RRDHOST *host) {
         else if(pthread_detach(host->rrdpush_thread))
             error("STREAM %s [send]: cannot request detach newly created thread.", host->hostname);
 
+        rrdhost_flag_clear(host, RRDHOST_ORPHAN);
         host->rrdpush_spawn = 1;
     }
 

diff --git a/src/rrdset.c b/src/rrdset.c
index 63aeacebd04fb42d0e3ac06fc110f398616295be..1918b5cb2ee8e3e45df900f2c8eb51aa98682a58 100644 (file)
--- a/src/rrdset.c
+++ b/src/rrdset.c
@@ -305,6 +305,26 @@ void rrdset_save(RRDSET *st) {
     }
 }
 
+void rrdset_delete(RRDSET *st) {
+    RRDDIM *rd;
+
+    rrdset_check_rdlock(st);
+
+    // info("Deleting chart '%s' ('%s')", st->id, st->name);
+
+    if(st->rrd_memory_mode == RRD_MEMORY_MODE_SAVE) {
+        debug(D_RRD_STATS, "Deleting stats '%s' to '%s'.", st->name, st->cache_filename);
+        unlink(st->cache_filename);
+    }
+
+    rrddim_foreach_read(rd, st) {
+        if(likely(rd->rrd_memory_mode == RRD_MEMORY_MODE_SAVE)) {
+            debug(D_RRD_STATS, "Deleting dimension '%s' to '%s'.", rd->name, rd->cache_filename);
+            unlink(rd->cache_filename);
+        }
+    }
+}
+
 // ----------------------------------------------------------------------------
 // RRDSET - create a chart
 
@@ -535,7 +555,7 @@ RRDSET *rrdset_create(
     rrdsetcalc_link_matching(st);
     rrdcalctemplate_link_matching(st);
 
-    rrdhost_cleanup(host);
+    rrdhost_cleanup_obsolete(host);
 
     rrdhost_unlock(host);