Add VFS Node release() and release nodes during restore, after traversal.

Add a release() method to Node that will drop resources that can (and
will) be automatically restored when required -- though restoring the
resources may have a non-trivial cost.  For now, drop the node's
metadata and its children.

Call node.release() in restore-cmd.py after restoring a node.  This
substantially decreases the memory required by a restore because the
whole tree is no longer retained in RAM.

Thanks to Patrick Rouleau <prouleau72@gmail.com> for helping track
down the problem, and proposing a slightly different initial patch to
fix it.

Signed-off-by: Rob Browning <rlb@defaultvalue.org>
Reviewed-by: Patrick Rouleau <prouleau72@gmail.com>

diff --git a/cmd/restore-cmd.py b/cmd/restore-cmd.py
index 158a707a8532210fb7246b6a1ad6de67816c852a..d1dc1cf6b3a186ecc67f9761584bbbab6bd56ddb 100755 (executable)
--- a/cmd/restore-cmd.py
+++ b/cmd/restore-cmd.py
@@ -265,6 +265,7 @@ def do_node(top, n, owner_map, meta = None):
     finally:
         if meta_stream:
             meta_stream.close()
+        n.release()
 
 
 handle_ctrl_c()

diff --git a/lib/bup/vfs.py b/lib/bup/vfs.py
index e1de1fe6bb19d12caab951de3884b668cc164cbd..07c5e372feb35f0ceb70d83ae12b2453ddb6db50 100644 (file)
--- a/lib/bup/vfs.py
+++ b/lib/bup/vfs.py
@@ -163,7 +163,7 @@ class _FileReader(object):
         pass
 
 
-class Node:
+class Node(object):
     """Base class for file representation."""
     def __init__(self, parent, name, mode, hash):
         self.parent = parent
@@ -300,16 +300,21 @@ class Node:
         """Open the current node. It is an error to open a non-file node."""
         raise NotFile('%s is not a regular file' % self.name)
 
-    def _populate_metadata(self):
+    def _populate_metadata(self, force=False):
         # Only Dirs contain .bupm files, so by default, do nothing.
         pass
 
     def metadata(self):
         """Return this Node's Metadata() object, if any."""
-        if self.parent:
-            self.parent._populate_metadata()
+        if not self._metadata and self.parent:
+            self.parent._populate_metadata(force=True)
         return self._metadata
 
+    def release(self):
+        """Release resources that can be automatically restored (at a cost)."""
+        self._metadata = None
+        self._subs = None
+
 
 class File(Node):
     """A normal file from bup's repository."""
@@ -401,8 +406,8 @@ class Dir(Node):
         Node.__init__(self, *args, **kwargs)
         self._bupm = None
 
-    def _populate_metadata(self):
-        if self._metadata:
+    def _populate_metadata(self, force=False):
+        if self._metadata and not force:
             return
         if not self._subs:
             self._mksubs()
@@ -452,6 +457,11 @@ class Dir(Node):
             self._mksubs()
         return self._bupm
 
+    def release(self):
+        """Release restorable resources held by this node."""
+        self._bupm = None
+        super(Dir, self).release()
+
 
 class CommitDir(Node):
     """A directory that contains all commits that are reachable by a ref.