Add optional dumb-server mode

In dumb server mode, the server tells the client to load all .idx files
up front.  Puts the burden of deciding what .idxs a client should work
from more squarely in the server side.  This mode is activated by
putting a bup-dumb-server file in the bup repodir.

Signed-off-by: Brandon Low <lostlogic@lostlogicx.com>

diff --git a/Documentation/bup-server.md b/Documentation/bup-server.md
index a8c8a4c11d3086cd4af428f13c8de45ea2cc102b..8badd335cf4fb806e56ec01988f1afe4908a5f0c 100644 (file)
--- a/Documentation/bup-server.md
+++ b/Documentation/bup-server.md
@@ -19,6 +19,28 @@ server` to receive the transmitted objects.
 
 There is normally no reason to run `bup server` yourself.
 
+# MODES
+
+smart
+:   In this mode, the server checks each incoming object
+    against the idx files in its repository.  If any object
+    already exists, it tells the client about the idx file
+    it was found in, allowing the client to download that
+    idx and avoid sending duplicate data.
+
+dumb
+:   In this mode, the server will not check its local index
+    before writing an object.  To avoid writing duplicate
+    objects, the server will tell the client to download all
+    of its .idx files at the start of the session.  This
+    mode is useful on low powered server hardware (ie
+    router/slow NAS).
+
+# FILES
+
+$BUP_DIR/bup-dumb-server
+:   Activate dumb server mode, as discussed above.
+
 # SEE ALSO
 
 `bup-save`(1), `bup-split`(1)

diff --git a/cmd/server-cmd.py b/cmd/server-cmd.py
index e44f302c7d7a7c8332b13141a107374702d1c022..6ec2de3cb9a48a200899426d94b476774a87c0b6 100755 (executable)
--- a/cmd/server-cmd.py
+++ b/cmd/server-cmd.py
@@ -1,28 +1,43 @@
 #!/usr/bin/env python
-import sys, struct
+import os, sys, struct
 from bup import options, git
 from bup.helpers import *
 
 suspended_w = None
+server_mode = 'smart'
+
+def _set_mode():
+    global server_mode
+    if os.path.exists(git.repo('bup-dumb-server')):
+        server_mode = 'dumb'
+    else:
+        server_mode = 'smart'
+    debug1('bup server: serving in %s mode\n' % server_mode)
 
 
 def init_dir(conn, arg):
     git.init_repo(arg)
     debug1('bup server: bupdir initialized: %r\n' % git.repodir)
+    _set_mode()
     conn.ok()
 
 
 def set_dir(conn, arg):
     git.check_repo_or_die(arg)
     debug1('bup server: bupdir is %r\n' % git.repodir)
+    _set_mode()
     conn.ok()
 
     
 def list_indexes(conn, junk):
+    global server_mode
     git.check_repo_or_die()
+    suffix = ''
+    if server_mode == 'dumb':
+        suffix = ' load'
     for f in os.listdir(git.repo('objects/pack')):
         if f.endswith('.idx'):
-            conn.write('%s\n' % f)
+            conn.write('%s%s\n' % (f, suffix))
     conn.ok()
 
 
@@ -76,7 +91,10 @@ def receive_objects_v2(conn, junk):
             w.abort()
             raise Exception('object read: expected %d bytes, got %d\n'
                             % (n, len(buf)))
-        oldpack = w.exists(shar)
+        if server_mode == 'smart':
+            oldpack = w.exists(shar)
+        else:
+            oldpack = None
         # FIXME: we only suggest a single index per cycle, because the client
         # is currently too dumb to download more than one per cycle anyway.
         # Actually we should fix the client, but this is a minor optimization

diff --git a/lib/bup/client.py b/lib/bup/client.py
index c65ec0cef8caa9e6fbc90ade9446be4a91265dcb..d47cc7537f6711dfe16eda4460d5b943b1b09ae7 100644 (file)
--- a/lib/bup/client.py
+++ b/lib/bup/client.py
@@ -66,7 +66,7 @@ class Client:
             else:
                 self.conn.write('set-dir %s\n' % dir)
             self.check_ok()
-        self.sync_indexes_del()
+        self.sync_indexes()
 
     def __del__(self):
         try:
@@ -115,27 +115,38 @@ class Client:
     def _not_busy(self):
         self._busy = None
 
-    def sync_indexes_del(self):
+    def sync_indexes(self):
         self.check_busy()
         conn = self.conn
+        mkdirp(self.cachedir)
+        # All cached idxs are extra until proven otherwise
+        extra = set()
+        for f in os.listdir(self.cachedir):
+            debug1('%s\n' % f)
+            if f.endswith('.idx'):
+                extra.add(f)
+        needed = set()
         conn.write('list-indexes\n')
-        packdir = git.repo('objects/pack')
-        all = {}
-        needed = {}
         for line in linereader(conn):
             if not line:
                 break
-            all[line] = 1
             assert(line.find('/') < 0)
-            if not os.path.exists(os.path.join(self.cachedir, line)):
-                needed[line] = 1
-        self.check_ok()
+            parts = line.split(' ')
+            idx = parts[0]
+            if len(parts) == 2 and parts[1] == 'load' and idx not in extra:
+                # If the server requests that we load an idx and we don't
+                # already have a copy of it, it is needed
+                needed.add(idx)
+            # Any idx that the server has heard of is proven not extra
+            extra.discard(idx)
 
-        mkdirp(self.cachedir)
-        for f in os.listdir(self.cachedir):
-            if f.endswith('.idx') and not f in all:
-                debug1('client: pruning old index: %r\n' % f)
-                os.unlink(os.path.join(self.cachedir, f))
+        self.check_ok()
+        debug1('client: removing extra indexes: %s\n' % extra)
+        for idx in extra:
+            os.unlink(os.path.join(self.cachedir, idx))
+        debug1('client: server requested load of: %s\n' % needed)
+        for idx in needed:
+            self.sync_index(idx)
 
     def sync_index(self, name):
         #debug1('requesting %r\n' % name)

diff --git a/lib/bup/t/tclient.py b/lib/bup/t/tclient.py
index 280937112c8a07feaa08e60cf0fd334a2ea24a59..a3553b69a4b0dace317165808415efdfb50570cf 100644 (file)
--- a/lib/bup/t/tclient.py
+++ b/lib/bup/t/tclient.py
@@ -29,6 +29,28 @@ def test_server_split_with_indexes():
     rw.new_blob(s1)
     
 
+@wvtest
+def test_dumb_client_server():
+    os.environ['BUP_MAIN_EXE'] = '../../../bup'
+    os.environ['BUP_DIR'] = bupdir = 'buptest_tclient.tmp'
+    subprocess.call(['rm', '-rf', bupdir])
+    git.init_repo(bupdir)
+    os.mknod(git.repo('bup-dumb-server'))
+
+    lw = git.PackWriter()
+    lw.new_blob(s1)
+    lw.close()
+
+    c = client.Client(bupdir, create=True)
+    rw = c.new_packwriter()
+    WVPASSEQ(len(os.listdir(c.cachedir)), 1)
+    rw.new_blob(s1)
+    WVPASSEQ(len(os.listdir(c.cachedir)), 1)
+    rw.new_blob(s2)
+    rw.close()
+    WVPASSEQ(len(os.listdir(c.cachedir)), 2)
+
+
 @wvtest
 def test_midx_refreshing():
     os.environ['BUP_MAIN_EXE'] = bupmain = '../../../bup'