cmd/split: add a --git-ids option.

[bup.git] / cmd / split-cmd.py

diff --git a/cmd/split-cmd.py b/cmd/split-cmd.py
index e8df4d30c66551b2b0984967ddf5c619641a4167..2a72bd673fb912d47b89f8713578342c0d5849b2 100755 (executable)
--- a/cmd/split-cmd.py
+++ b/cmd/split-cmd.py
@@ -1,8 +1,7 @@
 #!/usr/bin/env python
 #!/usr/bin/env python

-import sys, time, struct
+import sys, time

 from bup import hashsplit, git, options, client
 from bup.helpers import *
 from bup import hashsplit, git, options, client
 from bup.helpers import *

-from subprocess import PIPE

 
 
 optspec = """
 
 
 optspec = """


@@ -13,25 +12,32 @@ b,blobs    output a series of blob ids
 t,tree     output a tree id
 c,commit   output a commit id
 n,name=    name of backup set to update (if any)
 t,tree     output a tree id
 c,commit   output a commit id
 n,name=    name of backup set to update (if any)

-N,noop     don't actually save the data anywhere
+d,date=    date for the commit (seconds since the epoch)

 q,quiet    don't print progress messages
 v,verbose  increase log output (can be used more than once)
 q,quiet    don't print progress messages
 v,verbose  increase log output (can be used more than once)

+git-ids    read a list of git object ids from stdin and split their contents
+keep-boundaries  don't let one chunk span two input files
+noop       don't actually save the data anywhere

 copy       just copy input to output, hashsplitting along the way
 bench      print benchmark timings to stderr
 max-pack-size=  maximum bytes in a single pack
 max-pack-objects=  maximum number of objects in a single pack
 copy       just copy input to output, hashsplitting along the way
 bench      print benchmark timings to stderr
 max-pack-size=  maximum bytes in a single pack
 max-pack-objects=  maximum number of objects in a single pack

-fanout=  maximum number of blobs in a single tree
+fanout=    maximum number of blobs in a single tree
+bwlimit=   maximum bytes/sec to transmit to server

 """
 o = options.Options('bup split', optspec)
 (opt, flags, extra) = o.parse(sys.argv[1:])
 
 """
 o = options.Options('bup split', optspec)
 (opt, flags, extra) = o.parse(sys.argv[1:])
 

+handle_ctrl_c()

 git.check_repo_or_die()
 if not (opt.blobs or opt.tree or opt.commit or opt.name or
         opt.noop or opt.copy):
     o.fatal("use one or more of -b, -t, -c, -n, -N, --copy")
 if (opt.noop or opt.copy) and (opt.blobs or opt.tree or 
                                opt.commit or opt.name):
 git.check_repo_or_die()
 if not (opt.blobs or opt.tree or opt.commit or opt.name or
         opt.noop or opt.copy):
     o.fatal("use one or more of -b, -t, -c, -n, -N, --copy")
 if (opt.noop or opt.copy) and (opt.blobs or opt.tree or 
                                opt.commit or opt.name):

-    o.fatal('-N is incompatible with -b, -t, -c, -n')
+    o.fatal('-N and --copy are incompatible with -b, -t, -c, -n')
+if extra and opt.git_ids:
+    o.fatal("don't provide filenames when using --git-ids")

 
 if opt.verbose >= 2:
     git.verbose = opt.verbose - 1
 
 if opt.verbose >= 2:
     git.verbose = opt.verbose - 1


@@ -44,6 +50,13 @@ if opt.fanout:
     hashsplit.fanout = parse_num(opt.fanout)
 if opt.blobs:
     hashsplit.fanout = 0
     hashsplit.fanout = parse_num(opt.fanout)
 if opt.blobs:
     hashsplit.fanout = 0

+if opt.bwlimit:
+    client.bwlimit = parse_num(opt.bwlimit)
+if opt.date:
+    date = parse_date_or_fatal(opt.date, o.fatal)
+else:
+    date = time.time()
+

 
 is_reverse = os.environ.get('BUP_SERVER_REVERSE')
 if is_reverse and opt.remote:
 
 is_reverse = os.environ.get('BUP_SERVER_REVERSE')
 if is_reverse and opt.remote:


@@ -52,23 +65,64 @@ start_time = time.time()
 
 refname = opt.name and 'refs/heads/%s' % opt.name or None
 if opt.noop or opt.copy:
 
 refname = opt.name and 'refs/heads/%s' % opt.name or None
 if opt.noop or opt.copy:

-    cli = w = oldref = None
+    cli = pack_writer = oldref = None

 elif opt.remote or is_reverse:
 elif opt.remote or is_reverse:

-    cli = client.Client(opt.remote)
+    if opt.remote and opt.remote.find(":") == -1:
+        o.fatal("--remote argument must contain a colon")
+    try:
+        cli = client.Client(opt.remote)
+    except client.ClientError:
+        o.fatal("server exited unexpectedly; see errors above")

     oldref = refname and cli.read_ref(refname) or None
     oldref = refname and cli.read_ref(refname) or None

-    w = cli.new_packwriter()
+    pack_writer = cli.new_packwriter()

 else:
     cli = None
     oldref = refname and git.read_ref(refname) or None
 else:
     cli = None
     oldref = refname and git.read_ref(refname) or None

-    w = git.PackWriter()
+    pack_writer = git.PackWriter()
+
+if opt.git_ids:
+    # the input is actually a series of git object ids that we should retrieve
+    # and split.
+    #
+    # This is a bit messy, but basically it converts from a series of
+    # CatPipe.get() iterators into a series of file-type objects.
+    # It would be less ugly if either CatPipe.get() returned a file-like object
+    # (not very efficient), or split_to_shalist() expected an iterator instead
+    # of a file.
+    cp = git.CatPipe()
+    class IterToFile:
+        def __init__(self, it):
+            self.it = iter(it)
+        def read(self, size):
+            v = next(self.it)
+            return v or ''
+    def read_ids():
+        while 1:
+            line = sys.stdin.readline()
+            if not line:
+                break
+            if line:
+                line = line.strip()
+            try:
+                it = cp.get(line.strip())
+                next(it)  # skip the file type
+            except KeyError, e:
+                add_error('error: %s' % e)
+                continue
+            yield IterToFile(it)
+    files = read_ids()
+else:
+    # the input either comes from a series of files or from stdin.
+    files = extra and (open(fn) for fn in extra) or [sys.stdin]

 
 

-files = extra and (open(fn) for fn in extra) or [sys.stdin]
-if w:
-    shalist = hashsplit.split_to_shalist(w, files)
-    tree = w.new_tree(shalist)
+if pack_writer:
+    shalist = hashsplit.split_to_shalist(pack_writer, files,
+                                         keep_boundaries=opt.keep_boundaries)
+    tree = pack_writer.new_tree(shalist)

 else:
     last = 0
 else:
     last = 0

-    for (blob, bits) in hashsplit.hashsplit_iter(files):
+    for (blob, bits) in hashsplit.hashsplit_iter(files,
+                                    keep_boundaries=opt.keep_boundaries):

         hashsplit.total_split += len(blob)
         if opt.copy:
             sys.stdout.write(str(blob))
         hashsplit.total_split += len(blob)
         if opt.copy:
             sys.stdout.write(str(blob))


@@ -88,13 +142,13 @@ if opt.tree:
 if opt.commit or opt.name:
     msg = 'bup split\n\nGenerated by command:\n%r' % sys.argv
     ref = opt.name and ('refs/heads/%s' % opt.name) or None
 if opt.commit or opt.name:
     msg = 'bup split\n\nGenerated by command:\n%r' % sys.argv
     ref = opt.name and ('refs/heads/%s' % opt.name) or None

-    commit = w.new_commit(oldref, tree, msg)
+    commit = pack_writer.new_commit(oldref, tree, date, msg)

     if opt.commit:
         print commit.encode('hex')
 
     if opt.commit:
         print commit.encode('hex')
 

-if w:
-    w.close()  # must close before we can update the ref
-        
+if pack_writer:
+    pack_writer.close()  # must close before we can update the ref
+

 if opt.name:
     if cli:
         cli.update_ref(refname, commit, oldref)
 if opt.name:
     if cli:
         cli.update_ref(refname, commit, oldref)


@@ -109,3 +163,7 @@ size = hashsplit.total_split
 if opt.bench:
     log('\nbup: %.2fkbytes in %.2f secs = %.2f kbytes/sec\n'
         % (size/1024., secs, size/1024./secs))
 if opt.bench:
     log('\nbup: %.2fkbytes in %.2f secs = %.2f kbytes/sec\n'
         % (size/1024., secs, size/1024./secs))

+
+if saved_errors:
+    log('WARNING: %d errors encountered while saving.\n' % len(saved_errors))
+    sys.exit(1)