save: make save tree collisions a deferred error

[bup.git] / cmd / save-cmd.py

diff --git a/cmd/save-cmd.py b/cmd/save-cmd.py
index 3f1649a5e74c87f875f96c91d4bbff76439554f2..fc93408c2a5dcc61923d2f7bc18356856c4fe926 100755 (executable)
--- a/cmd/save-cmd.py
+++ b/cmd/save-cmd.py
@@ -1,5 +1,8 @@
 #!/usr/bin/env python
 import sys, stat, time, math
+from cStringIO import StringIO
+from errno import EACCES
+
 from bup import hashsplit, git, options, index, client, metadata, hlinkdb
 from bup.helpers import *
 from bup.hashsplit import GIT_MODE_TREE, GIT_MODE_FILE, GIT_MODE_SYMLINK
@@ -71,9 +74,13 @@ if opt.name and opt.name.startswith('.'):
     o.fatal("'%s' is not a valid branch name" % opt.name)
 refname = opt.name and 'refs/heads/%s' % opt.name or None
 if opt.remote or is_reverse:
-    cli = client.Client(opt.remote)
+    try:
+        cli = client.Client(opt.remote)
+    except client.ClientError, e:
+        log('error: %s' % e)
+        sys.exit(1)
     oldref = refname and cli.read_ref(refname) or None
-    w = cli.new_packwriter()
+    w = cli.new_packwriter(compression_level=opt.compress)
 else:
     cli = None
     oldref = refname and git.read_ref(refname) or None
@@ -124,13 +131,32 @@ def _pop(force_tree, dir_metadata=None):
     part = parts.pop()
     shalist = shalists.pop()
     metalist = metalists.pop()
-    if metalist:
+    if metalist and not force_tree:
         if dir_metadata: # Override the original metadata pushed for this dir.
             metalist = [('', dir_metadata)] + metalist[1:]
         sorted_metalist = sorted(metalist, key = lambda x : x[0])
         metadata = ''.join([m[1].encode() for m in sorted_metalist])
-        shalist.append((0100644, '.bupm', w.new_blob(metadata)))
-    tree = force_tree or w.new_tree(shalist)
+        metadata_f = StringIO(metadata)
+        mode, id = hashsplit.split_to_blob_or_tree(w.new_blob, w.new_tree,
+                                                   [metadata_f],
+                                                   keep_boundaries=False)
+        shalist.append((mode, '.bupm', id))
+    # FIXME: only test if collision is possible (i.e. given --strip, etc.)?
+    if force_tree:
+        tree = force_tree
+    else:
+        names_seen = set()
+        clean_list = []
+        for x in shalist:
+            name = x[1]
+            if name in names_seen:
+                parent_path = '/'.join(parts) + '/'
+                add_error('error: ignoring duplicate path %r in %r'
+                          % (name, parent_path))
+            else:
+                names_seen.add(name)
+                clean_list.append(x)
+        tree = w.new_tree(clean_list)
     if shalists:
         shalists[-1].append((GIT_MODE_TREE,
                              git.mangle_name(part,
@@ -180,10 +206,13 @@ def progress_report(n):
 
 indexfile = opt.indexfile or git.repo('bupindex')
 r = index.Reader(indexfile)
-if not os.access(indexfile + '.meta', os.W_OK|os.R_OK):
-    log('error: cannot access "%s"; have you run bup index?' % indexfile)
+try:
+    msr = index.MetaStoreReader(indexfile + '.meta')
+except IOError, ex:
+    if ex.errno != EACCES:
+        raise
+    log('error: cannot access %r; have you run bup index?' % indexfile)
     sys.exit(1)
-msr = index.MetaStoreReader(indexfile + '.meta')
 hlink_db = hlinkdb.HLinkDB(indexfile + '.hlink')
 
 def already_saved(ent):
@@ -264,7 +293,8 @@ for (transname,ent) in r.filter(extra, wantrecurse=wantrecurse_during):
         continue
     if opt.smaller and ent.size >= opt.smaller:
         if exists and not hashvalid:
-            add_error('skipping large file "%s"' % ent.name)
+            if opt.verbose:
+                log('skipping large file "%s"\n' % ent.name)
             lastskip_name = ent.name
         continue
 
@@ -293,11 +323,7 @@ for (transname,ent) in r.filter(extra, wantrecurse=wantrecurse_during):
     # ...].
 
     if first_root == None:
-        dir_name, fs_path = dirp[0]
         first_root = dirp[0]
-        # Not indexed, so just grab the FS metadata or use empty metadata.
-        meta = metadata.from_path(fs_path) if fs_path else metadata.Metadata()
-        _push(dir_name, meta)
     elif first_root != dirp[0]:
         root_collision = True
 
@@ -309,7 +335,12 @@ for (transname,ent) in r.filter(extra, wantrecurse=wantrecurse_during):
     for path_component in dirp[len(parts):]:
         dir_name, fs_path = path_component
         # Not indexed, so just grab the FS metadata or use empty metadata.
-        meta = metadata.from_path(fs_path) if fs_path else metadata.Metadata()
+        try:
+           meta = metadata.from_path(fs_path) if fs_path else metadata.Metadata()
+        except (OSError, IOError), e:
+            add_error(e)
+            lastskip_name = dir_name
+            meta = metadata.Metadata()
         _push(dir_name, meta)
 
     if not file:
@@ -381,9 +412,14 @@ for (transname,ent) in r.filter(extra, wantrecurse=wantrecurse_during):
             shalists[-1].append(git_info)
             sort_key = git.shalist_item_sort_key((ent.mode, file, id))
             hlink = find_hardlink_target(hlink_db, ent)
-            metalists[-1].append((sort_key,
-                                  metadata.from_path(ent.name,
-                                                     hardlink_target=hlink)))
+            try:
+                meta = metadata.from_path(ent.name, hardlink_target=hlink)
+            except (OSError, IOError), e:
+                add_error(e)
+                lastskip_name = ent.name
+            else:
+                metalists[-1].append((sort_key, meta))
+
     if exists and wasmissing:
         count += oldsize
         subcount = 0
@@ -407,7 +443,7 @@ tree = _pop(force_tree = None,
 if opt.tree:
     print tree.encode('hex')
 if opt.commit or opt.name:
-    msg = 'bup save\n\nGenerated by command:\n%r' % sys.argv
+    msg = 'bup save\n\nGenerated by command:\n%r\n' % sys.argv
     commit = w.new_commit(oldref, tree, date, msg)
     if opt.commit:
         print commit.encode('hex')