chmod -x lib/bup/cmd/*.py

[bup.git] / lib / bup / cmd / save.py

diff --git a/lib/bup/cmd/save.py b/lib/bup/cmd/save.py
old mode 100755 (executable)
new mode 100644 (file)
index 21a49ad..88f74d7
--- a/lib/bup/cmd/save.py
+++ b/lib/bup/cmd/save.py
@@ -1,13 +1,13 @@
 
 from __future__ import absolute_import, print_function
 from binascii import hexlify
 
 from __future__ import absolute_import, print_function
 from binascii import hexlify

-from errno import EACCES
+from errno import ENOENT

 from io import BytesIO
 import math, os, stat, sys, time
 
 from bup import compat, hashsplit, git, options, index, client, metadata
 from bup import hlinkdb
 from io import BytesIO
 import math, os, stat, sys, time
 
 from bup import compat, hashsplit, git, options, index, client, metadata
 from bup import hlinkdb

-from bup.compat import argv_bytes, environ
+from bup.compat import argv_bytes, environ, nullcontext

 from bup.hashsplit import GIT_MODE_TREE, GIT_MODE_FILE, GIT_MODE_SYMLINK
 from bup.helpers import (add_error, grafted_path_components, handle_ctrl_c,
                          hostname, istty2, log, parse_date_or_fatal, parse_num,
 from bup.hashsplit import GIT_MODE_TREE, GIT_MODE_FILE, GIT_MODE_SYMLINK
 from bup.helpers import (add_error, grafted_path_components, handle_ctrl_c,
                          hostname, istty2, log, parse_date_or_fatal, parse_num,


@@ -16,6 +16,7 @@ from bup.helpers import (add_error, grafted_path_components, handle_ctrl_c,
                          valid_save_name)
 from bup.io import byte_stream, path_msg
 from bup.pwdgrp import userfullname, username
                          valid_save_name)
 from bup.io import byte_stream, path_msg
 from bup.pwdgrp import userfullname, username

+from bup.tree import StackDir

 
 
 optspec = """
 
 
 optspec = """


@@ -37,11 +38,16 @@ graft=     a graft point *old_path*=*new_path* (can be used more than once)
 #,compress=  set compression level to # (0-9, 9 is highest) [1]
 """
 
 #,compress=  set compression level to # (0-9, 9 is highest) [1]
 """
 

-def main(argv):

 
 

-    # Hack around lack of nonlocal vars in python 2
-    _nonlocal = {}
+### Test hooks
+
+after_nondir_metadata_stat = None

 
 

+def before_saving_regular_file(name):
+    return
+
+
+def opts_from_cmdline(argv):

     o = options.Options(optspec)
     opt, flags, extra = o.parse_bytes(argv[1:])
 
     o = options.Options(optspec)
     opt, flags, extra = o.parse_bytes(argv[1:])
 


@@ -53,29 +59,27 @@ def main(argv):
         opt.remote = argv_bytes(opt.remote)
     if opt.strip_path:
         opt.strip_path = argv_bytes(opt.strip_path)
         opt.remote = argv_bytes(opt.remote)
     if opt.strip_path:
         opt.strip_path = argv_bytes(opt.strip_path)

-
-    git.check_repo_or_die()

     if not (opt.tree or opt.commit or opt.name):
         o.fatal("use one or more of -t, -c, -n")
     if not extra:
         o.fatal("no filenames given")
     if not (opt.tree or opt.commit or opt.name):
         o.fatal("use one or more of -t, -c, -n")
     if not extra:
         o.fatal("no filenames given")

-
-    extra = [argv_bytes(x) for x in extra]
+    if opt.date:
+        opt.date = parse_date_or_fatal(opt.date, o.fatal)
+    else:
+        opt.date = time.time()

 
     opt.progress = (istty2 and not opt.quiet)
     opt.smaller = parse_num(opt.smaller or 0)
 
     opt.progress = (istty2 and not opt.quiet)
     opt.smaller = parse_num(opt.smaller or 0)

-    if opt.bwlimit:
-        client.bwlimit = parse_num(opt.bwlimit)

 
 

-    if opt.date:
-        date = parse_date_or_fatal(opt.date, o.fatal)
-    else:
-        date = time.time()
+    if opt.bwlimit:
+        opt.bwlimit = parse_num(opt.bwlimit)

 
     if opt.strip and opt.strip_path:
         o.fatal("--strip is incompatible with --strip-path")
 
 
     if opt.strip and opt.strip_path:
         o.fatal("--strip is incompatible with --strip-path")
 

-    graft_points = []
+    opt.sources = [argv_bytes(x) for x in extra]
+
+    grafts = []

     if opt.graft:
         if opt.strip:
             o.fatal("--strip is incompatible with --graft")
     if opt.graft:
         if opt.strip:
             o.fatal("--strip is incompatible with --graft")


@@ -92,33 +96,20 @@ def main(argv):
                 old_path, new_path = splitted_parameter
                 if not (old_path and new_path):
                     o.fatal("a graft point cannot be empty")
                 old_path, new_path = splitted_parameter
                 if not (old_path and new_path):
                     o.fatal("a graft point cannot be empty")

-                graft_points.append((resolve_parent(old_path),
-                                     resolve_parent(new_path)))
+                grafts.append((resolve_parent(old_path),
+                               resolve_parent(new_path)))
+    opt.grafts = grafts

 
 

-    is_reverse = environ.get(b'BUP_SERVER_REVERSE')
-    if is_reverse and opt.remote:
+    opt.is_reverse = environ.get(b'BUP_SERVER_REVERSE')
+    if opt.is_reverse and opt.remote:

         o.fatal("don't use -r in reverse mode; it's automatic")
 
         o.fatal("don't use -r in reverse mode; it's automatic")
 

-    name = opt.name
-    if name and not valid_save_name(name):
-        o.fatal("'%s' is not a valid branch name" % path_msg(name))
-    refname = name and b'refs/heads/%s' % name or None
-    if opt.remote or is_reverse:
-        try:
-            cli = client.Client(opt.remote)
-        except client.ClientError as e:
-            log('error: %s' % e)
-            sys.exit(1)
-        oldref = refname and cli.read_ref(refname) or None
-        w = cli.new_packwriter(compression_level=opt.compress)
-    else:
-        cli = None
-        oldref = refname and git.read_ref(refname) or None
-        w = git.PackWriter(compression_level=opt.compress)
-
-    handle_ctrl_c()
+    if opt.name and not valid_save_name(opt.name):
+        o.fatal("'%s' is not a valid branch name" % path_msg(opt.name))

 
 

+    return opt

 
 

+def save_tree(opt, reader, hlink_db, msr, w):

     # Metadata is stored in a file named .bupm in each directory.  The
     # first metadata entry will be the metadata for the current directory.
     # The remaining entries will be for each of the other directory
     # Metadata is stored in a file named .bupm in each directory.  The
     # first metadata entry will be the metadata for the current directory.
     # The remaining entries will be for each of the other directory


@@ -135,68 +126,64 @@ def main(argv):
 
     # Maintain a stack of information representing the current location in
     # the archive being constructed.  The current path is recorded in
 
     # Maintain a stack of information representing the current location in
     # the archive being constructed.  The current path is recorded in

-    # parts, which will be something like ['', 'home', 'someuser'], and
-    # the accumulated content and metadata for of the dirs in parts is
-    # stored in parallel stacks in shalists and metalists.
-
-    parts = [] # Current archive position (stack of dir names).
-    shalists = [] # Hashes for each dir in paths.
-    metalists = [] # Metadata for each dir in paths.
+    # parts, which will be something like
+    #      [StackDir(name=''), StackDir(name='home'), StackDir(name='someuser')],
+    # and the accumulated content and metadata for files in the dirs is stored
+    # in the .items member of the StackDir.

 
 

+    stack = []

 
     def _push(part, metadata):
         # Enter a new archive directory -- make it the current directory.
 
     def _push(part, metadata):
         # Enter a new archive directory -- make it the current directory.

-        parts.append(part)
-        shalists.append([])
-        metalists.append([(b'', metadata)]) # This dir's metadata (no name).
+        item = StackDir(part, metadata)
+        stack.append(item)

 
 
 
 

-    def _pop(force_tree, dir_metadata=None):
+    def _pop(force_tree=None, dir_metadata=None):

         # Leave the current archive directory and add its tree to its parent.
         # Leave the current archive directory and add its tree to its parent.

-        assert(len(parts) >= 1)
-        part = parts.pop()
-        shalist = shalists.pop()
-        metalist = metalists.pop()
+        item = stack.pop()

         # FIXME: only test if collision is possible (i.e. given --strip, etc.)?
         if force_tree:
             tree = force_tree
         else:
             names_seen = set()
             clean_list = []
         # FIXME: only test if collision is possible (i.e. given --strip, etc.)?
         if force_tree:
             tree = force_tree
         else:
             names_seen = set()
             clean_list = []

-            metaidx = 1 # entry at 0 is for the dir
-            for x in shalist:
-                name = x[1]
+            for x in item.items:
+                name = x.name

                 if name in names_seen:
                 if name in names_seen:

-                    parent_path = b'/'.join(parts) + b'/'
+                    parent_path = b'/'.join(x.name for x in stack) + b'/'

                     add_error('error: ignoring duplicate path %s in %s'
                               % (path_msg(name), path_msg(parent_path)))
                     add_error('error: ignoring duplicate path %s in %s'
                               % (path_msg(name), path_msg(parent_path)))

-                    if not stat.S_ISDIR(x[0]):
-                        del metalist[metaidx]

                 else:
                     names_seen.add(name)
                     clean_list.append(x)
                 else:
                     names_seen.add(name)
                     clean_list.append(x)

-                    if not stat.S_ISDIR(x[0]):
-                        metaidx += 1
-
-            if dir_metadata: # Override the original metadata pushed for this dir.
-                metalist = [(b'', dir_metadata)] + metalist[1:]
-            sorted_metalist = sorted(metalist, key = lambda x : x[0])
-            metadata = b''.join([m[1].encode() for m in sorted_metalist])
-            metadata_f = BytesIO(metadata)
+
+            # if set, overrides the original metadata pushed for this dir.
+            if dir_metadata is None:
+                dir_metadata = item.meta
+            metalist = [(b'', dir_metadata)]
+            metalist += [(git.shalist_item_sort_key((entry.mode, entry.name, None)),
+                          entry.meta)
+                         for entry in clean_list if entry.mode != GIT_MODE_TREE]
+            metalist.sort(key = lambda x: x[0])
+            metadata = BytesIO(b''.join(m[1].encode() for m in metalist))

             mode, id = hashsplit.split_to_blob_or_tree(w.new_blob, w.new_tree,
             mode, id = hashsplit.split_to_blob_or_tree(w.new_blob, w.new_tree,

-                                                       [metadata_f],
+                                                       [metadata],

                                                        keep_boundaries=False)
                                                        keep_boundaries=False)

-            clean_list.append((mode, b'.bupm', id))
-
-            tree = w.new_tree(clean_list)
-        if shalists:
-            shalists[-1].append((GIT_MODE_TREE,
-                                 git.mangle_name(part,
-                                                 GIT_MODE_TREE, GIT_MODE_TREE),
-                                 tree))
+            shalist = [(mode, b'.bupm', id)]
+            shalist += [(entry.gitmode,
+                         git.mangle_name(entry.name, entry.mode, entry.gitmode),
+                         entry.oid)
+                        for entry in clean_list]
+
+            tree = w.new_tree(shalist)
+        if stack:
+            stack[-1].append(item.name, GIT_MODE_TREE, GIT_MODE_TREE, tree, None)

         return tree
 
 
         return tree
 
 

+    # Hack around lack of nonlocal vars in python 2
+    _nonlocal = {}

     _nonlocal['count'] = 0
     _nonlocal['subcount'] = 0
     _nonlocal['lastremain'] = None
     _nonlocal['count'] = 0
     _nonlocal['subcount'] = 0
     _nonlocal['lastremain'] = None


@@ -238,18 +225,6 @@ def main(argv):
                      remainstr, kpsstr))
 
 
                      remainstr, kpsstr))
 
 

-    indexfile = opt.indexfile or git.repo(b'bupindex')
-    r = index.Reader(indexfile)
-    try:
-        msr = index.MetaStoreReader(indexfile + b'.meta')
-    except IOError as ex:
-        if ex.errno != EACCES:
-            raise
-        log('error: cannot access %r; have you run bup index?'
-            % path_msg(indexfile))
-        sys.exit(1)
-    hlink_db = hlinkdb.HLinkDB(indexfile + b'.hlink')
-

     def already_saved(ent):
         return ent.is_valid() and w.exists(ent.sha) and ent.sha
 
     def already_saved(ent):
         return ent.is_valid() and w.exists(ent.sha) and ent.sha
 


@@ -264,10 +239,12 @@ def main(argv):
             link_paths = hlink_db.node_paths(ent.dev, ent.ino)
             if link_paths:
                 return link_paths[0]
             link_paths = hlink_db.node_paths(ent.dev, ent.ino)
             if link_paths:
                 return link_paths[0]

+        return None

 
     total = ftotal = 0
     if opt.progress:
 
     total = ftotal = 0
     if opt.progress:

-        for (transname,ent) in r.filter(extra, wantrecurse=wantrecurse_pre):
+        for transname, ent in reader.filter(opt.sources,
+                                            wantrecurse=wantrecurse_pre):

             if not (ftotal % 10024):
                 qprogress('Reading index: %d\r' % ftotal)
             exists = ent.exists()
             if not (ftotal % 10024):
                 qprogress('Reading index: %d\r' % ftotal)
             exists = ent.exists()


@@ -297,7 +274,8 @@ def main(argv):
     fcount = 0
     lastskip_name = None
     lastdir = b''
     fcount = 0
     lastskip_name = None
     lastdir = b''

-    for (transname,ent) in r.filter(extra, wantrecurse=wantrecurse_during):
+    for transname, ent in reader.filter(opt.sources,
+                                        wantrecurse=wantrecurse_during):

         (dir, file) = os.path.split(ent.name)
         exists = (ent.flags & index.IX_EXISTS)
         hashvalid = already_saved(ent)
         (dir, file) = os.path.split(ent.name)
         exists = (ent.flags & index.IX_EXISTS)
         hashvalid = already_saved(ent)


@@ -335,11 +313,11 @@ def main(argv):
 
         assert(dir.startswith(b'/'))
         if opt.strip:
 
         assert(dir.startswith(b'/'))
         if opt.strip:

-            dirp = stripped_path_components(dir, extra)
+            dirp = stripped_path_components(dir, opt.sources)

         elif opt.strip_path:
             dirp = stripped_path_components(dir, [opt.strip_path])
         elif opt.strip_path:
             dirp = stripped_path_components(dir, [opt.strip_path])

-        elif graft_points:
-            dirp = grafted_path_components(graft_points, dir)
+        elif opt.grafts:
+            dirp = grafted_path_components(opt.grafts, dir)

         else:
             dirp = path_components(dir)
 
         else:
             dirp = path_components(dir)
 


@@ -363,11 +341,11 @@ def main(argv):
             root_collision = True
 
         # If switching to a new sub-tree, finish the current sub-tree.
             root_collision = True
 
         # If switching to a new sub-tree, finish the current sub-tree.

-        while parts > [x[0] for x in dirp]:
-            _pop(force_tree = None)
+        while [x.name for x in stack] > [x[0] for x in dirp]:
+            _pop()

 
         # If switching to a new sub-tree, start a new sub-tree.
 
         # If switching to a new sub-tree, start a new sub-tree.

-        for path_component in dirp[len(parts):]:
+        for path_component in dirp[len(stack):]:

             dir_name, fs_path = path_component
             # Not indexed, so just grab the FS metadata or use empty metadata.
             try:
             dir_name, fs_path = path_component
             # Not indexed, so just grab the FS metadata or use empty metadata.
             try:


@@ -380,7 +358,7 @@ def main(argv):
             _push(dir_name, meta)
 
         if not file:
             _push(dir_name, meta)
 
         if not file:

-            if len(parts) == 1:
+            if len(stack) == 1:

                 continue # We're at the top level -- keep the current root dir
             # Since there's no filename, this is a subdir -- finish it.
             oldtree = already_saved(ent) # may be None
                 continue # We're at the top level -- keep the current root dir
             # Since there's no filename, this is a subdir -- finish it.
             oldtree = already_saved(ent) # may be None


@@ -397,22 +375,18 @@ def main(argv):
 
         # it's not a directory
         if hashvalid:
 
         # it's not a directory
         if hashvalid:

-            id = ent.sha
-            git_name = git.mangle_name(file, ent.mode, ent.gitmode)
-            git_info = (ent.gitmode, git_name, id)
-            shalists[-1].append(git_info)
-            sort_key = git.shalist_item_sort_key((ent.mode, file, id))

             meta = msr.metadata_at(ent.meta_ofs)
             meta.hardlink_target = find_hardlink_target(hlink_db, ent)
             # Restore the times that were cleared to 0 in the metastore.
             (meta.atime, meta.mtime, meta.ctime) = (ent.atime, ent.mtime, ent.ctime)
             meta = msr.metadata_at(ent.meta_ofs)
             meta.hardlink_target = find_hardlink_target(hlink_db, ent)
             # Restore the times that were cleared to 0 in the metastore.
             (meta.atime, meta.mtime, meta.ctime) = (ent.atime, ent.mtime, ent.ctime)

-            metalists[-1].append((sort_key, meta))
+            stack[-1].append(file, ent.mode, ent.gitmode, ent.sha, meta)

         else:
             id = None
             hlink = find_hardlink_target(hlink_db, ent)
             try:
                 meta = metadata.from_path(ent.name, hardlink_target=hlink,
         else:
             id = None
             hlink = find_hardlink_target(hlink_db, ent)
             try:
                 meta = metadata.from_path(ent.name, hardlink_target=hlink,

-                                          normalized=True)
+                                          normalized=True,
+                                          after_stat=after_nondir_metadata_stat)

             except (OSError, IOError) as e:
                 add_error(e)
                 lastskip_name = ent.name
             except (OSError, IOError) as e:
                 add_error(e)
                 lastskip_name = ent.name


@@ -445,6 +419,7 @@ def main(argv):
                     def new_blob(data):
                         meta.size += len(data)
                         return w.new_blob(data)
                     def new_blob(data):
                         meta.size += len(data)
                         return w.new_blob(data)

+                    before_saving_regular_file(ent.name)

                     with hashsplit.open_noatime(ent.name) as f:
                         (mode, id) = hashsplit.split_to_blob_or_tree(
                                                 new_blob, w.new_tree, [f],
                     with hashsplit.open_noatime(ent.name) as f:
                         (mode, id) = hashsplit.split_to_blob_or_tree(
                                                 new_blob, w.new_tree, [f],


@@ -465,11 +440,7 @@ def main(argv):
             if id:
                 ent.validate(mode, id)
                 ent.repack()
             if id:
                 ent.validate(mode, id)
                 ent.repack()

-                git_name = git.mangle_name(file, ent.mode, ent.gitmode)
-                git_info = (mode, git_name, id)
-                shalists[-1].append(git_info)
-                sort_key = git.shalist_item_sort_key((ent.mode, file, id))
-                metalists[-1].append((sort_key, meta))
+                stack[-1].append(file, ent.mode, ent.gitmode, id, meta)

 
         if exists and wasmissing:
             _nonlocal['count'] += oldsize
 
         if exists and wasmissing:
             _nonlocal['count'] += oldsize


@@ -481,48 +452,88 @@ def main(argv):
         progress('Saving: %.2f%% (%d/%dk, %d/%d files), done.    \n'
                  % (pct, _nonlocal['count']/1024, total/1024, fcount, ftotal))
 
         progress('Saving: %.2f%% (%d/%dk, %d/%d files), done.    \n'
                  % (pct, _nonlocal['count']/1024, total/1024, fcount, ftotal))
 

-    while len(parts) > 1: # _pop() all the parts above the root
-        _pop(force_tree = None)
-    assert(len(shalists) == 1)
-    assert(len(metalists) == 1)
+    while len(stack) > 1: # _pop() all the parts above the root
+        _pop()

 
     # Finish the root directory.
 
     # Finish the root directory.

-    tree = _pop(force_tree = None,
-                # When there's a collision, use empty metadata for the root.
-                dir_metadata = metadata.Metadata() if root_collision else None)
-
-    sys.stdout.flush()
-    out = byte_stream(sys.stdout)
-
-    if opt.tree:
-        out.write(hexlify(tree))
-        out.write(b'\n')
-    if opt.commit or name:
-        if compat.py_maj > 2:
-            # Strip b prefix from python 3 bytes reprs to preserve previous format
-             msgcmd = b'[%s]' % b', '.join([repr(argv_bytes(x))[1:].encode('ascii')
-                                           for x in argv])
-        else:
-            msgcmd = repr(argv)
-        msg = b'bup save\n\nGenerated by command:\n%s\n' % msgcmd
-        userline = (b'%s <%s@%s>' % (userfullname(), username(), hostname()))
-        commit = w.new_commit(tree, oldref, userline, date, None,
-                              userline, date, None, msg)
-        if opt.commit:
-            out.write(hexlify(commit))
-            out.write(b'\n')
-
-    msr.close()
-    w.close()  # must close before we can update the ref
+    # When there's a collision, use empty metadata for the root.
+    tree = _pop(dir_metadata = metadata.Metadata() if root_collision else None)

 
 

-    if opt.name:
-        if cli:
-            cli.update_ref(refname, commit, oldref)
+    return tree
+
+
+def commit_tree(tree, parent, date, argv, writer):
+    if compat.py_maj > 2:
+        # Strip b prefix from python 3 bytes reprs to preserve previous format
+         msgcmd = b'[%s]' % b', '.join([repr(argv_bytes(x))[1:].encode('ascii')
+                                       for x in argv])
+    else:
+        msgcmd = repr(argv)
+    msg = b'bup save\n\nGenerated by command:\n%s\n' % msgcmd
+    userline = (b'%s <%s@%s>' % (userfullname(), username(), hostname()))
+    return writer.new_commit(tree, parent, userline, date, None,
+                             userline, date, None, msg)
+
+
+def main(argv):
+    handle_ctrl_c()
+    opt = opts_from_cmdline(argv)
+    client.bwlimit = opt.bwlimit
+    git.check_repo_or_die()
+
+    remote_dest = opt.remote or opt.is_reverse
+    if not remote_dest:
+        repo = git
+        cli = nullcontext()
+    else:
+        try:
+            cli = repo = client.Client(opt.remote)
+        except client.ClientError as e:
+            log('error: %s' % e)
+            sys.exit(1)
+
+    # cli creation must be last nontrivial command in each if clause above
+    with cli:
+        if not remote_dest:
+            w = git.PackWriter(compression_level=opt.compress)

         else:
         else:

-            git.update_ref(refname, commit, oldref)
+            w = cli.new_packwriter(compression_level=opt.compress)
+
+        with w:
+            sys.stdout.flush()
+            out = byte_stream(sys.stdout)

 
 

-    if cli:
-        cli.close()
+            if opt.name:
+                refname = b'refs/heads/%s' % opt.name
+                parent = repo.read_ref(refname)
+            else:
+                refname = parent = None
+
+            indexfile = opt.indexfile or git.repo(b'bupindex')
+            try:
+                msr = index.MetaStoreReader(indexfile + b'.meta')
+            except IOError as ex:
+                if ex.errno != ENOENT:
+                    raise
+                log('error: cannot access %r; have you run bup index?'
+                    % path_msg(indexfile))
+                sys.exit(1)
+            with msr, \
+                 hlinkdb.HLinkDB(indexfile + b'.hlink') as hlink_db, \
+                 index.Reader(indexfile) as reader:
+                tree = save_tree(opt, reader, hlink_db, msr, w)
+            if opt.tree:
+                out.write(hexlify(tree))
+                out.write(b'\n')
+            if opt.commit or opt.name:
+                commit = commit_tree(tree, parent, opt.date, argv, w)
+                if opt.commit:
+                    out.write(hexlify(commit))
+                    out.write(b'\n')
+
+        # packwriter must be closed before we can update the ref
+        if opt.name:
+            repo.update_ref(refname, commit, parent)

 
     if saved_errors:
         log('WARNING: %d errors encountered while saving.\n' % len(saved_errors))
 
     if saved_errors:
         log('WARNING: %d errors encountered while saving.\n' % len(saved_errors))