Port import-duplicity to python

Additionally, specify an --archive-dir to all duplicity invocations so
that we don't scribble in ~/.cache/duplicity during the imports, and
unconditionally clean up temporary dirs.

Signed-off-by: Rob Browning <rlb@defaultvalue.org>

diff --git a/Documentation/bup-import-duplicity.md b/Documentation/bup-import-duplicity.md
index 6fbcf6a88b2c9d986c24e208327b067bc9d05844..0826b6a74ef7750193e5c16034d48afe619c165b 100644 (file)
--- a/Documentation/bup-import-duplicity.md
+++ b/Documentation/bup-import-duplicity.md
@@ -1,30 +1,48 @@
 % bup-import-duplicity(1) Bup %BUP_VERSION%
-% Zoran Zaric <zz@zoranzaric.de>
+% Zoran Zaric <zz@zoranzaric.de>, Rob Browning <rlb@defaultvalue.org>
 % %BUP_DATE%
 
 # NAME
 
-bup-import-duplicity - import a duplicity archive
+bup-import-duplicity - import duplicity backups
 
 # SYNOPSIS
 
-bup import-duplicity [-n] <duplicity target url> <backup name>
+bup import-duplicity [-n] \<source-url\> \<save-name\>
 
 # DESCRIPTION
 
-`bup import-duplicity` imports a duplicity archive. The
-timestamps for the backups are preserved and the path to
-the duplicity archive is stripped from the paths.
+`bup import-duplicity` imports all of the duplicity backups at
+`source-url` into `bup` via `bup save -n save-name`.  The bup saves
+will have the same timestamps (via `bup save --date`) as the original
+backups.
+
+Because this command operates by restoring each duplicity backup to a
+temporary directory, the extent to which the metadata is preserved
+will depend on the characteristics of the underlying filesystem,
+whether or not you run `import-duplicity` as root (or under
+`fakeroot`(1)), etc.
+
+Note that this command will use [`mkdtemp`][mkdtemp] to create
+temporary directories, which means that it should respect any
+`TEMPDIR`, `TEMP`, or `TMP` environment variable settings.  Make sure
+that the relevant filesystem has enough space for the largest
+duplicity backup being imported.
+
+Since all invocations of duplicity use a temporary `--archive-dir`,
+`import-duplicity` should not affect ~/.cache/duplicity.
 
 # OPTIONS
 
 -n,--dry-run
-:   don't do anything just print out what would be done
+:   don't do anything; just print out what would be done
 
 # EXAMPLES
 
-    $ bup import-duplicty file:///DUPLICITY legacy-duplicity
+    $ bup import-duplicity file:///duplicity/src/ legacy-duplicity
 
 # BUP
 
 Part of the `bup`(1) suite.
+
+[mkdtemp]: https://docs.python.org/2/library/tempfile.html#tempfile.mkdtemp

diff --git a/cmd/import-duplicity-cmd.py b/cmd/import-duplicity-cmd.py
new file mode 100755 (executable)
index 0000000..3798d75
--- /dev/null
+++ b/cmd/import-duplicity-cmd.py
@@ -0,0 +1,91 @@
+#!/usr/bin/env python
+
+from calendar import timegm
+from pipes import quote
+from subprocess import check_call, check_output
+from time import strftime, strptime
+import sys
+import tempfile
+
+from bup import git, options, vfs
+from bup.helpers import handle_ctrl_c, log, saved_errors, unlink
+import bup.path
+
+optspec = """
+bup import-duplicity [-n] <duplicity-source-url> <bup-save-name>
+--
+n,dry-run  don't do anything; just print what would be done
+"""
+
+
+def logcmd(cmd):
+    if isinstance(cmd, basestring):
+        log(cmd + '\n')
+    else:
+        log(' '.join(map(quote, cmd)) + '\n')
+
+
+def exc(cmd, shell=False):
+    global opt
+    logcmd(cmd)
+    if not opt.dry_run:
+        check_call(cmd, shell=shell)
+
+def exo(cmd, shell=False):
+    global opt
+    logcmd(cmd)
+    if not opt.dry_run:
+        return check_output(cmd, shell=shell)
+
+
+handle_ctrl_c()
+
+o = options.Options(optspec)
+opt, flags, extra = o.parse(sys.argv[1:])
+
+if len(extra) < 1 or not extra[0]:
+    o.fatal('duplicity source URL required')
+if len(extra) < 2 or not extra[1]:
+    o.fatal('bup destination save name required')
+if len(extra) > 2:
+    o.fatal('too many arguments')
+
+source_url, save_name = extra
+bup = bup.path.exe()
+
+git.check_repo_or_die()
+top = vfs.RefList(None)
+
+tmpdir = tempfile.mkdtemp(prefix='bup-import-dup-')
+try:
+    dup = ['duplicity', '--archive-dir', tmpdir + '/dup-cache']
+    restoredir = tmpdir + '/restore'
+    tmpidx = tmpdir + '/index'
+    collection_status = \
+        exo(' '.join(map(quote, dup))
+            + ' collection-status --log-fd=3 %s 3>&1 1>&2' % quote(source_url),
+            shell=True)
+    # Duplicity output lines of interest look like this (one leading space):
+    #  full 20150222T073111Z 1 noenc
+    #  inc 20150222T073233Z 1 noenc
+    dup_timestamps = []
+    for line in collection_status.splitlines():
+        if line.startswith(' inc '):
+            assert(len(line) >= len(' inc 20150222T073233Z'))
+            dup_timestamps.append(line[5:21])
+        elif line.startswith(' full '):
+            assert(len(line) >= len(' full 20150222T073233Z'))
+            dup_timestamps.append(line[6:22])
+    for i, dup_ts in enumerate(dup_timestamps):
+        tm = strptime(dup_ts, '%Y%m%dT%H%M%SZ')
+        exc(['rm', '-rf', restoredir])
+        exc(dup + ['restore', '-t', dup_ts, source_url, restoredir])
+        exc([bup, 'index', '-uxf', tmpidx, restoredir])
+        exc([bup, 'save', '--strip', '--date', str(timegm(tm)), '-f', tmpidx,
+             '-n', save_name, restoredir])
+finally:
+    exc(['rm', '-rf', tmpdir])
+
+if saved_errors:
+    log('warning: %d errors encountered\n' % len(saved_errors))
+    sys.exit(1)

diff --git a/cmd/import-duplicity-cmd.sh b/cmd/import-duplicity-cmd.sh
deleted file mode 100755 (executable)
index 61fa16b..0000000
--- a/cmd/import-duplicity-cmd.sh
+++ /dev/null
@@ -1,66 +0,0 @@
-#!/bin/sh
-
-set -e
-
-usage() {
-    echo "Usage: bup import-duplicity [-n]" \
-        "<duplicity target url> <backup name>"
-    echo "-n,--dry-run: just print what would be done"
-    exit -1
-}
-
-control_c() {
-    echo "bup import-duplicity: signal 2 received" 1>&2
-    exit 128
-}
-
-trap control_c INT
-
-dry_run=
-while [ "$1" = "-n" -o "$1" = "--dry-run" ]; do
-    dry_run=echo
-    shift
-done
-
-bup()
-{
-    $dry_run "${BUP_MAIN_EXE:=bup}" "$@"
-}
-
-duplicity_target_url="$1"
-branch="$2"
-
-[ -n "$duplicity_target_url" -a "$#" = 2 ] || usage
-
-dup_timestamps=$(duplicity collection-status --log-fd=3 \
-                 "$duplicity_target_url" 3>&1 1>/dev/null 2>/dev/null |
-                 grep "[[:digit:]][[:digit:]]T" |
-                 cut -d" " -f 3)
-backups_count=$(echo "$dup_timestamps" | wc -l)
-counter=1
-echo "$dup_timestamps" |
-while read dup_timestamp; do
-  timestamp=$(python -c "import time,calendar; " \
-      "print str(int(calendar.timegm(time.strptime('$dup_timestamp', " \
-      "'%Y%m%dT%H%M%SZ'))))")
-  echo "Importing backup from $(date --date=@$timestamp +%c) " \
-      "($counter / $backups_count)" 1>&2
-  echo 1>&2
-
-  tmpdir=$(mktemp -d)
-
-  echo "Restoring from duplicity..." 1>&2
-  duplicity restore -t "$dup_timestamp" "$duplicity_target_url" "$tmpdir"
-  echo 1>&2
-
-  echo "Importing into bup..." 1>&2
-  tmpidx=$(mktemp -u)
-  bup index -ux -f "$tmpidx" "$tmpdir"
-  bup save --strip --date="$timestamp" -f "$tmpidx" -n "$branch" "$tmpdir"
-  rm -f "$tmpidx"
-
-  rm -rf "$tmpdir"
-  counter=$((counter+1))
-  echo 1>&2
-  echo 1>&2
-done