-/cmd/bup-*
+\#*#
+.#*
randomgen
memtest
*.o
/build
*.swp
nbproject
+/lib/cmd/bup-*
/t/sampledata/var/
/t/tmp/
/lib/bup/_checkout.py
-cmd/bup
\ No newline at end of file
+lib/cmd/bup
\ No newline at end of file
--- /dev/null
+lib/cmd
\ No newline at end of file
+++ /dev/null
-#!/bin/sh
-"""": # -*-python-*-
-bup_python="$(dirname "$0")/bup-python" || exit $?
-exec "$bup_python" "$0" ${1+"$@"}
-"""
-# end of bup preamble
-
-from __future__ import absolute_import
-import glob, os, sys, tempfile
-
-from bup import options, git, bloom
-from bup.compat import argv_bytes, hexstr
-from bup.helpers import (add_error, debug1, handle_ctrl_c, log, progress, qprogress,
- saved_errors)
-from bup.io import path_msg
-
-
-optspec = """
-bup bloom [options...]
---
-ruin ruin the specified bloom file (clearing the bitfield)
-f,force ignore existing bloom file and regenerate it from scratch
-o,output= output bloom filename (default: auto)
-d,dir= input directory to look for idx files (default: auto)
-k,hashes= number of hash functions to use (4 or 5) (default: auto)
-c,check= check the given .idx file against the bloom filter
-"""
-
-
-def ruin_bloom(bloomfilename):
- rbloomfilename = git.repo_rel(bloomfilename)
- if not os.path.exists(bloomfilename):
- log(path_msg(bloomfilename) + '\n')
- add_error('bloom: %s not found to ruin\n' % path_msg(rbloomfilename))
- return
- b = bloom.ShaBloom(bloomfilename, readwrite=True, expected=1)
- b.map[16 : 16 + 2**b.bits] = b'\0' * 2**b.bits
-
-
-def check_bloom(path, bloomfilename, idx):
- rbloomfilename = git.repo_rel(bloomfilename)
- ridx = git.repo_rel(idx)
- if not os.path.exists(bloomfilename):
- log('bloom: %s: does not exist.\n' % path_msg(rbloomfilename))
- return
- b = bloom.ShaBloom(bloomfilename)
- if not b.valid():
- add_error('bloom: %r is invalid.\n' % path_msg(rbloomfilename))
- return
- base = os.path.basename(idx)
- if base not in b.idxnames:
- log('bloom: %s does not contain the idx.\n' % path_msg(rbloomfilename))
- return
- if base == idx:
- idx = os.path.join(path, idx)
- log('bloom: bloom file: %s\n' % path_msg(rbloomfilename))
- log('bloom: checking %s\n' % path_msg(ridx))
- for objsha in git.open_idx(idx):
- if not b.exists(objsha):
- add_error('bloom: ERROR: object %s missing' % hexstr(objsha))
-
-
-_first = None
-def do_bloom(path, outfilename, k):
- global _first
- assert k in (None, 4, 5)
- b = None
- if os.path.exists(outfilename) and not opt.force:
- b = bloom.ShaBloom(outfilename)
- if not b.valid():
- debug1("bloom: Existing invalid bloom found, regenerating.\n")
- b = None
-
- add = []
- rest = []
- add_count = 0
- rest_count = 0
- for i, name in enumerate(glob.glob(b'%s/*.idx' % path)):
- progress('bloom: counting: %d\r' % i)
- ix = git.open_idx(name)
- ixbase = os.path.basename(name)
- if b and (ixbase in b.idxnames):
- rest.append(name)
- rest_count += len(ix)
- else:
- add.append(name)
- add_count += len(ix)
-
- if not add:
- debug1("bloom: nothing to do.\n")
- return
-
- if b:
- if len(b) != rest_count:
- debug1("bloom: size %d != idx total %d, regenerating\n"
- % (len(b), rest_count))
- b = None
- elif k is not None and k != b.k:
- debug1("bloom: new k %d != existing k %d, regenerating\n"
- % (k, b.k))
- b = None
- elif (b.bits < bloom.MAX_BLOOM_BITS[b.k] and
- b.pfalse_positive(add_count) > bloom.MAX_PFALSE_POSITIVE):
- debug1("bloom: regenerating: adding %d entries gives "
- "%.2f%% false positives.\n"
- % (add_count, b.pfalse_positive(add_count)))
- b = None
- else:
- b = bloom.ShaBloom(outfilename, readwrite=True, expected=add_count)
- if not b: # Need all idxs to build from scratch
- add += rest
- add_count += rest_count
- del rest
- del rest_count
-
- msg = b is None and 'creating from' or 'adding'
- if not _first: _first = path
- dirprefix = (_first != path) and git.repo_rel(path) + b': ' or b''
- progress('bloom: %s%s %d file%s (%d object%s).\r'
- % (path_msg(dirprefix), msg,
- len(add), len(add)!=1 and 's' or '',
- add_count, add_count!=1 and 's' or ''))
-
- tfname = None
- if b is None:
- tfname = os.path.join(path, b'bup.tmp.bloom')
- b = bloom.create(tfname, expected=add_count, k=k)
- count = 0
- icount = 0
- for name in add:
- ix = git.open_idx(name)
- qprogress('bloom: writing %.2f%% (%d/%d objects)\r'
- % (icount*100.0/add_count, icount, add_count))
- b.add_idx(ix)
- count += 1
- icount += len(ix)
-
- # Currently, there's an open file object for tfname inside b.
- # Make sure it's closed before rename.
- b.close()
-
- if tfname:
- os.rename(tfname, outfilename)
-
-
-handle_ctrl_c()
-
-o = options.Options(optspec)
-(opt, flags, extra) = o.parse(sys.argv[1:])
-
-if extra:
- o.fatal('no positional parameters expected')
-
-if not opt.check and opt.k and opt.k not in (4,5):
- o.fatal('only k values of 4 and 5 are supported')
-
-if opt.check:
- opt.check = argv_bytes(opt.check)
-
-git.check_repo_or_die()
-
-output = argv_bytes(opt.output) if opt.output else None
-paths = opt.dir and [argv_bytes(opt.dir)] or git.all_packdirs()
-for path in paths:
- debug1('bloom: scanning %s\n' % path_msg(path))
- outfilename = output or os.path.join(path, b'bup.bloom')
- if opt.check:
- check_bloom(path, outfilename, opt.check)
- elif opt.ruin:
- ruin_bloom(outfilename)
- else:
- do_bloom(path, outfilename, opt.k)
-
-if saved_errors:
- log('WARNING: %d errors encountered during bloom.\n' % len(saved_errors))
- sys.exit(1)
-elif opt.check:
- log('All tests passed.\n')
+++ /dev/null
-#!/bin/sh
-"""": # -*-python-*- # -*-python-*-
-set -e
-top="$(pwd)"
-cmdpath="$0"
-# loop because macos doesn't have recursive readlink/realpath utils
-while test -L "$cmdpath"; do
- link="$(readlink "$cmdpath")"
- cd "$(dirname "$cmdpath")"
- cmdpath="$link"
-done
-script_home="$(cd "$(dirname "$cmdpath")" && pwd -P)"
-cd "$top"
-exec "$script_home/bup-python" "$0" ${1+"$@"}
-"""
-# end of bup preamble
-
-from __future__ import absolute_import, print_function
-import errno, getopt, os, re, select, signal, subprocess, sys
-from subprocess import PIPE
-
-from bup.compat import environ, restore_lc_env
-from bup.io import path_msg
-
-if sys.version_info[0] != 2 \
- and not environ.get(b'BUP_ALLOW_UNEXPECTED_PYTHON_VERSION') == b'true':
- print('error: bup may crash with python versions other than 2, or eat your data',
- file=sys.stderr)
- sys.exit(2)
-
-restore_lc_env()
-
-from bup import compat, path, helpers
-from bup.compat import add_ex_tb, add_ex_ctx, argv_bytes, wrap_main
-from bup.helpers import atoi, columnate, debug1, log, merge_dict, tty_width
-from bup.io import byte_stream, path_msg
-
-cmdpath = path.cmddir()
-
-# We manipulate the subcmds here as strings, but they must be ASCII
-# compatible, since we're going to be looking for exactly
-# b'bup-SUBCMD' to exec.
-
-def usage(msg=""):
- log('Usage: bup [-?|--help] [-d BUP_DIR] [--debug] [--profile] '
- '<command> [options...]\n\n')
- common = dict(
- ftp = 'Browse backup sets using an ftp-like client',
- fsck = 'Check backup sets for damage and add redundancy information',
- fuse = 'Mount your backup sets as a filesystem',
- help = 'Print detailed help for the given command',
- index = 'Create or display the index of files to back up',
- on = 'Backup a remote machine to the local one',
- restore = 'Extract files from a backup set',
- save = 'Save files into a backup set (note: run "bup index" first)',
- tag = 'Tag commits for easier access',
- web = 'Launch a web server to examine backup sets',
- )
-
- log('Common commands:\n')
- for cmd,synopsis in sorted(common.items()):
- log(' %-10s %s\n' % (cmd, synopsis))
- log('\n')
-
- log('Other available commands:\n')
- cmds = []
- for c in sorted(os.listdir(cmdpath)):
- if c.startswith(b'bup-') and c.find(b'.') < 0:
- cname = c[4:].decode('iso-8859-1')
- if cname not in common:
- cmds.append(c[4:])
- log(columnate(cmds, ' '))
- log('\n')
-
- log("See 'bup help COMMAND' for more information on " +
- "a specific command.\n")
- if msg:
- log("\n%s\n" % msg)
- sys.exit(99)
-
-
-if len(sys.argv) < 2:
- usage()
-
-# Handle global options.
-try:
- optspec = ['help', 'version', 'debug', 'profile', 'bup-dir=']
- global_args, subcmd = getopt.getopt(sys.argv[1:], '?VDd:', optspec)
-except getopt.GetoptError as ex:
- usage('error: %s' % ex.msg)
-
-subcmd = [argv_bytes(x) for x in subcmd]
-help_requested = None
-do_profile = False
-bup_dir = None
-
-for opt in global_args:
- if opt[0] in ['-?', '--help']:
- help_requested = True
- elif opt[0] in ['-V', '--version']:
- subcmd = [b'version']
- elif opt[0] in ['-D', '--debug']:
- helpers.buglvl += 1
- environ[b'BUP_DEBUG'] = b'%d' % helpers.buglvl
- elif opt[0] in ['--profile']:
- do_profile = True
- elif opt[0] in ['-d', '--bup-dir']:
- bup_dir = argv_bytes(opt[1])
- else:
- usage('error: unexpected option "%s"' % opt[0])
-
-if bup_dir:
- bup_dir = argv_bytes(bup_dir)
-
-# Make BUP_DIR absolute, so we aren't affected by chdir (i.e. save -C, etc.).
-if bup_dir:
- environ[b'BUP_DIR'] = os.path.abspath(bup_dir)
-
-if len(subcmd) == 0:
- if help_requested:
- subcmd = [b'help']
- else:
- usage()
-
-if help_requested and subcmd[0] != b'help':
- subcmd = [b'help'] + subcmd
-
-if len(subcmd) > 1 and subcmd[1] == b'--help' and subcmd[0] != b'help':
- subcmd = [b'help', subcmd[0]] + subcmd[2:]
-
-subcmd_name = subcmd[0]
-if not subcmd_name:
- usage()
-
-def subpath(subcmd):
- return os.path.join(cmdpath, b'bup-' + subcmd)
-
-subcmd[0] = subpath(subcmd_name)
-if not os.path.exists(subcmd[0]):
- usage('error: unknown command "%s"' % path_msg(subcmd_name))
-
-already_fixed = atoi(environ.get(b'BUP_FORCE_TTY'))
-if subcmd_name in [b'mux', b'ftp', b'help']:
- already_fixed = True
-fix_stdout = not already_fixed and os.isatty(1)
-fix_stderr = not already_fixed and os.isatty(2)
-
-if fix_stdout or fix_stderr:
- tty_env = merge_dict(environ,
- {b'BUP_FORCE_TTY': (b'%d'
- % ((fix_stdout and 1 or 0)
- + (fix_stderr and 2 or 0)))})
-else:
- tty_env = environ
-
-
-sep_rx = re.compile(br'([\r\n])')
-
-def print_clean_line(dest, content, width, sep=None):
- """Write some or all of content, followed by sep, to the dest fd after
- padding the content with enough spaces to fill the current
- terminal width or truncating it to the terminal width if sep is a
- carriage return."""
- global sep_rx
- assert sep in (b'\r', b'\n', None)
- if not content:
- if sep:
- os.write(dest, sep)
- return
- for x in content:
- assert not sep_rx.match(x)
- content = b''.join(content)
- if sep == b'\r' and len(content) > width:
- content = content[width:]
- os.write(dest, content)
- if len(content) < width:
- os.write(dest, b' ' * (width - len(content)))
- if sep:
- os.write(dest, sep)
-
-def filter_output(src_out, src_err, dest_out, dest_err):
- """Transfer data from src_out to dest_out and src_err to dest_err via
- print_clean_line until src_out and src_err close."""
- global sep_rx
- assert not isinstance(src_out, bool)
- assert not isinstance(src_err, bool)
- assert not isinstance(dest_out, bool)
- assert not isinstance(dest_err, bool)
- assert src_out is not None or src_err is not None
- assert (src_out is None) == (dest_out is None)
- assert (src_err is None) == (dest_err is None)
- pending = {}
- pending_ex = None
- try:
- fds = tuple([x for x in (src_out, src_err) if x is not None])
- while fds:
- ready_fds, _, _ = select.select(fds, [], [])
- width = tty_width()
- for fd in ready_fds:
- buf = os.read(fd, 4096)
- dest = dest_out if fd == src_out else dest_err
- if not buf:
- fds = tuple([x for x in fds if x is not fd])
- print_clean_line(dest, pending.pop(fd, []), width)
- else:
- split = sep_rx.split(buf)
- while len(split) > 1:
- content, sep = split[:2]
- split = split[2:]
- print_clean_line(dest,
- pending.pop(fd, []) + [content],
- width,
- sep)
- assert(len(split) == 1)
- if split[0]:
- pending.setdefault(fd, []).extend(split)
- except BaseException as ex:
- pending_ex = add_ex_ctx(add_ex_tb(ex), pending_ex)
- try:
- # Try to finish each of the streams
- for fd, pending_items in compat.items(pending):
- dest = dest_out if fd == src_out else dest_err
- try:
- print_clean_line(dest, pending_items, width)
- except (EnvironmentError, EOFError) as ex:
- pending_ex = add_ex_ctx(add_ex_tb(ex), pending_ex)
- except BaseException as ex:
- pending_ex = add_ex_ctx(add_ex_tb(ex), pending_ex)
- if pending_ex:
- raise pending_ex
-
-def run_subcmd(subcmd):
-
- c = (do_profile and [sys.executable, b'-m', b'cProfile'] or []) + subcmd
- if not (fix_stdout or fix_stderr):
- os.execvp(c[0], c)
-
- sys.stdout.flush()
- sys.stderr.flush()
- out = byte_stream(sys.stdout)
- err = byte_stream(sys.stderr)
- p = None
- try:
- p = subprocess.Popen(c,
- stdout=PIPE if fix_stdout else out,
- stderr=PIPE if fix_stderr else err,
- env=tty_env, bufsize=4096, close_fds=True)
- # Assume p will receive these signals and quit, which will
- # then cause us to quit.
- for sig in (signal.SIGINT, signal.SIGTERM, signal.SIGQUIT):
- signal.signal(sig, signal.SIG_IGN)
-
- filter_output(fix_stdout and p.stdout.fileno() or None,
- fix_stderr and p.stderr.fileno() or None,
- fix_stdout and out.fileno() or None,
- fix_stderr and err.fileno() or None)
- return p.wait()
- except BaseException as ex:
- add_ex_tb(ex)
- try:
- if p and p.poll() == None:
- os.kill(p.pid, signal.SIGTERM)
- p.wait()
- except BaseException as kill_ex:
- raise add_ex_ctx(add_ex_tb(kill_ex), ex)
- raise ex
-
-wrap_main(lambda : run_subcmd(subcmd))
+++ /dev/null
-#!/bin/sh
-"""": # -*-python-*-
-bup_python="$(dirname "$0")/bup-python" || exit $?
-exec "$bup_python" "$0" ${1+"$@"}
-"""
-# end of bup preamble
-
-from __future__ import absolute_import
-import re, stat, sys
-
-from bup import options, git, vfs
-from bup.compat import argv_bytes
-from bup.helpers import chunkyreader, handle_ctrl_c, log, saved_errors
-from bup.io import byte_stream
-from bup.repo import LocalRepo
-
-optspec = """
-bup cat-file [--meta|--bupm] /branch/revision/[path]
---
-meta print the target's metadata entry (decoded then reencoded) to stdout
-bupm print the target directory's .bupm file directly to stdout
-"""
-
-handle_ctrl_c()
-
-o = options.Options(optspec)
-(opt, flags, extra) = o.parse(sys.argv[1:])
-
-git.check_repo_or_die()
-
-if not extra:
- o.fatal('must specify a target')
-if len(extra) > 1:
- o.fatal('only one target file allowed')
-if opt.bupm and opt.meta:
- o.fatal('--meta and --bupm are incompatible')
-
-target = argv_bytes(extra[0])
-
-if not re.match(br'/*[^/]+/[^/]+', target):
- o.fatal("path %r doesn't include a branch and revision" % target)
-
-repo = LocalRepo()
-resolved = vfs.resolve(repo, target, follow=False)
-leaf_name, leaf_item = resolved[-1]
-if not leaf_item:
- log('error: cannot access %r in %r\n'
- % ('/'.join(name for name, item in resolved), path))
- sys.exit(1)
-
-mode = vfs.item_mode(leaf_item)
-
-sys.stdout.flush()
-out = byte_stream(sys.stdout)
-
-if opt.bupm:
- if not stat.S_ISDIR(mode):
- o.fatal('%r is not a directory' % target)
- _, bupm_oid = vfs.tree_data_and_bupm(repo, leaf_item.oid)
- if bupm_oid:
- with vfs.tree_data_reader(repo, bupm_oid) as meta_stream:
- out.write(meta_stream.read())
-elif opt.meta:
- augmented = vfs.augment_item_meta(repo, leaf_item, include_size=True)
- out.write(augmented.meta.encode())
-else:
- if stat.S_ISREG(mode):
- with vfs.fopen(repo, leaf_item) as f:
- for b in chunkyreader(f):
- out.write(b)
- else:
- o.fatal('%r is not a plain file' % target)
-
-if saved_errors:
- log('warning: %d errors encountered\n' % len(saved_errors))
- sys.exit(1)
+++ /dev/null
-#!/bin/sh
-"""": # -*-python-*-
-bup_python="$(dirname "$0")/bup-python" || exit $?
-exec "$bup_python" "$0" ${1+"$@"}
-"""
-# end of bup preamble
-
-from __future__ import absolute_import
-import sys, getopt, socket, subprocess, fcntl
-from bup import options, path
-from bup.helpers import *
-
-optspec = """
-bup daemon [options...] -- [bup-server options...]
---
-l,listen ip address to listen on, defaults to *
-p,port port to listen on, defaults to 1982
-"""
-o = options.Options(optspec, optfunc=getopt.getopt)
-(opt, flags, extra) = o.parse(sys.argv[1:])
-
-host = opt.listen
-port = opt.port and int(opt.port) or 1982
-
-import socket
-import sys
-
-socks = []
-e = None
-for res in socket.getaddrinfo(host, port, socket.AF_UNSPEC,
- socket.SOCK_STREAM, 0, socket.AI_PASSIVE):
- af, socktype, proto, canonname, sa = res
- try:
- s = socket.socket(af, socktype, proto)
- except socket.error as e:
- continue
- try:
- if af == socket.AF_INET6:
- log("bup daemon: listening on [%s]:%s\n" % sa[:2])
- else:
- log("bup daemon: listening on %s:%s\n" % sa[:2])
- s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
- s.bind(sa)
- s.listen(1)
- fcntl.fcntl(s.fileno(), fcntl.F_SETFD, fcntl.FD_CLOEXEC)
- except socket.error as e:
- s.close()
- continue
- socks.append(s)
-
-if not socks:
- log('bup daemon: listen socket: %s\n' % e.args[1])
- sys.exit(1)
-
-try:
- while True:
- [rl,wl,xl] = select.select(socks, [], [], 60)
- for l in rl:
- s, src = l.accept()
- try:
- log("Socket accepted connection from %s\n" % (src,))
- fd1 = os.dup(s.fileno())
- fd2 = os.dup(s.fileno())
- s.close()
- sp = subprocess.Popen([path.exe(), 'mux', '--',
- path.exe(), 'server']
- + extra, stdin=fd1, stdout=fd2)
- finally:
- os.close(fd1)
- os.close(fd2)
-finally:
- for l in socks:
- l.shutdown(socket.SHUT_RDWR)
- l.close()
-
-debug1("bup daemon: done")
+++ /dev/null
-#!/bin/sh
-"""": # -*-python-*-
-bup_python="$(dirname "$0")/bup-python" || exit $?
-exec "$bup_python" "$0" ${1+"$@"}
-"""
-# end of bup preamble
-
-from __future__ import absolute_import
-import sys, os, random
-
-from bup import options
-from bup.compat import argv_bytes, bytes_from_uint, range
-from bup.helpers import log
-from bup.io import path_msg
-
-
-def randblock(n):
- return b''.join(bytes_from_uint(random.randrange(0,256)) for i in range(n))
-
-
-optspec = """
-bup damage [-n count] [-s maxsize] [-S seed] <filenames...>
---
- WARNING: THIS COMMAND IS EXTREMELY DANGEROUS
-n,num= number of blocks to damage
-s,size= maximum size of each damaged block
-percent= maximum size of each damaged block (as a percent of entire file)
-equal spread damage evenly throughout the file
-S,seed= random number seed (for repeatable tests)
-"""
-o = options.Options(optspec)
-(opt, flags, extra) = o.parse(sys.argv[1:])
-
-if not extra:
- o.fatal('filenames expected')
-
-if opt.seed != None:
- random.seed(opt.seed)
-
-for name in extra:
- name = argv_bytes(name)
- log('Damaging "%s"...\n' % path_msg(name))
- with open(name, 'r+b') as f:
- st = os.fstat(f.fileno())
- size = st.st_size
- if opt.percent or opt.size:
- ms1 = int(float(opt.percent or 0)/100.0*size) or size
- ms2 = opt.size or size
- maxsize = min(ms1, ms2)
- else:
- maxsize = 1
- chunks = opt.num or 10
- chunksize = size // chunks
- for r in range(chunks):
- sz = random.randrange(1, maxsize+1)
- if sz > size:
- sz = size
- if opt.equal:
- ofs = r*chunksize
- else:
- ofs = random.randrange(0, size - sz + 1)
- log(' %6d bytes at %d\n' % (sz, ofs))
- f.seek(ofs)
- f.write(randblock(sz))
+++ /dev/null
-#!/bin/sh
-"""": # -*-python-*-
-bup_python="$(dirname "$0")/bup-python" || exit $?
-exec "$bup_python" "$0" ${1+"$@"}
-"""
-# end of bup preamble
-
-from __future__ import absolute_import, print_function
-from os.path import relpath
-import sys
-
-from bup import options, drecurse
-from bup.compat import argv_bytes
-from bup.helpers import log, parse_excludes, parse_rx_excludes, saved_errors
-from bup.io import byte_stream
-
-
-optspec = """
-bup drecurse <path>
---
-x,xdev,one-file-system don't cross filesystem boundaries
-exclude= a path to exclude from the backup (can be used more than once)
-exclude-from= a file that contains exclude paths (can be used more than once)
-exclude-rx= skip paths matching the unanchored regex (may be repeated)
-exclude-rx-from= skip --exclude-rx patterns in file (may be repeated)
-q,quiet don't actually print filenames
-profile run under the python profiler
-"""
-o = options.Options(optspec)
-(opt, flags, extra) = o.parse(sys.argv[1:])
-
-if len(extra) != 1:
- o.fatal("exactly one filename expected")
-
-drecurse_top = argv_bytes(extra[0])
-excluded_paths = parse_excludes(flags, o.fatal)
-if not drecurse_top.startswith(b'/'):
- excluded_paths = [relpath(x) for x in excluded_paths]
-exclude_rxs = parse_rx_excludes(flags, o.fatal)
-it = drecurse.recursive_dirlist([drecurse_top], opt.xdev,
- excluded_paths=excluded_paths,
- exclude_rxs=exclude_rxs)
-if opt.profile:
- import cProfile
- def do_it():
- for i in it:
- pass
- cProfile.run('do_it()')
-else:
- if opt.quiet:
- for i in it:
- pass
- else:
- sys.stdout.flush()
- out = byte_stream(sys.stdout)
- for (name,st) in it:
- out.write(name + b'\n')
-
-if saved_errors:
- log('WARNING: %d errors encountered.\n' % len(saved_errors))
- sys.exit(1)
+++ /dev/null
-#!/bin/sh
-"""": # -*-python-*-
-bup_python="$(dirname "$0")/bup-python" || exit $?
-exec "$bup_python" "$0" ${1+"$@"}
-"""
-# end of bup preamble
-
-from __future__ import absolute_import, print_function
-import sys, os, glob, subprocess
-from shutil import rmtree
-from subprocess import PIPE, Popen
-from tempfile import mkdtemp
-from binascii import hexlify
-
-from bup import options, git
-from bup.compat import argv_bytes
-from bup.helpers import Sha1, chunkyreader, istty2, log, progress
-from bup.io import byte_stream
-
-
-par2_ok = 0
-nullf = open(os.devnull, 'wb+')
-
-def debug(s):
- if opt.verbose > 1:
- log(s)
-
-def run(argv):
- # at least in python 2.5, using "stdout=2" or "stdout=sys.stderr" below
- # doesn't actually work, because subprocess closes fd #2 right before
- # execing for some reason. So we work around it by duplicating the fd
- # first.
- fd = os.dup(2) # copy stderr
- try:
- p = subprocess.Popen(argv, stdout=fd, close_fds=False)
- return p.wait()
- finally:
- os.close(fd)
-
-def par2_setup():
- global par2_ok
- rv = 1
- try:
- p = subprocess.Popen([b'par2', b'--help'],
- stdout=nullf, stderr=nullf, stdin=nullf)
- rv = p.wait()
- except OSError:
- log('fsck: warning: par2 not found; disabling recovery features.\n')
- else:
- par2_ok = 1
-
-def is_par2_parallel():
- # A true result means it definitely allows -t1; a false result is
- # technically inconclusive, but likely means no.
- tmpdir = mkdtemp(prefix=b'bup-fsck')
- try:
- canary = tmpdir + b'/canary'
- with open(canary, 'wb') as f:
- f.write(b'canary\n')
- p = subprocess.Popen((b'par2', b'create', b'-qq', b'-t1', canary),
- stderr=PIPE, stdin=nullf)
- _, err = p.communicate()
- parallel = p.returncode == 0
- if opt.verbose:
- if len(err) > 0 and err != b'Invalid option specified: -t1\n':
- log('Unexpected par2 error output\n')
- log(repr(err) + '\n')
- if parallel:
- log('Assuming par2 supports parallel processing\n')
- else:
- log('Assuming par2 does not support parallel processing\n')
- return parallel
- finally:
- rmtree(tmpdir)
-
-_par2_parallel = None
-
-def par2(action, args, verb_floor=0):
- global _par2_parallel
- if _par2_parallel is None:
- _par2_parallel = is_par2_parallel()
- cmd = [b'par2', action]
- if opt.verbose >= verb_floor and not istty2:
- cmd.append(b'-q')
- else:
- cmd.append(b'-qq')
- if _par2_parallel:
- cmd.append(b'-t1')
- cmd.extend(args)
- return run(cmd)
-
-def par2_generate(base):
- return par2(b'create',
- [b'-n1', b'-c200', b'--', base, base + b'.pack', base + b'.idx'],
- verb_floor=2)
-
-def par2_verify(base):
- return par2(b'verify', [b'--', base], verb_floor=3)
-
-def par2_repair(base):
- return par2(b'repair', [b'--', base], verb_floor=2)
-
-def quick_verify(base):
- f = open(base + b'.pack', 'rb')
- f.seek(-20, 2)
- wantsum = f.read(20)
- assert(len(wantsum) == 20)
- f.seek(0)
- sum = Sha1()
- for b in chunkyreader(f, os.fstat(f.fileno()).st_size - 20):
- sum.update(b)
- if sum.digest() != wantsum:
- raise ValueError('expected %r, got %r' % (hexlify(wantsum),
- sum.hexdigest()))
-
-
-def git_verify(base):
- if opt.quick:
- try:
- quick_verify(base)
- except Exception as e:
- log('error: %s\n' % e)
- return 1
- return 0
- else:
- return run([b'git', b'verify-pack', b'--', base])
-
-
-def do_pack(base, last, par2_exists, out):
- code = 0
- if par2_ok and par2_exists and (opt.repair or not opt.generate):
- vresult = par2_verify(base)
- if vresult != 0:
- if opt.repair:
- rresult = par2_repair(base)
- if rresult != 0:
- action_result = b'failed'
- log('%s par2 repair: failed (%d)\n' % (last, rresult))
- code = rresult
- else:
- action_result = b'repaired'
- log('%s par2 repair: succeeded (0)\n' % last)
- code = 100
- else:
- action_result = b'failed'
- log('%s par2 verify: failed (%d)\n' % (last, vresult))
- code = vresult
- else:
- action_result = b'ok'
- elif not opt.generate or (par2_ok and not par2_exists):
- gresult = git_verify(base)
- if gresult != 0:
- action_result = b'failed'
- log('%s git verify: failed (%d)\n' % (last, gresult))
- code = gresult
- else:
- if par2_ok and opt.generate:
- presult = par2_generate(base)
- if presult != 0:
- action_result = b'failed'
- log('%s par2 create: failed (%d)\n' % (last, presult))
- code = presult
- else:
- action_result = b'generated'
- else:
- action_result = b'ok'
- else:
- assert(opt.generate and (not par2_ok or par2_exists))
- action_result = b'exists' if par2_exists else b'skipped'
- if opt.verbose:
- out.write(last + b' ' + action_result + b'\n')
- return code
-
-
-optspec = """
-bup fsck [options...] [filenames...]
---
-r,repair attempt to repair errors using par2 (dangerous!)
-g,generate generate auto-repair information using par2
-v,verbose increase verbosity (can be used more than once)
-quick just check pack sha1sum, don't use git verify-pack
-j,jobs= run 'n' jobs in parallel
-par2-ok immediately return 0 if par2 is ok, 1 if not
-disable-par2 ignore par2 even if it is available
-"""
-o = options.Options(optspec)
-(opt, flags, extra) = o.parse(sys.argv[1:])
-opt.verbose = opt.verbose or 0
-
-par2_setup()
-if opt.par2_ok:
- if par2_ok:
- sys.exit(0) # 'true' in sh
- else:
- sys.exit(1)
-if opt.disable_par2:
- par2_ok = 0
-
-git.check_repo_or_die()
-
-if extra:
- extra = [argv_byes(x) for x in extra]
-else:
- debug('fsck: No filenames given: checking all packs.\n')
- extra = glob.glob(git.repo(b'objects/pack/*.pack'))
-
-sys.stdout.flush()
-out = byte_stream(sys.stdout)
-code = 0
-count = 0
-outstanding = {}
-for name in extra:
- if name.endswith(b'.pack'):
- base = name[:-5]
- elif name.endswith(b'.idx'):
- base = name[:-4]
- elif name.endswith(b'.par2'):
- base = name[:-5]
- elif os.path.exists(name + b'.pack'):
- base = name
- else:
- raise Exception('%r is not a pack file!' % name)
- (dir,last) = os.path.split(base)
- par2_exists = os.path.exists(base + b'.par2')
- if par2_exists and os.stat(base + b'.par2').st_size == 0:
- par2_exists = 0
- sys.stdout.flush() # Not sure we still need this, but it'll flush out too
- debug('fsck: checking %r (%s)\n'
- % (last, par2_ok and par2_exists and 'par2' or 'git'))
- if not opt.verbose:
- progress('fsck (%d/%d)\r' % (count, len(extra)))
-
- if not opt.jobs:
- nc = do_pack(base, last, par2_exists, out)
- code = code or nc
- count += 1
- else:
- while len(outstanding) >= opt.jobs:
- (pid,nc) = os.wait()
- nc >>= 8
- if pid in outstanding:
- del outstanding[pid]
- code = code or nc
- count += 1
- pid = os.fork()
- if pid: # parent
- outstanding[pid] = 1
- else: # child
- try:
- sys.exit(do_pack(base, last, par2_exists, out))
- except Exception as e:
- log('exception: %r\n' % e)
- sys.exit(99)
-
-while len(outstanding):
- (pid,nc) = os.wait()
- nc >>= 8
- if pid in outstanding:
- del outstanding[pid]
- code = code or nc
- count += 1
- if not opt.verbose:
- progress('fsck (%d/%d)\r' % (count, len(extra)))
-
-if istty2:
- debug('fsck done. \n')
-sys.exit(code)
+++ /dev/null
-#!/bin/sh
-"""": # -*-python-*-
-bup_python="$(dirname "$0")/bup-python" || exit $?
-exec "$bup_python" "$0" ${1+"$@"}
-"""
-# end of bup preamble
-
-# For now, this completely relies on the assumption that the current
-# encoding (LC_CTYPE, etc.) is ASCII compatible, and that it returns
-# the exact same bytes from a decode/encode round-trip (or the reverse
-# (e.g. ISO-8859-1).
-
-from __future__ import absolute_import, print_function
-import sys, os, stat, fnmatch
-
-from bup import options, git, shquote, ls, vfs
-from bup.compat import argv_bytes, input
-from bup.helpers import chunkyreader, handle_ctrl_c, log
-from bup.io import byte_stream, path_msg
-from bup.repo import LocalRepo
-
-handle_ctrl_c()
-
-
-class OptionError(Exception):
- pass
-
-
-def input_bytes(s):
- return s.encode('iso-8859-1')
-
-
-def do_ls(repo, args, out):
- try:
- opt = ls.opts_from_cmdline(args, onabort=OptionError)
- except OptionError as e:
- log('error: %s' % e)
- return
- return ls.within_repo(repo, opt, out)
-
-
-def write_to_file(inf, outf):
- for blob in chunkyreader(inf):
- outf.write(blob)
-
-
-def inputiter():
- if os.isatty(sys.stdin.fileno()):
- while 1:
- try:
- yield input('bup> ')
- except EOFError:
- print() # Clear the line for the terminal's next prompt
- break
- else:
- for line in sys.stdin:
- yield line
-
-
-def _completer_get_subs(repo, line):
- (qtype, lastword) = shquote.unfinished_word(line)
- dir, name = os.path.split(lastword.encode('iso-8859-1'))
- dir_path = vfs.resolve(repo, dir or b'/')
- _, dir_item = dir_path[-1]
- if not dir_item:
- subs = tuple()
- else:
- subs = tuple(dir_path + (entry,)
- for entry in vfs.contents(repo, dir_item)
- if (entry[0] != b'.' and entry[0].startswith(name)))
- return qtype, lastword, subs
-
-
-_last_line = None
-_last_res = None
-def completer(text, iteration):
- global repo
- global _last_line
- global _last_res
- try:
- line = readline.get_line_buffer()[:readline.get_endidx()]
- if _last_line != line:
- _last_res = _completer_get_subs(repo, line)
- _last_line = line
- qtype, lastword, subs = _last_res
- if iteration < len(subs):
- path = subs[iteration]
- leaf_name, leaf_item = path[-1]
- res = vfs.try_resolve(repo, leaf_name, parent=path[:-1])
- leaf_name, leaf_item = res[-1]
- fullname = os.path.join(*(name for name, item in res))
- if stat.S_ISDIR(vfs.item_mode(leaf_item)):
- ret = shquote.what_to_add(qtype, lastword,
- fullname.decode('iso-8859-1') + '/',
- terminate=False)
- else:
- ret = shquote.what_to_add(qtype, lastword,
- fullname.decode('iso-8859-1'),
- terminate=True) + b' '
- return text + ret
- except Exception as e:
- log('\n')
- try:
- import traceback
- traceback.print_tb(sys.exc_traceback)
- except Exception as e2:
- log('Error printing traceback: %s\n' % e2)
- log('\nError in completion: %s\n' % e)
-
-
-optspec = """
-bup ftp [commands...]
-"""
-o = options.Options(optspec)
-(opt, flags, extra) = o.parse(sys.argv[1:])
-
-git.check_repo_or_die()
-
-sys.stdout.flush()
-out = byte_stream(sys.stdout)
-repo = LocalRepo()
-pwd = vfs.resolve(repo, b'/')
-rv = 0
-
-if extra:
- lines = extra
-else:
- try:
- import readline
- except ImportError:
- log('* readline module not available: line editing disabled.\n')
- readline = None
-
- if readline:
- readline.set_completer_delims(' \t\n\r/')
- readline.set_completer(completer)
- if sys.platform.startswith('darwin'):
- # MacOS uses a slightly incompatible clone of libreadline
- readline.parse_and_bind('bind ^I rl_complete')
- readline.parse_and_bind('tab: complete')
- lines = inputiter()
-
-for line in lines:
- if not line.strip():
- continue
- words = [word for (wordstart,word) in shquote.quotesplit(line)]
- cmd = words[0].lower()
- #log('execute: %r %r\n' % (cmd, parm))
- try:
- if cmd == 'ls':
- # FIXME: respect pwd (perhaps via ls accepting resolve path/parent)
- do_ls(repo, words[1:], out)
- elif cmd == 'cd':
- np = pwd
- for parm in words[1:]:
- res = vfs.resolve(repo, input_bytes(parm), parent=np)
- _, leaf_item = res[-1]
- if not leaf_item:
- raise Exception('%s does not exist'
- % path_msg(b'/'.join(name for name, item
- in res)))
- if not stat.S_ISDIR(vfs.item_mode(leaf_item)):
- raise Exception('%s is not a directory' % path_msg(parm))
- np = res
- pwd = np
- elif cmd == 'pwd':
- if len(pwd) == 1:
- out.write(b'/')
- out.write(b'/'.join(name for name, item in pwd) + b'\n')
- elif cmd == 'cat':
- for parm in words[1:]:
- res = vfs.resolve(repo, input_bytes(parm), parent=pwd)
- _, leaf_item = res[-1]
- if not leaf_item:
- raise Exception('%s does not exist' %
- path_msg(b'/'.join(name for name, item
- in res)))
- with vfs.fopen(repo, leaf_item) as srcfile:
- write_to_file(srcfile, out)
- elif cmd == 'get':
- if len(words) not in [2,3]:
- rv = 1
- raise Exception('Usage: get <filename> [localname]')
- rname = input_bytes(words[1])
- (dir,base) = os.path.split(rname)
- lname = input_bytes(len(words) > 2 and words[2] or base)
- res = vfs.resolve(repo, rname, parent=pwd)
- _, leaf_item = res[-1]
- if not leaf_item:
- raise Exception('%s does not exist' %
- path_msg(b'/'.join(name for name, item in res)))
- with vfs.fopen(repo, leaf_item) as srcfile:
- with open(lname, 'wb') as destfile:
- log('Saving %s\n' % path_msg(lname))
- write_to_file(srcfile, destfile)
- elif cmd == 'mget':
- for parm in words[1:]:
- dir, base = os.path.split(input_bytes(parm))
-
- res = vfs.resolve(repo, dir, parent=pwd)
- _, dir_item = res[-1]
- if not dir_item:
- raise Exception('%s does not exist' % path_msg(dir))
- for name, item in vfs.contents(repo, dir_item):
- if name == b'.':
- continue
- if fnmatch.fnmatch(name, base):
- if stat.S_ISLNK(vfs.item_mode(item)):
- deref = vfs.resolve(repo, name, parent=res)
- deref_name, deref_item = deref[-1]
- if not deref_item:
- raise Exception('%s does not exist' %
- path_msg('/'.join(name for name, item
- in deref)))
- item = deref_item
- with vfs.fopen(repo, item) as srcfile:
- with open(name, 'wb') as destfile:
- log('Saving %s\n' % path_msg(name))
- write_to_file(srcfile, destfile)
- elif cmd == 'help' or cmd == '?':
- # FIXME: move to stdout
- log('Commands: ls cd pwd cat get mget help quit\n')
- elif cmd in ('quit', 'exit', 'bye'):
- break
- else:
- rv = 1
- raise Exception('no such command %r' % cmd)
- except Exception as e:
- rv = 1
- log('error: %s\n' % e)
- raise
-
-sys.exit(rv)
+++ /dev/null
-#!/bin/sh
-"""": # -*-python-*-
-bup_python="$(dirname "$0")/bup-python" || exit $?
-exec "$bup_python" "$0" ${1+"$@"}
-"""
-# end of bup preamble
-
-from __future__ import absolute_import, print_function
-import sys, os, errno
-
-try:
- import fuse
-except ImportError:
- print('error: cannot find the python "fuse" module; please install it',
- file=sys.stderr)
- sys.exit(2)
-if not hasattr(fuse, '__version__'):
- print('error: fuse module is too old for fuse.__version__', file=sys.stderr)
- sys.exit(2)
-fuse.fuse_python_api = (0, 2)
-
-if sys.version_info[0] > 2:
- try:
- fuse_ver = fuse.__version__.split('.')
- fuse_ver_maj = int(fuse_ver[0])
- except:
- log('error: cannot determine the fuse major version; please report',
- file=sys.stderr)
- sys.exit(2)
- if len(fuse_ver) < 3 or fuse_ver_maj < 1:
- print("error: fuse module can't handle binary data; please upgrade to 1.0+\n",
- file=sys.stderr)
- sys.exit(2)
-
-from bup import options, git, vfs, xstat
-from bup.compat import argv_bytes, fsdecode, py_maj
-from bup.helpers import log
-from bup.repo import LocalRepo
-
-
-# FIXME: self.meta and want_meta?
-
-# The path handling is just wrong, but the current fuse module can't
-# handle bytes paths.
-
-class BupFs(fuse.Fuse):
- def __init__(self, repo, verbose=0, fake_metadata=False):
- fuse.Fuse.__init__(self)
- self.repo = repo
- self.verbose = verbose
- self.fake_metadata = fake_metadata
-
- def getattr(self, path):
- path = argv_bytes(path)
- global opt
- if self.verbose > 0:
- log('--getattr(%r)\n' % path)
- res = vfs.resolve(self.repo, path, want_meta=(not self.fake_metadata),
- follow=False)
- name, item = res[-1]
- if not item:
- return -errno.ENOENT
- if self.fake_metadata:
- item = vfs.augment_item_meta(self.repo, item, include_size=True)
- else:
- item = vfs.ensure_item_has_metadata(self.repo, item,
- include_size=True)
- meta = item.meta
- # FIXME: do we want/need to do anything more with nlink?
- st = fuse.Stat(st_mode=meta.mode, st_nlink=1, st_size=meta.size)
- st.st_mode = meta.mode
- st.st_uid = meta.uid or 0
- st.st_gid = meta.gid or 0
- st.st_atime = max(0, xstat.fstime_floor_secs(meta.atime))
- st.st_mtime = max(0, xstat.fstime_floor_secs(meta.mtime))
- st.st_ctime = max(0, xstat.fstime_floor_secs(meta.ctime))
- return st
-
- def readdir(self, path, offset):
- path = argv_bytes(path)
- assert not offset # We don't return offsets, so offset should be unused
- res = vfs.resolve(self.repo, path, follow=False)
- dir_name, dir_item = res[-1]
- if not dir_item:
- yield -errno.ENOENT
- yield fuse.Direntry('..')
- # FIXME: make sure want_meta=False is being completely respected
- for ent_name, ent_item in vfs.contents(repo, dir_item, want_meta=False):
- fusename = fsdecode(ent_name.replace(b'/', b'-'))
- yield fuse.Direntry(fusename)
-
- def readlink(self, path):
- path = argv_bytes(path)
- if self.verbose > 0:
- log('--readlink(%r)\n' % path)
- res = vfs.resolve(self.repo, path, follow=False)
- name, item = res[-1]
- if not item:
- return -errno.ENOENT
- return fsdecode(vfs.readlink(repo, item))
-
- def open(self, path, flags):
- path = argv_bytes(path)
- if self.verbose > 0:
- log('--open(%r)\n' % path)
- res = vfs.resolve(self.repo, path, follow=False)
- name, item = res[-1]
- if not item:
- return -errno.ENOENT
- accmode = os.O_RDONLY | os.O_WRONLY | os.O_RDWR
- if (flags & accmode) != os.O_RDONLY:
- return -errno.EACCES
- # Return None since read doesn't need the file atm...
- # If we *do* return the file, it'll show up as the last argument
- #return vfs.fopen(repo, item)
-
- def read(self, path, size, offset):
- path = argv_bytes(path)
- if self.verbose > 0:
- log('--read(%r)\n' % path)
- res = vfs.resolve(self.repo, path, follow=False)
- name, item = res[-1]
- if not item:
- return -errno.ENOENT
- with vfs.fopen(repo, item) as f:
- f.seek(offset)
- return f.read(size)
-
-
-optspec = """
-bup fuse [-d] [-f] <mountpoint>
---
-f,foreground run in foreground
-d,debug run in the foreground and display FUSE debug information
-o,allow-other allow other users to access the filesystem
-meta report original metadata for paths when available
-v,verbose increase log output (can be used more than once)
-"""
-o = options.Options(optspec)
-opt, flags, extra = o.parse(sys.argv[1:])
-if not opt.verbose:
- opt.verbose = 0
-
-# Set stderr to be line buffered, even if it's not connected to the console
-# so that we'll be able to see diagnostics in a timely fashion.
-errfd = sys.stderr.fileno()
-sys.stderr.flush()
-sys.stderr = os.fdopen(errfd, 'w', 1)
-
-if len(extra) != 1:
- o.fatal('only one mount point argument expected')
-
-git.check_repo_or_die()
-repo = LocalRepo()
-f = BupFs(repo=repo, verbose=opt.verbose, fake_metadata=(not opt.meta))
-
-# This is likely wrong, but the fuse module doesn't currently accept bytes
-f.fuse_args.mountpoint = extra[0]
-
-if opt.debug:
- f.fuse_args.add('debug')
-if opt.foreground:
- f.fuse_args.setmod('foreground')
-f.multithreaded = False
-if opt.allow_other:
- f.fuse_args.add('allow_other')
-f.main()
+++ /dev/null
-#!/bin/sh
-"""": # -*-python-*-
-bup_python="$(dirname "$0")/bup-python" || exit $?
-exec "$bup_python" "$0" ${1+"$@"}
-"""
-# end of bup preamble
-
-from __future__ import absolute_import
-import sys
-
-from bup import git, options
-from bup.gc import bup_gc
-from bup.helpers import die_if_errors, handle_ctrl_c, log
-
-
-optspec = """
-bup gc [options...]
---
-v,verbose increase log output (can be used more than once)
-threshold= only rewrite a packfile if it's over this percent garbage [10]
-#,compress= set compression level to # (0-9, 9 is highest) [1]
-unsafe use the command even though it may be DANGEROUS
-"""
-
-# FIXME: server mode?
-# FIXME: make sure client handles server-side changes reasonably
-
-handle_ctrl_c()
-
-o = options.Options(optspec)
-(opt, flags, extra) = o.parse(sys.argv[1:])
-
-if not opt.unsafe:
- o.fatal('refusing to run dangerous, experimental command without --unsafe')
-
-if extra:
- o.fatal('no positional parameters expected')
-
-if opt.threshold:
- try:
- opt.threshold = int(opt.threshold)
- except ValueError:
- o.fatal('threshold must be an integer percentage value')
- if opt.threshold < 0 or opt.threshold > 100:
- o.fatal('threshold must be an integer percentage value')
-
-git.check_repo_or_die()
-
-bup_gc(threshold=opt.threshold,
- compression=opt.compress,
- verbosity=opt.verbose)
-
-die_if_errors()
+++ /dev/null
-#!/bin/sh
-"""": # -*-python-*-
-bup_python="$(dirname "$0")/bup-python" || exit $?
-exec "$bup_python" "$0" ${1+"$@"}
-"""
-# end of bup preamble
-
-from __future__ import absolute_import, print_function
-import os, re, stat, sys, textwrap, time
-from binascii import hexlify, unhexlify
-from collections import namedtuple
-from functools import partial
-from stat import S_ISDIR
-
-from bup import git, client, helpers, vfs
-from bup.compat import argv_bytes, environ, hexstr, items, wrap_main
-from bup.git import get_cat_data, parse_commit, walk_object
-from bup.helpers import add_error, debug1, handle_ctrl_c, log, saved_errors
-from bup.helpers import hostname, shstr, tty_width
-from bup.io import path_msg
-from bup.pwdgrp import userfullname, username
-from bup.repo import LocalRepo, RemoteRepo
-
-argspec = (
- "usage: bup get [-s source] [-r remote] (<--ff|--append|...> REF [DEST])...",
-
- """Transfer data from a source repository to a destination repository
- according to the methods specified (--ff, --ff:, --append, etc.).
- Both repositories default to BUP_DIR. A remote destination may be
- specified with -r, and data may be pulled from a remote repository
- with the related "bup on HOST get ..." command.""",
-
- ('optional arguments:',
- (('-h, --help', 'show this help message and exit'),
- ('-v, --verbose',
- 'increase log output (can be specified more than once)'),
- ('-q, --quiet', "don't show progress meter"),
- ('-s SOURCE, --source SOURCE',
- 'path to the source repository (defaults to BUP_DIR)'),
- ('-r REMOTE, --remote REMOTE',
- 'hostname:/path/to/repo of remote destination repository'),
- ('-t --print-trees', 'output a tree id for each ref set'),
- ('-c, --print-commits', 'output a commit id for each ref set'),
- ('--print-tags', 'output an id for each tag'),
- ('--bwlimit BWLIMIT', 'maximum bytes/sec to transmit to server'),
- ('-0, -1, -2, -3, -4, -5, -6, -7, -8, -9, --compress LEVEL',
- 'set compression LEVEL (default: 1)'))),
-
- ('transfer methods:',
- (('--ff REF, --ff: REF DEST',
- 'fast-forward dest REF (or DEST) to match source REF'),
- ('--append REF, --append: REF DEST',
- 'append REF (treeish or committish) to dest REF (or DEST)'),
- ('--pick REF, --pick: REF DEST',
- 'append single source REF commit to dest REF (or DEST)'),
- ('--force-pick REF, --force-pick: REF DEST',
- '--pick, overwriting REF (or DEST)'),
- ('--new-tag REF, --new-tag: REF DEST',
- 'tag source ref REF as REF (or DEST) in dest unless it already exists'),
- ('--replace, --replace: REF DEST',
- 'overwrite REF (or DEST) in dest with source REF'),
- ('--unnamed REF',
- 'fetch REF anonymously (without destination ref)'))))
-
-def render_opts(opts, width=None):
- if not width:
- width = tty_width()
- result = []
- for args, desc in opts:
- result.append(textwrap.fill(args, width=width,
- initial_indent=(' ' * 2),
- subsequent_indent=(' ' * 4)))
- result.append('\n')
- result.append(textwrap.fill(desc, width=width,
- initial_indent=(' ' * 6),
- subsequent_indent=(' ' * 6)))
- result.append('\n')
- return result
-
-def usage(argspec, width=None):
- if not width:
- width = tty_width()
- usage, preamble, groups = argspec[0], argspec[1], argspec[2:]
- msg = []
- msg.append(textwrap.fill(usage, width=width, subsequent_indent=' '))
- msg.append('\n\n')
- msg.append(textwrap.fill(preamble.replace('\n', ' '), width=width))
- msg.append('\n')
- for group_name, group_args in groups:
- msg.extend(['\n', group_name, '\n'])
- msg.extend(render_opts(group_args, width=width))
- return ''.join(msg)
-
-def misuse(message=None):
- sys.stderr.write(usage(argspec))
- if message:
- sys.stderr.write('\nerror: ')
- sys.stderr.write(message)
- sys.stderr.write('\n')
- sys.exit(1)
-
-def require_n_args_or_die(n, args):
- if len(args) < n + 1:
- misuse('%s argument requires %d %s'
- % (n, 'values' if n == 1 else 'value'))
- result = args[1:1+n], args[1+n:]
- assert len(result[0]) == n
- return result
-
-Spec = namedtuple('Spec', ('method', 'src', 'dest'))
-
-def spec_msg(s):
- if not s.dest:
- return '--%s %s' % (s.method, path_msg(s.src))
- return '--%s: %s %s' % (s.method, path_msg(s.src), path_msg(s.dest))
-
-def parse_args(args):
- class GetOpts:
- pass
- opt = GetOpts()
- opt.help = False
- opt.verbose = 0
- opt.quiet = False
- opt.print_commits = opt.print_trees = opt.print_tags = False
- opt.bwlimit = None
- opt.compress = 1
- opt.source = opt.remote = None
- opt.target_specs = []
-
- remaining = args[1:] # Skip argv[0]
- while remaining:
- arg = remaining[0]
- if arg in ('-h', '--help'):
- sys.stdout.write(usage(argspec))
- sys.exit(0)
- elif arg in ('-v', '--verbose'):
- opt.verbose += 1
- remaining = remaining[1:]
- elif arg in ('--ff', '--append', '--pick', '--force-pick',
- '--new-tag', '--replace', '--unnamed'):
- (ref,), remaining = require_n_args_or_die(1, remaining)
- ref = argv_bytes(ref)
- opt.target_specs.append(Spec(method=arg[2:], src=ref, dest=None))
- elif arg in ('--ff:', '--append:', '--pick:', '--force-pick:',
- '--new-tag:', '--replace:'):
- (ref, dest), remaining = require_n_args_or_die(2, remaining)
- ref, dest = argv_bytes(ref), argv_bytes(dest)
- opt.target_specs.append(Spec(method=arg[2:-1], src=ref, dest=dest))
- elif arg in ('-s', '--source'):
- (opt.source,), remaining = require_n_args_or_die(1, remaining)
- elif arg in ('-r', '--remote'):
- (opt.remote,), remaining = require_n_args_or_die(1, remaining)
- elif arg in ('-c', '--print-commits'):
- opt.print_commits, remaining = True, remaining[1:]
- elif arg in ('-t', '--print-trees'):
- opt.print_trees, remaining = True, remaining[1:]
- elif arg == '--print-tags':
- opt.print_tags, remaining = True, remaining[1:]
- elif arg in ('-0', '-1', '-2', '-3', '-4', '-5', '-6', '-7', '-8', '-9'):
- opt.compress = int(arg[1:])
- remaining = remaining[1:]
- elif arg == '--compress':
- (opt.compress,), remaining = require_n_args_or_die(1, remaining)
- opt.compress = int(opt.compress)
- elif arg == '--bwlimit':
- (opt.bwlimit,), remaining = require_n_args_or_die(1, remaining)
- opt.bwlimit = long(opt.bwlimit)
- elif arg.startswith('-') and len(arg) > 2 and arg[1] != '-':
- # Try to interpret this as -xyz, i.e. "-xyz -> -x -y -z".
- # We do this last so that --foo -bar is valid if --foo
- # requires a value.
- remaining[0:1] = ('-' + c for c in arg[1:])
- # FIXME
- continue
- else:
- misuse()
- return opt
-
-# FIXME: client error handling (remote exceptions, etc.)
-
-# FIXME: walk_object in in git.py doesn't support opt.verbose. Do we
-# need to adjust for that here?
-def get_random_item(name, hash, repo, writer, opt):
- def already_seen(oid):
- return writer.exists(unhexlify(oid))
- for item in walk_object(repo.cat, hash, stop_at=already_seen,
- include_data=True):
- # already_seen ensures that writer.exists(id) is false.
- # Otherwise, just_write() would fail.
- writer.just_write(item.oid, item.type, item.data)
-
-
-def append_commit(name, hash, parent, src_repo, writer, opt):
- now = time.time()
- items = parse_commit(get_cat_data(src_repo.cat(hash), b'commit'))
- tree = unhexlify(items.tree)
- author = b'%s <%s>' % (items.author_name, items.author_mail)
- author_time = (items.author_sec, items.author_offset)
- committer = b'%s <%s@%s>' % (userfullname(), username(), hostname())
- get_random_item(name, hexlify(tree), src_repo, writer, opt)
- c = writer.new_commit(tree, parent,
- author, items.author_sec, items.author_offset,
- committer, now, None,
- items.message)
- return c, tree
-
-
-def append_commits(commits, src_name, dest_hash, src_repo, writer, opt):
- last_c, tree = dest_hash, None
- for commit in commits:
- last_c, tree = append_commit(src_name, commit, last_c,
- src_repo, writer, opt)
- assert(tree is not None)
- return last_c, tree
-
-Loc = namedtuple('Loc', ['type', 'hash', 'path'])
-default_loc = Loc(None, None, None)
-
-def find_vfs_item(name, repo):
- res = repo.resolve(name, follow=False, want_meta=False)
- leaf_name, leaf_item = res[-1]
- if not leaf_item:
- return None
- kind = type(leaf_item)
- if kind == vfs.Root:
- kind = 'root'
- elif kind == vfs.Tags:
- kind = 'tags'
- elif kind == vfs.RevList:
- kind = 'branch'
- elif kind == vfs.Commit:
- if len(res) > 1 and type(res[-2][1]) == vfs.RevList:
- kind = 'save'
- else:
- kind = 'commit'
- elif kind == vfs.Item:
- if S_ISDIR(vfs.item_mode(leaf_item)):
- kind = 'tree'
- else:
- kind = 'blob'
- elif kind == vfs.Chunky:
- kind = 'tree'
- elif kind == vfs.FakeLink:
- # Don't have to worry about ELOOP, excepting malicious
- # remotes, since "latest" is the only FakeLink.
- assert leaf_name == b'latest'
- res = repo.resolve(leaf_item.target, parent=res[:-1],
- follow=False, want_meta=False)
- leaf_name, leaf_item = res[-1]
- assert leaf_item
- assert type(leaf_item) == vfs.Commit
- name = b'/'.join(x[0] for x in res)
- kind = 'save'
- else:
- raise Exception('unexpected resolution for %s: %r'
- % (path_msg(name), res))
- path = b'/'.join(name for name, item in res)
- if hasattr(leaf_item, 'coid'):
- result = Loc(type=kind, hash=leaf_item.coid, path=path)
- elif hasattr(leaf_item, 'oid'):
- result = Loc(type=kind, hash=leaf_item.oid, path=path)
- else:
- result = Loc(type=kind, hash=None, path=path)
- return result
-
-
-Target = namedtuple('Target', ['spec', 'src', 'dest'])
-
-def loc_desc(loc):
- if loc and loc.hash:
- loc = loc._replace(hash=hexlify(loc.hash))
- return repr(loc)
-
-
-# FIXME: see if resolve() means we can drop the vfs path cleanup
-
-def cleanup_vfs_path(p):
- result = os.path.normpath(p)
- if result.startswith(b'/'):
- return result
- return b'/' + result
-
-
-def validate_vfs_path(p):
- if p.startswith(b'/.') \
- and not p.startswith(b'/.tag/'):
- misuse('unsupported destination path %s in %s'
- % (path_msg(dest.path), spec_msg(spec)))
- return p
-
-
-def resolve_src(spec, src_repo):
- src = find_vfs_item(spec.src, src_repo)
- spec_args = spec_msg(spec)
- if not src:
- misuse('cannot find source for %s' % spec_args)
- if src.type == 'root':
- misuse('cannot fetch entire repository for %s' % spec_args)
- if src.type == 'tags':
- misuse('cannot fetch entire /.tag directory for %s' % spec_args)
- debug1('src: %s\n' % loc_desc(src))
- return src
-
-
-def get_save_branch(repo, path):
- res = repo.resolve(path, follow=False, want_meta=False)
- leaf_name, leaf_item = res[-1]
- if not leaf_item:
- misuse('error: cannot access %r in %r' % (leaf_name, path))
- assert len(res) == 3
- res_path = b'/'.join(name for name, item in res[:-1])
- return res_path
-
-
-def resolve_branch_dest(spec, src, src_repo, dest_repo):
- # Resulting dest must be treeish, or not exist.
- if not spec.dest:
- # Pick a default dest.
- if src.type == 'branch':
- spec = spec._replace(dest=spec.src)
- elif src.type == 'save':
- spec = spec._replace(dest=get_save_branch(src_repo, spec.src))
- elif src.path.startswith(b'/.tag/'): # Dest defaults to the same.
- spec = spec._replace(dest=spec.src)
-
- spec_args = spec_msg(spec)
- if not spec.dest:
- misuse('no destination (implicit or explicit) for %s', spec_args)
-
- dest = find_vfs_item(spec.dest, dest_repo)
- if dest:
- if dest.type == 'commit':
- misuse('destination for %s is a tagged commit, not a branch'
- % spec_args)
- if dest.type != 'branch':
- misuse('destination for %s is a %s, not a branch'
- % (spec_args, dest.type))
- else:
- dest = default_loc._replace(path=cleanup_vfs_path(spec.dest))
-
- if dest.path.startswith(b'/.'):
- misuse('destination for %s must be a valid branch name' % spec_args)
-
- debug1('dest: %s\n' % loc_desc(dest))
- return spec, dest
-
-
-def resolve_ff(spec, src_repo, dest_repo):
- src = resolve_src(spec, src_repo)
- spec_args = spec_msg(spec)
- if src.type == 'tree':
- misuse('%s is impossible; can only --append a tree to a branch'
- % spec_args)
- if src.type not in ('branch', 'save', 'commit'):
- misuse('source for %s must be a branch, save, or commit, not %s'
- % (spec_args, src.type))
- spec, dest = resolve_branch_dest(spec, src, src_repo, dest_repo)
- return Target(spec=spec, src=src, dest=dest)
-
-
-def handle_ff(item, src_repo, writer, opt):
- assert item.spec.method == 'ff'
- assert item.src.type in ('branch', 'save', 'commit')
- src_oidx = hexlify(item.src.hash)
- dest_oidx = hexlify(item.dest.hash) if item.dest.hash else None
- if not dest_oidx or dest_oidx in src_repo.rev_list(src_oidx):
- # Can fast forward.
- get_random_item(item.spec.src, src_oidx, src_repo, writer, opt)
- commit_items = parse_commit(get_cat_data(src_repo.cat(src_oidx), b'commit'))
- return item.src.hash, unhexlify(commit_items.tree)
- misuse('destination is not an ancestor of source for %s'
- % spec_msg(item.spec))
-
-
-def resolve_append(spec, src_repo, dest_repo):
- src = resolve_src(spec, src_repo)
- if src.type not in ('branch', 'save', 'commit', 'tree'):
- misuse('source for %s must be a branch, save, commit, or tree, not %s'
- % (spec_msg(spec), src.type))
- spec, dest = resolve_branch_dest(spec, src, src_repo, dest_repo)
- return Target(spec=spec, src=src, dest=dest)
-
-
-def handle_append(item, src_repo, writer, opt):
- assert item.spec.method == 'append'
- assert item.src.type in ('branch', 'save', 'commit', 'tree')
- assert item.dest.type == 'branch' or not item.dest.type
- src_oidx = hexlify(item.src.hash)
- if item.src.type == 'tree':
- get_random_item(item.spec.src, src_oidx, src_repo, writer, opt)
- parent = item.dest.hash
- msg = b'bup save\n\nGenerated by command:\n%r\n' % sys.argv
- userline = b'%s <%s@%s>' % (userfullname(), username(), hostname())
- now = time.time()
- commit = writer.new_commit(item.src.hash, parent,
- userline, now, None,
- userline, now, None, msg)
- return commit, item.src.hash
- commits = list(src_repo.rev_list(src_oidx))
- commits.reverse()
- return append_commits(commits, item.spec.src, item.dest.hash,
- src_repo, writer, opt)
-
-
-def resolve_pick(spec, src_repo, dest_repo):
- src = resolve_src(spec, src_repo)
- spec_args = spec_msg(spec)
- if src.type == 'tree':
- misuse('%s is impossible; can only --append a tree' % spec_args)
- if src.type not in ('commit', 'save'):
- misuse('%s impossible; can only pick a commit or save, not %s'
- % (spec_args, src.type))
- if not spec.dest:
- if src.path.startswith(b'/.tag/'):
- spec = spec._replace(dest=spec.src)
- elif src.type == 'save':
- spec = spec._replace(dest=get_save_branch(src_repo, spec.src))
- if not spec.dest:
- misuse('no destination provided for %s', spec_args)
- dest = find_vfs_item(spec.dest, dest_repo)
- if not dest:
- cp = validate_vfs_path(cleanup_vfs_path(spec.dest))
- dest = default_loc._replace(path=cp)
- else:
- if not dest.type == 'branch' and not dest.path.startswith(b'/.tag/'):
- misuse('%s destination is not a tag or branch' % spec_args)
- if spec.method == 'pick' \
- and dest.hash and dest.path.startswith(b'/.tag/'):
- misuse('cannot overwrite existing tag for %s (requires --force-pick)'
- % spec_args)
- return Target(spec=spec, src=src, dest=dest)
-
-
-def handle_pick(item, src_repo, writer, opt):
- assert item.spec.method in ('pick', 'force-pick')
- assert item.src.type in ('save', 'commit')
- src_oidx = hexlify(item.src.hash)
- if item.dest.hash:
- return append_commit(item.spec.src, src_oidx, item.dest.hash,
- src_repo, writer, opt)
- return append_commit(item.spec.src, src_oidx, None, src_repo, writer, opt)
-
-
-def resolve_new_tag(spec, src_repo, dest_repo):
- src = resolve_src(spec, src_repo)
- spec_args = spec_msg(spec)
- if not spec.dest and src.path.startswith(b'/.tag/'):
- spec = spec._replace(dest=src.path)
- if not spec.dest:
- misuse('no destination (implicit or explicit) for %s', spec_args)
- dest = find_vfs_item(spec.dest, dest_repo)
- if not dest:
- dest = default_loc._replace(path=cleanup_vfs_path(spec.dest))
- if not dest.path.startswith(b'/.tag/'):
- misuse('destination for %s must be a VFS tag' % spec_args)
- if dest.hash:
- misuse('cannot overwrite existing tag for %s (requires --replace)'
- % spec_args)
- return Target(spec=spec, src=src, dest=dest)
-
-
-def handle_new_tag(item, src_repo, writer, opt):
- assert item.spec.method == 'new-tag'
- assert item.dest.path.startswith(b'/.tag/')
- get_random_item(item.spec.src, hexlify(item.src.hash),
- src_repo, writer, opt)
- return (item.src.hash,)
-
-
-def resolve_replace(spec, src_repo, dest_repo):
- src = resolve_src(spec, src_repo)
- spec_args = spec_msg(spec)
- if not spec.dest:
- if src.path.startswith(b'/.tag/') or src.type == 'branch':
- spec = spec._replace(dest=spec.src)
- if not spec.dest:
- misuse('no destination provided for %s', spec_args)
- dest = find_vfs_item(spec.dest, dest_repo)
- if dest:
- if not dest.type == 'branch' and not dest.path.startswith(b'/.tag/'):
- misuse('%s impossible; can only overwrite branch or tag'
- % spec_args)
- else:
- cp = validate_vfs_path(cleanup_vfs_path(spec.dest))
- dest = default_loc._replace(path=cp)
- if not dest.path.startswith(b'/.tag/') \
- and not src.type in ('branch', 'save', 'commit'):
- misuse('cannot overwrite branch with %s for %s' % (src.type, spec_args))
- return Target(spec=spec, src=src, dest=dest)
-
-
-def handle_replace(item, src_repo, writer, opt):
- assert(item.spec.method == 'replace')
- if item.dest.path.startswith(b'/.tag/'):
- get_random_item(item.spec.src, hexlify(item.src.hash),
- src_repo, writer, opt)
- return (item.src.hash,)
- assert(item.dest.type == 'branch' or not item.dest.type)
- src_oidx = hexlify(item.src.hash)
- get_random_item(item.spec.src, src_oidx, src_repo, writer, opt)
- commit_items = parse_commit(get_cat_data(src_repo.cat(src_oidx), b'commit'))
- return item.src.hash, unhexlify(commit_items.tree)
-
-
-def resolve_unnamed(spec, src_repo, dest_repo):
- if spec.dest:
- misuse('destination name given for %s' % spec_msg(spec))
- src = resolve_src(spec, src_repo)
- return Target(spec=spec, src=src, dest=None)
-
-
-def handle_unnamed(item, src_repo, writer, opt):
- get_random_item(item.spec.src, hexlify(item.src.hash),
- src_repo, writer, opt)
- return (None,)
-
-
-def resolve_targets(specs, src_repo, dest_repo):
- resolved_items = []
- common_args = src_repo, dest_repo
- for spec in specs:
- debug1('initial-spec: %r\n' % (spec,))
- if spec.method == 'ff':
- resolved_items.append(resolve_ff(spec, *common_args))
- elif spec.method == 'append':
- resolved_items.append(resolve_append(spec, *common_args))
- elif spec.method in ('pick', 'force-pick'):
- resolved_items.append(resolve_pick(spec, *common_args))
- elif spec.method == 'new-tag':
- resolved_items.append(resolve_new_tag(spec, *common_args))
- elif spec.method == 'replace':
- resolved_items.append(resolve_replace(spec, *common_args))
- elif spec.method == 'unnamed':
- resolved_items.append(resolve_unnamed(spec, *common_args))
- else: # Should be impossible -- prevented by the option parser.
- assert(False)
-
- # FIXME: check for prefix overlap? i.e.:
- # bup get --ff foo --ff: baz foo/bar
- # bup get --new-tag .tag/foo --new-tag: bar .tag/foo/bar
-
- # Now that we have all the items, check for duplicate tags.
- tags_targeted = set()
- for item in resolved_items:
- dest_path = item.dest and item.dest.path
- if dest_path:
- assert(dest_path.startswith(b'/'))
- if dest_path.startswith(b'/.tag/'):
- if dest_path in tags_targeted:
- if item.spec.method not in ('replace', 'force-pick'):
- misuse('cannot overwrite tag %s via %s' \
- % (path_msg(dest_path), spec_msg(item.spec)))
- else:
- tags_targeted.add(dest_path)
- return resolved_items
-
-
-def log_item(name, type, opt, tree=None, commit=None, tag=None):
- if tag and opt.print_tags:
- print(hexstr(tag))
- if tree and opt.print_trees:
- print(hexstr(tree))
- if commit and opt.print_commits:
- print(hexstr(commit))
- if opt.verbose:
- last = ''
- if type in ('root', 'branch', 'save', 'commit', 'tree'):
- if not name.endswith(b'/'):
- last = '/'
- log('%s%s\n' % (path_msg(name), last))
-
-def main():
- handle_ctrl_c()
- is_reverse = environ.get(b'BUP_SERVER_REVERSE')
- opt = parse_args(sys.argv)
- git.check_repo_or_die()
- if opt.source:
- opt.source = argv_bytes(opt.source)
- src_dir = opt.source or git.repo()
- if opt.bwlimit:
- client.bwlimit = parse_num(opt.bwlimit)
- if is_reverse and opt.remote:
- misuse("don't use -r in reverse mode; it's automatic")
- if opt.remote:
- opt.remote = argv_bytes(opt.remote)
- if opt.remote or is_reverse:
- dest_repo = RemoteRepo(opt.remote)
- else:
- dest_repo = LocalRepo()
-
- with dest_repo as dest_repo:
- with LocalRepo(repo_dir=src_dir) as src_repo:
- with dest_repo.new_packwriter(compression_level=opt.compress) as writer:
- # Resolve and validate all sources and destinations,
- # implicit or explicit, and do it up-front, so we can
- # fail before we start writing (for any obviously
- # broken cases).
- target_items = resolve_targets(opt.target_specs,
- src_repo, dest_repo)
-
- updated_refs = {} # ref_name -> (original_ref, tip_commit(bin))
- no_ref_info = (None, None)
-
- handlers = {'ff': handle_ff,
- 'append': handle_append,
- 'force-pick': handle_pick,
- 'pick': handle_pick,
- 'new-tag': handle_new_tag,
- 'replace': handle_replace,
- 'unnamed': handle_unnamed}
-
- for item in target_items:
- debug1('get-spec: %r\n' % (item.spec,))
- debug1('get-src: %s\n' % loc_desc(item.src))
- debug1('get-dest: %s\n' % loc_desc(item.dest))
- dest_path = item.dest and item.dest.path
- if dest_path:
- if dest_path.startswith(b'/.tag/'):
- dest_ref = b'refs/tags/%s' % dest_path[6:]
- else:
- dest_ref = b'refs/heads/%s' % dest_path[1:]
- else:
- dest_ref = None
-
- dest_hash = item.dest and item.dest.hash
- orig_ref, cur_ref = updated_refs.get(dest_ref, no_ref_info)
- orig_ref = orig_ref or dest_hash
- cur_ref = cur_ref or dest_hash
-
- handler = handlers[item.spec.method]
- item_result = handler(item, src_repo, writer, opt)
- if len(item_result) > 1:
- new_id, tree = item_result
- else:
- new_id = item_result[0]
-
- if not dest_ref:
- log_item(item.spec.src, item.src.type, opt)
- else:
- updated_refs[dest_ref] = (orig_ref, new_id)
- if dest_ref.startswith(b'refs/tags/'):
- log_item(item.spec.src, item.src.type, opt, tag=new_id)
- else:
- log_item(item.spec.src, item.src.type, opt,
- tree=tree, commit=new_id)
-
- # Only update the refs at the very end, once the writer is
- # closed, so that if something goes wrong above, the old refs
- # will be undisturbed.
- for ref_name, info in items(updated_refs):
- orig_ref, new_ref = info
- try:
- dest_repo.update_ref(ref_name, new_ref, orig_ref)
- if opt.verbose:
- new_hex = hexlify(new_ref)
- if orig_ref:
- orig_hex = hexlify(orig_ref)
- log('updated %r (%s -> %s)\n' % (ref_name, orig_hex, new_hex))
- else:
- log('updated %r (%s)\n' % (ref_name, new_hex))
- except (git.GitError, client.ClientError) as ex:
- add_error('unable to update ref %r: %s' % (ref_name, ex))
-
- if saved_errors:
- log('WARNING: %d errors encountered while saving.\n' % len(saved_errors))
- sys.exit(1)
-
-wrap_main(main)
+++ /dev/null
-#!/bin/sh
-"""": # -*-python-*-
-bup_python="$(dirname "$0")/bup-python" || exit $?
-exec "$bup_python" "$0" ${1+"$@"}
-"""
-# end of bup preamble
-
-from __future__ import absolute_import
-import sys, os, glob
-from bup import options, path
-
-optspec = """
-bup help <command>
-"""
-o = options.Options(optspec)
-(opt, flags, extra) = o.parse(sys.argv[1:])
-
-if len(extra) == 0:
- # the wrapper program provides the default usage string
- os.execvp(path.exe(), ['bup'])
-elif len(extra) == 1:
- docname = (extra[0]=='bup' and 'bup' or ('bup-%s' % extra[0]))
- manpath = os.path.join(path.exedir(),
- 'Documentation/' + docname + '.[1-9]')
- g = glob.glob(manpath)
- try:
- if g:
- os.execvp('man', ['man', '-l', g[0]])
- else:
- os.execvp('man', ['man', docname])
- except OSError as e:
- sys.stderr.write('Unable to run man command: %s\n' % e)
- sys.exit(1)
-else:
- o.fatal("exactly one command name expected")
+++ /dev/null
-#!/bin/sh
-"""": # -*-python-*-
-bup_python="$(dirname "$0")/bup-python" || exit $?
-exec "$bup_python" "$0" ${1+"$@"}
-"""
-# end of bup preamble
-
-from __future__ import absolute_import
-from calendar import timegm
-from pipes import quote
-from subprocess import check_call
-from time import strftime, strptime
-import os
-import sys
-import tempfile
-
-from bup import git, helpers, options
-from bup.compat import argv_bytes, str_type
-from bup.helpers import (handle_ctrl_c,
- log,
- readpipe,
- shstr,
- saved_errors,
- unlink)
-import bup.path
-
-optspec = """
-bup import-duplicity [-n] <duplicity-source-url> <bup-save-name>
---
-n,dry-run don't do anything; just print what would be done
-"""
-
-def logcmd(cmd):
- log(shstr(cmd).decode('iso-8859-1', errors='replace') + '\n')
-
-def exc(cmd, shell=False):
- global opt
- logcmd(cmd)
- if not opt.dry_run:
- check_call(cmd, shell=shell)
-
-def exo(cmd, shell=False, preexec_fn=None, close_fds=True):
- global opt
- logcmd(cmd)
- if not opt.dry_run:
- return helpers.exo(cmd, shell=shell, preexec_fn=preexec_fn,
- close_fds=close_fds)[0]
-
-def redirect_dup_output():
- os.dup2(1, 3)
- os.dup2(1, 2)
-
-
-handle_ctrl_c()
-
-log('\nbup: import-duplicity is EXPERIMENTAL (proceed with caution)\n\n')
-
-o = options.Options(optspec)
-opt, flags, extra = o.parse(sys.argv[1:])
-
-if len(extra) < 1 or not extra[0]:
- o.fatal('duplicity source URL required')
-if len(extra) < 2 or not extra[1]:
- o.fatal('bup destination save name required')
-if len(extra) > 2:
- o.fatal('too many arguments')
-
-source_url, save_name = extra
-source_url = argv_bytes(source_url)
-save_name = argv_bytes(save_name)
-bup = bup.path.exe()
-
-git.check_repo_or_die()
-
-tmpdir = tempfile.mkdtemp(prefix=b'bup-import-dup-')
-try:
- dup = [b'duplicity', b'--archive-dir', tmpdir + b'/dup-cache']
- restoredir = tmpdir + b'/restore'
- tmpidx = tmpdir + b'/index'
-
- collection_status = \
- exo(dup + [b'collection-status', b'--log-fd=3', source_url],
- close_fds=False, preexec_fn=redirect_dup_output) # i.e. 3>&1 1>&2
- # Duplicity output lines of interest look like this (one leading space):
- # full 20150222T073111Z 1 noenc
- # inc 20150222T073233Z 1 noenc
- dup_timestamps = []
- for line in collection_status.splitlines():
- if line.startswith(b' inc '):
- assert(len(line) >= len(b' inc 20150222T073233Z'))
- dup_timestamps.append(line[5:21])
- elif line.startswith(b' full '):
- assert(len(line) >= len(b' full 20150222T073233Z'))
- dup_timestamps.append(line[6:22])
- for i, dup_ts in enumerate(dup_timestamps):
- tm = strptime(dup_ts.decode('ascii'), '%Y%m%dT%H%M%SZ')
- exc([b'rm', b'-rf', restoredir])
- exc(dup + [b'restore', b'-t', dup_ts, source_url, restoredir])
- exc([bup, b'index', b'-uxf', tmpidx, restoredir])
- exc([bup, b'save', b'--strip', b'--date', b'%d' % timegm(tm),
- b'-f', tmpidx, b'-n', save_name, restoredir])
- sys.stderr.flush()
-finally:
- exc([b'rm', b'-rf', tmpdir])
-
-if saved_errors:
- log('warning: %d errors encountered\n' % len(saved_errors))
- sys.exit(1)
+++ /dev/null
-#!/usr/bin/env bash
-
-cmd_dir="$(cd "$(dirname "$0")" && pwd)" || exit $?
-
-set -o pipefail
-
-must() {
- local file=${BASH_SOURCE[0]}
- local line=${BASH_LINENO[0]}
- "$@"
- local rc=$?
- if test $rc -ne 0; then
- echo "Failed at line $line in $file" 1>&2
- exit $rc
- fi
-}
-
-usage() {
- echo "Usage: bup import-rdiff-backup [-n]" \
- "<path to rdiff-backup root> <backup name>"
- echo "-n,--dry-run: just print what would be done"
- exit 1
-}
-
-control_c() {
- echo "bup import-rdiff-backup: signal 2 received" 1>&2
- exit 128
-}
-
-must trap control_c INT
-
-dry_run=
-while [ "$1" = "-n" -o "$1" = "--dry-run" ]; do
- dry_run=echo
- shift
-done
-
-bup()
-{
- $dry_run "$cmd_dir/bup" "$@"
-}
-
-snapshot_root="$1"
-branch="$2"
-
-[ -n "$snapshot_root" -a "$#" = 2 ] || usage
-
-if [ ! -e "$snapshot_root/." ]; then
- echo "'$snapshot_root' isn't a directory!"
- exit 1
-fi
-
-
-backups=$(must rdiff-backup --list-increments --parsable-output "$snapshot_root") \
- || exit $?
-backups_count=$(echo "$backups" | must wc -l) || exit $?
-counter=1
-echo "$backups" |
-while read timestamp type; do
- tmpdir=$(must mktemp -d import-rdiff-backup-XXXXXXX) || exit $?
-
- echo "Importing backup from $(date -d @$timestamp +%c) " \
- "($counter / $backups_count)" 1>&2
- echo 1>&2
-
- echo "Restoring from rdiff-backup..." 1>&2
- must rdiff-backup -r $timestamp "$snapshot_root" "$tmpdir"
- echo 1>&2
-
- echo "Importing into bup..." 1>&2
- TMPIDX=$(must mktemp -u import-rdiff-backup-idx-XXXXXXX) || exit $?
- must bup index -ux -f "$tmpidx" "$tmpdir"
- must bup save --strip --date="$timestamp" -f "$tmpidx" -n "$branch" "$tmpdir"
- must rm -f "$tmpidx"
-
- must rm -rf "$tmpdir"
- counter=$((counter+1))
- echo 1>&2
- echo 1>&2
-done
+++ /dev/null
-#!/bin/sh
-# Does an import of a rsnapshot archive.
-
-cmd_dir="$(cd "$(dirname "$0")" && pwd)" || exit $?
-
-usage() {
- echo "Usage: bup import-rsnapshot [-n]" \
- "<path to snapshot_root> [<backuptarget>]"
- echo "-n,--dry-run: just print what would be done"
- exit 1
-}
-
-DRY_RUN=
-while [ "$1" = "-n" -o "$1" = "--dry-run" ]; do
- DRY_RUN=echo
- shift
-done
-
-bup()
-{
- $DRY_RUN "$cmd_dir/bup" "$@"
-}
-
-SNAPSHOT_ROOT=$1
-TARGET=$2
-
-[ -n "$SNAPSHOT_ROOT" -a "$#" -le 2 ] || usage
-
-if [ ! -e "$SNAPSHOT_ROOT/." ]; then
- echo "'$SNAPSHOT_ROOT' isn't a directory!"
- exit 1
-fi
-
-
-cd "$SNAPSHOT_ROOT" || exit 2
-
-for SNAPSHOT in *; do
- [ -e "$SNAPSHOT/." ] || continue
- echo "snapshot='$SNAPSHOT'" >&2
- for BRANCH_PATH in "$SNAPSHOT/"*; do
- BRANCH=$(basename "$BRANCH_PATH") || exit $?
- [ -e "$BRANCH_PATH/." ] || continue
- [ -z "$TARGET" -o "$TARGET" = "$BRANCH" ] || continue
-
- echo "snapshot='$SNAPSHOT' branch='$BRANCH'" >&2
-
- # Get the snapshot's ctime
- DATE=$(perl -e '@a=stat($ARGV[0]) or die "$ARGV[0]: $!";
- print $a[10];' "$BRANCH_PATH")
- [ -n "$DATE" ] || exit 3
-
- TMPIDX=bupindex.$BRANCH.tmp
- bup index -ux -f "$TMPIDX" "$BRANCH_PATH/" || exit $?
- bup save --strip --date="$DATE" \
- -f "$TMPIDX" -n "$BRANCH" \
- "$BRANCH_PATH/" || exit $?
- rm "$TMPIDX" || exit $?
- done
-done
+++ /dev/null
-#!/bin/sh
-"""": # -*-python-*-
-bup_python="$(dirname "$0")/bup-python" || exit $?
-exec "$bup_python" "$0" ${1+"$@"}
-"""
-# end of bup preamble
-
-from __future__ import absolute_import, print_function
-from binascii import hexlify
-import sys, stat, time, os, errno, re
-
-from bup import metadata, options, git, index, drecurse, hlinkdb
-from bup.compat import argv_bytes
-from bup.drecurse import recursive_dirlist
-from bup.hashsplit import GIT_MODE_TREE, GIT_MODE_FILE
-from bup.helpers import (add_error, handle_ctrl_c, log, parse_excludes, parse_rx_excludes,
- progress, qprogress, saved_errors)
-from bup.io import byte_stream, path_msg
-
-
-class IterHelper:
- def __init__(self, l):
- self.i = iter(l)
- self.cur = None
- self.next()
-
- def __next__(self):
- self.cur = next(self.i, None)
- return self.cur
-
- next = __next__
-
-def check_index(reader):
- try:
- log('check: checking forward iteration...\n')
- e = None
- d = {}
- for e in reader.forward_iter():
- if e.children_n:
- if opt.verbose:
- log('%08x+%-4d %r\n' % (e.children_ofs, e.children_n,
- path_msg(e.name)))
- assert(e.children_ofs)
- assert e.name.endswith(b'/')
- assert(not d.get(e.children_ofs))
- d[e.children_ofs] = 1
- if e.flags & index.IX_HASHVALID:
- assert(e.sha != index.EMPTY_SHA)
- assert(e.gitmode)
- assert not e or bytes(e.name) == b'/' # last entry is *always* /
- log('check: checking normal iteration...\n')
- last = None
- for e in reader:
- if last:
- assert(last > e.name)
- last = e.name
- except:
- log('index error! at %r\n' % e)
- raise
- log('check: passed.\n')
-
-
-def clear_index(indexfile):
- indexfiles = [indexfile, indexfile + b'.meta', indexfile + b'.hlink']
- for indexfile in indexfiles:
- path = git.repo(indexfile)
- try:
- os.remove(path)
- if opt.verbose:
- log('clear: removed %s\n' % path_msg(path))
- except OSError as e:
- if e.errno != errno.ENOENT:
- raise
-
-
-def update_index(top, excluded_paths, exclude_rxs, xdev_exceptions, out=None):
- # tmax and start must be epoch nanoseconds.
- tmax = (time.time() - 1) * 10**9
- ri = index.Reader(indexfile)
- msw = index.MetaStoreWriter(indexfile + b'.meta')
- wi = index.Writer(indexfile, msw, tmax)
- rig = IterHelper(ri.iter(name=top))
- tstart = int(time.time()) * 10**9
-
- hlinks = hlinkdb.HLinkDB(indexfile + b'.hlink')
-
- fake_hash = None
- if opt.fake_valid:
- def fake_hash(name):
- return (GIT_MODE_FILE, index.FAKE_SHA)
-
- total = 0
- bup_dir = os.path.abspath(git.repo())
- index_start = time.time()
- for path, pst in recursive_dirlist([top],
- xdev=opt.xdev,
- bup_dir=bup_dir,
- excluded_paths=excluded_paths,
- exclude_rxs=exclude_rxs,
- xdev_exceptions=xdev_exceptions):
- if opt.verbose>=2 or (opt.verbose==1 and stat.S_ISDIR(pst.st_mode)):
- out.write(b'%s\n' % path)
- out.flush()
- elapsed = time.time() - index_start
- paths_per_sec = total / elapsed if elapsed else 0
- qprogress('Indexing: %d (%d paths/s)\r' % (total, paths_per_sec))
- elif not (total % 128):
- elapsed = time.time() - index_start
- paths_per_sec = total / elapsed if elapsed else 0
- qprogress('Indexing: %d (%d paths/s)\r' % (total, paths_per_sec))
- total += 1
-
- while rig.cur and rig.cur.name > path: # deleted paths
- if rig.cur.exists():
- rig.cur.set_deleted()
- rig.cur.repack()
- if rig.cur.nlink > 1 and not stat.S_ISDIR(rig.cur.mode):
- hlinks.del_path(rig.cur.name)
- rig.next()
-
- if rig.cur and rig.cur.name == path: # paths that already existed
- need_repack = False
- if(rig.cur.stale(pst, tstart, check_device=opt.check_device)):
- try:
- meta = metadata.from_path(path, statinfo=pst)
- except (OSError, IOError) as e:
- add_error(e)
- rig.next()
- continue
- if not stat.S_ISDIR(rig.cur.mode) and rig.cur.nlink > 1:
- hlinks.del_path(rig.cur.name)
- if not stat.S_ISDIR(pst.st_mode) and pst.st_nlink > 1:
- hlinks.add_path(path, pst.st_dev, pst.st_ino)
- # Clear these so they don't bloat the store -- they're
- # already in the index (since they vary a lot and they're
- # fixed length). If you've noticed "tmax", you might
- # wonder why it's OK to do this, since that code may
- # adjust (mangle) the index mtime and ctime -- producing
- # fake values which must not end up in a .bupm. However,
- # it looks like that shouldn't be possible: (1) When
- # "save" validates the index entry, it always reads the
- # metadata from the filesytem. (2) Metadata is only
- # read/used from the index if hashvalid is true. (3)
- # "faked" entries will be stale(), and so we'll invalidate
- # them below.
- meta.ctime = meta.mtime = meta.atime = 0
- meta_ofs = msw.store(meta)
- rig.cur.update_from_stat(pst, meta_ofs)
- rig.cur.invalidate()
- need_repack = True
- if not (rig.cur.flags & index.IX_HASHVALID):
- if fake_hash:
- if rig.cur.sha == index.EMPTY_SHA:
- rig.cur.gitmode, rig.cur.sha = fake_hash(path)
- rig.cur.flags |= index.IX_HASHVALID
- need_repack = True
- if opt.fake_invalid:
- rig.cur.invalidate()
- need_repack = True
- if need_repack:
- rig.cur.repack()
- rig.next()
- else: # new paths
- try:
- meta = metadata.from_path(path, statinfo=pst)
- except (OSError, IOError) as e:
- add_error(e)
- continue
- # See same assignment to 0, above, for rationale.
- meta.atime = meta.mtime = meta.ctime = 0
- meta_ofs = msw.store(meta)
- wi.add(path, pst, meta_ofs, hashgen=fake_hash)
- if not stat.S_ISDIR(pst.st_mode) and pst.st_nlink > 1:
- hlinks.add_path(path, pst.st_dev, pst.st_ino)
-
- elapsed = time.time() - index_start
- paths_per_sec = total / elapsed if elapsed else 0
- progress('Indexing: %d, done (%d paths/s).\n' % (total, paths_per_sec))
-
- hlinks.prepare_save()
-
- if ri.exists():
- ri.save()
- wi.flush()
- if wi.count:
- wr = wi.new_reader()
- if opt.check:
- log('check: before merging: oldfile\n')
- check_index(ri)
- log('check: before merging: newfile\n')
- check_index(wr)
- mi = index.Writer(indexfile, msw, tmax)
-
- for e in index.merge(ri, wr):
- # FIXME: shouldn't we remove deleted entries eventually? When?
- mi.add_ixentry(e)
-
- ri.close()
- mi.close()
- wr.close()
- wi.abort()
- else:
- wi.close()
-
- msw.close()
- hlinks.commit_save()
-
-
-optspec = """
-bup index <-p|-m|-s|-u|--clear|--check> [options...] <filenames...>
---
- Modes:
-p,print print the index entries for the given names (also works with -u)
-m,modified print only added/deleted/modified files (implies -p)
-s,status print each filename with a status char (A/M/D) (implies -p)
-u,update recursively update the index entries for the given file/dir names (default if no mode is specified)
-check carefully check index file integrity
-clear clear the default index
- Options:
-H,hash print the hash for each object next to its name
-l,long print more information about each file
-no-check-device don't invalidate an entry if the containing device changes
-fake-valid mark all index entries as up-to-date even if they aren't
-fake-invalid mark all index entries as invalid
-f,indexfile= the name of the index file (normally BUP_DIR/bupindex)
-exclude= a path to exclude from the backup (may be repeated)
-exclude-from= skip --exclude paths in file (may be repeated)
-exclude-rx= skip paths matching the unanchored regex (may be repeated)
-exclude-rx-from= skip --exclude-rx patterns in file (may be repeated)
-v,verbose increase log output (can be used more than once)
-x,xdev,one-file-system don't cross filesystem boundaries
-"""
-o = options.Options(optspec)
-(opt, flags, extra) = o.parse(sys.argv[1:])
-
-if not (opt.modified or \
- opt['print'] or \
- opt.status or \
- opt.update or \
- opt.check or \
- opt.clear):
- opt.update = 1
-if (opt.fake_valid or opt.fake_invalid) and not opt.update:
- o.fatal('--fake-{in,}valid are meaningless without -u')
-if opt.fake_valid and opt.fake_invalid:
- o.fatal('--fake-valid is incompatible with --fake-invalid')
-if opt.clear and opt.indexfile:
- o.fatal('cannot clear an external index (via -f)')
-
-# FIXME: remove this once we account for timestamp races, i.e. index;
-# touch new-file; index. It's possible for this to happen quickly
-# enough that new-file ends up with the same timestamp as the first
-# index, and then bup will ignore it.
-tick_start = time.time()
-time.sleep(1 - (tick_start - int(tick_start)))
-
-git.check_repo_or_die()
-
-handle_ctrl_c()
-
-if opt.verbose is None:
- opt.verbose = 0
-
-if opt.indexfile:
- indexfile = argv_bytes(opt.indexfile)
-else:
- indexfile = git.repo(b'bupindex')
-
-if opt.check:
- log('check: starting initial check.\n')
- check_index(index.Reader(indexfile))
-
-if opt.clear:
- log('clear: clearing index.\n')
- clear_index(indexfile)
-
-sys.stdout.flush()
-out = byte_stream(sys.stdout)
-
-if opt.update:
- if not extra:
- o.fatal('update mode (-u) requested but no paths given')
- extra = [argv_bytes(x) for x in extra]
- excluded_paths = parse_excludes(flags, o.fatal)
- exclude_rxs = parse_rx_excludes(flags, o.fatal)
- xexcept = index.unique_resolved_paths(extra)
- for rp, path in index.reduce_paths(extra):
- update_index(rp, excluded_paths, exclude_rxs, xdev_exceptions=xexcept,
- out=out)
-
-if opt['print'] or opt.status or opt.modified:
- extra = [argv_bytes(x) for x in extra]
- for name, ent in index.Reader(indexfile).filter(extra or [b'']):
- if (opt.modified
- and (ent.is_valid() or ent.is_deleted() or not ent.mode)):
- continue
- line = b''
- if opt.status:
- if ent.is_deleted():
- line += b'D '
- elif not ent.is_valid():
- if ent.sha == index.EMPTY_SHA:
- line += b'A '
- else:
- line += b'M '
- else:
- line += b' '
- if opt.hash:
- line += hexlify(ent.sha) + b' '
- if opt.long:
- line += b'%7s %7s ' % (oct(ent.mode), oct(ent.gitmode))
- out.write(line + (name or b'./') + b'\n')
-
-if opt.check and (opt['print'] or opt.status or opt.modified or opt.update):
- log('check: starting final check.\n')
- check_index(index.Reader(indexfile))
-
-if saved_errors:
- log('WARNING: %d errors encountered.\n' % len(saved_errors))
- sys.exit(1)
+++ /dev/null
-#!/bin/sh
-"""": # -*-python-*-
-bup_python="$(dirname "$0")/bup-python" || exit $?
-exec "$bup_python" "$0" ${1+"$@"}
-"""
-# end of bup preamble
-
-from __future__ import absolute_import
-import sys
-
-from bup import git, options, client
-from bup.helpers import log, saved_errors
-from bup.compat import argv_bytes
-
-
-optspec = """
-[BUP_DIR=...] bup init [-r host:path]
---
-r,remote= remote repository path
-"""
-o = options.Options(optspec)
-(opt, flags, extra) = o.parse(sys.argv[1:])
-
-if extra:
- o.fatal("no arguments expected")
-
-
-try:
- git.init_repo() # local repo
-except git.GitError as e:
- log("bup: error: could not init repository: %s" % e)
- sys.exit(1)
-
-if opt.remote:
- git.check_repo_or_die()
- cli = client.Client(argv_bytes(opt.remote), create=True)
- cli.close()
+++ /dev/null
-#!/bin/sh
-"""": # -*-python-*-
-bup_python="$(dirname "$0")/bup-python" || exit $?
-exec "$bup_python" "$0" ${1+"$@"}
-"""
-# end of bup preamble
-
-from __future__ import absolute_import
-import sys
-
-from bup import git, options
-from bup.compat import argv_bytes
-from bup.helpers import linereader, log
-from bup.io import byte_stream
-from bup.repo import LocalRepo, RemoteRepo
-
-
-optspec = """
-bup join [-r host:path] [refs or hashes...]
---
-r,remote= remote repository path
-o= output filename
-"""
-o = options.Options(optspec)
-(opt, flags, extra) = o.parse(sys.argv[1:])
-if opt.remote:
- opt.remote = argv_bytes(opt.remote)
-
-git.check_repo_or_die()
-
-stdin = byte_stream(sys.stdin)
-
-if not extra:
- extra = linereader(stdin)
-
-ret = 0
-repo = RemoteRepo(opt.remote) if opt.remote else LocalRepo()
-
-if opt.o:
- outfile = open(opt.o, 'wb')
-else:
- sys.stdout.flush()
- outfile = byte_stream(sys.stdout)
-
-for ref in [argv_bytes(x) for x in extra]:
- try:
- for blob in repo.join(ref):
- outfile.write(blob)
- except KeyError as e:
- outfile.flush()
- log('error: %s\n' % e)
- ret = 1
-
-sys.exit(ret)
+++ /dev/null
-#!/bin/sh
-"""": # -*-python-*-
-bup_python="$(dirname "$0")/bup-python" || exit $?
-exec "$bup_python" "$0" ${1+"$@"}
-"""
-# end of bup preamble
-
-from __future__ import absolute_import, print_function
-from binascii import hexlify, unhexlify
-import sys, os
-
-from bup import git, options
-from bup.compat import argv_bytes
-from bup.helpers import add_error, handle_ctrl_c, log, qprogress, saved_errors
-from bup.io import byte_stream
-
-optspec = """
-bup list-idx [--find=<prefix>] <idxfilenames...>
---
-find= display only objects that start with <prefix>
-"""
-o = options.Options(optspec)
-(opt, flags, extra) = o.parse(sys.argv[1:])
-
-handle_ctrl_c()
-
-opt.find = argv_bytes(opt.find) if opt.find else b''
-
-if not extra:
- o.fatal('you must provide at least one filename')
-
-if len(opt.find) > 40:
- o.fatal('--find parameter must be <= 40 chars long')
-else:
- if len(opt.find) % 2:
- s = opt.find + b'0'
- else:
- s = opt.find
- try:
- bin = unhexlify(s)
- except TypeError:
- o.fatal('--find parameter is not a valid hex string')
-
-sys.stdout.flush()
-out = byte_stream(sys.stdout)
-find = opt.find.lower()
-count = 0
-idxfiles = [argv_bytes(x) for x in extra]
-for name in idxfiles:
- try:
- ix = git.open_idx(name)
- except git.GitError as e:
- add_error('%r: %s' % (name, e))
- continue
- if len(opt.find) == 40:
- if ix.exists(bin):
- out.write(b'%s %s\n' % (name, find))
- else:
- # slow, exhaustive search
- for _i in ix:
- i = hexlify(_i)
- if i.startswith(find):
- out.write(b'%s %s\n' % (name, i))
- qprogress('Searching: %d\r' % count)
- count += 1
-
-if saved_errors:
- log('WARNING: %d errors encountered while saving.\n' % len(saved_errors))
- sys.exit(1)
+++ /dev/null
-#!/bin/sh
-"""": # -*-python-*-
-bup_python="$(dirname "$0")/bup-python" || exit $?
-exec "$bup_python" "$0" ${1+"$@"}
-"""
-# end of bup preamble
-
-from __future__ import absolute_import
-import sys
-
-from bup import git, ls
-from bup.io import byte_stream
-
-
-git.check_repo_or_die()
-
-sys.stdout.flush()
-out = byte_stream(sys.stdout)
-# Check out lib/bup/ls.py for the opt spec
-rc = ls.via_cmdline(sys.argv[1:], out=out)
-sys.exit(rc)
+++ /dev/null
-#!/bin/sh
-"""": # -*-python-*-
-bup_python="$(dirname "$0")/bup-python" || exit $?
-exec "$bup_python" "$0" ${1+"$@"}
-"""
-# end of bup preamble
-
-from __future__ import absolute_import
-import sys, struct, math
-
-from bup import options, git, _helpers
-from bup.helpers import log
-from bup.io import byte_stream
-
-POPULATION_OF_EARTH=6.7e9 # as of September, 2010
-
-optspec = """
-bup margin
---
-predict Guess object offsets and report the maximum deviation
-ignore-midx Don't use midx files; use only plain pack idx files.
-"""
-o = options.Options(optspec)
-(opt, flags, extra) = o.parse(sys.argv[1:])
-
-if extra:
- o.fatal("no arguments expected")
-
-git.check_repo_or_die()
-
-mi = git.PackIdxList(git.repo(b'objects/pack'), ignore_midx=opt.ignore_midx)
-
-def do_predict(ix, out):
- total = len(ix)
- maxdiff = 0
- for count,i in enumerate(ix):
- prefix = struct.unpack('!Q', i[:8])[0]
- expected = prefix * total // (1 << 64)
- diff = count - expected
- maxdiff = max(maxdiff, abs(diff))
- out.write(b'%d of %d (%.3f%%) '
- % (maxdiff, len(ix), maxdiff * 100.0 / len(ix)))
- out.flush()
- assert(count+1 == len(ix))
-
-sys.stdout.flush()
-out = byte_stream(sys.stdout)
-
-if opt.predict:
- if opt.ignore_midx:
- for pack in mi.packs:
- do_predict(pack, out)
- else:
- do_predict(mi, out)
-else:
- # default mode: find longest matching prefix
- last = b'\0'*20
- longmatch = 0
- for i in mi:
- if i == last:
- continue
- #assert(str(i) >= last)
- pm = _helpers.bitmatch(last, i)
- longmatch = max(longmatch, pm)
- last = i
- out.write(b'%d\n' % longmatch)
- log('%d matching prefix bits\n' % longmatch)
- doublings = math.log(len(mi), 2)
- bpd = longmatch / doublings
- log('%.2f bits per doubling\n' % bpd)
- remain = 160 - longmatch
- rdoublings = remain / bpd
- log('%d bits (%.2f doublings) remaining\n' % (remain, rdoublings))
- larger = 2**rdoublings
- log('%g times larger is possible\n' % larger)
- perperson = larger/POPULATION_OF_EARTH
- log('\nEveryone on earth could have %d data sets like yours, all in one\n'
- 'repository, and we would expect 1 object collision.\n'
- % int(perperson))
+++ /dev/null
-#!/bin/sh
-"""": # -*-python-*-
-bup_python="$(dirname "$0")/bup-python" || exit $?
-exec "$bup_python" "$0" ${1+"$@"}
-"""
-# end of bup preamble
-
-from __future__ import absolute_import, print_function
-import sys, re, struct, time, resource
-
-from bup import git, bloom, midx, options, _helpers
-from bup.compat import range
-from bup.helpers import handle_ctrl_c
-from bup.io import byte_stream
-
-
-handle_ctrl_c()
-
-
-_linux_warned = 0
-def linux_memstat():
- global _linux_warned
- #fields = ['VmSize', 'VmRSS', 'VmData', 'VmStk', 'ms']
- d = {}
- try:
- f = open(b'/proc/self/status', 'rb')
- except IOError as e:
- if not _linux_warned:
- log('Warning: %s\n' % e)
- _linux_warned = 1
- return {}
- for line in f:
- # Note that on Solaris, this file exists but is binary. If that
- # happens, this split() might not return two elements. We don't
- # really need to care about the binary format since this output
- # isn't used for much and report() can deal with missing entries.
- t = re.split(br':\s*', line.strip(), 1)
- if len(t) == 2:
- k,v = t
- d[k] = v
- return d
-
-
-last = last_u = last_s = start = 0
-def report(count, out):
- global last, last_u, last_s, start
- headers = ['RSS', 'MajFlt', 'user', 'sys', 'ms']
- ru = resource.getrusage(resource.RUSAGE_SELF)
- now = time.time()
- rss = int(ru.ru_maxrss // 1024)
- if not rss:
- rss = linux_memstat().get(b'VmRSS', b'??')
- fields = [rss,
- ru.ru_majflt,
- int((ru.ru_utime - last_u) * 1000),
- int((ru.ru_stime - last_s) * 1000),
- int((now - last) * 1000)]
- fmt = '%9s ' + ('%10s ' * len(fields))
- if count >= 0:
- line = fmt % tuple([count] + fields)
- out.write(line.encode('ascii') + b'\n')
- else:
- start = now
- out.write((fmt % tuple([''] + headers)).encode('ascii') + b'\n')
- out.flush()
-
- # don't include time to run report() in usage counts
- ru = resource.getrusage(resource.RUSAGE_SELF)
- last_u = ru.ru_utime
- last_s = ru.ru_stime
- last = time.time()
-
-
-optspec = """
-bup memtest [-n elements] [-c cycles]
---
-n,number= number of objects per cycle [10000]
-c,cycles= number of cycles to run [100]
-ignore-midx ignore .midx files, use only .idx files
-existing test with existing objects instead of fake ones
-"""
-o = options.Options(optspec)
-(opt, flags, extra) = o.parse(sys.argv[1:])
-
-if extra:
- o.fatal('no arguments expected')
-
-git.check_repo_or_die()
-m = git.PackIdxList(git.repo(b'objects/pack'), ignore_midx=opt.ignore_midx)
-
-sys.stdout.flush()
-out = byte_stream(sys.stdout)
-
-report(-1, out)
-_helpers.random_sha()
-report(0, out)
-
-if opt.existing:
- def foreverit(mi):
- while 1:
- for e in mi:
- yield e
- objit = iter(foreverit(m))
-
-for c in range(opt.cycles):
- for n in range(opt.number):
- if opt.existing:
- bin = next(objit)
- assert(m.exists(bin))
- else:
- bin = _helpers.random_sha()
-
- # technically, a randomly generated object id might exist.
- # but the likelihood of that is the likelihood of finding
- # a collision in sha-1 by accident, which is so unlikely that
- # we don't care.
- assert(not m.exists(bin))
- report((c+1)*opt.number, out)
-
-if bloom._total_searches:
- out.write(b'bloom: %d objects searched in %d steps: avg %.3f steps/object\n'
- % (bloom._total_searches, bloom._total_steps,
- bloom._total_steps*1.0/bloom._total_searches))
-if midx._total_searches:
- out.write(b'midx: %d objects searched in %d steps: avg %.3f steps/object\n'
- % (midx._total_searches, midx._total_steps,
- midx._total_steps*1.0/midx._total_searches))
-if git._total_searches:
- out.write(b'idx: %d objects searched in %d steps: avg %.3f steps/object\n'
- % (git._total_searches, git._total_steps,
- git._total_steps*1.0/git._total_searches))
-out.write(b'Total time: %.3fs\n' % (time.time() - start))
+++ /dev/null
-#!/bin/sh
-"""": # -*-python-*-
-bup_python="$(dirname "$0")/bup-python" || exit $?
-exec "$bup_python" "$0" ${1+"$@"}
-"""
-# end of bup preamble
-
-# Copyright (C) 2010 Rob Browning
-#
-# This code is covered under the terms of the GNU Library General
-# Public License as described in the bup LICENSE file.
-
-# TODO: Add tar-like -C option.
-
-from __future__ import absolute_import
-import sys
-from bup import metadata
-from bup import options
-from bup.compat import argv_bytes
-from bup.io import byte_stream
-from bup.helpers import handle_ctrl_c, log, saved_errors
-
-
-def open_input(name):
- if not name or name == b'-':
- return byte_stream(sys.stdin)
- return open(name, 'rb')
-
-
-def open_output(name):
- if not name or name == b'-':
- sys.stdout.flush()
- return byte_stream(sys.stdout)
- return open(name, 'wb')
-
-
-optspec = """
-bup meta --create [OPTION ...] <PATH ...>
-bup meta --list [OPTION ...]
-bup meta --extract [OPTION ...]
-bup meta --start-extract [OPTION ...]
-bup meta --finish-extract [OPTION ...]
-bup meta --edit [OPTION ...] <PATH ...>
---
-c,create write metadata for PATHs to stdout (or --file)
-t,list display metadata
-x,extract perform --start-extract followed by --finish-extract
-start-extract build tree matching metadata provided on standard input (or --file)
-finish-extract finish applying standard input (or --file) metadata to filesystem
-edit alter metadata; write to stdout (or --file)
-f,file= specify source or destination file
-R,recurse recurse into subdirectories
-xdev,one-file-system don't cross filesystem boundaries
-numeric-ids apply numeric IDs (user, group, etc.) rather than names
-symlinks handle symbolic links (default is true)
-paths include paths in metadata (default is true)
-set-uid= set metadata uid (via --edit)
-set-gid= set metadata gid (via --edit)
-set-user= set metadata user (via --edit)
-unset-user remove metadata user (via --edit)
-set-group= set metadata group (via --edit)
-unset-group remove metadata group (via --edit)
-v,verbose increase log output (can be used more than once)
-q,quiet don't show progress meter
-"""
-
-handle_ctrl_c()
-
-o = options.Options(optspec)
-(opt, flags, remainder) = o.parse(['--paths', '--symlinks', '--recurse']
- + sys.argv[1:])
-
-opt.verbose = opt.verbose or 0
-opt.quiet = opt.quiet or 0
-metadata.verbose = opt.verbose - opt.quiet
-opt.file = argv_bytes(opt.file) if opt.file else None
-
-action_count = sum([bool(x) for x in [opt.create, opt.list, opt.extract,
- opt.start_extract, opt.finish_extract,
- opt.edit]])
-if action_count > 1:
- o.fatal("bup: only one action permitted: --create --list --extract --edit")
-if action_count == 0:
- o.fatal("bup: no action specified")
-
-if opt.create:
- if len(remainder) < 1:
- o.fatal("no paths specified for create")
- output_file = open_output(opt.file)
- metadata.save_tree(output_file,
- [argv_bytes(r) for r in remainder],
- recurse=opt.recurse,
- write_paths=opt.paths,
- save_symlinks=opt.symlinks,
- xdev=opt.xdev)
-elif opt.list:
- if len(remainder) > 0:
- o.fatal("cannot specify paths for --list")
- src = open_input(opt.file)
- metadata.display_archive(src, open_output(b'-'))
-elif opt.start_extract:
- if len(remainder) > 0:
- o.fatal("cannot specify paths for --start-extract")
- src = open_input(opt.file)
- metadata.start_extract(src, create_symlinks=opt.symlinks)
-elif opt.finish_extract:
- if len(remainder) > 0:
- o.fatal("cannot specify paths for --finish-extract")
- src = open_input(opt.file)
- metadata.finish_extract(src, restore_numeric_ids=opt.numeric_ids)
-elif opt.extract:
- if len(remainder) > 0:
- o.fatal("cannot specify paths for --extract")
- src = open_input(opt.file)
- metadata.extract(src,
- restore_numeric_ids=opt.numeric_ids,
- create_symlinks=opt.symlinks)
-elif opt.edit:
- if len(remainder) < 1:
- o.fatal("no paths specified for edit")
- output_file = open_output(opt.file)
-
- unset_user = False # True if --unset-user was the last relevant option.
- unset_group = False # True if --unset-group was the last relevant option.
- for flag in flags:
- if flag[0] == '--set-user':
- unset_user = False
- elif flag[0] == '--unset-user':
- unset_user = True
- elif flag[0] == '--set-group':
- unset_group = False
- elif flag[0] == '--unset-group':
- unset_group = True
-
- for path in remainder:
- f = open(argv_bytes(path), 'rb')
- try:
- for m in metadata._ArchiveIterator(f):
- if opt.set_uid is not None:
- try:
- m.uid = int(opt.set_uid)
- except ValueError:
- o.fatal("uid must be an integer")
-
- if opt.set_gid is not None:
- try:
- m.gid = int(opt.set_gid)
- except ValueError:
- o.fatal("gid must be an integer")
-
- if unset_user:
- m.user = b''
- elif opt.set_user is not None:
- m.user = argv_bytes(opt.set_user)
-
- if unset_group:
- m.group = b''
- elif opt.set_group is not None:
- m.group = argv_bytes(opt.set_group)
-
- m.write(output_file)
- finally:
- f.close()
-
-
-if saved_errors:
- log('WARNING: %d errors encountered.\n' % len(saved_errors))
- sys.exit(1)
-else:
- sys.exit(0)
+++ /dev/null
-#!/bin/sh
-"""": # -*-python-*-
-bup_python="$(dirname "$0")/bup-python" || exit $?
-exec "$bup_python" "$0" ${1+"$@"}
-"""
-# end of bup preamble
-
-from __future__ import absolute_import, print_function
-from binascii import hexlify
-import glob, math, os, resource, struct, sys, tempfile
-
-from bup import options, git, midx, _helpers, xstat
-from bup.compat import argv_bytes, hexstr, range
-from bup.helpers import (Sha1, add_error, atomically_replaced_file, debug1, fdatasync,
- handle_ctrl_c, log, mmap_readwrite, qprogress,
- saved_errors, unlink)
-from bup.io import byte_stream, path_msg
-
-
-PAGE_SIZE=4096
-SHA_PER_PAGE=PAGE_SIZE/20.
-
-optspec = """
-bup midx [options...] <idxnames...>
---
-o,output= output midx filename (default: auto-generated)
-a,auto automatically use all existing .midx/.idx files as input
-f,force merge produce exactly one .midx containing all objects
-p,print print names of generated midx files
-check validate contents of the given midx files (with -a, all midx files)
-max-files= maximum number of idx files to open at once [-1]
-d,dir= directory containing idx/midx files
-"""
-
-merge_into = _helpers.merge_into
-
-
-def _group(l, count):
- for i in range(0, len(l), count):
- yield l[i:i+count]
-
-
-def max_files():
- mf = min(resource.getrlimit(resource.RLIMIT_NOFILE))
- if mf > 32:
- mf -= 20 # just a safety margin
- else:
- mf -= 6 # minimum safety margin
- return mf
-
-
-def check_midx(name):
- nicename = git.repo_rel(name)
- log('Checking %s.\n' % path_msg(nicename))
- try:
- ix = git.open_idx(name)
- except git.GitError as e:
- add_error('%s: %s' % (pathmsg(name), e))
- return
- for count,subname in enumerate(ix.idxnames):
- sub = git.open_idx(os.path.join(os.path.dirname(name), subname))
- for ecount,e in enumerate(sub):
- if not (ecount % 1234):
- qprogress(' %d/%d: %s %d/%d\r'
- % (count, len(ix.idxnames),
- git.shorten_hash(subname).decode('ascii'),
- ecount, len(sub)))
- if not sub.exists(e):
- add_error("%s: %s: %s missing from idx"
- % (path_msg(nicename),
- git.shorten_hash(subname).decode('ascii'),
- hexstr(e)))
- if not ix.exists(e):
- add_error("%s: %s: %s missing from midx"
- % (path_msg(nicename),
- git.shorten_hash(subname).decode('ascii'),
- hexstr(e)))
- prev = None
- for ecount,e in enumerate(ix):
- if not (ecount % 1234):
- qprogress(' Ordering: %d/%d\r' % (ecount, len(ix)))
- if e and prev and not e >= prev:
- add_error('%s: ordering error: %s < %s'
- % (nicename, hexstr(e), hexstr(prev)))
- prev = e
-
-
-_first = None
-def _do_midx(outdir, outfilename, infilenames, prefixstr):
- global _first
- if not outfilename:
- assert(outdir)
- sum = hexlify(Sha1(b'\0'.join(infilenames)).digest())
- outfilename = b'%s/midx-%s.midx' % (outdir, sum)
-
- inp = []
- total = 0
- allfilenames = []
- midxs = []
- try:
- for name in infilenames:
- ix = git.open_idx(name)
- midxs.append(ix)
- inp.append((
- ix.map,
- len(ix),
- ix.sha_ofs,
- isinstance(ix, midx.PackMidx) and ix.which_ofs or 0,
- len(allfilenames),
- ))
- for n in ix.idxnames:
- allfilenames.append(os.path.basename(n))
- total += len(ix)
- inp.sort(reverse=True, key=lambda x: x[0][x[2] : x[2] + 20])
-
- if not _first: _first = outdir
- dirprefix = (_first != outdir) and git.repo_rel(outdir) + b': ' or b''
- debug1('midx: %s%screating from %d files (%d objects).\n'
- % (dirprefix, prefixstr, len(infilenames), total))
- if (opt.auto and (total < 1024 and len(infilenames) < 3)) \
- or ((opt.auto or opt.force) and len(infilenames) < 2) \
- or (opt.force and not total):
- debug1('midx: nothing to do.\n')
- return
-
- pages = int(total/SHA_PER_PAGE) or 1
- bits = int(math.ceil(math.log(pages, 2)))
- entries = 2**bits
- debug1('midx: table size: %d (%d bits)\n' % (entries*4, bits))
-
- unlink(outfilename)
- with atomically_replaced_file(outfilename, 'wb') as f:
- f.write(b'MIDX')
- f.write(struct.pack('!II', midx.MIDX_VERSION, bits))
- assert(f.tell() == 12)
-
- f.truncate(12 + 4*entries + 20*total + 4*total)
- f.flush()
- fdatasync(f.fileno())
-
- fmap = mmap_readwrite(f, close=False)
- count = merge_into(fmap, bits, total, inp)
- del fmap # Assume this calls msync() now.
- f.seek(0, os.SEEK_END)
- f.write(b'\0'.join(allfilenames))
- finally:
- for ix in midxs:
- if isinstance(ix, midx.PackMidx):
- ix.close()
- midxs = None
- inp = None
-
-
- # This is just for testing (if you enable this, don't clear inp above)
- if 0:
- p = midx.PackMidx(outfilename)
- assert(len(p.idxnames) == len(infilenames))
- log(repr(p.idxnames) + '\n')
- assert(len(p) == total)
- for pe, e in p, git.idxmerge(inp, final_progress=False):
- pin = next(pi)
- assert(i == pin)
- assert(p.exists(i))
-
- return total, outfilename
-
-
-def do_midx(outdir, outfilename, infilenames, prefixstr, prout):
- rv = _do_midx(outdir, outfilename, infilenames, prefixstr)
- if rv and opt['print']:
- prout.write(rv[1] + b'\n')
-
-
-def do_midx_dir(path, outfilename, prout):
- already = {}
- sizes = {}
- if opt.force and not opt.auto:
- midxs = [] # don't use existing midx files
- else:
- midxs = glob.glob(b'%s/*.midx' % path)
- contents = {}
- for mname in midxs:
- m = git.open_idx(mname)
- contents[mname] = [(b'%s/%s' % (path,i)) for i in m.idxnames]
- sizes[mname] = len(m)
-
- # sort the biggest+newest midxes first, so that we can eliminate
- # smaller (or older) redundant ones that come later in the list
- midxs.sort(key=lambda ix: (-sizes[ix], -xstat.stat(ix).st_mtime))
-
- for mname in midxs:
- any = 0
- for iname in contents[mname]:
- if not already.get(iname):
- already[iname] = 1
- any = 1
- if not any:
- debug1('%r is redundant\n' % mname)
- unlink(mname)
- already[mname] = 1
-
- midxs = [k for k in midxs if not already.get(k)]
- idxs = [k for k in glob.glob(b'%s/*.idx' % path) if not already.get(k)]
-
- for iname in idxs:
- i = git.open_idx(iname)
- sizes[iname] = len(i)
-
- all = [(sizes[n],n) for n in (midxs + idxs)]
-
- # FIXME: what are the optimal values? Does this make sense?
- DESIRED_HWM = opt.force and 1 or 5
- DESIRED_LWM = opt.force and 1 or 2
- existed = dict((name,1) for sz,name in all)
- debug1('midx: %d indexes; want no more than %d.\n'
- % (len(all), DESIRED_HWM))
- if len(all) <= DESIRED_HWM:
- debug1('midx: nothing to do.\n')
- while len(all) > DESIRED_HWM:
- all.sort()
- part1 = [name for sz,name in all[:len(all)-DESIRED_LWM+1]]
- part2 = all[len(all)-DESIRED_LWM+1:]
- all = list(do_midx_group(path, outfilename, part1)) + part2
- if len(all) > DESIRED_HWM:
- debug1('\nStill too many indexes (%d > %d). Merging again.\n'
- % (len(all), DESIRED_HWM))
-
- if opt['print']:
- for sz,name in all:
- if not existed.get(name):
- prout.write(name + b'\n')
-
-
-def do_midx_group(outdir, outfilename, infiles):
- groups = list(_group(infiles, opt.max_files))
- gprefix = ''
- for n,sublist in enumerate(groups):
- if len(groups) != 1:
- gprefix = 'Group %d: ' % (n+1)
- rv = _do_midx(outdir, outfilename, sublist, gprefix)
- if rv:
- yield rv
-
-
-handle_ctrl_c()
-
-o = options.Options(optspec)
-(opt, flags, extra) = o.parse(sys.argv[1:])
-opt.dir = argv_bytes(opt.dir) if opt.dir else None
-opt.output = argv_bytes(opt.output) if opt.output else None
-
-if extra and (opt.auto or opt.force):
- o.fatal("you can't use -f/-a and also provide filenames")
-if opt.check and (not extra and not opt.auto):
- o.fatal("if using --check, you must provide filenames or -a")
-
-git.check_repo_or_die()
-
-if opt.max_files < 0:
- opt.max_files = max_files()
-assert(opt.max_files >= 5)
-
-extra = [argv_bytes(x) for x in extra]
-
-if opt.check:
- # check existing midx files
- if extra:
- midxes = extra
- else:
- midxes = []
- paths = opt.dir and [opt.dir] or git.all_packdirs()
- for path in paths:
- debug1('midx: scanning %s\n' % path)
- midxes += glob.glob(os.path.join(path, b'*.midx'))
- for name in midxes:
- check_midx(name)
- if not saved_errors:
- log('All tests passed.\n')
-else:
- if extra:
- sys.stdout.flush()
- do_midx(git.repo(b'objects/pack'), opt.output, extra, b'',
- byte_stream(sys.stdout))
- elif opt.auto or opt.force:
- sys.stdout.flush()
- paths = opt.dir and [opt.dir] or git.all_packdirs()
- for path in paths:
- debug1('midx: scanning %s\n' % path_msg(path))
- do_midx_dir(path, opt.output, byte_stream(sys.stdout))
- else:
- o.fatal("you must use -f or -a or provide input filenames")
-
-if saved_errors:
- log('WARNING: %d errors encountered.\n' % len(saved_errors))
- sys.exit(1)
+++ /dev/null
-#!/bin/sh
-"""": # -*-python-*-
-bup_python="$(dirname "$0")/bup-python" || exit $?
-exec "$bup_python" "$0" ${1+"$@"}
-"""
-# end of bup preamble
-
-from __future__ import absolute_import
-import os, sys, subprocess, struct
-
-from bup import options
-from bup.helpers import debug1, debug2, mux
-from bup.io import byte_stream
-
-# Give the subcommand exclusive access to stdin.
-orig_stdin = os.dup(0)
-devnull = os.open(os.devnull, os.O_RDONLY)
-os.dup2(devnull, 0)
-os.close(devnull)
-
-optspec = """
-bup mux command [arguments...]
---
-"""
-o = options.Options(optspec)
-(opt, flags, extra) = o.parse(sys.argv[1:])
-if len(extra) < 1:
- o.fatal('command is required')
-
-subcmd = extra
-
-debug2('bup mux: starting %r\n' % (extra,))
-
-outr, outw = os.pipe()
-errr, errw = os.pipe()
-def close_fds():
- os.close(outr)
- os.close(errr)
-
-p = subprocess.Popen(subcmd, stdin=orig_stdin, stdout=outw, stderr=errw,
- close_fds=False, preexec_fn=close_fds)
-os.close(outw)
-os.close(errw)
-sys.stdout.flush()
-out = byte_stream(sys.stdout)
-out.write(b'BUPMUX')
-out.flush()
-mux(p, out.fileno(), outr, errr)
-os.close(outr)
-os.close(errr)
-prv = p.wait()
-
-if prv:
- debug1('%s exited with code %d\n' % (extra[0], prv))
-
-debug1('bup mux: done\n')
-
-sys.exit(prv)
+++ /dev/null
-#!/bin/sh
-"""": # -*-python-*-
-bup_python="$(dirname "$0")/bup-python" || exit $?
-exec "$bup_python" "$0" ${1+"$@"}
-"""
-# end of bup preamble
-
-from __future__ import absolute_import
-import sys, os, struct
-
-from bup import options, helpers, path
-from bup.compat import environ, py_maj
-from bup.io import byte_stream
-
-optspec = """
-bup on--server
---
- This command is run automatically by 'bup on'
-"""
-o = options.Options(optspec)
-(opt, flags, extra) = o.parse(sys.argv[1:])
-if extra:
- o.fatal('no arguments expected')
-
-# get the subcommand's argv.
-# Normally we could just pass this on the command line, but since we'll often
-# be getting called on the other end of an ssh pipe, which tends to mangle
-# argv (by sending it via the shell), this way is much safer.
-
-stdin = byte_stream(sys.stdin)
-buf = stdin.read(4)
-sz = struct.unpack('!I', buf)[0]
-assert(sz > 0)
-assert(sz < 1000000)
-buf = stdin.read(sz)
-assert(len(buf) == sz)
-argv = buf.split(b'\0')
-argv[0] = path.exe()
-argv = [argv[0], b'mux', b'--'] + argv
-
-
-# stdin/stdout are supposedly connected to 'bup server' that the caller
-# started for us (often on the other end of an ssh tunnel), so we don't want
-# to misuse them. Move them out of the way, then replace stdout with
-# a pointer to stderr in case our subcommand wants to do something with it.
-#
-# It might be nice to do the same with stdin, but my experiments showed that
-# ssh seems to make its child's stderr a readable-but-never-reads-anything
-# socket. They really should have used shutdown(SHUT_WR) on the other end
-# of it, but probably didn't. Anyway, it's too messy, so let's just make sure
-# anyone reading from stdin is disappointed.
-#
-# (You can't just leave stdin/stdout "not open" by closing the file
-# descriptors. Then the next file that opens is automatically assigned 0 or 1,
-# and people *trying* to read/write stdin/stdout get screwed.)
-os.dup2(0, 3)
-os.dup2(1, 4)
-os.dup2(2, 1)
-fd = os.open(os.devnull, os.O_RDONLY)
-os.dup2(fd, 0)
-os.close(fd)
-
-environ[b'BUP_SERVER_REVERSE'] = helpers.hostname()
-os.execvp(argv[0], argv)
-sys.exit(99)
+++ /dev/null
-#!/bin/sh
-"""": # -*-python-*-
-bup_python="$(dirname "$0")/bup-python" || exit $?
-exec "$bup_python" "$0" ${1+"$@"}
-"""
-# end of bup preamble
-
-from __future__ import absolute_import
-from subprocess import PIPE
-import sys, os, struct, getopt, subprocess, signal
-
-from bup import options, ssh, path
-from bup.compat import argv_bytes
-from bup.helpers import DemuxConn, log
-from bup.io import byte_stream
-
-
-optspec = """
-bup on <hostname> index ...
-bup on <hostname> save ...
-bup on <hostname> split ...
-bup on <hostname> get ...
-"""
-o = options.Options(optspec, optfunc=getopt.getopt)
-(opt, flags, extra) = o.parse(sys.argv[1:])
-if len(extra) < 2:
- o.fatal('arguments expected')
-
-class SigException(Exception):
- def __init__(self, signum):
- self.signum = signum
- Exception.__init__(self, 'signal %d received' % signum)
-def handler(signum, frame):
- raise SigException(signum)
-
-signal.signal(signal.SIGTERM, handler)
-signal.signal(signal.SIGINT, handler)
-
-sys.stdout.flush()
-out = byte_stream(sys.stdout)
-
-try:
- sp = None
- p = None
- ret = 99
-
- hp = argv_bytes(extra[0]).split(b':')
- if len(hp) == 1:
- (hostname, port) = (hp[0], None)
- else:
- (hostname, port) = hp
- argv = [argv_bytes(x) for x in extra[1:]]
- p = ssh.connect(hostname, port, b'on--server', stderr=PIPE)
-
- try:
- argvs = b'\0'.join([b'bup'] + argv)
- p.stdin.write(struct.pack('!I', len(argvs)) + argvs)
- p.stdin.flush()
- sp = subprocess.Popen([path.exe(), b'server'],
- stdin=p.stdout, stdout=p.stdin)
- p.stdin.close()
- p.stdout.close()
- # Demultiplex remote client's stderr (back to stdout/stderr).
- dmc = DemuxConn(p.stderr.fileno(), open(os.devnull, "wb"))
- for line in iter(dmc.readline, b''):
- out.write(line)
- finally:
- while 1:
- # if we get a signal while waiting, we have to keep waiting, just
- # in case our child doesn't die.
- try:
- ret = p.wait()
- if sp:
- sp.wait()
- break
- except SigException as e:
- log('\nbup on: %s\n' % e)
- os.kill(p.pid, e.signum)
- ret = 84
-except SigException as e:
- if ret == 0:
- ret = 99
- log('\nbup on: %s\n' % e)
-
-sys.exit(ret)
+++ /dev/null
-#!/bin/sh
-"""": # -*-python-*-
-bup_python="$(dirname "$0")/bup-python" || exit $?
-exec "$bup_python" "$0" ${1+"$@"}
-"""
-# end of bup preamble
-
-from __future__ import absolute_import, print_function
-from binascii import hexlify, unhexlify
-from collections import defaultdict
-from itertools import groupby
-from sys import stderr
-from time import localtime, strftime, time
-import re, sys
-
-from bup import git, options
-from bup.compat import argv_bytes, int_types
-from bup.gc import bup_gc
-from bup.helpers import die_if_errors, log, partition, period_as_secs
-from bup.io import byte_stream
-from bup.repo import LocalRepo
-from bup.rm import bup_rm
-
-
-def branches(refnames=tuple()):
- return ((name[11:], hexlify(sha)) for (name,sha)
- in git.list_refs(patterns=(b'refs/heads/' + n for n in refnames),
- limit_to_heads=True))
-
-def save_name(branch, utc):
- return branch + b'/' \
- + strftime('%Y-%m-%d-%H%M%S', localtime(utc)).encode('ascii')
-
-def classify_saves(saves, period_start):
- """For each (utc, id) in saves, yield (True, (utc, id)) if the save
- should be kept and (False, (utc, id)) if the save should be removed.
- The ids are binary hashes.
- """
-
- def retain_newest_in_region(region):
- for save in region[0:1]:
- yield True, save
- for save in region[1:]:
- yield False, save
-
- matches, rest = partition(lambda s: s[0] >= period_start['all'], saves)
- for save in matches:
- yield True, save
-
- tm_ranges = ((period_start['dailies'], lambda s: localtime(s[0]).tm_yday),
- (period_start['monthlies'], lambda s: localtime(s[0]).tm_mon),
- (period_start['yearlies'], lambda s: localtime(s[0]).tm_year))
-
- # Break the decreasing utc sorted saves up into the respective
- # period ranges (dailies, monthlies, ...). Within each range,
- # group the saves by the period scale (days, months, ...), and
- # then yield a "keep" action (True, utc) for the newest save in
- # each group, and a "drop" action (False, utc) for the rest.
- for pstart, time_region_id in tm_ranges:
- matches, rest = partition(lambda s: s[0] >= pstart, rest)
- for region_id, region_saves in groupby(matches, time_region_id):
- for action in retain_newest_in_region(list(region_saves)):
- yield action
-
- # Finally, drop any saves older than the specified periods
- for save in rest:
- yield False, save
-
-
-optspec = """
-bup prune-older [options...] [BRANCH...]
---
-keep-all-for= retain all saves within the PERIOD
-keep-dailies-for= retain the newest save per day within the PERIOD
-keep-monthlies-for= retain the newest save per month within the PERIOD
-keep-yearlies-for= retain the newest save per year within the PERIOD
-wrt= end all periods at this number of seconds since the epoch
-pretend don't prune, just report intended actions to standard output
-gc collect garbage after removals [1]
-gc-threshold= only rewrite a packfile if it's over this percent garbage [10]
-#,compress= set compression level to # (0-9, 9 is highest) [1]
-v,verbose increase log output (can be used more than once)
-unsafe use the command even though it may be DANGEROUS
-"""
-
-o = options.Options(optspec)
-opt, flags, roots = o.parse(sys.argv[1:])
-roots = [argv_bytes(x) for x in roots]
-
-if not opt.unsafe:
- o.fatal('refusing to run dangerous, experimental command without --unsafe')
-
-now = int(time()) if opt.wrt is None else opt.wrt
-if not isinstance(now, int_types):
- o.fatal('--wrt value ' + str(now) + ' is not an integer')
-
-period_start = {}
-for period, extent in (('all', opt.keep_all_for),
- ('dailies', opt.keep_dailies_for),
- ('monthlies', opt.keep_monthlies_for),
- ('yearlies', opt.keep_yearlies_for)):
- if extent:
- secs = period_as_secs(extent.encode('ascii'))
- if not secs:
- o.fatal('%r is not a valid period' % extent)
- period_start[period] = now - secs
-
-if not period_start:
- o.fatal('at least one keep argument is required')
-
-period_start = defaultdict(lambda: float('inf'), period_start)
-
-if opt.verbose:
- epoch_ymd = strftime('%Y-%m-%d-%H%M%S', localtime(0))
- for kind in ['all', 'dailies', 'monthlies', 'yearlies']:
- period_utc = period_start[kind]
- if period_utc != float('inf'):
- if not (period_utc > float('-inf')):
- log('keeping all ' + kind)
- else:
- try:
- when = strftime('%Y-%m-%d-%H%M%S', localtime(period_utc))
- log('keeping ' + kind + ' since ' + when + '\n')
- except ValueError as ex:
- if period_utc < 0:
- log('keeping %s since %d seconds before %s\n'
- %(kind, abs(period_utc), epoch_ymd))
- elif period_utc > 0:
- log('keeping %s since %d seconds after %s\n'
- %(kind, period_utc, epoch_ymd))
- else:
- log('keeping %s since %s\n' % (kind, epoch_ymd))
-
-git.check_repo_or_die()
-
-# This could be more efficient, but for now just build the whole list
-# in memory and let bup_rm() do some redundant work.
-
-def parse_info(f):
- author_secs = f.readline().strip()
- return int(author_secs)
-
-sys.stdout.flush()
-out = byte_stream(sys.stdout)
-
-removals = []
-for branch, branch_id in branches(roots):
- die_if_errors()
- saves = ((utc, unhexlify(oidx)) for (oidx, utc) in
- git.rev_list(branch_id, format=b'%at', parse=parse_info))
- for keep_save, (utc, id) in classify_saves(saves, period_start):
- assert(keep_save in (False, True))
- # FIXME: base removals on hashes
- if opt.pretend:
- out.write(b'+ ' if keep_save else b'- '
- + save_name(branch, utc) + b'\n')
- elif not keep_save:
- removals.append(save_name(branch, utc))
-
-if not opt.pretend:
- die_if_errors()
- repo = LocalRepo()
- bup_rm(repo, removals, compression=opt.compress, verbosity=opt.verbose)
- if opt.gc:
- die_if_errors()
- bup_gc(threshold=opt.gc_threshold,
- compression=opt.compress,
- verbosity=opt.verbose)
-
-die_if_errors()
+++ /dev/null
-#!/bin/sh
-
-set -e
-
-top="$(pwd)"
-cmdpath="$0"
-# loop because macos has no recursive resolution
-while test -L "$cmdpath"; do
- link="$(readlink "$cmdpath")"
- cd "$(dirname "$cmdpath")"
- cmdpath="$link"
-done
-script_home="$(cd "$(dirname "$cmdpath")" && pwd -P)"
-cd "$top"
-
-bup_libdir="$script_home/../lib" # bup_libdir will be adjusted during install
-export PYTHONPATH="$bup_libdir${PYTHONPATH:+:$PYTHONPATH}"
-
-# Force python to use ISO-8859-1 (aka Latin 1), a single-byte
-# encoding, to help avoid any manipulation of data from system APIs
-# (paths, users, groups, command line arguments, etc.)
-
-export PYTHONCOERCECLOCALE=0 # Perhaps not necessary, but shouldn't hurt
-
-# We can't just export LC_CTYPE directly here because the locale might
-# not exist outside python, and then bash (at least) may be cranky.
-
-if [ "${LC_ALL+x}" ]; then
- unset LC_ALL
- exec env \
- BUP_LC_ALL="$LC_ALL" \
- LC_COLLATE="$LC_ALL" \
- LC_MONETARY="$LC_ALL" \
- LC_NUMERIC="$LC_ALL" \
- LC_TIME="$LC_ALL" \
- LC_MESSAGES="$LC_ALL" \
- LC_CTYPE=ISO-8859-1 \
- @bup_python@ "$@"
-elif [ "${LC_CTYPE+x}" ]; then
- exec env \
- BUP_LC_CTYPE="$LC_CTYPE" \
- LC_CTYPE=ISO-8859-1 \
- @bup_python@ "$@"
-else
- exec env \
- LC_CTYPE=ISO-8859-1 \
- @bup_python@ "$@"
-fi
+++ /dev/null
-#!/bin/sh
-"""": # -*-python-*-
-bup_python="$(dirname "$0")/bup-python" || exit $?
-exec "$bup_python" "$0" ${1+"$@"}
-"""
-# end of bup preamble
-
-from __future__ import absolute_import
-import os, sys
-
-from bup import options, _helpers
-from bup.helpers import atoi, handle_ctrl_c, log, parse_num
-
-
-optspec = """
-bup random [-S seed] <numbytes>
---
-S,seed= optional random number seed [1]
-f,force print random data to stdout even if it's a tty
-v,verbose print byte counter to stderr
-"""
-o = options.Options(optspec)
-(opt, flags, extra) = o.parse(sys.argv[1:])
-
-if len(extra) != 1:
- o.fatal("exactly one argument expected")
-
-total = parse_num(extra[0])
-
-handle_ctrl_c()
-
-if opt.force or (not os.isatty(1) and
- not atoi(os.environ.get('BUP_FORCE_TTY')) & 1):
- _helpers.write_random(sys.stdout.fileno(), total, opt.seed,
- opt.verbose and 1 or 0)
-else:
- log('error: not writing binary data to a terminal. Use -f to force.\n')
- sys.exit(1)
+++ /dev/null
-#!/bin/sh
-"""": # -*-python-*-
-bup_python="$(dirname "$0")/bup-python" || exit $?
-exec "$bup_python" "$0" ${1+"$@"}
-"""
-# end of bup preamble
-
-from __future__ import absolute_import
-from stat import S_ISDIR
-import copy, errno, os, sys, stat, re
-
-from bup import options, git, metadata, vfs
-from bup._helpers import write_sparsely
-from bup.compat import argv_bytes, fsencode, wrap_main
-from bup.helpers import (add_error, chunkyreader, die_if_errors, handle_ctrl_c,
- log, mkdirp, parse_rx_excludes, progress, qprogress,
- saved_errors, should_rx_exclude_path, unlink)
-from bup.io import byte_stream
-from bup.repo import LocalRepo, RemoteRepo
-
-
-optspec = """
-bup restore [-r host:path] [-C outdir] </branch/revision/path/to/dir ...>
---
-r,remote= remote repository path
-C,outdir= change to given outdir before extracting files
-numeric-ids restore numeric IDs (user, group, etc.) rather than names
-exclude-rx= skip paths matching the unanchored regex (may be repeated)
-exclude-rx-from= skip --exclude-rx patterns in file (may be repeated)
-sparse create sparse files
-v,verbose increase log output (can be used more than once)
-map-user= given OLD=NEW, restore OLD user as NEW user
-map-group= given OLD=NEW, restore OLD group as NEW group
-map-uid= given OLD=NEW, restore OLD uid as NEW uid
-map-gid= given OLD=NEW, restore OLD gid as NEW gid
-q,quiet don't show progress meter
-"""
-
-total_restored = 0
-
-# stdout should be flushed after each line, even when not connected to a tty
-stdoutfd = sys.stdout.fileno()
-sys.stdout.flush()
-sys.stdout = os.fdopen(stdoutfd, 'w', 1)
-out = byte_stream(sys.stdout)
-
-def valid_restore_path(path):
- path = os.path.normpath(path)
- if path.startswith(b'/'):
- path = path[1:]
- if b'/' in path:
- return True
-
-def parse_owner_mappings(type, options, fatal):
- """Traverse the options and parse all --map-TYPEs, or call Option.fatal()."""
- opt_name = '--map-' + type
- if type in ('uid', 'gid'):
- value_rx = re.compile(br'^(-?[0-9]+)=(-?[0-9]+)$')
- else:
- value_rx = re.compile(br'^([^=]+)=([^=]*)$')
- owner_map = {}
- for flag in options:
- (option, parameter) = flag
- if option != opt_name:
- continue
- parameter = argv_bytes(parameter)
- match = value_rx.match(parameter)
- if not match:
- raise fatal("couldn't parse %r as %s mapping" % (parameter, type))
- old_id, new_id = match.groups()
- if type in ('uid', 'gid'):
- old_id = int(old_id)
- new_id = int(new_id)
- owner_map[old_id] = new_id
- return owner_map
-
-def apply_metadata(meta, name, restore_numeric_ids, owner_map):
- m = copy.deepcopy(meta)
- m.user = owner_map['user'].get(m.user, m.user)
- m.group = owner_map['group'].get(m.group, m.group)
- m.uid = owner_map['uid'].get(m.uid, m.uid)
- m.gid = owner_map['gid'].get(m.gid, m.gid)
- m.apply_to_path(name, restore_numeric_ids = restore_numeric_ids)
-
-def hardlink_compatible(prev_path, prev_item, new_item, top):
- prev_candidate = top + prev_path
- if not os.path.exists(prev_candidate):
- return False
- prev_meta, new_meta = prev_item.meta, new_item.meta
- if new_item.oid != prev_item.oid \
- or new_meta.mtime != prev_meta.mtime \
- or new_meta.ctime != prev_meta.ctime \
- or new_meta.mode != prev_meta.mode:
- return False
- # FIXME: should we be checking the path on disk, or the recorded metadata?
- # The exists() above might seem to suggest the former.
- if not new_meta.same_file(prev_meta):
- return False
- return True
-
-def hardlink_if_possible(fullname, item, top, hardlinks):
- """Find a suitable hardlink target, link to it, and return true,
- otherwise return false."""
- # The cwd will be dirname(fullname), and fullname will be
- # absolute, i.e. /foo/bar, and the caller is expected to handle
- # restoring the metadata if hardlinking isn't possible.
-
- # FIXME: we can probably replace the target_vfs_path with the
- # relevant vfs item
-
- # hardlinks tracks a list of (restore_path, vfs_path, meta)
- # triples for each path we've written for a given hardlink_target.
- # This allows us to handle the case where we restore a set of
- # hardlinks out of order (with respect to the original save
- # call(s)) -- i.e. when we don't restore the hardlink_target path
- # first. This data also allows us to attempt to handle other
- # situations like hardlink sets that change on disk during a save,
- # or between index and save.
-
- target = item.meta.hardlink_target
- assert(target)
- assert(fullname.startswith(b'/'))
- target_versions = hardlinks.get(target)
- if target_versions:
- # Check every path in the set that we've written so far for a match.
- for prev_path, prev_item in target_versions:
- if hardlink_compatible(prev_path, prev_item, item, top):
- try:
- os.link(top + prev_path, top + fullname)
- return True
- except OSError as e:
- if e.errno != errno.EXDEV:
- raise
- else:
- target_versions = []
- hardlinks[target] = target_versions
- target_versions.append((fullname, item))
- return False
-
-def write_file_content(repo, dest_path, vfs_file):
- with vfs.fopen(repo, vfs_file) as inf:
- with open(dest_path, 'wb') as outf:
- for b in chunkyreader(inf):
- outf.write(b)
-
-def write_file_content_sparsely(repo, dest_path, vfs_file):
- with vfs.fopen(repo, vfs_file) as inf:
- outfd = os.open(dest_path, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600)
- try:
- trailing_zeros = 0;
- for b in chunkyreader(inf):
- trailing_zeros = write_sparsely(outfd, b, 512, trailing_zeros)
- pos = os.lseek(outfd, trailing_zeros, os.SEEK_END)
- os.ftruncate(outfd, pos)
- finally:
- os.close(outfd)
-
-def restore(repo, parent_path, name, item, top, sparse, numeric_ids, owner_map,
- exclude_rxs, verbosity, hardlinks):
- global total_restored
- mode = vfs.item_mode(item)
- treeish = S_ISDIR(mode)
- fullname = parent_path + b'/' + name
- # Match behavior of index --exclude-rx with respect to paths.
- if should_rx_exclude_path(fullname + (b'/' if treeish else b''),
- exclude_rxs):
- return
-
- if not treeish:
- # Do this now so we'll have meta.symlink_target for verbose output
- item = vfs.augment_item_meta(repo, item, include_size=True)
- meta = item.meta
- assert(meta.mode == mode)
-
- if stat.S_ISDIR(mode):
- if verbosity >= 1:
- out.write(b'%s/\n' % fullname)
- elif stat.S_ISLNK(mode):
- assert(meta.symlink_target)
- if verbosity >= 2:
- out.write(b'%s@ -> %s\n' % (fullname, meta.symlink_target))
- else:
- if verbosity >= 2:
- out.write(fullname + '\n')
-
- orig_cwd = os.getcwd()
- try:
- if treeish:
- # Assumes contents() returns '.' with the full metadata first
- sub_items = vfs.contents(repo, item, want_meta=True)
- dot, item = next(sub_items, None)
- assert(dot == b'.')
- item = vfs.augment_item_meta(repo, item, include_size=True)
- meta = item.meta
- meta.create_path(name)
- os.chdir(name)
- total_restored += 1
- if verbosity >= 0:
- qprogress('Restoring: %d\r' % total_restored)
- for sub_name, sub_item in sub_items:
- restore(repo, fullname, sub_name, sub_item, top, sparse,
- numeric_ids, owner_map, exclude_rxs, verbosity,
- hardlinks)
- os.chdir(b'..')
- apply_metadata(meta, name, numeric_ids, owner_map)
- else:
- created_hardlink = False
- if meta.hardlink_target:
- created_hardlink = hardlink_if_possible(fullname, item, top,
- hardlinks)
- if not created_hardlink:
- meta.create_path(name)
- if stat.S_ISREG(meta.mode):
- if sparse:
- write_file_content_sparsely(repo, name, item)
- else:
- write_file_content(repo, name, item)
- total_restored += 1
- if verbosity >= 0:
- qprogress('Restoring: %d\r' % total_restored)
- if not created_hardlink:
- apply_metadata(meta, name, numeric_ids, owner_map)
- finally:
- os.chdir(orig_cwd)
-
-def main():
- o = options.Options(optspec)
- opt, flags, extra = o.parse(sys.argv[1:])
- verbosity = (opt.verbose or 0) if not opt.quiet else -1
- if opt.remote:
- opt.remote = argv_bytes(opt.remote)
- if opt.outdir:
- opt.outdir = argv_bytes(opt.outdir)
-
- git.check_repo_or_die()
-
- if not extra:
- o.fatal('must specify at least one filename to restore')
-
- exclude_rxs = parse_rx_excludes(flags, o.fatal)
-
- owner_map = {}
- for map_type in ('user', 'group', 'uid', 'gid'):
- owner_map[map_type] = parse_owner_mappings(map_type, flags, o.fatal)
-
- if opt.outdir:
- mkdirp(opt.outdir)
- os.chdir(opt.outdir)
-
- repo = RemoteRepo(opt.remote) if opt.remote else LocalRepo()
- top = fsencode(os.getcwd())
- hardlinks = {}
- for path in [argv_bytes(x) for x in extra]:
- if not valid_restore_path(path):
- add_error("path %r doesn't include a branch and revision" % path)
- continue
- try:
- resolved = vfs.resolve(repo, path, want_meta=True, follow=False)
- except vfs.IOError as e:
- add_error(e)
- continue
- if len(resolved) == 3 and resolved[2][0] == b'latest':
- # Follow latest symlink to the actual save
- try:
- resolved = vfs.resolve(repo, b'latest', parent=resolved[:-1],
- want_meta=True)
- except vfs.IOError as e:
- add_error(e)
- continue
- # Rename it back to 'latest'
- resolved = tuple(elt if i != 2 else (b'latest',) + elt[1:]
- for i, elt in enumerate(resolved))
- path_parent, path_name = os.path.split(path)
- leaf_name, leaf_item = resolved[-1]
- if not leaf_item:
- add_error('error: cannot access %r in %r'
- % ('/'.join(name for name, item in resolved),
- path))
- continue
- if not path_name or path_name == b'.':
- # Source is /foo/what/ever/ or /foo/what/ever/. -- extract
- # what/ever/* to the current directory, and if name == '.'
- # (i.e. /foo/what/ever/.), then also restore what/ever's
- # metadata to the current directory.
- treeish = vfs.item_mode(leaf_item)
- if not treeish:
- add_error('%r cannot be restored as a directory' % path)
- else:
- items = vfs.contents(repo, leaf_item, want_meta=True)
- dot, leaf_item = next(items, None)
- assert dot == b'.'
- for sub_name, sub_item in items:
- restore(repo, b'', sub_name, sub_item, top,
- opt.sparse, opt.numeric_ids, owner_map,
- exclude_rxs, verbosity, hardlinks)
- if path_name == b'.':
- leaf_item = vfs.augment_item_meta(repo, leaf_item,
- include_size=True)
- apply_metadata(leaf_item.meta, b'.',
- opt.numeric_ids, owner_map)
- else:
- restore(repo, b'', leaf_name, leaf_item, top,
- opt.sparse, opt.numeric_ids, owner_map,
- exclude_rxs, verbosity, hardlinks)
-
- if verbosity >= 0:
- progress('Restoring: %d, done.\n' % total_restored)
- die_if_errors()
-
-wrap_main(main)
+++ /dev/null
-#!/bin/sh
-"""": # -*-python-*-
-bup_python="$(dirname "$0")/bup-python" || exit $?
-exec "$bup_python" "$0" ${1+"$@"}
-"""
-# end of bup preamble
-
-from __future__ import absolute_import
-import sys
-
-from bup.compat import argv_bytes
-from bup.git import check_repo_or_die
-from bup.options import Options
-from bup.helpers import die_if_errors, handle_ctrl_c, log
-from bup.repo import LocalRepo
-from bup.rm import bup_rm
-
-optspec = """
-bup rm <branch|save...>
---
-#,compress= set compression level to # (0-9, 9 is highest) [6]
-v,verbose increase verbosity (can be specified multiple times)
-unsafe use the command even though it may be DANGEROUS
-"""
-
-handle_ctrl_c()
-
-o = Options(optspec)
-opt, flags, extra = o.parse(sys.argv[1:])
-
-if not opt.unsafe:
- o.fatal('refusing to run dangerous, experimental command without --unsafe')
-
-if len(extra) < 1:
- o.fatal('no paths specified')
-
-check_repo_or_die()
-repo = LocalRepo()
-bup_rm(repo, [argv_bytes(x) for x in extra],
- compression=opt.compress, verbosity=opt.verbose)
-die_if_errors()
+++ /dev/null
-#!/bin/sh
-"""": # -*-python-*-
-bup_python="$(dirname "$0")/bup-python" || exit $?
-exec "$bup_python" "$0" ${1+"$@"}
-"""
-# end of bup preamble
-
-from __future__ import absolute_import, print_function
-from binascii import hexlify
-from errno import EACCES
-from io import BytesIO
-import os, sys, stat, time, math
-
-from bup import hashsplit, git, options, index, client, metadata, hlinkdb
-from bup.compat import argv_bytes, environ
-from bup.hashsplit import GIT_MODE_TREE, GIT_MODE_FILE, GIT_MODE_SYMLINK
-from bup.helpers import (add_error, grafted_path_components, handle_ctrl_c,
- hostname, istty2, log, parse_date_or_fatal, parse_num,
- path_components, progress, qprogress, resolve_parent,
- saved_errors, stripped_path_components,
- valid_save_name)
-from bup.io import byte_stream, path_msg
-from bup.pwdgrp import userfullname, username
-
-
-optspec = """
-bup save [-tc] [-n name] <filenames...>
---
-r,remote= hostname:/path/to/repo of remote repository
-t,tree output a tree id
-c,commit output a commit id
-n,name= name of backup set to update (if any)
-d,date= date for the commit (seconds since the epoch)
-v,verbose increase log output (can be used more than once)
-q,quiet don't show progress meter
-smaller= only back up files smaller than n bytes
-bwlimit= maximum bytes/sec to transmit to server
-f,indexfile= the name of the index file (normally BUP_DIR/bupindex)
-strip strips the path to every filename given
-strip-path= path-prefix to be stripped when saving
-graft= a graft point *old_path*=*new_path* (can be used more than once)
-#,compress= set compression level to # (0-9, 9 is highest) [1]
-"""
-o = options.Options(optspec)
-(opt, flags, extra) = o.parse(sys.argv[1:])
-
-if opt.indexfile:
- opt.indexfile = argv_bytes(opt.indexfile)
-if opt.name:
- opt.name = argv_bytes(opt.name)
-if opt.remote:
- opt.remote = argv_bytes(opt.remote)
-if opt.strip_path:
- opt.strip_path = argv_bytes(opt.strip_path)
-
-git.check_repo_or_die()
-if not (opt.tree or opt.commit or opt.name):
- o.fatal("use one or more of -t, -c, -n")
-if not extra:
- o.fatal("no filenames given")
-
-extra = [argv_bytes(x) for x in extra]
-
-opt.progress = (istty2 and not opt.quiet)
-opt.smaller = parse_num(opt.smaller or 0)
-if opt.bwlimit:
- client.bwlimit = parse_num(opt.bwlimit)
-
-if opt.date:
- date = parse_date_or_fatal(opt.date, o.fatal)
-else:
- date = time.time()
-
-if opt.strip and opt.strip_path:
- o.fatal("--strip is incompatible with --strip-path")
-
-graft_points = []
-if opt.graft:
- if opt.strip:
- o.fatal("--strip is incompatible with --graft")
-
- if opt.strip_path:
- o.fatal("--strip-path is incompatible with --graft")
-
- for (option, parameter) in flags:
- if option == "--graft":
- parameter = argv_bytes(parameter)
- splitted_parameter = parameter.split(b'=')
- if len(splitted_parameter) != 2:
- o.fatal("a graft point must be of the form old_path=new_path")
- old_path, new_path = splitted_parameter
- if not (old_path and new_path):
- o.fatal("a graft point cannot be empty")
- graft_points.append((resolve_parent(old_path),
- resolve_parent(new_path)))
-
-is_reverse = environ.get(b'BUP_SERVER_REVERSE')
-if is_reverse and opt.remote:
- o.fatal("don't use -r in reverse mode; it's automatic")
-
-name = opt.name
-if name and not valid_save_name(name):
- o.fatal("'%s' is not a valid branch name" % path_msg(name))
-refname = name and b'refs/heads/%s' % name or None
-if opt.remote or is_reverse:
- try:
- cli = client.Client(opt.remote)
- except client.ClientError as e:
- log('error: %s' % e)
- sys.exit(1)
- oldref = refname and cli.read_ref(refname) or None
- w = cli.new_packwriter(compression_level=opt.compress)
-else:
- cli = None
- oldref = refname and git.read_ref(refname) or None
- w = git.PackWriter(compression_level=opt.compress)
-
-handle_ctrl_c()
-
-
-# Metadata is stored in a file named .bupm in each directory. The
-# first metadata entry will be the metadata for the current directory.
-# The remaining entries will be for each of the other directory
-# elements, in the order they're listed in the index.
-#
-# Since the git tree elements are sorted according to
-# git.shalist_item_sort_key, the metalist items are accumulated as
-# (sort_key, metadata) tuples, and then sorted when the .bupm file is
-# created. The sort_key should have been computed using the element's
-# mangled name and git mode (after hashsplitting), but the code isn't
-# actually doing that but rather uses the element's real name and mode.
-# This makes things a bit more difficult when reading it back, see
-# vfs.ordered_tree_entries().
-
-# Maintain a stack of information representing the current location in
-# the archive being constructed. The current path is recorded in
-# parts, which will be something like ['', 'home', 'someuser'], and
-# the accumulated content and metadata for of the dirs in parts is
-# stored in parallel stacks in shalists and metalists.
-
-parts = [] # Current archive position (stack of dir names).
-shalists = [] # Hashes for each dir in paths.
-metalists = [] # Metadata for each dir in paths.
-
-
-def _push(part, metadata):
- # Enter a new archive directory -- make it the current directory.
- parts.append(part)
- shalists.append([])
- metalists.append([(b'', metadata)]) # This dir's metadata (no name).
-
-
-def _pop(force_tree, dir_metadata=None):
- # Leave the current archive directory and add its tree to its parent.
- assert(len(parts) >= 1)
- part = parts.pop()
- shalist = shalists.pop()
- metalist = metalists.pop()
- # FIXME: only test if collision is possible (i.e. given --strip, etc.)?
- if force_tree:
- tree = force_tree
- else:
- names_seen = set()
- clean_list = []
- metaidx = 1 # entry at 0 is for the dir
- for x in shalist:
- name = x[1]
- if name in names_seen:
- parent_path = b'/'.join(parts) + b'/'
- add_error('error: ignoring duplicate path %s in %s'
- % (path_msg(name), path_msg(parent_path)))
- if not stat.S_ISDIR(x[0]):
- del metalist[metaidx]
- else:
- names_seen.add(name)
- clean_list.append(x)
- if not stat.S_ISDIR(x[0]):
- metaidx += 1
-
- if dir_metadata: # Override the original metadata pushed for this dir.
- metalist = [(b'', dir_metadata)] + metalist[1:]
- sorted_metalist = sorted(metalist, key = lambda x : x[0])
- metadata = b''.join([m[1].encode() for m in sorted_metalist])
- metadata_f = BytesIO(metadata)
- mode, id = hashsplit.split_to_blob_or_tree(w.new_blob, w.new_tree,
- [metadata_f],
- keep_boundaries=False)
- clean_list.append((mode, b'.bupm', id))
-
- tree = w.new_tree(clean_list)
- if shalists:
- shalists[-1].append((GIT_MODE_TREE,
- git.mangle_name(part,
- GIT_MODE_TREE, GIT_MODE_TREE),
- tree))
- return tree
-
-
-lastremain = None
-def progress_report(n):
- global count, subcount, lastremain
- subcount += n
- cc = count + subcount
- pct = total and (cc*100.0/total) or 0
- now = time.time()
- elapsed = now - tstart
- kps = elapsed and int(cc/1024./elapsed)
- kps_frac = 10 ** int(math.log(kps+1, 10) - 1)
- kps = int(kps/kps_frac)*kps_frac
- if cc:
- remain = elapsed*1.0/cc * (total-cc)
- else:
- remain = 0.0
- if (lastremain and (remain > lastremain)
- and ((remain - lastremain)/lastremain < 0.05)):
- remain = lastremain
- else:
- lastremain = remain
- hours = int(remain/60/60)
- mins = int(remain/60 - hours*60)
- secs = int(remain - hours*60*60 - mins*60)
- if elapsed < 30:
- remainstr = ''
- kpsstr = ''
- else:
- kpsstr = '%dk/s' % kps
- if hours:
- remainstr = '%dh%dm' % (hours, mins)
- elif mins:
- remainstr = '%dm%d' % (mins, secs)
- else:
- remainstr = '%ds' % secs
- qprogress('Saving: %.2f%% (%d/%dk, %d/%d files) %s %s\r'
- % (pct, cc/1024, total/1024, fcount, ftotal,
- remainstr, kpsstr))
-
-
-indexfile = opt.indexfile or git.repo(b'bupindex')
-r = index.Reader(indexfile)
-try:
- msr = index.MetaStoreReader(indexfile + b'.meta')
-except IOError as ex:
- if ex.errno != EACCES:
- raise
- log('error: cannot access %r; have you run bup index?'
- % path_msg(indexfile))
- sys.exit(1)
-hlink_db = hlinkdb.HLinkDB(indexfile + b'.hlink')
-
-def already_saved(ent):
- return ent.is_valid() and w.exists(ent.sha) and ent.sha
-
-def wantrecurse_pre(ent):
- return not already_saved(ent)
-
-def wantrecurse_during(ent):
- return not already_saved(ent) or ent.sha_missing()
-
-def find_hardlink_target(hlink_db, ent):
- if hlink_db and not stat.S_ISDIR(ent.mode) and ent.nlink > 1:
- link_paths = hlink_db.node_paths(ent.dev, ent.ino)
- if link_paths:
- return link_paths[0]
-
-total = ftotal = 0
-if opt.progress:
- for (transname,ent) in r.filter(extra, wantrecurse=wantrecurse_pre):
- if not (ftotal % 10024):
- qprogress('Reading index: %d\r' % ftotal)
- exists = ent.exists()
- hashvalid = already_saved(ent)
- ent.set_sha_missing(not hashvalid)
- if not opt.smaller or ent.size < opt.smaller:
- if exists and not hashvalid:
- total += ent.size
- ftotal += 1
- progress('Reading index: %d, done.\n' % ftotal)
- hashsplit.progress_callback = progress_report
-
-# Root collisions occur when strip or graft options map more than one
-# path to the same directory (paths which originally had separate
-# parents). When that situation is detected, use empty metadata for
-# the parent. Otherwise, use the metadata for the common parent.
-# Collision example: "bup save ... --strip /foo /foo/bar /bar".
-
-# FIXME: Add collision tests, or handle collisions some other way.
-
-# FIXME: Detect/handle strip/graft name collisions (other than root),
-# i.e. if '/foo/bar' and '/bar' both map to '/'.
-
-first_root = None
-root_collision = None
-tstart = time.time()
-count = subcount = fcount = 0
-lastskip_name = None
-lastdir = b''
-for (transname,ent) in r.filter(extra, wantrecurse=wantrecurse_during):
- (dir, file) = os.path.split(ent.name)
- exists = (ent.flags & index.IX_EXISTS)
- hashvalid = already_saved(ent)
- wasmissing = ent.sha_missing()
- oldsize = ent.size
- if opt.verbose:
- if not exists:
- status = 'D'
- elif not hashvalid:
- if ent.sha == index.EMPTY_SHA:
- status = 'A'
- else:
- status = 'M'
- else:
- status = ' '
- if opt.verbose >= 2:
- log('%s %-70s\n' % (status, path_msg(ent.name)))
- elif not stat.S_ISDIR(ent.mode) and lastdir != dir:
- if not lastdir.startswith(dir):
- log('%s %-70s\n' % (status, path_msg(os.path.join(dir, b''))))
- lastdir = dir
-
- if opt.progress:
- progress_report(0)
- fcount += 1
-
- if not exists:
- continue
- if opt.smaller and ent.size >= opt.smaller:
- if exists and not hashvalid:
- if opt.verbose:
- log('skipping large file "%s"\n' % path_msg(ent.name))
- lastskip_name = ent.name
- continue
-
- assert(dir.startswith(b'/'))
- if opt.strip:
- dirp = stripped_path_components(dir, extra)
- elif opt.strip_path:
- dirp = stripped_path_components(dir, [opt.strip_path])
- elif graft_points:
- dirp = grafted_path_components(graft_points, dir)
- else:
- dirp = path_components(dir)
-
- # At this point, dirp contains a representation of the archive
- # path that looks like [(archive_dir_name, real_fs_path), ...].
- # So given "bup save ... --strip /foo/bar /foo/bar/baz", dirp
- # might look like this at some point:
- # [('', '/foo/bar'), ('baz', '/foo/bar/baz'), ...].
-
- # This dual representation supports stripping/grafting, where the
- # archive path may not have a direct correspondence with the
- # filesystem. The root directory is represented by an initial
- # component named '', and any component that doesn't have a
- # corresponding filesystem directory (due to grafting, for
- # example) will have a real_fs_path of None, i.e. [('', None),
- # ...].
-
- if first_root == None:
- first_root = dirp[0]
- elif first_root != dirp[0]:
- root_collision = True
-
- # If switching to a new sub-tree, finish the current sub-tree.
- while parts > [x[0] for x in dirp]:
- _pop(force_tree = None)
-
- # If switching to a new sub-tree, start a new sub-tree.
- for path_component in dirp[len(parts):]:
- dir_name, fs_path = path_component
- # Not indexed, so just grab the FS metadata or use empty metadata.
- try:
- meta = metadata.from_path(fs_path, normalized=True) \
- if fs_path else metadata.Metadata()
- except (OSError, IOError) as e:
- add_error(e)
- lastskip_name = dir_name
- meta = metadata.Metadata()
- _push(dir_name, meta)
-
- if not file:
- if len(parts) == 1:
- continue # We're at the top level -- keep the current root dir
- # Since there's no filename, this is a subdir -- finish it.
- oldtree = already_saved(ent) # may be None
- newtree = _pop(force_tree = oldtree)
- if not oldtree:
- if lastskip_name and lastskip_name.startswith(ent.name):
- ent.invalidate()
- else:
- ent.validate(GIT_MODE_TREE, newtree)
- ent.repack()
- if exists and wasmissing:
- count += oldsize
- continue
-
- # it's not a directory
- if hashvalid:
- id = ent.sha
- git_name = git.mangle_name(file, ent.mode, ent.gitmode)
- git_info = (ent.gitmode, git_name, id)
- shalists[-1].append(git_info)
- sort_key = git.shalist_item_sort_key((ent.mode, file, id))
- meta = msr.metadata_at(ent.meta_ofs)
- meta.hardlink_target = find_hardlink_target(hlink_db, ent)
- # Restore the times that were cleared to 0 in the metastore.
- (meta.atime, meta.mtime, meta.ctime) = (ent.atime, ent.mtime, ent.ctime)
- metalists[-1].append((sort_key, meta))
- else:
- id = None
- if stat.S_ISREG(ent.mode):
- try:
- with hashsplit.open_noatime(ent.name) as f:
- (mode, id) = hashsplit.split_to_blob_or_tree(
- w.new_blob, w.new_tree, [f],
- keep_boundaries=False)
- except (IOError, OSError) as e:
- add_error('%s: %s' % (ent.name, e))
- lastskip_name = ent.name
- elif stat.S_ISDIR(ent.mode):
- assert(0) # handled above
- elif stat.S_ISLNK(ent.mode):
- try:
- rl = os.readlink(ent.name)
- except (OSError, IOError) as e:
- add_error(e)
- lastskip_name = ent.name
- else:
- (mode, id) = (GIT_MODE_SYMLINK, w.new_blob(rl))
- else:
- # Everything else should be fully described by its
- # metadata, so just record an empty blob, so the paths
- # in the tree and .bupm will match up.
- (mode, id) = (GIT_MODE_FILE, w.new_blob(b''))
-
- if id:
- ent.validate(mode, id)
- ent.repack()
- git_name = git.mangle_name(file, ent.mode, ent.gitmode)
- git_info = (mode, git_name, id)
- shalists[-1].append(git_info)
- sort_key = git.shalist_item_sort_key((ent.mode, file, id))
- hlink = find_hardlink_target(hlink_db, ent)
- try:
- meta = metadata.from_path(ent.name, hardlink_target=hlink,
- normalized=True)
- except (OSError, IOError) as e:
- add_error(e)
- lastskip_name = ent.name
- meta = metadata.Metadata()
- metalists[-1].append((sort_key, meta))
-
- if exists and wasmissing:
- count += oldsize
- subcount = 0
-
-
-if opt.progress:
- pct = total and count*100.0/total or 100
- progress('Saving: %.2f%% (%d/%dk, %d/%d files), done. \n'
- % (pct, count/1024, total/1024, fcount, ftotal))
-
-while len(parts) > 1: # _pop() all the parts above the root
- _pop(force_tree = None)
-assert(len(shalists) == 1)
-assert(len(metalists) == 1)
-
-# Finish the root directory.
-tree = _pop(force_tree = None,
- # When there's a collision, use empty metadata for the root.
- dir_metadata = metadata.Metadata() if root_collision else None)
-
-sys.stdout.flush()
-out = byte_stream(sys.stdout)
-
-if opt.tree:
- out.write(hexlify(tree))
- out.write(b'\n')
-if opt.commit or name:
- msg = (b'bup save\n\nGenerated by command:\n%r\n'
- % [argv_bytes(x) for x in sys.argv])
- userline = (b'%s <%s@%s>' % (userfullname(), username(), hostname()))
- commit = w.new_commit(tree, oldref, userline, date, None,
- userline, date, None, msg)
- if opt.commit:
- out.write(hexlify(commit))
- out.write(b'\n')
-
-msr.close()
-w.close() # must close before we can update the ref
-
-if opt.name:
- if cli:
- cli.update_ref(refname, commit, oldref)
- else:
- git.update_ref(refname, commit, oldref)
-
-if cli:
- cli.close()
-
-if saved_errors:
- log('WARNING: %d errors encountered while saving.\n' % len(saved_errors))
- sys.exit(1)
+++ /dev/null
-#!/bin/sh
-"""": # -*-python-*-
-bup_python="$(dirname "$0")/bup-python" || exit $?
-exec "$bup_python" "$0" ${1+"$@"}
-"""
-# end of bup preamble
-
-from __future__ import absolute_import
-from binascii import hexlify, unhexlify
-import os, sys, struct, subprocess
-
-from bup import options, git, vfs, vint
-from bup.compat import environ, hexstr
-from bup.git import MissingObject
-from bup.helpers import (Conn, debug1, debug2, linereader, lines_until_sentinel,
- log)
-from bup.io import byte_stream, path_msg
-from bup.repo import LocalRepo
-
-
-suspended_w = None
-dumb_server_mode = False
-repo = None
-
-def do_help(conn, junk):
- conn.write(b'Commands:\n %s\n' % b'\n '.join(sorted(commands)))
- conn.ok()
-
-
-def _set_mode():
- global dumb_server_mode
- dumb_server_mode = os.path.exists(git.repo(b'bup-dumb-server'))
- debug1('bup server: serving in %s mode\n'
- % (dumb_server_mode and 'dumb' or 'smart'))
-
-
-def _init_session(reinit_with_new_repopath=None):
- global repo
- if reinit_with_new_repopath is None and git.repodir:
- if not repo:
- repo = LocalRepo()
- return
- git.check_repo_or_die(reinit_with_new_repopath)
- if repo:
- repo.close()
- repo = LocalRepo()
- # OK. we now know the path is a proper repository. Record this path in the
- # environment so that subprocesses inherit it and know where to operate.
- environ[b'BUP_DIR'] = git.repodir
- debug1('bup server: bupdir is %s\n' % path_msg(git.repodir))
- _set_mode()
-
-
-def init_dir(conn, arg):
- git.init_repo(arg)
- debug1('bup server: bupdir initialized: %s\n' % path_msg(git.repodir))
- _init_session(arg)
- conn.ok()
-
-
-def set_dir(conn, arg):
- _init_session(arg)
- conn.ok()
-
-
-def list_indexes(conn, junk):
- _init_session()
- suffix = b''
- if dumb_server_mode:
- suffix = b' load'
- for f in os.listdir(git.repo(b'objects/pack')):
- if f.endswith(b'.idx'):
- conn.write(b'%s%s\n' % (f, suffix))
- conn.ok()
-
-
-def send_index(conn, name):
- _init_session()
- assert name.find(b'/') < 0
- assert name.endswith(b'.idx')
- idx = git.open_idx(git.repo(b'objects/pack/%s' % name))
- conn.write(struct.pack('!I', len(idx.map)))
- conn.write(idx.map)
- conn.ok()
-
-
-def receive_objects_v2(conn, junk):
- global suspended_w
- _init_session()
- suggested = set()
- if suspended_w:
- w = suspended_w
- suspended_w = None
- else:
- if dumb_server_mode:
- w = git.PackWriter(objcache_maker=None)
- else:
- w = git.PackWriter()
- while 1:
- ns = conn.read(4)
- if not ns:
- w.abort()
- raise Exception('object read: expected length header, got EOF\n')
- n = struct.unpack('!I', ns)[0]
- #debug2('expecting %d bytes\n' % n)
- if not n:
- debug1('bup server: received %d object%s.\n'
- % (w.count, w.count!=1 and "s" or ''))
- fullpath = w.close(run_midx=not dumb_server_mode)
- if fullpath:
- (dir, name) = os.path.split(fullpath)
- conn.write(b'%s.idx\n' % name)
- conn.ok()
- return
- elif n == 0xffffffff:
- debug2('bup server: receive-objects suspended.\n')
- suspended_w = w
- conn.ok()
- return
-
- shar = conn.read(20)
- crcr = struct.unpack('!I', conn.read(4))[0]
- n -= 20 + 4
- buf = conn.read(n) # object sizes in bup are reasonably small
- #debug2('read %d bytes\n' % n)
- _check(w, n, len(buf), 'object read: expected %d bytes, got %d\n')
- if not dumb_server_mode:
- oldpack = w.exists(shar, want_source=True)
- if oldpack:
- assert(not oldpack == True)
- assert(oldpack.endswith(b'.idx'))
- (dir,name) = os.path.split(oldpack)
- if not (name in suggested):
- debug1("bup server: suggesting index %s\n"
- % git.shorten_hash(name).decode('ascii'))
- debug1("bup server: because of object %s\n"
- % hexstr(shar))
- conn.write(b'index %s\n' % name)
- suggested.add(name)
- continue
- nw, crc = w._raw_write((buf,), sha=shar)
- _check(w, crcr, crc, 'object read: expected crc %d, got %d\n')
- # NOTREACHED
-
-
-def _check(w, expected, actual, msg):
- if expected != actual:
- w.abort()
- raise Exception(msg % (expected, actual))
-
-
-def read_ref(conn, refname):
- _init_session()
- r = git.read_ref(refname)
- conn.write(b'%s\n' % hexlify(r) if r else b'')
- conn.ok()
-
-
-def update_ref(conn, refname):
- _init_session()
- newval = conn.readline().strip()
- oldval = conn.readline().strip()
- git.update_ref(refname, unhexlify(newval), unhexlify(oldval))
- conn.ok()
-
-def join(conn, id):
- _init_session()
- try:
- for blob in git.cp().join(id):
- conn.write(struct.pack('!I', len(blob)))
- conn.write(blob)
- except KeyError as e:
- log('server: error: %s\n' % e)
- conn.write(b'\0\0\0\0')
- conn.error(e)
- else:
- conn.write(b'\0\0\0\0')
- conn.ok()
-
-def cat_batch(conn, dummy):
- _init_session()
- cat_pipe = git.cp()
- # For now, avoid potential deadlock by just reading them all
- for ref in tuple(lines_until_sentinel(conn, b'\n', Exception)):
- ref = ref[:-1]
- it = cat_pipe.get(ref)
- info = next(it)
- if not info[0]:
- conn.write(b'missing\n')
- continue
- conn.write(b'%s %s %d\n' % info)
- for buf in it:
- conn.write(buf)
- conn.ok()
-
-def refs(conn, args):
- limit_to_heads, limit_to_tags = args.split()
- assert limit_to_heads in (b'0', b'1')
- assert limit_to_tags in (b'0', b'1')
- limit_to_heads = int(limit_to_heads)
- limit_to_tags = int(limit_to_tags)
- _init_session()
- patterns = tuple(x[:-1] for x in lines_until_sentinel(conn, b'\n', Exception))
- for name, oid in git.list_refs(patterns=patterns,
- limit_to_heads=limit_to_heads,
- limit_to_tags=limit_to_tags):
- assert b'\n' not in name
- conn.write(b'%s %s\n' % (hexlify(oid), name))
- conn.write(b'\n')
- conn.ok()
-
-def rev_list(conn, _):
- _init_session()
- count = conn.readline()
- if not count:
- raise Exception('Unexpected EOF while reading rev-list count')
- assert count == b'\n'
- count = None
- fmt = conn.readline()
- if not fmt:
- raise Exception('Unexpected EOF while reading rev-list format')
- fmt = None if fmt == b'\n' else fmt[:-1]
- refs = tuple(x[:-1] for x in lines_until_sentinel(conn, b'\n', Exception))
- args = git.rev_list_invocation(refs, format=fmt)
- p = subprocess.Popen(args, env=git._gitenv(git.repodir),
- stdout=subprocess.PIPE)
- while True:
- out = p.stdout.read(64 * 1024)
- if not out:
- break
- conn.write(out)
- conn.write(b'\n')
- rv = p.wait() # not fatal
- if rv:
- msg = 'git rev-list returned error %d' % rv
- conn.error(msg)
- raise GitError(msg)
- conn.ok()
-
-def resolve(conn, args):
- _init_session()
- (flags,) = args.split()
- flags = int(flags)
- want_meta = bool(flags & 1)
- follow = bool(flags & 2)
- have_parent = bool(flags & 4)
- parent = vfs.read_resolution(conn) if have_parent else None
- path = vint.read_bvec(conn)
- if not len(path):
- raise Exception('Empty resolve path')
- try:
- res = list(vfs.resolve(repo, path, parent=parent, want_meta=want_meta,
- follow=follow))
- except vfs.IOError as ex:
- res = ex
- if isinstance(res, vfs.IOError):
- conn.write(b'\x00') # error
- vfs.write_ioerror(conn, res)
- else:
- conn.write(b'\x01') # success
- vfs.write_resolution(conn, res)
- conn.ok()
-
-optspec = """
-bup server
-"""
-o = options.Options(optspec)
-(opt, flags, extra) = o.parse(sys.argv[1:])
-
-if extra:
- o.fatal('no arguments expected')
-
-debug2('bup server: reading from stdin.\n')
-
-commands = {
- b'quit': None,
- b'help': do_help,
- b'init-dir': init_dir,
- b'set-dir': set_dir,
- b'list-indexes': list_indexes,
- b'send-index': send_index,
- b'receive-objects-v2': receive_objects_v2,
- b'read-ref': read_ref,
- b'update-ref': update_ref,
- b'join': join,
- b'cat': join, # apocryphal alias
- b'cat-batch' : cat_batch,
- b'refs': refs,
- b'rev-list': rev_list,
- b'resolve': resolve
-}
-
-# FIXME: this protocol is totally lame and not at all future-proof.
-# (Especially since we abort completely as soon as *anything* bad happens)
-sys.stdout.flush()
-conn = Conn(byte_stream(sys.stdin), byte_stream(sys.stdout))
-lr = linereader(conn)
-for _line in lr:
- line = _line.strip()
- if not line:
- continue
- debug1('bup server: command: %r\n' % line)
- words = line.split(b' ', 1)
- cmd = words[0]
- rest = len(words)>1 and words[1] or b''
- if cmd == b'quit':
- break
- else:
- cmd = commands.get(cmd)
- if cmd:
- cmd(conn, rest)
- else:
- raise Exception('unknown server command: %r\n' % line)
-
-debug1('bup server: done\n')
+++ /dev/null
-#!/bin/sh
-"""": # -*-python-*-
-bup_python="$(dirname "$0")/bup-python" || exit $?
-exec "$bup_python" "$0" ${1+"$@"}
-"""
-# end of bup preamble
-
-from __future__ import absolute_import, division, print_function
-from binascii import hexlify
-import os, sys, time
-
-from bup import hashsplit, git, options, client
-from bup.compat import argv_bytes, environ
-from bup.helpers import (add_error, handle_ctrl_c, hostname, log, parse_num,
- qprogress, reprogress, saved_errors,
- valid_save_name,
- parse_date_or_fatal)
-from bup.io import byte_stream
-from bup.pwdgrp import userfullname, username
-
-
-optspec = """
-bup split [-t] [-c] [-n name] OPTIONS [--git-ids | filenames...]
-bup split -b OPTIONS [--git-ids | filenames...]
-bup split --copy OPTIONS [--git-ids | filenames...]
-bup split --noop [-b|-t] OPTIONS [--git-ids | filenames...]
---
- Modes:
-b,blobs output a series of blob ids. Implies --fanout=0.
-t,tree output a tree id
-c,commit output a commit id
-n,name= save the result under the given name
-noop split the input, but throw away the result
-copy split the input, copy it to stdout, don't save to repo
- Options:
-r,remote= remote repository path
-d,date= date for the commit (seconds since the epoch)
-q,quiet don't print progress messages
-v,verbose increase log output (can be used more than once)
-git-ids read a list of git object ids from stdin and split their contents
-keep-boundaries don't let one chunk span two input files
-bench print benchmark timings to stderr
-max-pack-size= maximum bytes in a single pack
-max-pack-objects= maximum number of objects in a single pack
-fanout= average number of blobs in a single tree
-bwlimit= maximum bytes/sec to transmit to server
-#,compress= set compression level to # (0-9, 9 is highest) [1]
-"""
-handle_ctrl_c()
-
-o = options.Options(optspec)
-(opt, flags, extra) = o.parse(sys.argv[1:])
-if opt.name: opt.name = argv_bytes(opt.name)
-if opt.remote: opt.remote = argv_bytes(opt.remote)
-if opt.verbose is None: opt.verbose = 0
-
-if not (opt.blobs or opt.tree or opt.commit or opt.name or
- opt.noop or opt.copy):
- o.fatal("use one or more of -b, -t, -c, -n, --noop, --copy")
-if opt.copy and (opt.blobs or opt.tree):
- o.fatal('--copy is incompatible with -b, -t')
-if (opt.noop or opt.copy) and (opt.commit or opt.name):
- o.fatal('--noop and --copy are incompatible with -c, -n')
-if opt.blobs and (opt.tree or opt.commit or opt.name):
- o.fatal('-b is incompatible with -t, -c, -n')
-if extra and opt.git_ids:
- o.fatal("don't provide filenames when using --git-ids")
-
-if opt.verbose >= 2:
- git.verbose = opt.verbose - 1
- opt.bench = 1
-
-max_pack_size = None
-if opt.max_pack_size:
- max_pack_size = parse_num(opt.max_pack_size)
-max_pack_objects = None
-if opt.max_pack_objects:
- max_pack_objects = parse_num(opt.max_pack_objects)
-
-if opt.fanout:
- hashsplit.fanout = parse_num(opt.fanout)
-if opt.blobs:
- hashsplit.fanout = 0
-if opt.bwlimit:
- client.bwlimit = parse_num(opt.bwlimit)
-if opt.date:
- date = parse_date_or_fatal(opt.date, o.fatal)
-else:
- date = time.time()
-
-total_bytes = 0
-def prog(filenum, nbytes):
- global total_bytes
- total_bytes += nbytes
- if filenum > 0:
- qprogress('Splitting: file #%d, %d kbytes\r'
- % (filenum+1, total_bytes // 1024))
- else:
- qprogress('Splitting: %d kbytes\r' % (total_bytes // 1024))
-
-
-is_reverse = environ.get(b'BUP_SERVER_REVERSE')
-if is_reverse and opt.remote:
- o.fatal("don't use -r in reverse mode; it's automatic")
-start_time = time.time()
-
-if opt.name and not valid_save_name(opt.name):
- o.fatal("'%r' is not a valid branch name." % opt.name)
-refname = opt.name and b'refs/heads/%s' % opt.name or None
-
-if opt.noop or opt.copy:
- cli = pack_writer = oldref = None
-elif opt.remote or is_reverse:
- git.check_repo_or_die()
- cli = client.Client(opt.remote)
- oldref = refname and cli.read_ref(refname) or None
- pack_writer = cli.new_packwriter(compression_level=opt.compress,
- max_pack_size=max_pack_size,
- max_pack_objects=max_pack_objects)
-else:
- git.check_repo_or_die()
- cli = None
- oldref = refname and git.read_ref(refname) or None
- pack_writer = git.PackWriter(compression_level=opt.compress,
- max_pack_size=max_pack_size,
- max_pack_objects=max_pack_objects)
-
-input = byte_stream(sys.stdin)
-
-if opt.git_ids:
- # the input is actually a series of git object ids that we should retrieve
- # and split.
- #
- # This is a bit messy, but basically it converts from a series of
- # CatPipe.get() iterators into a series of file-type objects.
- # It would be less ugly if either CatPipe.get() returned a file-like object
- # (not very efficient), or split_to_shalist() expected an iterator instead
- # of a file.
- cp = git.CatPipe()
- class IterToFile:
- def __init__(self, it):
- self.it = iter(it)
- def read(self, size):
- v = next(self.it, None)
- return v or b''
- def read_ids():
- while 1:
- line = input.readline()
- if not line:
- break
- if line:
- line = line.strip()
- try:
- it = cp.get(line.strip())
- next(it, None) # skip the file info
- except KeyError as e:
- add_error('error: %s' % e)
- continue
- yield IterToFile(it)
- files = read_ids()
-else:
- # the input either comes from a series of files or from stdin.
- files = extra and (open(argv_bytes(fn), 'rb') for fn in extra) or [input]
-
-if pack_writer:
- new_blob = pack_writer.new_blob
- new_tree = pack_writer.new_tree
-elif opt.blobs or opt.tree:
- # --noop mode
- new_blob = lambda content: git.calc_hash(b'blob', content)
- new_tree = lambda shalist: git.calc_hash(b'tree', git.tree_encode(shalist))
-
-sys.stdout.flush()
-out = byte_stream(sys.stdout)
-
-if opt.blobs:
- shalist = hashsplit.split_to_blobs(new_blob, files,
- keep_boundaries=opt.keep_boundaries,
- progress=prog)
- for (sha, size, level) in shalist:
- out.write(hexlify(sha) + b'\n')
- reprogress()
-elif opt.tree or opt.commit or opt.name:
- if opt.name: # insert dummy_name which may be used as a restore target
- mode, sha = \
- hashsplit.split_to_blob_or_tree(new_blob, new_tree, files,
- keep_boundaries=opt.keep_boundaries,
- progress=prog)
- splitfile_name = git.mangle_name(b'data', hashsplit.GIT_MODE_FILE, mode)
- shalist = [(mode, splitfile_name, sha)]
- else:
- shalist = hashsplit.split_to_shalist(
- new_blob, new_tree, files,
- keep_boundaries=opt.keep_boundaries, progress=prog)
- tree = new_tree(shalist)
-else:
- last = 0
- it = hashsplit.hashsplit_iter(files,
- keep_boundaries=opt.keep_boundaries,
- progress=prog)
- for (blob, level) in it:
- hashsplit.total_split += len(blob)
- if opt.copy:
- sys.stdout.write(str(blob))
- megs = hashsplit.total_split // 1024 // 1024
- if not opt.quiet and last != megs:
- last = megs
-
-if opt.verbose:
- log('\n')
-if opt.tree:
- out.write(hexlify(tree) + b'\n')
-if opt.commit or opt.name:
- msg = b'bup split\n\nGenerated by command:\n%r\n' % sys.argv
- ref = opt.name and (b'refs/heads/%s' % opt.name) or None
- userline = b'%s <%s@%s>' % (userfullname(), username(), hostname())
- commit = pack_writer.new_commit(tree, oldref, userline, date, None,
- userline, date, None, msg)
- if opt.commit:
- out.write(hexlify(commit) + b'\n')
-
-if pack_writer:
- pack_writer.close() # must close before we can update the ref
-
-if opt.name:
- if cli:
- cli.update_ref(refname, commit, oldref)
- else:
- git.update_ref(refname, commit, oldref)
-
-if cli:
- cli.close()
-
-secs = time.time() - start_time
-size = hashsplit.total_split
-if opt.bench:
- log('bup: %.2f kbytes in %.2f secs = %.2f kbytes/sec\n'
- % (size / 1024, secs, size / 1024 / secs))
-
-if saved_errors:
- log('WARNING: %d errors encountered while saving.\n' % len(saved_errors))
- sys.exit(1)
+++ /dev/null
-#!/bin/sh
-"""": # -*-python-*-
-bup_python="$(dirname "$0")/bup-python" || exit $?
-exec "$bup_python" "$0" ${1+"$@"}
-"""
-# end of bup preamble
-
-from __future__ import absolute_import
-from binascii import hexlify
-import os, sys
-
-from bup import git, options
-from bup.compat import argv_bytes
-from bup.helpers import debug1, handle_ctrl_c, log
-from bup.io import byte_stream, path_msg
-
-# FIXME: review for safe writes.
-
-handle_ctrl_c()
-
-optspec = """
-bup tag
-bup tag [-f] <tag name> <commit>
-bup tag [-f] -d <tag name>
---
-d,delete= Delete a tag
-f,force Overwrite existing tag, or ignore missing tag when deleting
-"""
-
-o = options.Options(optspec)
-(opt, flags, extra) = o.parse(sys.argv[1:])
-
-git.check_repo_or_die()
-
-tags = [t for sublist in git.tags().values() for t in sublist]
-
-if opt.delete:
- # git.delete_ref() doesn't complain if a ref doesn't exist. We
- # could implement this verification but we'd need to read in the
- # contents of the tag file and pass the hash, and we already know
- # about the tag's existance via "tags".
- tag_name = argv_bytes(opt.delete)
- if not opt.force and tag_name not in tags:
- log("error: tag '%s' doesn't exist\n" % path_msg(tag_name))
- sys.exit(1)
- tag_file = b'refs/tags/%s' % tag_name
- git.delete_ref(tag_file)
- sys.exit(0)
-
-if not extra:
- for t in tags:
- sys.stdout.flush()
- out = byte_stream(sys.stdout)
- out.write(t)
- out.write(b'\n')
- sys.exit(0)
-elif len(extra) != 2:
- o.fatal('expected commit ref and hash')
-
-tag_name, commit = map(argv_bytes, extra[:2])
-if not tag_name:
- o.fatal("tag name must not be empty.")
-debug1("args: tag name = %s; commit = %s\n"
- % (path_msg(tag_name), commit.decode('ascii')))
-
-if tag_name in tags and not opt.force:
- log("bup: error: tag '%s' already exists\n" % path_msg(tag_name))
- sys.exit(1)
-
-if tag_name.startswith(b'.'):
- o.fatal("'%s' is not a valid tag name." % path_msg(tag_name))
-
-try:
- hash = git.rev_parse(commit)
-except git.GitError as e:
- log("bup: error: %s" % e)
- sys.exit(2)
-
-if not hash:
- log("bup: error: commit %s not found.\n" % commit.decode('ascii'))
- sys.exit(2)
-
-pL = git.PackIdxList(git.repo(b'objects/pack'))
-if not pL.exists(hash):
- log("bup: error: commit %s not found.\n" % commit.decode('ascii'))
- sys.exit(2)
-
-tag_file = git.repo(b'refs/tags/' + tag_name)
-try:
- tag = open(tag_file, 'wb')
-except OSError as e:
- log("bup: error: could not create tag '%s': %s" % (path_msg(tag_name), e))
- sys.exit(3)
-with tag as tag:
- tag.write(hexlify(hash))
- tag.write(b'\n')
+++ /dev/null
-#!/bin/sh
-"""": # -*-python-*-
-bup_python="$(dirname "$0")/bup-python" || exit $?
-exec "$bup_python" "$0" ${1+"$@"}
-"""
-# end of bup preamble
-
-from __future__ import absolute_import
-import sys, time
-from bup import options
-
-optspec = """
-bup tick
-"""
-o = options.Options(optspec)
-(opt, flags, extra) = o.parse(sys.argv[1:])
-
-if extra:
- o.fatal("no arguments expected")
-
-t = time.time()
-tleft = 1 - (t - int(t))
-time.sleep(tleft)
+++ /dev/null
-#!/bin/sh
-"""": # -*-python-*-
-bup_python="$(dirname "$0")/bup-python" || exit $?
-exec "$bup_python" "$0" ${1+"$@"}
-"""
-# end of bup preamble
-
-from __future__ import absolute_import, print_function
-import re, sys
-
-from bup import options
-from bup import version
-
-version_rx = re.compile(r'^[0-9]+\.[0-9]+(\.[0-9]+)?(-[0-9]+-g[0-9abcdef]+)?$')
-
-optspec = """
-bup version [--date|--commit|--tag]
---
-date display the date this version of bup was created
-commit display the git commit id of this version of bup
-tag display the tag name of this version. If no tag is available, display the commit id
-"""
-o = options.Options(optspec)
-(opt, flags, extra) = o.parse(sys.argv[1:])
-
-
-total = (opt.date or 0) + (opt.commit or 0) + (opt.tag or 0)
-if total > 1:
- o.fatal('at most one option expected')
-
-
-def version_date():
- """Format bup's version date string for output."""
- return version.DATE.split(' ')[0]
-
-
-def version_commit():
- """Get the commit hash of bup's current version."""
- return version.COMMIT
-
-
-def version_tag():
- """Format bup's version tag (the official version number).
-
- When generated from a commit other than one pointed to with a tag, the
- returned string will be "unknown-" followed by the first seven positions of
- the commit hash.
- """
- names = version.NAMES.strip()
- assert(names[0] == '(')
- assert(names[-1] == ')')
- names = names[1:-1]
- l = [n.strip() for n in names.split(',')]
- for n in l:
- if n.startswith('tag: ') and version_rx.match(n[5:]):
- return n[5:]
- return 'unknown-%s' % version.COMMIT[:7]
-
-
-if opt.date:
- print(version_date())
-elif opt.commit:
- print(version_commit())
-else:
- print(version_tag())
+++ /dev/null
-#!/bin/sh
-"""": # -*-python-*-
-bup_python="$(dirname "$0")/bup-python" || exit $?
-exec "$bup_python" "$0" ${1+"$@"}
-"""
-# end of bup preamble
-
-from __future__ import absolute_import, print_function
-from collections import namedtuple
-import mimetypes, os, posixpath, signal, stat, sys, time, urllib, webbrowser
-
-from bup import options, git, vfs
-from bup.helpers import (chunkyreader, debug1, format_filesize, handle_ctrl_c,
- log, saved_errors)
-from bup.metadata import Metadata
-from bup.path import resource_path
-from bup.repo import LocalRepo
-
-try:
- from tornado import gen
- from tornado.httpserver import HTTPServer
- from tornado.ioloop import IOLoop
- from tornado.netutil import bind_unix_socket
- import tornado.web
-except ImportError:
- log('error: cannot find the python "tornado" module; please install it\n')
- sys.exit(1)
-
-
-# FIXME: right now the way hidden files are handled causes every
-# directory to be traversed twice.
-
-handle_ctrl_c()
-
-
-def http_date_from_utc_ns(utc_ns):
- return time.strftime('%a, %d %b %Y %H:%M:%S', time.gmtime(utc_ns / 10**9))
-
-
-def _compute_breadcrumbs(path, show_hidden=False):
- """Returns a list of breadcrumb objects for a path."""
- breadcrumbs = []
- breadcrumbs.append(('[root]', '/'))
- path_parts = path.split('/')[1:-1]
- full_path = '/'
- for part in path_parts:
- full_path += part + "/"
- url_append = ""
- if show_hidden:
- url_append = '?hidden=1'
- breadcrumbs.append((part, full_path+url_append))
- return breadcrumbs
-
-
-def _contains_hidden_files(repo, dir_item):
- """Return true if the directory contains items with names other than
- '.' and '..' that begin with '.'
-
- """
- for name, item in vfs.contents(repo, dir_item, want_meta=False):
- if name in ('.', '..'):
- continue
- if name.startswith('.'):
- return True
- return False
-
-
-def _dir_contents(repo, resolution, show_hidden=False):
- """Yield the display information for the contents of dir_item."""
-
- url_query = '?hidden=1' if show_hidden else ''
-
- def display_info(name, item, resolved_item, display_name=None):
- # link should be based on fully resolved type to avoid extra
- # HTTP redirect.
- if stat.S_ISDIR(vfs.item_mode(resolved_item)):
- link = urllib.quote(name) + '/'
- else:
- link = urllib.quote(name)
-
- size = vfs.item_size(repo, item)
- if opt.human_readable:
- display_size = format_filesize(size)
- else:
- display_size = size
-
- if not display_name:
- mode = vfs.item_mode(item)
- if stat.S_ISDIR(mode):
- display_name = name + '/'
- elif stat.S_ISLNK(mode):
- display_name = name + '@'
- else:
- display_name = name
-
- return display_name, link + url_query, display_size
-
- dir_item = resolution[-1][1]
- for name, item in vfs.contents(repo, dir_item):
- if not show_hidden:
- if (name not in ('.', '..')) and name.startswith('.'):
- continue
- if name == '.':
- yield display_info(name, item, item, '.')
- parent_item = resolution[-2][1] if len(resolution) > 1 else dir_item
- yield display_info('..', parent_item, parent_item, '..')
- continue
- res = vfs.try_resolve(repo, name, parent=resolution, want_meta=False)
- res_name, res_item = res[-1]
- yield display_info(name, item, res_item)
-
-
-class BupRequestHandler(tornado.web.RequestHandler):
-
- def initialize(self, repo=None):
- self.repo = repo
-
- def decode_argument(self, value, name=None):
- if name == 'path':
- return value
- return super(BupRequestHandler, self).decode_argument(value, name)
-
- def get(self, path):
- return self._process_request(path)
-
- def head(self, path):
- return self._process_request(path)
-
- def _process_request(self, path):
- path = urllib.unquote(path)
- print('Handling request for %s' % path)
- # Set want_meta because dir metadata won't be fetched, and if
- # it's not a dir, then we're going to want the metadata.
- res = vfs.resolve(self.repo, path, want_meta=True)
- leaf_name, leaf_item = res[-1]
- if not leaf_item:
- self.send_error(404)
- return
- mode = vfs.item_mode(leaf_item)
- if stat.S_ISDIR(mode):
- self._list_directory(path, res)
- else:
- self._get_file(self.repo, path, res)
-
- def _list_directory(self, path, resolution):
- """Helper to produce a directory listing.
-
- Return value is either a file object, or None (indicating an
- error). In either case, the headers are sent.
- """
- if not path.endswith('/') and len(path) > 0:
- print('Redirecting from %s to %s' % (path, path + '/'))
- return self.redirect(path + '/', permanent=True)
-
- hidden_arg = self.request.arguments.get('hidden', [0])[-1]
- try:
- show_hidden = int(hidden_arg)
- except ValueError as e:
- show_hidden = False
-
- self.render(
- 'list-directory.html',
- path=path,
- breadcrumbs=_compute_breadcrumbs(path, show_hidden),
- files_hidden=_contains_hidden_files(self.repo, resolution[-1][1]),
- hidden_shown=show_hidden,
- dir_contents=_dir_contents(self.repo, resolution,
- show_hidden=show_hidden))
-
- @gen.coroutine
- def _get_file(self, repo, path, resolved):
- """Process a request on a file.
-
- Return value is either a file object, or None (indicating an error).
- In either case, the headers are sent.
- """
- file_item = resolved[-1][1]
- file_item = vfs.augment_item_meta(repo, file_item, include_size=True)
- meta = file_item.meta
- ctype = self._guess_type(path)
- self.set_header("Last-Modified", http_date_from_utc_ns(meta.mtime))
- self.set_header("Content-Type", ctype)
-
- self.set_header("Content-Length", str(meta.size))
- assert len(file_item.oid) == 20
- self.set_header("Etag", file_item.oid.encode('hex'))
- if self.request.method != 'HEAD':
- with vfs.fopen(self.repo, file_item) as f:
- it = chunkyreader(f)
- for blob in chunkyreader(f):
- self.write(blob)
- raise gen.Return()
-
- def _guess_type(self, path):
- """Guess the type of a file.
-
- Argument is a PATH (a filename).
-
- Return value is a string of the form type/subtype,
- usable for a MIME Content-type header.
-
- The default implementation looks the file's extension
- up in the table self.extensions_map, using application/octet-stream
- as a default; however it would be permissible (if
- slow) to look inside the data to make a better guess.
- """
- base, ext = posixpath.splitext(path)
- if ext in self.extensions_map:
- return self.extensions_map[ext]
- ext = ext.lower()
- if ext in self.extensions_map:
- return self.extensions_map[ext]
- else:
- return self.extensions_map['']
-
- if not mimetypes.inited:
- mimetypes.init() # try to read system mime.types
- extensions_map = mimetypes.types_map.copy()
- extensions_map.update({
- '': 'text/plain', # Default
- '.py': 'text/plain',
- '.c': 'text/plain',
- '.h': 'text/plain',
- })
-
-
-io_loop = None
-
-def handle_sigterm(signum, frame):
- global io_loop
- debug1('\nbup-web: signal %d received\n' % signum)
- log('Shutdown requested\n')
- if not io_loop:
- sys.exit(0)
- io_loop.stop()
-
-
-signal.signal(signal.SIGTERM, handle_sigterm)
-
-UnixAddress = namedtuple('UnixAddress', ['path'])
-InetAddress = namedtuple('InetAddress', ['host', 'port'])
-
-optspec = """
-bup web [[hostname]:port]
-bup web unix://path
---
-human-readable display human readable file sizes (i.e. 3.9K, 4.7M)
-browser show repository in default browser (incompatible with unix://)
-"""
-o = options.Options(optspec)
-(opt, flags, extra) = o.parse(sys.argv[1:])
-
-if len(extra) > 1:
- o.fatal("at most one argument expected")
-
-if len(extra) == 0:
- address = InetAddress(host='127.0.0.1', port=8080)
-else:
- bind_url = extra[0]
- if bind_url.startswith('unix://'):
- address = UnixAddress(path=bind_url[len('unix://'):])
- else:
- addr_parts = extra[0].split(':', 1)
- if len(addr_parts) == 1:
- host = '127.0.0.1'
- port = addr_parts[0]
- else:
- host, port = addr_parts
- try:
- port = int(port)
- except (TypeError, ValueError) as ex:
- o.fatal('port must be an integer, not %r' % port)
- address = InetAddress(host=host, port=port)
-
-git.check_repo_or_die()
-
-settings = dict(
- debug = 1,
- template_path = resource_path('web'),
- static_path = resource_path('web/static')
-)
-
-# Disable buffering on stdout, for debug messages
-sys.stdout = os.fdopen(sys.stdout.fileno(), 'w', 0)
-
-application = tornado.web.Application([
- (r"(?P<path>/.*)", BupRequestHandler, dict(repo=LocalRepo())),
-], **settings)
-
-http_server = HTTPServer(application)
-io_loop_pending = IOLoop.instance()
-
-if isinstance(address, InetAddress):
- http_server.listen(address.port, address.host)
- try:
- sock = http_server._socket # tornado < 2.0
- except AttributeError as e:
- sock = http_server._sockets.values()[0]
- print('Serving HTTP on %s:%d...' % sock.getsockname())
- if opt.browser:
- browser_addr = 'http://' + address[0] + ':' + str(address[1])
- io_loop_pending.add_callback(lambda : webbrowser.open(browser_addr))
-elif isinstance(address, UnixAddress):
- unix_socket = bind_unix_socket(address.path)
- http_server.add_socket(unix_socket)
- print('Serving HTTP on filesystem socket %r' % address.path)
-else:
- log('error: unexpected address %r', address)
- sys.exit(1)
-
-io_loop = io_loop_pending
-io_loop.start()
-
-if saved_errors:
- log('WARNING: %d errors encountered while saving.\n' % len(saved_errors))
- sys.exit(1)
+++ /dev/null
-#!/bin/sh
-"""": # -*-python-*-
-bup_python="$(dirname "$0")/bup-python" || exit $?
-exec "$bup_python" "$0" ${1+"$@"}
-"""
-# end of bup preamble
-
-# Copyright (C) 2010 Rob Browning
-#
-# This code is covered under the terms of the GNU Library General
-# Public License as described in the bup LICENSE file.
-
-from __future__ import absolute_import, print_function
-import sys, stat, errno
-
-from bup import metadata, options, xstat
-from bup.compat import argv_bytes
-from bup.helpers import add_error, handle_ctrl_c, parse_timestamp, saved_errors, \
- add_error, log
-from bup.io import byte_stream
-
-
-def parse_timestamp_arg(field, value):
- res = str(value) # Undo autoconversion.
- try:
- res = parse_timestamp(res)
- except ValueError as ex:
- if ex.args:
- o.fatal('unable to parse %s resolution "%s" (%s)'
- % (field, value, ex))
- else:
- o.fatal('unable to parse %s resolution "%s"' % (field, value))
-
- if res != 1 and res % 10:
- o.fatal('%s resolution "%s" must be a power of 10' % (field, value))
- return res
-
-
-optspec = """
-bup xstat pathinfo [OPTION ...] <PATH ...>
---
-v,verbose increase log output (can be used more than once)
-q,quiet don't show progress meter
-exclude-fields= exclude comma-separated fields
-include-fields= include comma-separated fields (definitive if first)
-atime-resolution= limit s, ms, us, ns, 10ns (value must be a power of 10) [ns]
-mtime-resolution= limit s, ms, us, ns, 10ns (value must be a power of 10) [ns]
-ctime-resolution= limit s, ms, us, ns, 10ns (value must be a power of 10) [ns]
-"""
-
-target_filename = b''
-active_fields = metadata.all_fields
-
-handle_ctrl_c()
-
-o = options.Options(optspec)
-(opt, flags, remainder) = o.parse(sys.argv[1:])
-
-atime_resolution = parse_timestamp_arg('atime', opt.atime_resolution)
-mtime_resolution = parse_timestamp_arg('mtime', opt.mtime_resolution)
-ctime_resolution = parse_timestamp_arg('ctime', opt.ctime_resolution)
-
-treat_include_fields_as_definitive = True
-for flag, value in flags:
- if flag == '--exclude-fields':
- exclude_fields = frozenset(value.split(','))
- for f in exclude_fields:
- if not f in metadata.all_fields:
- o.fatal(f + ' is not a valid field name')
- active_fields = active_fields - exclude_fields
- treat_include_fields_as_definitive = False
- elif flag == '--include-fields':
- include_fields = frozenset(value.split(','))
- for f in include_fields:
- if not f in metadata.all_fields:
- o.fatal(f + ' is not a valid field name')
- if treat_include_fields_as_definitive:
- active_fields = include_fields
- treat_include_fields_as_definitive = False
- else:
- active_fields = active_fields | include_fields
-
-opt.verbose = opt.verbose or 0
-opt.quiet = opt.quiet or 0
-metadata.verbose = opt.verbose - opt.quiet
-
-sys.stdout.flush()
-out = byte_stream(sys.stdout)
-
-first_path = True
-for path in remainder:
- path = argv_bytes(path)
- try:
- m = metadata.from_path(path, archive_path = path)
- except (OSError,IOError) as e:
- if e.errno == errno.ENOENT:
- add_error(e)
- continue
- else:
- raise
- if metadata.verbose >= 0:
- if not first_path:
- out.write(b'\n')
- if atime_resolution != 1:
- m.atime = (m.atime / atime_resolution) * atime_resolution
- if mtime_resolution != 1:
- m.mtime = (m.mtime / mtime_resolution) * mtime_resolution
- if ctime_resolution != 1:
- m.ctime = (m.ctime / ctime_resolution) * ctime_resolution
- out.write(metadata.detailed_bytes(m, active_fields))
- out.write(b'\n')
- first_path = False
-
-if saved_errors:
- log('WARNING: %d errors encountered.\n' % len(saved_errors))
- sys.exit(1)
-else:
- sys.exit(0)
+++ /dev/null
-../cmd
\ No newline at end of file
--- /dev/null
+#!/bin/sh
+"""": # -*-python-*-
+bup_python="$(dirname "$0")/bup-python" || exit $?
+exec "$bup_python" "$0" ${1+"$@"}
+"""
+# end of bup preamble
+
+from __future__ import absolute_import
+import glob, os, sys, tempfile
+
+from bup import options, git, bloom
+from bup.compat import argv_bytes, hexstr
+from bup.helpers import (add_error, debug1, handle_ctrl_c, log, progress, qprogress,
+ saved_errors)
+from bup.io import path_msg
+
+
+optspec = """
+bup bloom [options...]
+--
+ruin ruin the specified bloom file (clearing the bitfield)
+f,force ignore existing bloom file and regenerate it from scratch
+o,output= output bloom filename (default: auto)
+d,dir= input directory to look for idx files (default: auto)
+k,hashes= number of hash functions to use (4 or 5) (default: auto)
+c,check= check the given .idx file against the bloom filter
+"""
+
+
+def ruin_bloom(bloomfilename):
+ rbloomfilename = git.repo_rel(bloomfilename)
+ if not os.path.exists(bloomfilename):
+ log(path_msg(bloomfilename) + '\n')
+ add_error('bloom: %s not found to ruin\n' % path_msg(rbloomfilename))
+ return
+ b = bloom.ShaBloom(bloomfilename, readwrite=True, expected=1)
+ b.map[16 : 16 + 2**b.bits] = b'\0' * 2**b.bits
+
+
+def check_bloom(path, bloomfilename, idx):
+ rbloomfilename = git.repo_rel(bloomfilename)
+ ridx = git.repo_rel(idx)
+ if not os.path.exists(bloomfilename):
+ log('bloom: %s: does not exist.\n' % path_msg(rbloomfilename))
+ return
+ b = bloom.ShaBloom(bloomfilename)
+ if not b.valid():
+ add_error('bloom: %r is invalid.\n' % path_msg(rbloomfilename))
+ return
+ base = os.path.basename(idx)
+ if base not in b.idxnames:
+ log('bloom: %s does not contain the idx.\n' % path_msg(rbloomfilename))
+ return
+ if base == idx:
+ idx = os.path.join(path, idx)
+ log('bloom: bloom file: %s\n' % path_msg(rbloomfilename))
+ log('bloom: checking %s\n' % path_msg(ridx))
+ for objsha in git.open_idx(idx):
+ if not b.exists(objsha):
+ add_error('bloom: ERROR: object %s missing' % hexstr(objsha))
+
+
+_first = None
+def do_bloom(path, outfilename, k):
+ global _first
+ assert k in (None, 4, 5)
+ b = None
+ if os.path.exists(outfilename) and not opt.force:
+ b = bloom.ShaBloom(outfilename)
+ if not b.valid():
+ debug1("bloom: Existing invalid bloom found, regenerating.\n")
+ b = None
+
+ add = []
+ rest = []
+ add_count = 0
+ rest_count = 0
+ for i, name in enumerate(glob.glob(b'%s/*.idx' % path)):
+ progress('bloom: counting: %d\r' % i)
+ ix = git.open_idx(name)
+ ixbase = os.path.basename(name)
+ if b and (ixbase in b.idxnames):
+ rest.append(name)
+ rest_count += len(ix)
+ else:
+ add.append(name)
+ add_count += len(ix)
+
+ if not add:
+ debug1("bloom: nothing to do.\n")
+ return
+
+ if b:
+ if len(b) != rest_count:
+ debug1("bloom: size %d != idx total %d, regenerating\n"
+ % (len(b), rest_count))
+ b = None
+ elif k is not None and k != b.k:
+ debug1("bloom: new k %d != existing k %d, regenerating\n"
+ % (k, b.k))
+ b = None
+ elif (b.bits < bloom.MAX_BLOOM_BITS[b.k] and
+ b.pfalse_positive(add_count) > bloom.MAX_PFALSE_POSITIVE):
+ debug1("bloom: regenerating: adding %d entries gives "
+ "%.2f%% false positives.\n"
+ % (add_count, b.pfalse_positive(add_count)))
+ b = None
+ else:
+ b = bloom.ShaBloom(outfilename, readwrite=True, expected=add_count)
+ if not b: # Need all idxs to build from scratch
+ add += rest
+ add_count += rest_count
+ del rest
+ del rest_count
+
+ msg = b is None and 'creating from' or 'adding'
+ if not _first: _first = path
+ dirprefix = (_first != path) and git.repo_rel(path) + b': ' or b''
+ progress('bloom: %s%s %d file%s (%d object%s).\r'
+ % (path_msg(dirprefix), msg,
+ len(add), len(add)!=1 and 's' or '',
+ add_count, add_count!=1 and 's' or ''))
+
+ tfname = None
+ if b is None:
+ tfname = os.path.join(path, b'bup.tmp.bloom')
+ b = bloom.create(tfname, expected=add_count, k=k)
+ count = 0
+ icount = 0
+ for name in add:
+ ix = git.open_idx(name)
+ qprogress('bloom: writing %.2f%% (%d/%d objects)\r'
+ % (icount*100.0/add_count, icount, add_count))
+ b.add_idx(ix)
+ count += 1
+ icount += len(ix)
+
+ # Currently, there's an open file object for tfname inside b.
+ # Make sure it's closed before rename.
+ b.close()
+
+ if tfname:
+ os.rename(tfname, outfilename)
+
+
+handle_ctrl_c()
+
+o = options.Options(optspec)
+(opt, flags, extra) = o.parse(sys.argv[1:])
+
+if extra:
+ o.fatal('no positional parameters expected')
+
+if not opt.check and opt.k and opt.k not in (4,5):
+ o.fatal('only k values of 4 and 5 are supported')
+
+if opt.check:
+ opt.check = argv_bytes(opt.check)
+
+git.check_repo_or_die()
+
+output = argv_bytes(opt.output) if opt.output else None
+paths = opt.dir and [argv_bytes(opt.dir)] or git.all_packdirs()
+for path in paths:
+ debug1('bloom: scanning %s\n' % path_msg(path))
+ outfilename = output or os.path.join(path, b'bup.bloom')
+ if opt.check:
+ check_bloom(path, outfilename, opt.check)
+ elif opt.ruin:
+ ruin_bloom(outfilename)
+ else:
+ do_bloom(path, outfilename, opt.k)
+
+if saved_errors:
+ log('WARNING: %d errors encountered during bloom.\n' % len(saved_errors))
+ sys.exit(1)
+elif opt.check:
+ log('All tests passed.\n')
--- /dev/null
+#!/bin/sh
+"""": # -*-python-*- # -*-python-*-
+set -e
+top="$(pwd)"
+cmdpath="$0"
+# loop because macos doesn't have recursive readlink/realpath utils
+while test -L "$cmdpath"; do
+ link="$(readlink "$cmdpath")"
+ cd "$(dirname "$cmdpath")"
+ cmdpath="$link"
+done
+script_home="$(cd "$(dirname "$cmdpath")" && pwd -P)"
+cd "$top"
+exec "$script_home/bup-python" "$0" ${1+"$@"}
+"""
+# end of bup preamble
+
+from __future__ import absolute_import, print_function
+import errno, getopt, os, re, select, signal, subprocess, sys
+from subprocess import PIPE
+
+from bup.compat import environ, restore_lc_env
+from bup.io import path_msg
+
+if sys.version_info[0] != 2 \
+ and not environ.get(b'BUP_ALLOW_UNEXPECTED_PYTHON_VERSION') == b'true':
+ print('error: bup may crash with python versions other than 2, or eat your data',
+ file=sys.stderr)
+ sys.exit(2)
+
+restore_lc_env()
+
+from bup import compat, path, helpers
+from bup.compat import add_ex_tb, add_ex_ctx, argv_bytes, wrap_main
+from bup.helpers import atoi, columnate, debug1, log, merge_dict, tty_width
+from bup.io import byte_stream, path_msg
+
+cmdpath = path.cmddir()
+
+# We manipulate the subcmds here as strings, but they must be ASCII
+# compatible, since we're going to be looking for exactly
+# b'bup-SUBCMD' to exec.
+
+def usage(msg=""):
+ log('Usage: bup [-?|--help] [-d BUP_DIR] [--debug] [--profile] '
+ '<command> [options...]\n\n')
+ common = dict(
+ ftp = 'Browse backup sets using an ftp-like client',
+ fsck = 'Check backup sets for damage and add redundancy information',
+ fuse = 'Mount your backup sets as a filesystem',
+ help = 'Print detailed help for the given command',
+ index = 'Create or display the index of files to back up',
+ on = 'Backup a remote machine to the local one',
+ restore = 'Extract files from a backup set',
+ save = 'Save files into a backup set (note: run "bup index" first)',
+ tag = 'Tag commits for easier access',
+ web = 'Launch a web server to examine backup sets',
+ )
+
+ log('Common commands:\n')
+ for cmd,synopsis in sorted(common.items()):
+ log(' %-10s %s\n' % (cmd, synopsis))
+ log('\n')
+
+ log('Other available commands:\n')
+ cmds = []
+ for c in sorted(os.listdir(cmdpath)):
+ if c.startswith(b'bup-') and c.find(b'.') < 0:
+ cname = c[4:].decode('iso-8859-1')
+ if cname not in common:
+ cmds.append(c[4:])
+ log(columnate(cmds, ' '))
+ log('\n')
+
+ log("See 'bup help COMMAND' for more information on " +
+ "a specific command.\n")
+ if msg:
+ log("\n%s\n" % msg)
+ sys.exit(99)
+
+
+if len(sys.argv) < 2:
+ usage()
+
+# Handle global options.
+try:
+ optspec = ['help', 'version', 'debug', 'profile', 'bup-dir=']
+ global_args, subcmd = getopt.getopt(sys.argv[1:], '?VDd:', optspec)
+except getopt.GetoptError as ex:
+ usage('error: %s' % ex.msg)
+
+subcmd = [argv_bytes(x) for x in subcmd]
+help_requested = None
+do_profile = False
+bup_dir = None
+
+for opt in global_args:
+ if opt[0] in ['-?', '--help']:
+ help_requested = True
+ elif opt[0] in ['-V', '--version']:
+ subcmd = [b'version']
+ elif opt[0] in ['-D', '--debug']:
+ helpers.buglvl += 1
+ environ[b'BUP_DEBUG'] = b'%d' % helpers.buglvl
+ elif opt[0] in ['--profile']:
+ do_profile = True
+ elif opt[0] in ['-d', '--bup-dir']:
+ bup_dir = argv_bytes(opt[1])
+ else:
+ usage('error: unexpected option "%s"' % opt[0])
+
+if bup_dir:
+ bup_dir = argv_bytes(bup_dir)
+
+# Make BUP_DIR absolute, so we aren't affected by chdir (i.e. save -C, etc.).
+if bup_dir:
+ environ[b'BUP_DIR'] = os.path.abspath(bup_dir)
+
+if len(subcmd) == 0:
+ if help_requested:
+ subcmd = [b'help']
+ else:
+ usage()
+
+if help_requested and subcmd[0] != b'help':
+ subcmd = [b'help'] + subcmd
+
+if len(subcmd) > 1 and subcmd[1] == b'--help' and subcmd[0] != b'help':
+ subcmd = [b'help', subcmd[0]] + subcmd[2:]
+
+subcmd_name = subcmd[0]
+if not subcmd_name:
+ usage()
+
+def subpath(subcmd):
+ return os.path.join(cmdpath, b'bup-' + subcmd)
+
+subcmd[0] = subpath(subcmd_name)
+if not os.path.exists(subcmd[0]):
+ usage('error: unknown command "%s"' % path_msg(subcmd_name))
+
+already_fixed = atoi(environ.get(b'BUP_FORCE_TTY'))
+if subcmd_name in [b'mux', b'ftp', b'help']:
+ already_fixed = True
+fix_stdout = not already_fixed and os.isatty(1)
+fix_stderr = not already_fixed and os.isatty(2)
+
+if fix_stdout or fix_stderr:
+ tty_env = merge_dict(environ,
+ {b'BUP_FORCE_TTY': (b'%d'
+ % ((fix_stdout and 1 or 0)
+ + (fix_stderr and 2 or 0)))})
+else:
+ tty_env = environ
+
+
+sep_rx = re.compile(br'([\r\n])')
+
+def print_clean_line(dest, content, width, sep=None):
+ """Write some or all of content, followed by sep, to the dest fd after
+ padding the content with enough spaces to fill the current
+ terminal width or truncating it to the terminal width if sep is a
+ carriage return."""
+ global sep_rx
+ assert sep in (b'\r', b'\n', None)
+ if not content:
+ if sep:
+ os.write(dest, sep)
+ return
+ for x in content:
+ assert not sep_rx.match(x)
+ content = b''.join(content)
+ if sep == b'\r' and len(content) > width:
+ content = content[width:]
+ os.write(dest, content)
+ if len(content) < width:
+ os.write(dest, b' ' * (width - len(content)))
+ if sep:
+ os.write(dest, sep)
+
+def filter_output(src_out, src_err, dest_out, dest_err):
+ """Transfer data from src_out to dest_out and src_err to dest_err via
+ print_clean_line until src_out and src_err close."""
+ global sep_rx
+ assert not isinstance(src_out, bool)
+ assert not isinstance(src_err, bool)
+ assert not isinstance(dest_out, bool)
+ assert not isinstance(dest_err, bool)
+ assert src_out is not None or src_err is not None
+ assert (src_out is None) == (dest_out is None)
+ assert (src_err is None) == (dest_err is None)
+ pending = {}
+ pending_ex = None
+ try:
+ fds = tuple([x for x in (src_out, src_err) if x is not None])
+ while fds:
+ ready_fds, _, _ = select.select(fds, [], [])
+ width = tty_width()
+ for fd in ready_fds:
+ buf = os.read(fd, 4096)
+ dest = dest_out if fd == src_out else dest_err
+ if not buf:
+ fds = tuple([x for x in fds if x is not fd])
+ print_clean_line(dest, pending.pop(fd, []), width)
+ else:
+ split = sep_rx.split(buf)
+ while len(split) > 1:
+ content, sep = split[:2]
+ split = split[2:]
+ print_clean_line(dest,
+ pending.pop(fd, []) + [content],
+ width,
+ sep)
+ assert(len(split) == 1)
+ if split[0]:
+ pending.setdefault(fd, []).extend(split)
+ except BaseException as ex:
+ pending_ex = add_ex_ctx(add_ex_tb(ex), pending_ex)
+ try:
+ # Try to finish each of the streams
+ for fd, pending_items in compat.items(pending):
+ dest = dest_out if fd == src_out else dest_err
+ try:
+ print_clean_line(dest, pending_items, width)
+ except (EnvironmentError, EOFError) as ex:
+ pending_ex = add_ex_ctx(add_ex_tb(ex), pending_ex)
+ except BaseException as ex:
+ pending_ex = add_ex_ctx(add_ex_tb(ex), pending_ex)
+ if pending_ex:
+ raise pending_ex
+
+def run_subcmd(subcmd):
+
+ c = (do_profile and [sys.executable, b'-m', b'cProfile'] or []) + subcmd
+ if not (fix_stdout or fix_stderr):
+ os.execvp(c[0], c)
+
+ sys.stdout.flush()
+ sys.stderr.flush()
+ out = byte_stream(sys.stdout)
+ err = byte_stream(sys.stderr)
+ p = None
+ try:
+ p = subprocess.Popen(c,
+ stdout=PIPE if fix_stdout else out,
+ stderr=PIPE if fix_stderr else err,
+ env=tty_env, bufsize=4096, close_fds=True)
+ # Assume p will receive these signals and quit, which will
+ # then cause us to quit.
+ for sig in (signal.SIGINT, signal.SIGTERM, signal.SIGQUIT):
+ signal.signal(sig, signal.SIG_IGN)
+
+ filter_output(fix_stdout and p.stdout.fileno() or None,
+ fix_stderr and p.stderr.fileno() or None,
+ fix_stdout and out.fileno() or None,
+ fix_stderr and err.fileno() or None)
+ return p.wait()
+ except BaseException as ex:
+ add_ex_tb(ex)
+ try:
+ if p and p.poll() == None:
+ os.kill(p.pid, signal.SIGTERM)
+ p.wait()
+ except BaseException as kill_ex:
+ raise add_ex_ctx(add_ex_tb(kill_ex), ex)
+ raise ex
+
+wrap_main(lambda : run_subcmd(subcmd))
--- /dev/null
+#!/bin/sh
+"""": # -*-python-*-
+bup_python="$(dirname "$0")/bup-python" || exit $?
+exec "$bup_python" "$0" ${1+"$@"}
+"""
+# end of bup preamble
+
+from __future__ import absolute_import
+import re, stat, sys
+
+from bup import options, git, vfs
+from bup.compat import argv_bytes
+from bup.helpers import chunkyreader, handle_ctrl_c, log, saved_errors
+from bup.io import byte_stream
+from bup.repo import LocalRepo
+
+optspec = """
+bup cat-file [--meta|--bupm] /branch/revision/[path]
+--
+meta print the target's metadata entry (decoded then reencoded) to stdout
+bupm print the target directory's .bupm file directly to stdout
+"""
+
+handle_ctrl_c()
+
+o = options.Options(optspec)
+(opt, flags, extra) = o.parse(sys.argv[1:])
+
+git.check_repo_or_die()
+
+if not extra:
+ o.fatal('must specify a target')
+if len(extra) > 1:
+ o.fatal('only one target file allowed')
+if opt.bupm and opt.meta:
+ o.fatal('--meta and --bupm are incompatible')
+
+target = argv_bytes(extra[0])
+
+if not re.match(br'/*[^/]+/[^/]+', target):
+ o.fatal("path %r doesn't include a branch and revision" % target)
+
+repo = LocalRepo()
+resolved = vfs.resolve(repo, target, follow=False)
+leaf_name, leaf_item = resolved[-1]
+if not leaf_item:
+ log('error: cannot access %r in %r\n'
+ % ('/'.join(name for name, item in resolved), path))
+ sys.exit(1)
+
+mode = vfs.item_mode(leaf_item)
+
+sys.stdout.flush()
+out = byte_stream(sys.stdout)
+
+if opt.bupm:
+ if not stat.S_ISDIR(mode):
+ o.fatal('%r is not a directory' % target)
+ _, bupm_oid = vfs.tree_data_and_bupm(repo, leaf_item.oid)
+ if bupm_oid:
+ with vfs.tree_data_reader(repo, bupm_oid) as meta_stream:
+ out.write(meta_stream.read())
+elif opt.meta:
+ augmented = vfs.augment_item_meta(repo, leaf_item, include_size=True)
+ out.write(augmented.meta.encode())
+else:
+ if stat.S_ISREG(mode):
+ with vfs.fopen(repo, leaf_item) as f:
+ for b in chunkyreader(f):
+ out.write(b)
+ else:
+ o.fatal('%r is not a plain file' % target)
+
+if saved_errors:
+ log('warning: %d errors encountered\n' % len(saved_errors))
+ sys.exit(1)
--- /dev/null
+#!/bin/sh
+"""": # -*-python-*-
+bup_python="$(dirname "$0")/bup-python" || exit $?
+exec "$bup_python" "$0" ${1+"$@"}
+"""
+# end of bup preamble
+
+from __future__ import absolute_import
+import sys, getopt, socket, subprocess, fcntl
+from bup import options, path
+from bup.helpers import *
+
+optspec = """
+bup daemon [options...] -- [bup-server options...]
+--
+l,listen ip address to listen on, defaults to *
+p,port port to listen on, defaults to 1982
+"""
+o = options.Options(optspec, optfunc=getopt.getopt)
+(opt, flags, extra) = o.parse(sys.argv[1:])
+
+host = opt.listen
+port = opt.port and int(opt.port) or 1982
+
+import socket
+import sys
+
+socks = []
+e = None
+for res in socket.getaddrinfo(host, port, socket.AF_UNSPEC,
+ socket.SOCK_STREAM, 0, socket.AI_PASSIVE):
+ af, socktype, proto, canonname, sa = res
+ try:
+ s = socket.socket(af, socktype, proto)
+ except socket.error as e:
+ continue
+ try:
+ if af == socket.AF_INET6:
+ log("bup daemon: listening on [%s]:%s\n" % sa[:2])
+ else:
+ log("bup daemon: listening on %s:%s\n" % sa[:2])
+ s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
+ s.bind(sa)
+ s.listen(1)
+ fcntl.fcntl(s.fileno(), fcntl.F_SETFD, fcntl.FD_CLOEXEC)
+ except socket.error as e:
+ s.close()
+ continue
+ socks.append(s)
+
+if not socks:
+ log('bup daemon: listen socket: %s\n' % e.args[1])
+ sys.exit(1)
+
+try:
+ while True:
+ [rl,wl,xl] = select.select(socks, [], [], 60)
+ for l in rl:
+ s, src = l.accept()
+ try:
+ log("Socket accepted connection from %s\n" % (src,))
+ fd1 = os.dup(s.fileno())
+ fd2 = os.dup(s.fileno())
+ s.close()
+ sp = subprocess.Popen([path.exe(), 'mux', '--',
+ path.exe(), 'server']
+ + extra, stdin=fd1, stdout=fd2)
+ finally:
+ os.close(fd1)
+ os.close(fd2)
+finally:
+ for l in socks:
+ l.shutdown(socket.SHUT_RDWR)
+ l.close()
+
+debug1("bup daemon: done")
--- /dev/null
+#!/bin/sh
+"""": # -*-python-*-
+bup_python="$(dirname "$0")/bup-python" || exit $?
+exec "$bup_python" "$0" ${1+"$@"}
+"""
+# end of bup preamble
+
+from __future__ import absolute_import
+import sys, os, random
+
+from bup import options
+from bup.compat import argv_bytes, bytes_from_uint, range
+from bup.helpers import log
+from bup.io import path_msg
+
+
+def randblock(n):
+ return b''.join(bytes_from_uint(random.randrange(0,256)) for i in range(n))
+
+
+optspec = """
+bup damage [-n count] [-s maxsize] [-S seed] <filenames...>
+--
+ WARNING: THIS COMMAND IS EXTREMELY DANGEROUS
+n,num= number of blocks to damage
+s,size= maximum size of each damaged block
+percent= maximum size of each damaged block (as a percent of entire file)
+equal spread damage evenly throughout the file
+S,seed= random number seed (for repeatable tests)
+"""
+o = options.Options(optspec)
+(opt, flags, extra) = o.parse(sys.argv[1:])
+
+if not extra:
+ o.fatal('filenames expected')
+
+if opt.seed != None:
+ random.seed(opt.seed)
+
+for name in extra:
+ name = argv_bytes(name)
+ log('Damaging "%s"...\n' % path_msg(name))
+ with open(name, 'r+b') as f:
+ st = os.fstat(f.fileno())
+ size = st.st_size
+ if opt.percent or opt.size:
+ ms1 = int(float(opt.percent or 0)/100.0*size) or size
+ ms2 = opt.size or size
+ maxsize = min(ms1, ms2)
+ else:
+ maxsize = 1
+ chunks = opt.num or 10
+ chunksize = size // chunks
+ for r in range(chunks):
+ sz = random.randrange(1, maxsize+1)
+ if sz > size:
+ sz = size
+ if opt.equal:
+ ofs = r*chunksize
+ else:
+ ofs = random.randrange(0, size - sz + 1)
+ log(' %6d bytes at %d\n' % (sz, ofs))
+ f.seek(ofs)
+ f.write(randblock(sz))
--- /dev/null
+#!/bin/sh
+"""": # -*-python-*-
+bup_python="$(dirname "$0")/bup-python" || exit $?
+exec "$bup_python" "$0" ${1+"$@"}
+"""
+# end of bup preamble
+
+from __future__ import absolute_import, print_function
+from os.path import relpath
+import sys
+
+from bup import options, drecurse
+from bup.compat import argv_bytes
+from bup.helpers import log, parse_excludes, parse_rx_excludes, saved_errors
+from bup.io import byte_stream
+
+
+optspec = """
+bup drecurse <path>
+--
+x,xdev,one-file-system don't cross filesystem boundaries
+exclude= a path to exclude from the backup (can be used more than once)
+exclude-from= a file that contains exclude paths (can be used more than once)
+exclude-rx= skip paths matching the unanchored regex (may be repeated)
+exclude-rx-from= skip --exclude-rx patterns in file (may be repeated)
+q,quiet don't actually print filenames
+profile run under the python profiler
+"""
+o = options.Options(optspec)
+(opt, flags, extra) = o.parse(sys.argv[1:])
+
+if len(extra) != 1:
+ o.fatal("exactly one filename expected")
+
+drecurse_top = argv_bytes(extra[0])
+excluded_paths = parse_excludes(flags, o.fatal)
+if not drecurse_top.startswith(b'/'):
+ excluded_paths = [relpath(x) for x in excluded_paths]
+exclude_rxs = parse_rx_excludes(flags, o.fatal)
+it = drecurse.recursive_dirlist([drecurse_top], opt.xdev,
+ excluded_paths=excluded_paths,
+ exclude_rxs=exclude_rxs)
+if opt.profile:
+ import cProfile
+ def do_it():
+ for i in it:
+ pass
+ cProfile.run('do_it()')
+else:
+ if opt.quiet:
+ for i in it:
+ pass
+ else:
+ sys.stdout.flush()
+ out = byte_stream(sys.stdout)
+ for (name,st) in it:
+ out.write(name + b'\n')
+
+if saved_errors:
+ log('WARNING: %d errors encountered.\n' % len(saved_errors))
+ sys.exit(1)
--- /dev/null
+#!/bin/sh
+"""": # -*-python-*-
+bup_python="$(dirname "$0")/bup-python" || exit $?
+exec "$bup_python" "$0" ${1+"$@"}
+"""
+# end of bup preamble
+
+from __future__ import absolute_import, print_function
+import sys, os, glob, subprocess
+from shutil import rmtree
+from subprocess import PIPE, Popen
+from tempfile import mkdtemp
+from binascii import hexlify
+
+from bup import options, git
+from bup.compat import argv_bytes
+from bup.helpers import Sha1, chunkyreader, istty2, log, progress
+from bup.io import byte_stream
+
+
+par2_ok = 0
+nullf = open(os.devnull, 'wb+')
+
+def debug(s):
+ if opt.verbose > 1:
+ log(s)
+
+def run(argv):
+ # at least in python 2.5, using "stdout=2" or "stdout=sys.stderr" below
+ # doesn't actually work, because subprocess closes fd #2 right before
+ # execing for some reason. So we work around it by duplicating the fd
+ # first.
+ fd = os.dup(2) # copy stderr
+ try:
+ p = subprocess.Popen(argv, stdout=fd, close_fds=False)
+ return p.wait()
+ finally:
+ os.close(fd)
+
+def par2_setup():
+ global par2_ok
+ rv = 1
+ try:
+ p = subprocess.Popen([b'par2', b'--help'],
+ stdout=nullf, stderr=nullf, stdin=nullf)
+ rv = p.wait()
+ except OSError:
+ log('fsck: warning: par2 not found; disabling recovery features.\n')
+ else:
+ par2_ok = 1
+
+def is_par2_parallel():
+ # A true result means it definitely allows -t1; a false result is
+ # technically inconclusive, but likely means no.
+ tmpdir = mkdtemp(prefix=b'bup-fsck')
+ try:
+ canary = tmpdir + b'/canary'
+ with open(canary, 'wb') as f:
+ f.write(b'canary\n')
+ p = subprocess.Popen((b'par2', b'create', b'-qq', b'-t1', canary),
+ stderr=PIPE, stdin=nullf)
+ _, err = p.communicate()
+ parallel = p.returncode == 0
+ if opt.verbose:
+ if len(err) > 0 and err != b'Invalid option specified: -t1\n':
+ log('Unexpected par2 error output\n')
+ log(repr(err) + '\n')
+ if parallel:
+ log('Assuming par2 supports parallel processing\n')
+ else:
+ log('Assuming par2 does not support parallel processing\n')
+ return parallel
+ finally:
+ rmtree(tmpdir)
+
+_par2_parallel = None
+
+def par2(action, args, verb_floor=0):
+ global _par2_parallel
+ if _par2_parallel is None:
+ _par2_parallel = is_par2_parallel()
+ cmd = [b'par2', action]
+ if opt.verbose >= verb_floor and not istty2:
+ cmd.append(b'-q')
+ else:
+ cmd.append(b'-qq')
+ if _par2_parallel:
+ cmd.append(b'-t1')
+ cmd.extend(args)
+ return run(cmd)
+
+def par2_generate(base):
+ return par2(b'create',
+ [b'-n1', b'-c200', b'--', base, base + b'.pack', base + b'.idx'],
+ verb_floor=2)
+
+def par2_verify(base):
+ return par2(b'verify', [b'--', base], verb_floor=3)
+
+def par2_repair(base):
+ return par2(b'repair', [b'--', base], verb_floor=2)
+
+def quick_verify(base):
+ f = open(base + b'.pack', 'rb')
+ f.seek(-20, 2)
+ wantsum = f.read(20)
+ assert(len(wantsum) == 20)
+ f.seek(0)
+ sum = Sha1()
+ for b in chunkyreader(f, os.fstat(f.fileno()).st_size - 20):
+ sum.update(b)
+ if sum.digest() != wantsum:
+ raise ValueError('expected %r, got %r' % (hexlify(wantsum),
+ sum.hexdigest()))
+
+
+def git_verify(base):
+ if opt.quick:
+ try:
+ quick_verify(base)
+ except Exception as e:
+ log('error: %s\n' % e)
+ return 1
+ return 0
+ else:
+ return run([b'git', b'verify-pack', b'--', base])
+
+
+def do_pack(base, last, par2_exists, out):
+ code = 0
+ if par2_ok and par2_exists and (opt.repair or not opt.generate):
+ vresult = par2_verify(base)
+ if vresult != 0:
+ if opt.repair:
+ rresult = par2_repair(base)
+ if rresult != 0:
+ action_result = b'failed'
+ log('%s par2 repair: failed (%d)\n' % (last, rresult))
+ code = rresult
+ else:
+ action_result = b'repaired'
+ log('%s par2 repair: succeeded (0)\n' % last)
+ code = 100
+ else:
+ action_result = b'failed'
+ log('%s par2 verify: failed (%d)\n' % (last, vresult))
+ code = vresult
+ else:
+ action_result = b'ok'
+ elif not opt.generate or (par2_ok and not par2_exists):
+ gresult = git_verify(base)
+ if gresult != 0:
+ action_result = b'failed'
+ log('%s git verify: failed (%d)\n' % (last, gresult))
+ code = gresult
+ else:
+ if par2_ok and opt.generate:
+ presult = par2_generate(base)
+ if presult != 0:
+ action_result = b'failed'
+ log('%s par2 create: failed (%d)\n' % (last, presult))
+ code = presult
+ else:
+ action_result = b'generated'
+ else:
+ action_result = b'ok'
+ else:
+ assert(opt.generate and (not par2_ok or par2_exists))
+ action_result = b'exists' if par2_exists else b'skipped'
+ if opt.verbose:
+ out.write(last + b' ' + action_result + b'\n')
+ return code
+
+
+optspec = """
+bup fsck [options...] [filenames...]
+--
+r,repair attempt to repair errors using par2 (dangerous!)
+g,generate generate auto-repair information using par2
+v,verbose increase verbosity (can be used more than once)
+quick just check pack sha1sum, don't use git verify-pack
+j,jobs= run 'n' jobs in parallel
+par2-ok immediately return 0 if par2 is ok, 1 if not
+disable-par2 ignore par2 even if it is available
+"""
+o = options.Options(optspec)
+(opt, flags, extra) = o.parse(sys.argv[1:])
+opt.verbose = opt.verbose or 0
+
+par2_setup()
+if opt.par2_ok:
+ if par2_ok:
+ sys.exit(0) # 'true' in sh
+ else:
+ sys.exit(1)
+if opt.disable_par2:
+ par2_ok = 0
+
+git.check_repo_or_die()
+
+if extra:
+ extra = [argv_byes(x) for x in extra]
+else:
+ debug('fsck: No filenames given: checking all packs.\n')
+ extra = glob.glob(git.repo(b'objects/pack/*.pack'))
+
+sys.stdout.flush()
+out = byte_stream(sys.stdout)
+code = 0
+count = 0
+outstanding = {}
+for name in extra:
+ if name.endswith(b'.pack'):
+ base = name[:-5]
+ elif name.endswith(b'.idx'):
+ base = name[:-4]
+ elif name.endswith(b'.par2'):
+ base = name[:-5]
+ elif os.path.exists(name + b'.pack'):
+ base = name
+ else:
+ raise Exception('%r is not a pack file!' % name)
+ (dir,last) = os.path.split(base)
+ par2_exists = os.path.exists(base + b'.par2')
+ if par2_exists and os.stat(base + b'.par2').st_size == 0:
+ par2_exists = 0
+ sys.stdout.flush() # Not sure we still need this, but it'll flush out too
+ debug('fsck: checking %r (%s)\n'
+ % (last, par2_ok and par2_exists and 'par2' or 'git'))
+ if not opt.verbose:
+ progress('fsck (%d/%d)\r' % (count, len(extra)))
+
+ if not opt.jobs:
+ nc = do_pack(base, last, par2_exists, out)
+ code = code or nc
+ count += 1
+ else:
+ while len(outstanding) >= opt.jobs:
+ (pid,nc) = os.wait()
+ nc >>= 8
+ if pid in outstanding:
+ del outstanding[pid]
+ code = code or nc
+ count += 1
+ pid = os.fork()
+ if pid: # parent
+ outstanding[pid] = 1
+ else: # child
+ try:
+ sys.exit(do_pack(base, last, par2_exists, out))
+ except Exception as e:
+ log('exception: %r\n' % e)
+ sys.exit(99)
+
+while len(outstanding):
+ (pid,nc) = os.wait()
+ nc >>= 8
+ if pid in outstanding:
+ del outstanding[pid]
+ code = code or nc
+ count += 1
+ if not opt.verbose:
+ progress('fsck (%d/%d)\r' % (count, len(extra)))
+
+if istty2:
+ debug('fsck done. \n')
+sys.exit(code)
--- /dev/null
+#!/bin/sh
+"""": # -*-python-*-
+bup_python="$(dirname "$0")/bup-python" || exit $?
+exec "$bup_python" "$0" ${1+"$@"}
+"""
+# end of bup preamble
+
+# For now, this completely relies on the assumption that the current
+# encoding (LC_CTYPE, etc.) is ASCII compatible, and that it returns
+# the exact same bytes from a decode/encode round-trip (or the reverse
+# (e.g. ISO-8859-1).
+
+from __future__ import absolute_import, print_function
+import sys, os, stat, fnmatch
+
+from bup import options, git, shquote, ls, vfs
+from bup.compat import argv_bytes, input
+from bup.helpers import chunkyreader, handle_ctrl_c, log
+from bup.io import byte_stream, path_msg
+from bup.repo import LocalRepo
+
+handle_ctrl_c()
+
+
+class OptionError(Exception):
+ pass
+
+
+def input_bytes(s):
+ return s.encode('iso-8859-1')
+
+
+def do_ls(repo, args, out):
+ try:
+ opt = ls.opts_from_cmdline(args, onabort=OptionError)
+ except OptionError as e:
+ log('error: %s' % e)
+ return
+ return ls.within_repo(repo, opt, out)
+
+
+def write_to_file(inf, outf):
+ for blob in chunkyreader(inf):
+ outf.write(blob)
+
+
+def inputiter():
+ if os.isatty(sys.stdin.fileno()):
+ while 1:
+ try:
+ yield input('bup> ')
+ except EOFError:
+ print() # Clear the line for the terminal's next prompt
+ break
+ else:
+ for line in sys.stdin:
+ yield line
+
+
+def _completer_get_subs(repo, line):
+ (qtype, lastword) = shquote.unfinished_word(line)
+ dir, name = os.path.split(lastword.encode('iso-8859-1'))
+ dir_path = vfs.resolve(repo, dir or b'/')
+ _, dir_item = dir_path[-1]
+ if not dir_item:
+ subs = tuple()
+ else:
+ subs = tuple(dir_path + (entry,)
+ for entry in vfs.contents(repo, dir_item)
+ if (entry[0] != b'.' and entry[0].startswith(name)))
+ return qtype, lastword, subs
+
+
+_last_line = None
+_last_res = None
+def completer(text, iteration):
+ global repo
+ global _last_line
+ global _last_res
+ try:
+ line = readline.get_line_buffer()[:readline.get_endidx()]
+ if _last_line != line:
+ _last_res = _completer_get_subs(repo, line)
+ _last_line = line
+ qtype, lastword, subs = _last_res
+ if iteration < len(subs):
+ path = subs[iteration]
+ leaf_name, leaf_item = path[-1]
+ res = vfs.try_resolve(repo, leaf_name, parent=path[:-1])
+ leaf_name, leaf_item = res[-1]
+ fullname = os.path.join(*(name for name, item in res))
+ if stat.S_ISDIR(vfs.item_mode(leaf_item)):
+ ret = shquote.what_to_add(qtype, lastword,
+ fullname.decode('iso-8859-1') + '/',
+ terminate=False)
+ else:
+ ret = shquote.what_to_add(qtype, lastword,
+ fullname.decode('iso-8859-1'),
+ terminate=True) + b' '
+ return text + ret
+ except Exception as e:
+ log('\n')
+ try:
+ import traceback
+ traceback.print_tb(sys.exc_traceback)
+ except Exception as e2:
+ log('Error printing traceback: %s\n' % e2)
+ log('\nError in completion: %s\n' % e)
+
+
+optspec = """
+bup ftp [commands...]
+"""
+o = options.Options(optspec)
+(opt, flags, extra) = o.parse(sys.argv[1:])
+
+git.check_repo_or_die()
+
+sys.stdout.flush()
+out = byte_stream(sys.stdout)
+repo = LocalRepo()
+pwd = vfs.resolve(repo, b'/')
+rv = 0
+
+if extra:
+ lines = extra
+else:
+ try:
+ import readline
+ except ImportError:
+ log('* readline module not available: line editing disabled.\n')
+ readline = None
+
+ if readline:
+ readline.set_completer_delims(' \t\n\r/')
+ readline.set_completer(completer)
+ if sys.platform.startswith('darwin'):
+ # MacOS uses a slightly incompatible clone of libreadline
+ readline.parse_and_bind('bind ^I rl_complete')
+ readline.parse_and_bind('tab: complete')
+ lines = inputiter()
+
+for line in lines:
+ if not line.strip():
+ continue
+ words = [word for (wordstart,word) in shquote.quotesplit(line)]
+ cmd = words[0].lower()
+ #log('execute: %r %r\n' % (cmd, parm))
+ try:
+ if cmd == 'ls':
+ # FIXME: respect pwd (perhaps via ls accepting resolve path/parent)
+ do_ls(repo, words[1:], out)
+ elif cmd == 'cd':
+ np = pwd
+ for parm in words[1:]:
+ res = vfs.resolve(repo, input_bytes(parm), parent=np)
+ _, leaf_item = res[-1]
+ if not leaf_item:
+ raise Exception('%s does not exist'
+ % path_msg(b'/'.join(name for name, item
+ in res)))
+ if not stat.S_ISDIR(vfs.item_mode(leaf_item)):
+ raise Exception('%s is not a directory' % path_msg(parm))
+ np = res
+ pwd = np
+ elif cmd == 'pwd':
+ if len(pwd) == 1:
+ out.write(b'/')
+ out.write(b'/'.join(name for name, item in pwd) + b'\n')
+ elif cmd == 'cat':
+ for parm in words[1:]:
+ res = vfs.resolve(repo, input_bytes(parm), parent=pwd)
+ _, leaf_item = res[-1]
+ if not leaf_item:
+ raise Exception('%s does not exist' %
+ path_msg(b'/'.join(name for name, item
+ in res)))
+ with vfs.fopen(repo, leaf_item) as srcfile:
+ write_to_file(srcfile, out)
+ elif cmd == 'get':
+ if len(words) not in [2,3]:
+ rv = 1
+ raise Exception('Usage: get <filename> [localname]')
+ rname = input_bytes(words[1])
+ (dir,base) = os.path.split(rname)
+ lname = input_bytes(len(words) > 2 and words[2] or base)
+ res = vfs.resolve(repo, rname, parent=pwd)
+ _, leaf_item = res[-1]
+ if not leaf_item:
+ raise Exception('%s does not exist' %
+ path_msg(b'/'.join(name for name, item in res)))
+ with vfs.fopen(repo, leaf_item) as srcfile:
+ with open(lname, 'wb') as destfile:
+ log('Saving %s\n' % path_msg(lname))
+ write_to_file(srcfile, destfile)
+ elif cmd == 'mget':
+ for parm in words[1:]:
+ dir, base = os.path.split(input_bytes(parm))
+
+ res = vfs.resolve(repo, dir, parent=pwd)
+ _, dir_item = res[-1]
+ if not dir_item:
+ raise Exception('%s does not exist' % path_msg(dir))
+ for name, item in vfs.contents(repo, dir_item):
+ if name == b'.':
+ continue
+ if fnmatch.fnmatch(name, base):
+ if stat.S_ISLNK(vfs.item_mode(item)):
+ deref = vfs.resolve(repo, name, parent=res)
+ deref_name, deref_item = deref[-1]
+ if not deref_item:
+ raise Exception('%s does not exist' %
+ path_msg('/'.join(name for name, item
+ in deref)))
+ item = deref_item
+ with vfs.fopen(repo, item) as srcfile:
+ with open(name, 'wb') as destfile:
+ log('Saving %s\n' % path_msg(name))
+ write_to_file(srcfile, destfile)
+ elif cmd == 'help' or cmd == '?':
+ # FIXME: move to stdout
+ log('Commands: ls cd pwd cat get mget help quit\n')
+ elif cmd in ('quit', 'exit', 'bye'):
+ break
+ else:
+ rv = 1
+ raise Exception('no such command %r' % cmd)
+ except Exception as e:
+ rv = 1
+ log('error: %s\n' % e)
+ raise
+
+sys.exit(rv)
--- /dev/null
+#!/bin/sh
+"""": # -*-python-*-
+bup_python="$(dirname "$0")/bup-python" || exit $?
+exec "$bup_python" "$0" ${1+"$@"}
+"""
+# end of bup preamble
+
+from __future__ import absolute_import, print_function
+import sys, os, errno
+
+try:
+ import fuse
+except ImportError:
+ print('error: cannot find the python "fuse" module; please install it',
+ file=sys.stderr)
+ sys.exit(2)
+if not hasattr(fuse, '__version__'):
+ print('error: fuse module is too old for fuse.__version__', file=sys.stderr)
+ sys.exit(2)
+fuse.fuse_python_api = (0, 2)
+
+if sys.version_info[0] > 2:
+ try:
+ fuse_ver = fuse.__version__.split('.')
+ fuse_ver_maj = int(fuse_ver[0])
+ except:
+ log('error: cannot determine the fuse major version; please report',
+ file=sys.stderr)
+ sys.exit(2)
+ if len(fuse_ver) < 3 or fuse_ver_maj < 1:
+ print("error: fuse module can't handle binary data; please upgrade to 1.0+\n",
+ file=sys.stderr)
+ sys.exit(2)
+
+from bup import options, git, vfs, xstat
+from bup.compat import argv_bytes, fsdecode, py_maj
+from bup.helpers import log
+from bup.repo import LocalRepo
+
+
+# FIXME: self.meta and want_meta?
+
+# The path handling is just wrong, but the current fuse module can't
+# handle bytes paths.
+
+class BupFs(fuse.Fuse):
+ def __init__(self, repo, verbose=0, fake_metadata=False):
+ fuse.Fuse.__init__(self)
+ self.repo = repo
+ self.verbose = verbose
+ self.fake_metadata = fake_metadata
+
+ def getattr(self, path):
+ path = argv_bytes(path)
+ global opt
+ if self.verbose > 0:
+ log('--getattr(%r)\n' % path)
+ res = vfs.resolve(self.repo, path, want_meta=(not self.fake_metadata),
+ follow=False)
+ name, item = res[-1]
+ if not item:
+ return -errno.ENOENT
+ if self.fake_metadata:
+ item = vfs.augment_item_meta(self.repo, item, include_size=True)
+ else:
+ item = vfs.ensure_item_has_metadata(self.repo, item,
+ include_size=True)
+ meta = item.meta
+ # FIXME: do we want/need to do anything more with nlink?
+ st = fuse.Stat(st_mode=meta.mode, st_nlink=1, st_size=meta.size)
+ st.st_mode = meta.mode
+ st.st_uid = meta.uid or 0
+ st.st_gid = meta.gid or 0
+ st.st_atime = max(0, xstat.fstime_floor_secs(meta.atime))
+ st.st_mtime = max(0, xstat.fstime_floor_secs(meta.mtime))
+ st.st_ctime = max(0, xstat.fstime_floor_secs(meta.ctime))
+ return st
+
+ def readdir(self, path, offset):
+ path = argv_bytes(path)
+ assert not offset # We don't return offsets, so offset should be unused
+ res = vfs.resolve(self.repo, path, follow=False)
+ dir_name, dir_item = res[-1]
+ if not dir_item:
+ yield -errno.ENOENT
+ yield fuse.Direntry('..')
+ # FIXME: make sure want_meta=False is being completely respected
+ for ent_name, ent_item in vfs.contents(repo, dir_item, want_meta=False):
+ fusename = fsdecode(ent_name.replace(b'/', b'-'))
+ yield fuse.Direntry(fusename)
+
+ def readlink(self, path):
+ path = argv_bytes(path)
+ if self.verbose > 0:
+ log('--readlink(%r)\n' % path)
+ res = vfs.resolve(self.repo, path, follow=False)
+ name, item = res[-1]
+ if not item:
+ return -errno.ENOENT
+ return fsdecode(vfs.readlink(repo, item))
+
+ def open(self, path, flags):
+ path = argv_bytes(path)
+ if self.verbose > 0:
+ log('--open(%r)\n' % path)
+ res = vfs.resolve(self.repo, path, follow=False)
+ name, item = res[-1]
+ if not item:
+ return -errno.ENOENT
+ accmode = os.O_RDONLY | os.O_WRONLY | os.O_RDWR
+ if (flags & accmode) != os.O_RDONLY:
+ return -errno.EACCES
+ # Return None since read doesn't need the file atm...
+ # If we *do* return the file, it'll show up as the last argument
+ #return vfs.fopen(repo, item)
+
+ def read(self, path, size, offset):
+ path = argv_bytes(path)
+ if self.verbose > 0:
+ log('--read(%r)\n' % path)
+ res = vfs.resolve(self.repo, path, follow=False)
+ name, item = res[-1]
+ if not item:
+ return -errno.ENOENT
+ with vfs.fopen(repo, item) as f:
+ f.seek(offset)
+ return f.read(size)
+
+
+optspec = """
+bup fuse [-d] [-f] <mountpoint>
+--
+f,foreground run in foreground
+d,debug run in the foreground and display FUSE debug information
+o,allow-other allow other users to access the filesystem
+meta report original metadata for paths when available
+v,verbose increase log output (can be used more than once)
+"""
+o = options.Options(optspec)
+opt, flags, extra = o.parse(sys.argv[1:])
+if not opt.verbose:
+ opt.verbose = 0
+
+# Set stderr to be line buffered, even if it's not connected to the console
+# so that we'll be able to see diagnostics in a timely fashion.
+errfd = sys.stderr.fileno()
+sys.stderr.flush()
+sys.stderr = os.fdopen(errfd, 'w', 1)
+
+if len(extra) != 1:
+ o.fatal('only one mount point argument expected')
+
+git.check_repo_or_die()
+repo = LocalRepo()
+f = BupFs(repo=repo, verbose=opt.verbose, fake_metadata=(not opt.meta))
+
+# This is likely wrong, but the fuse module doesn't currently accept bytes
+f.fuse_args.mountpoint = extra[0]
+
+if opt.debug:
+ f.fuse_args.add('debug')
+if opt.foreground:
+ f.fuse_args.setmod('foreground')
+f.multithreaded = False
+if opt.allow_other:
+ f.fuse_args.add('allow_other')
+f.main()
--- /dev/null
+#!/bin/sh
+"""": # -*-python-*-
+bup_python="$(dirname "$0")/bup-python" || exit $?
+exec "$bup_python" "$0" ${1+"$@"}
+"""
+# end of bup preamble
+
+from __future__ import absolute_import
+import sys
+
+from bup import git, options
+from bup.gc import bup_gc
+from bup.helpers import die_if_errors, handle_ctrl_c, log
+
+
+optspec = """
+bup gc [options...]
+--
+v,verbose increase log output (can be used more than once)
+threshold= only rewrite a packfile if it's over this percent garbage [10]
+#,compress= set compression level to # (0-9, 9 is highest) [1]
+unsafe use the command even though it may be DANGEROUS
+"""
+
+# FIXME: server mode?
+# FIXME: make sure client handles server-side changes reasonably
+
+handle_ctrl_c()
+
+o = options.Options(optspec)
+(opt, flags, extra) = o.parse(sys.argv[1:])
+
+if not opt.unsafe:
+ o.fatal('refusing to run dangerous, experimental command without --unsafe')
+
+if extra:
+ o.fatal('no positional parameters expected')
+
+if opt.threshold:
+ try:
+ opt.threshold = int(opt.threshold)
+ except ValueError:
+ o.fatal('threshold must be an integer percentage value')
+ if opt.threshold < 0 or opt.threshold > 100:
+ o.fatal('threshold must be an integer percentage value')
+
+git.check_repo_or_die()
+
+bup_gc(threshold=opt.threshold,
+ compression=opt.compress,
+ verbosity=opt.verbose)
+
+die_if_errors()
--- /dev/null
+#!/bin/sh
+"""": # -*-python-*-
+bup_python="$(dirname "$0")/bup-python" || exit $?
+exec "$bup_python" "$0" ${1+"$@"}
+"""
+# end of bup preamble
+
+from __future__ import absolute_import, print_function
+import os, re, stat, sys, textwrap, time
+from binascii import hexlify, unhexlify
+from collections import namedtuple
+from functools import partial
+from stat import S_ISDIR
+
+from bup import git, client, helpers, vfs
+from bup.compat import argv_bytes, environ, hexstr, items, wrap_main
+from bup.git import get_cat_data, parse_commit, walk_object
+from bup.helpers import add_error, debug1, handle_ctrl_c, log, saved_errors
+from bup.helpers import hostname, shstr, tty_width
+from bup.io import path_msg
+from bup.pwdgrp import userfullname, username
+from bup.repo import LocalRepo, RemoteRepo
+
+argspec = (
+ "usage: bup get [-s source] [-r remote] (<--ff|--append|...> REF [DEST])...",
+
+ """Transfer data from a source repository to a destination repository
+ according to the methods specified (--ff, --ff:, --append, etc.).
+ Both repositories default to BUP_DIR. A remote destination may be
+ specified with -r, and data may be pulled from a remote repository
+ with the related "bup on HOST get ..." command.""",
+
+ ('optional arguments:',
+ (('-h, --help', 'show this help message and exit'),
+ ('-v, --verbose',
+ 'increase log output (can be specified more than once)'),
+ ('-q, --quiet', "don't show progress meter"),
+ ('-s SOURCE, --source SOURCE',
+ 'path to the source repository (defaults to BUP_DIR)'),
+ ('-r REMOTE, --remote REMOTE',
+ 'hostname:/path/to/repo of remote destination repository'),
+ ('-t --print-trees', 'output a tree id for each ref set'),
+ ('-c, --print-commits', 'output a commit id for each ref set'),
+ ('--print-tags', 'output an id for each tag'),
+ ('--bwlimit BWLIMIT', 'maximum bytes/sec to transmit to server'),
+ ('-0, -1, -2, -3, -4, -5, -6, -7, -8, -9, --compress LEVEL',
+ 'set compression LEVEL (default: 1)'))),
+
+ ('transfer methods:',
+ (('--ff REF, --ff: REF DEST',
+ 'fast-forward dest REF (or DEST) to match source REF'),
+ ('--append REF, --append: REF DEST',
+ 'append REF (treeish or committish) to dest REF (or DEST)'),
+ ('--pick REF, --pick: REF DEST',
+ 'append single source REF commit to dest REF (or DEST)'),
+ ('--force-pick REF, --force-pick: REF DEST',
+ '--pick, overwriting REF (or DEST)'),
+ ('--new-tag REF, --new-tag: REF DEST',
+ 'tag source ref REF as REF (or DEST) in dest unless it already exists'),
+ ('--replace, --replace: REF DEST',
+ 'overwrite REF (or DEST) in dest with source REF'),
+ ('--unnamed REF',
+ 'fetch REF anonymously (without destination ref)'))))
+
+def render_opts(opts, width=None):
+ if not width:
+ width = tty_width()
+ result = []
+ for args, desc in opts:
+ result.append(textwrap.fill(args, width=width,
+ initial_indent=(' ' * 2),
+ subsequent_indent=(' ' * 4)))
+ result.append('\n')
+ result.append(textwrap.fill(desc, width=width,
+ initial_indent=(' ' * 6),
+ subsequent_indent=(' ' * 6)))
+ result.append('\n')
+ return result
+
+def usage(argspec, width=None):
+ if not width:
+ width = tty_width()
+ usage, preamble, groups = argspec[0], argspec[1], argspec[2:]
+ msg = []
+ msg.append(textwrap.fill(usage, width=width, subsequent_indent=' '))
+ msg.append('\n\n')
+ msg.append(textwrap.fill(preamble.replace('\n', ' '), width=width))
+ msg.append('\n')
+ for group_name, group_args in groups:
+ msg.extend(['\n', group_name, '\n'])
+ msg.extend(render_opts(group_args, width=width))
+ return ''.join(msg)
+
+def misuse(message=None):
+ sys.stderr.write(usage(argspec))
+ if message:
+ sys.stderr.write('\nerror: ')
+ sys.stderr.write(message)
+ sys.stderr.write('\n')
+ sys.exit(1)
+
+def require_n_args_or_die(n, args):
+ if len(args) < n + 1:
+ misuse('%s argument requires %d %s'
+ % (n, 'values' if n == 1 else 'value'))
+ result = args[1:1+n], args[1+n:]
+ assert len(result[0]) == n
+ return result
+
+Spec = namedtuple('Spec', ('method', 'src', 'dest'))
+
+def spec_msg(s):
+ if not s.dest:
+ return '--%s %s' % (s.method, path_msg(s.src))
+ return '--%s: %s %s' % (s.method, path_msg(s.src), path_msg(s.dest))
+
+def parse_args(args):
+ class GetOpts:
+ pass
+ opt = GetOpts()
+ opt.help = False
+ opt.verbose = 0
+ opt.quiet = False
+ opt.print_commits = opt.print_trees = opt.print_tags = False
+ opt.bwlimit = None
+ opt.compress = 1
+ opt.source = opt.remote = None
+ opt.target_specs = []
+
+ remaining = args[1:] # Skip argv[0]
+ while remaining:
+ arg = remaining[0]
+ if arg in ('-h', '--help'):
+ sys.stdout.write(usage(argspec))
+ sys.exit(0)
+ elif arg in ('-v', '--verbose'):
+ opt.verbose += 1
+ remaining = remaining[1:]
+ elif arg in ('--ff', '--append', '--pick', '--force-pick',
+ '--new-tag', '--replace', '--unnamed'):
+ (ref,), remaining = require_n_args_or_die(1, remaining)
+ ref = argv_bytes(ref)
+ opt.target_specs.append(Spec(method=arg[2:], src=ref, dest=None))
+ elif arg in ('--ff:', '--append:', '--pick:', '--force-pick:',
+ '--new-tag:', '--replace:'):
+ (ref, dest), remaining = require_n_args_or_die(2, remaining)
+ ref, dest = argv_bytes(ref), argv_bytes(dest)
+ opt.target_specs.append(Spec(method=arg[2:-1], src=ref, dest=dest))
+ elif arg in ('-s', '--source'):
+ (opt.source,), remaining = require_n_args_or_die(1, remaining)
+ elif arg in ('-r', '--remote'):
+ (opt.remote,), remaining = require_n_args_or_die(1, remaining)
+ elif arg in ('-c', '--print-commits'):
+ opt.print_commits, remaining = True, remaining[1:]
+ elif arg in ('-t', '--print-trees'):
+ opt.print_trees, remaining = True, remaining[1:]
+ elif arg == '--print-tags':
+ opt.print_tags, remaining = True, remaining[1:]
+ elif arg in ('-0', '-1', '-2', '-3', '-4', '-5', '-6', '-7', '-8', '-9'):
+ opt.compress = int(arg[1:])
+ remaining = remaining[1:]
+ elif arg == '--compress':
+ (opt.compress,), remaining = require_n_args_or_die(1, remaining)
+ opt.compress = int(opt.compress)
+ elif arg == '--bwlimit':
+ (opt.bwlimit,), remaining = require_n_args_or_die(1, remaining)
+ opt.bwlimit = long(opt.bwlimit)
+ elif arg.startswith('-') and len(arg) > 2 and arg[1] != '-':
+ # Try to interpret this as -xyz, i.e. "-xyz -> -x -y -z".
+ # We do this last so that --foo -bar is valid if --foo
+ # requires a value.
+ remaining[0:1] = ('-' + c for c in arg[1:])
+ # FIXME
+ continue
+ else:
+ misuse()
+ return opt
+
+# FIXME: client error handling (remote exceptions, etc.)
+
+# FIXME: walk_object in in git.py doesn't support opt.verbose. Do we
+# need to adjust for that here?
+def get_random_item(name, hash, repo, writer, opt):
+ def already_seen(oid):
+ return writer.exists(unhexlify(oid))
+ for item in walk_object(repo.cat, hash, stop_at=already_seen,
+ include_data=True):
+ # already_seen ensures that writer.exists(id) is false.
+ # Otherwise, just_write() would fail.
+ writer.just_write(item.oid, item.type, item.data)
+
+
+def append_commit(name, hash, parent, src_repo, writer, opt):
+ now = time.time()
+ items = parse_commit(get_cat_data(src_repo.cat(hash), b'commit'))
+ tree = unhexlify(items.tree)
+ author = b'%s <%s>' % (items.author_name, items.author_mail)
+ author_time = (items.author_sec, items.author_offset)
+ committer = b'%s <%s@%s>' % (userfullname(), username(), hostname())
+ get_random_item(name, hexlify(tree), src_repo, writer, opt)
+ c = writer.new_commit(tree, parent,
+ author, items.author_sec, items.author_offset,
+ committer, now, None,
+ items.message)
+ return c, tree
+
+
+def append_commits(commits, src_name, dest_hash, src_repo, writer, opt):
+ last_c, tree = dest_hash, None
+ for commit in commits:
+ last_c, tree = append_commit(src_name, commit, last_c,
+ src_repo, writer, opt)
+ assert(tree is not None)
+ return last_c, tree
+
+Loc = namedtuple('Loc', ['type', 'hash', 'path'])
+default_loc = Loc(None, None, None)
+
+def find_vfs_item(name, repo):
+ res = repo.resolve(name, follow=False, want_meta=False)
+ leaf_name, leaf_item = res[-1]
+ if not leaf_item:
+ return None
+ kind = type(leaf_item)
+ if kind == vfs.Root:
+ kind = 'root'
+ elif kind == vfs.Tags:
+ kind = 'tags'
+ elif kind == vfs.RevList:
+ kind = 'branch'
+ elif kind == vfs.Commit:
+ if len(res) > 1 and type(res[-2][1]) == vfs.RevList:
+ kind = 'save'
+ else:
+ kind = 'commit'
+ elif kind == vfs.Item:
+ if S_ISDIR(vfs.item_mode(leaf_item)):
+ kind = 'tree'
+ else:
+ kind = 'blob'
+ elif kind == vfs.Chunky:
+ kind = 'tree'
+ elif kind == vfs.FakeLink:
+ # Don't have to worry about ELOOP, excepting malicious
+ # remotes, since "latest" is the only FakeLink.
+ assert leaf_name == b'latest'
+ res = repo.resolve(leaf_item.target, parent=res[:-1],
+ follow=False, want_meta=False)
+ leaf_name, leaf_item = res[-1]
+ assert leaf_item
+ assert type(leaf_item) == vfs.Commit
+ name = b'/'.join(x[0] for x in res)
+ kind = 'save'
+ else:
+ raise Exception('unexpected resolution for %s: %r'
+ % (path_msg(name), res))
+ path = b'/'.join(name for name, item in res)
+ if hasattr(leaf_item, 'coid'):
+ result = Loc(type=kind, hash=leaf_item.coid, path=path)
+ elif hasattr(leaf_item, 'oid'):
+ result = Loc(type=kind, hash=leaf_item.oid, path=path)
+ else:
+ result = Loc(type=kind, hash=None, path=path)
+ return result
+
+
+Target = namedtuple('Target', ['spec', 'src', 'dest'])
+
+def loc_desc(loc):
+ if loc and loc.hash:
+ loc = loc._replace(hash=hexlify(loc.hash))
+ return repr(loc)
+
+
+# FIXME: see if resolve() means we can drop the vfs path cleanup
+
+def cleanup_vfs_path(p):
+ result = os.path.normpath(p)
+ if result.startswith(b'/'):
+ return result
+ return b'/' + result
+
+
+def validate_vfs_path(p):
+ if p.startswith(b'/.') \
+ and not p.startswith(b'/.tag/'):
+ misuse('unsupported destination path %s in %s'
+ % (path_msg(dest.path), spec_msg(spec)))
+ return p
+
+
+def resolve_src(spec, src_repo):
+ src = find_vfs_item(spec.src, src_repo)
+ spec_args = spec_msg(spec)
+ if not src:
+ misuse('cannot find source for %s' % spec_args)
+ if src.type == 'root':
+ misuse('cannot fetch entire repository for %s' % spec_args)
+ if src.type == 'tags':
+ misuse('cannot fetch entire /.tag directory for %s' % spec_args)
+ debug1('src: %s\n' % loc_desc(src))
+ return src
+
+
+def get_save_branch(repo, path):
+ res = repo.resolve(path, follow=False, want_meta=False)
+ leaf_name, leaf_item = res[-1]
+ if not leaf_item:
+ misuse('error: cannot access %r in %r' % (leaf_name, path))
+ assert len(res) == 3
+ res_path = b'/'.join(name for name, item in res[:-1])
+ return res_path
+
+
+def resolve_branch_dest(spec, src, src_repo, dest_repo):
+ # Resulting dest must be treeish, or not exist.
+ if not spec.dest:
+ # Pick a default dest.
+ if src.type == 'branch':
+ spec = spec._replace(dest=spec.src)
+ elif src.type == 'save':
+ spec = spec._replace(dest=get_save_branch(src_repo, spec.src))
+ elif src.path.startswith(b'/.tag/'): # Dest defaults to the same.
+ spec = spec._replace(dest=spec.src)
+
+ spec_args = spec_msg(spec)
+ if not spec.dest:
+ misuse('no destination (implicit or explicit) for %s', spec_args)
+
+ dest = find_vfs_item(spec.dest, dest_repo)
+ if dest:
+ if dest.type == 'commit':
+ misuse('destination for %s is a tagged commit, not a branch'
+ % spec_args)
+ if dest.type != 'branch':
+ misuse('destination for %s is a %s, not a branch'
+ % (spec_args, dest.type))
+ else:
+ dest = default_loc._replace(path=cleanup_vfs_path(spec.dest))
+
+ if dest.path.startswith(b'/.'):
+ misuse('destination for %s must be a valid branch name' % spec_args)
+
+ debug1('dest: %s\n' % loc_desc(dest))
+ return spec, dest
+
+
+def resolve_ff(spec, src_repo, dest_repo):
+ src = resolve_src(spec, src_repo)
+ spec_args = spec_msg(spec)
+ if src.type == 'tree':
+ misuse('%s is impossible; can only --append a tree to a branch'
+ % spec_args)
+ if src.type not in ('branch', 'save', 'commit'):
+ misuse('source for %s must be a branch, save, or commit, not %s'
+ % (spec_args, src.type))
+ spec, dest = resolve_branch_dest(spec, src, src_repo, dest_repo)
+ return Target(spec=spec, src=src, dest=dest)
+
+
+def handle_ff(item, src_repo, writer, opt):
+ assert item.spec.method == 'ff'
+ assert item.src.type in ('branch', 'save', 'commit')
+ src_oidx = hexlify(item.src.hash)
+ dest_oidx = hexlify(item.dest.hash) if item.dest.hash else None
+ if not dest_oidx or dest_oidx in src_repo.rev_list(src_oidx):
+ # Can fast forward.
+ get_random_item(item.spec.src, src_oidx, src_repo, writer, opt)
+ commit_items = parse_commit(get_cat_data(src_repo.cat(src_oidx), b'commit'))
+ return item.src.hash, unhexlify(commit_items.tree)
+ misuse('destination is not an ancestor of source for %s'
+ % spec_msg(item.spec))
+
+
+def resolve_append(spec, src_repo, dest_repo):
+ src = resolve_src(spec, src_repo)
+ if src.type not in ('branch', 'save', 'commit', 'tree'):
+ misuse('source for %s must be a branch, save, commit, or tree, not %s'
+ % (spec_msg(spec), src.type))
+ spec, dest = resolve_branch_dest(spec, src, src_repo, dest_repo)
+ return Target(spec=spec, src=src, dest=dest)
+
+
+def handle_append(item, src_repo, writer, opt):
+ assert item.spec.method == 'append'
+ assert item.src.type in ('branch', 'save', 'commit', 'tree')
+ assert item.dest.type == 'branch' or not item.dest.type
+ src_oidx = hexlify(item.src.hash)
+ if item.src.type == 'tree':
+ get_random_item(item.spec.src, src_oidx, src_repo, writer, opt)
+ parent = item.dest.hash
+ msg = b'bup save\n\nGenerated by command:\n%r\n' % sys.argv
+ userline = b'%s <%s@%s>' % (userfullname(), username(), hostname())
+ now = time.time()
+ commit = writer.new_commit(item.src.hash, parent,
+ userline, now, None,
+ userline, now, None, msg)
+ return commit, item.src.hash
+ commits = list(src_repo.rev_list(src_oidx))
+ commits.reverse()
+ return append_commits(commits, item.spec.src, item.dest.hash,
+ src_repo, writer, opt)
+
+
+def resolve_pick(spec, src_repo, dest_repo):
+ src = resolve_src(spec, src_repo)
+ spec_args = spec_msg(spec)
+ if src.type == 'tree':
+ misuse('%s is impossible; can only --append a tree' % spec_args)
+ if src.type not in ('commit', 'save'):
+ misuse('%s impossible; can only pick a commit or save, not %s'
+ % (spec_args, src.type))
+ if not spec.dest:
+ if src.path.startswith(b'/.tag/'):
+ spec = spec._replace(dest=spec.src)
+ elif src.type == 'save':
+ spec = spec._replace(dest=get_save_branch(src_repo, spec.src))
+ if not spec.dest:
+ misuse('no destination provided for %s', spec_args)
+ dest = find_vfs_item(spec.dest, dest_repo)
+ if not dest:
+ cp = validate_vfs_path(cleanup_vfs_path(spec.dest))
+ dest = default_loc._replace(path=cp)
+ else:
+ if not dest.type == 'branch' and not dest.path.startswith(b'/.tag/'):
+ misuse('%s destination is not a tag or branch' % spec_args)
+ if spec.method == 'pick' \
+ and dest.hash and dest.path.startswith(b'/.tag/'):
+ misuse('cannot overwrite existing tag for %s (requires --force-pick)'
+ % spec_args)
+ return Target(spec=spec, src=src, dest=dest)
+
+
+def handle_pick(item, src_repo, writer, opt):
+ assert item.spec.method in ('pick', 'force-pick')
+ assert item.src.type in ('save', 'commit')
+ src_oidx = hexlify(item.src.hash)
+ if item.dest.hash:
+ return append_commit(item.spec.src, src_oidx, item.dest.hash,
+ src_repo, writer, opt)
+ return append_commit(item.spec.src, src_oidx, None, src_repo, writer, opt)
+
+
+def resolve_new_tag(spec, src_repo, dest_repo):
+ src = resolve_src(spec, src_repo)
+ spec_args = spec_msg(spec)
+ if not spec.dest and src.path.startswith(b'/.tag/'):
+ spec = spec._replace(dest=src.path)
+ if not spec.dest:
+ misuse('no destination (implicit or explicit) for %s', spec_args)
+ dest = find_vfs_item(spec.dest, dest_repo)
+ if not dest:
+ dest = default_loc._replace(path=cleanup_vfs_path(spec.dest))
+ if not dest.path.startswith(b'/.tag/'):
+ misuse('destination for %s must be a VFS tag' % spec_args)
+ if dest.hash:
+ misuse('cannot overwrite existing tag for %s (requires --replace)'
+ % spec_args)
+ return Target(spec=spec, src=src, dest=dest)
+
+
+def handle_new_tag(item, src_repo, writer, opt):
+ assert item.spec.method == 'new-tag'
+ assert item.dest.path.startswith(b'/.tag/')
+ get_random_item(item.spec.src, hexlify(item.src.hash),
+ src_repo, writer, opt)
+ return (item.src.hash,)
+
+
+def resolve_replace(spec, src_repo, dest_repo):
+ src = resolve_src(spec, src_repo)
+ spec_args = spec_msg(spec)
+ if not spec.dest:
+ if src.path.startswith(b'/.tag/') or src.type == 'branch':
+ spec = spec._replace(dest=spec.src)
+ if not spec.dest:
+ misuse('no destination provided for %s', spec_args)
+ dest = find_vfs_item(spec.dest, dest_repo)
+ if dest:
+ if not dest.type == 'branch' and not dest.path.startswith(b'/.tag/'):
+ misuse('%s impossible; can only overwrite branch or tag'
+ % spec_args)
+ else:
+ cp = validate_vfs_path(cleanup_vfs_path(spec.dest))
+ dest = default_loc._replace(path=cp)
+ if not dest.path.startswith(b'/.tag/') \
+ and not src.type in ('branch', 'save', 'commit'):
+ misuse('cannot overwrite branch with %s for %s' % (src.type, spec_args))
+ return Target(spec=spec, src=src, dest=dest)
+
+
+def handle_replace(item, src_repo, writer, opt):
+ assert(item.spec.method == 'replace')
+ if item.dest.path.startswith(b'/.tag/'):
+ get_random_item(item.spec.src, hexlify(item.src.hash),
+ src_repo, writer, opt)
+ return (item.src.hash,)
+ assert(item.dest.type == 'branch' or not item.dest.type)
+ src_oidx = hexlify(item.src.hash)
+ get_random_item(item.spec.src, src_oidx, src_repo, writer, opt)
+ commit_items = parse_commit(get_cat_data(src_repo.cat(src_oidx), b'commit'))
+ return item.src.hash, unhexlify(commit_items.tree)
+
+
+def resolve_unnamed(spec, src_repo, dest_repo):
+ if spec.dest:
+ misuse('destination name given for %s' % spec_msg(spec))
+ src = resolve_src(spec, src_repo)
+ return Target(spec=spec, src=src, dest=None)
+
+
+def handle_unnamed(item, src_repo, writer, opt):
+ get_random_item(item.spec.src, hexlify(item.src.hash),
+ src_repo, writer, opt)
+ return (None,)
+
+
+def resolve_targets(specs, src_repo, dest_repo):
+ resolved_items = []
+ common_args = src_repo, dest_repo
+ for spec in specs:
+ debug1('initial-spec: %r\n' % (spec,))
+ if spec.method == 'ff':
+ resolved_items.append(resolve_ff(spec, *common_args))
+ elif spec.method == 'append':
+ resolved_items.append(resolve_append(spec, *common_args))
+ elif spec.method in ('pick', 'force-pick'):
+ resolved_items.append(resolve_pick(spec, *common_args))
+ elif spec.method == 'new-tag':
+ resolved_items.append(resolve_new_tag(spec, *common_args))
+ elif spec.method == 'replace':
+ resolved_items.append(resolve_replace(spec, *common_args))
+ elif spec.method == 'unnamed':
+ resolved_items.append(resolve_unnamed(spec, *common_args))
+ else: # Should be impossible -- prevented by the option parser.
+ assert(False)
+
+ # FIXME: check for prefix overlap? i.e.:
+ # bup get --ff foo --ff: baz foo/bar
+ # bup get --new-tag .tag/foo --new-tag: bar .tag/foo/bar
+
+ # Now that we have all the items, check for duplicate tags.
+ tags_targeted = set()
+ for item in resolved_items:
+ dest_path = item.dest and item.dest.path
+ if dest_path:
+ assert(dest_path.startswith(b'/'))
+ if dest_path.startswith(b'/.tag/'):
+ if dest_path in tags_targeted:
+ if item.spec.method not in ('replace', 'force-pick'):
+ misuse('cannot overwrite tag %s via %s' \
+ % (path_msg(dest_path), spec_msg(item.spec)))
+ else:
+ tags_targeted.add(dest_path)
+ return resolved_items
+
+
+def log_item(name, type, opt, tree=None, commit=None, tag=None):
+ if tag and opt.print_tags:
+ print(hexstr(tag))
+ if tree and opt.print_trees:
+ print(hexstr(tree))
+ if commit and opt.print_commits:
+ print(hexstr(commit))
+ if opt.verbose:
+ last = ''
+ if type in ('root', 'branch', 'save', 'commit', 'tree'):
+ if not name.endswith(b'/'):
+ last = '/'
+ log('%s%s\n' % (path_msg(name), last))
+
+def main():
+ handle_ctrl_c()
+ is_reverse = environ.get(b'BUP_SERVER_REVERSE')
+ opt = parse_args(sys.argv)
+ git.check_repo_or_die()
+ if opt.source:
+ opt.source = argv_bytes(opt.source)
+ src_dir = opt.source or git.repo()
+ if opt.bwlimit:
+ client.bwlimit = parse_num(opt.bwlimit)
+ if is_reverse and opt.remote:
+ misuse("don't use -r in reverse mode; it's automatic")
+ if opt.remote:
+ opt.remote = argv_bytes(opt.remote)
+ if opt.remote or is_reverse:
+ dest_repo = RemoteRepo(opt.remote)
+ else:
+ dest_repo = LocalRepo()
+
+ with dest_repo as dest_repo:
+ with LocalRepo(repo_dir=src_dir) as src_repo:
+ with dest_repo.new_packwriter(compression_level=opt.compress) as writer:
+ # Resolve and validate all sources and destinations,
+ # implicit or explicit, and do it up-front, so we can
+ # fail before we start writing (for any obviously
+ # broken cases).
+ target_items = resolve_targets(opt.target_specs,
+ src_repo, dest_repo)
+
+ updated_refs = {} # ref_name -> (original_ref, tip_commit(bin))
+ no_ref_info = (None, None)
+
+ handlers = {'ff': handle_ff,
+ 'append': handle_append,
+ 'force-pick': handle_pick,
+ 'pick': handle_pick,
+ 'new-tag': handle_new_tag,
+ 'replace': handle_replace,
+ 'unnamed': handle_unnamed}
+
+ for item in target_items:
+ debug1('get-spec: %r\n' % (item.spec,))
+ debug1('get-src: %s\n' % loc_desc(item.src))
+ debug1('get-dest: %s\n' % loc_desc(item.dest))
+ dest_path = item.dest and item.dest.path
+ if dest_path:
+ if dest_path.startswith(b'/.tag/'):
+ dest_ref = b'refs/tags/%s' % dest_path[6:]
+ else:
+ dest_ref = b'refs/heads/%s' % dest_path[1:]
+ else:
+ dest_ref = None
+
+ dest_hash = item.dest and item.dest.hash
+ orig_ref, cur_ref = updated_refs.get(dest_ref, no_ref_info)
+ orig_ref = orig_ref or dest_hash
+ cur_ref = cur_ref or dest_hash
+
+ handler = handlers[item.spec.method]
+ item_result = handler(item, src_repo, writer, opt)
+ if len(item_result) > 1:
+ new_id, tree = item_result
+ else:
+ new_id = item_result[0]
+
+ if not dest_ref:
+ log_item(item.spec.src, item.src.type, opt)
+ else:
+ updated_refs[dest_ref] = (orig_ref, new_id)
+ if dest_ref.startswith(b'refs/tags/'):
+ log_item(item.spec.src, item.src.type, opt, tag=new_id)
+ else:
+ log_item(item.spec.src, item.src.type, opt,
+ tree=tree, commit=new_id)
+
+ # Only update the refs at the very end, once the writer is
+ # closed, so that if something goes wrong above, the old refs
+ # will be undisturbed.
+ for ref_name, info in items(updated_refs):
+ orig_ref, new_ref = info
+ try:
+ dest_repo.update_ref(ref_name, new_ref, orig_ref)
+ if opt.verbose:
+ new_hex = hexlify(new_ref)
+ if orig_ref:
+ orig_hex = hexlify(orig_ref)
+ log('updated %r (%s -> %s)\n' % (ref_name, orig_hex, new_hex))
+ else:
+ log('updated %r (%s)\n' % (ref_name, new_hex))
+ except (git.GitError, client.ClientError) as ex:
+ add_error('unable to update ref %r: %s' % (ref_name, ex))
+
+ if saved_errors:
+ log('WARNING: %d errors encountered while saving.\n' % len(saved_errors))
+ sys.exit(1)
+
+wrap_main(main)
--- /dev/null
+#!/bin/sh
+"""": # -*-python-*-
+bup_python="$(dirname "$0")/bup-python" || exit $?
+exec "$bup_python" "$0" ${1+"$@"}
+"""
+# end of bup preamble
+
+from __future__ import absolute_import
+import sys, os, glob
+from bup import options, path
+
+optspec = """
+bup help <command>
+"""
+o = options.Options(optspec)
+(opt, flags, extra) = o.parse(sys.argv[1:])
+
+if len(extra) == 0:
+ # the wrapper program provides the default usage string
+ os.execvp(path.exe(), ['bup'])
+elif len(extra) == 1:
+ docname = (extra[0]=='bup' and 'bup' or ('bup-%s' % extra[0]))
+ manpath = os.path.join(path.exedir(),
+ 'Documentation/' + docname + '.[1-9]')
+ g = glob.glob(manpath)
+ try:
+ if g:
+ os.execvp('man', ['man', '-l', g[0]])
+ else:
+ os.execvp('man', ['man', docname])
+ except OSError as e:
+ sys.stderr.write('Unable to run man command: %s\n' % e)
+ sys.exit(1)
+else:
+ o.fatal("exactly one command name expected")
--- /dev/null
+#!/bin/sh
+"""": # -*-python-*-
+bup_python="$(dirname "$0")/bup-python" || exit $?
+exec "$bup_python" "$0" ${1+"$@"}
+"""
+# end of bup preamble
+
+from __future__ import absolute_import
+from calendar import timegm
+from pipes import quote
+from subprocess import check_call
+from time import strftime, strptime
+import os
+import sys
+import tempfile
+
+from bup import git, helpers, options
+from bup.compat import argv_bytes, str_type
+from bup.helpers import (handle_ctrl_c,
+ log,
+ readpipe,
+ shstr,
+ saved_errors,
+ unlink)
+import bup.path
+
+optspec = """
+bup import-duplicity [-n] <duplicity-source-url> <bup-save-name>
+--
+n,dry-run don't do anything; just print what would be done
+"""
+
+def logcmd(cmd):
+ log(shstr(cmd).decode('iso-8859-1', errors='replace') + '\n')
+
+def exc(cmd, shell=False):
+ global opt
+ logcmd(cmd)
+ if not opt.dry_run:
+ check_call(cmd, shell=shell)
+
+def exo(cmd, shell=False, preexec_fn=None, close_fds=True):
+ global opt
+ logcmd(cmd)
+ if not opt.dry_run:
+ return helpers.exo(cmd, shell=shell, preexec_fn=preexec_fn,
+ close_fds=close_fds)[0]
+
+def redirect_dup_output():
+ os.dup2(1, 3)
+ os.dup2(1, 2)
+
+
+handle_ctrl_c()
+
+log('\nbup: import-duplicity is EXPERIMENTAL (proceed with caution)\n\n')
+
+o = options.Options(optspec)
+opt, flags, extra = o.parse(sys.argv[1:])
+
+if len(extra) < 1 or not extra[0]:
+ o.fatal('duplicity source URL required')
+if len(extra) < 2 or not extra[1]:
+ o.fatal('bup destination save name required')
+if len(extra) > 2:
+ o.fatal('too many arguments')
+
+source_url, save_name = extra
+source_url = argv_bytes(source_url)
+save_name = argv_bytes(save_name)
+bup = bup.path.exe()
+
+git.check_repo_or_die()
+
+tmpdir = tempfile.mkdtemp(prefix=b'bup-import-dup-')
+try:
+ dup = [b'duplicity', b'--archive-dir', tmpdir + b'/dup-cache']
+ restoredir = tmpdir + b'/restore'
+ tmpidx = tmpdir + b'/index'
+
+ collection_status = \
+ exo(dup + [b'collection-status', b'--log-fd=3', source_url],
+ close_fds=False, preexec_fn=redirect_dup_output) # i.e. 3>&1 1>&2
+ # Duplicity output lines of interest look like this (one leading space):
+ # full 20150222T073111Z 1 noenc
+ # inc 20150222T073233Z 1 noenc
+ dup_timestamps = []
+ for line in collection_status.splitlines():
+ if line.startswith(b' inc '):
+ assert(len(line) >= len(b' inc 20150222T073233Z'))
+ dup_timestamps.append(line[5:21])
+ elif line.startswith(b' full '):
+ assert(len(line) >= len(b' full 20150222T073233Z'))
+ dup_timestamps.append(line[6:22])
+ for i, dup_ts in enumerate(dup_timestamps):
+ tm = strptime(dup_ts.decode('ascii'), '%Y%m%dT%H%M%SZ')
+ exc([b'rm', b'-rf', restoredir])
+ exc(dup + [b'restore', b'-t', dup_ts, source_url, restoredir])
+ exc([bup, b'index', b'-uxf', tmpidx, restoredir])
+ exc([bup, b'save', b'--strip', b'--date', b'%d' % timegm(tm),
+ b'-f', tmpidx, b'-n', save_name, restoredir])
+ sys.stderr.flush()
+finally:
+ exc([b'rm', b'-rf', tmpdir])
+
+if saved_errors:
+ log('warning: %d errors encountered\n' % len(saved_errors))
+ sys.exit(1)
--- /dev/null
+#!/usr/bin/env bash
+
+cmd_dir="$(cd "$(dirname "$0")" && pwd)" || exit $?
+
+set -o pipefail
+
+must() {
+ local file=${BASH_SOURCE[0]}
+ local line=${BASH_LINENO[0]}
+ "$@"
+ local rc=$?
+ if test $rc -ne 0; then
+ echo "Failed at line $line in $file" 1>&2
+ exit $rc
+ fi
+}
+
+usage() {
+ echo "Usage: bup import-rdiff-backup [-n]" \
+ "<path to rdiff-backup root> <backup name>"
+ echo "-n,--dry-run: just print what would be done"
+ exit 1
+}
+
+control_c() {
+ echo "bup import-rdiff-backup: signal 2 received" 1>&2
+ exit 128
+}
+
+must trap control_c INT
+
+dry_run=
+while [ "$1" = "-n" -o "$1" = "--dry-run" ]; do
+ dry_run=echo
+ shift
+done
+
+bup()
+{
+ $dry_run "$cmd_dir/bup" "$@"
+}
+
+snapshot_root="$1"
+branch="$2"
+
+[ -n "$snapshot_root" -a "$#" = 2 ] || usage
+
+if [ ! -e "$snapshot_root/." ]; then
+ echo "'$snapshot_root' isn't a directory!"
+ exit 1
+fi
+
+
+backups=$(must rdiff-backup --list-increments --parsable-output "$snapshot_root") \
+ || exit $?
+backups_count=$(echo "$backups" | must wc -l) || exit $?
+counter=1
+echo "$backups" |
+while read timestamp type; do
+ tmpdir=$(must mktemp -d import-rdiff-backup-XXXXXXX) || exit $?
+
+ echo "Importing backup from $(date -d @$timestamp +%c) " \
+ "($counter / $backups_count)" 1>&2
+ echo 1>&2
+
+ echo "Restoring from rdiff-backup..." 1>&2
+ must rdiff-backup -r $timestamp "$snapshot_root" "$tmpdir"
+ echo 1>&2
+
+ echo "Importing into bup..." 1>&2
+ TMPIDX=$(must mktemp -u import-rdiff-backup-idx-XXXXXXX) || exit $?
+ must bup index -ux -f "$tmpidx" "$tmpdir"
+ must bup save --strip --date="$timestamp" -f "$tmpidx" -n "$branch" "$tmpdir"
+ must rm -f "$tmpidx"
+
+ must rm -rf "$tmpdir"
+ counter=$((counter+1))
+ echo 1>&2
+ echo 1>&2
+done
--- /dev/null
+#!/bin/sh
+# Does an import of a rsnapshot archive.
+
+cmd_dir="$(cd "$(dirname "$0")" && pwd)" || exit $?
+
+usage() {
+ echo "Usage: bup import-rsnapshot [-n]" \
+ "<path to snapshot_root> [<backuptarget>]"
+ echo "-n,--dry-run: just print what would be done"
+ exit 1
+}
+
+DRY_RUN=
+while [ "$1" = "-n" -o "$1" = "--dry-run" ]; do
+ DRY_RUN=echo
+ shift
+done
+
+bup()
+{
+ $DRY_RUN "$cmd_dir/bup" "$@"
+}
+
+SNAPSHOT_ROOT=$1
+TARGET=$2
+
+[ -n "$SNAPSHOT_ROOT" -a "$#" -le 2 ] || usage
+
+if [ ! -e "$SNAPSHOT_ROOT/." ]; then
+ echo "'$SNAPSHOT_ROOT' isn't a directory!"
+ exit 1
+fi
+
+
+cd "$SNAPSHOT_ROOT" || exit 2
+
+for SNAPSHOT in *; do
+ [ -e "$SNAPSHOT/." ] || continue
+ echo "snapshot='$SNAPSHOT'" >&2
+ for BRANCH_PATH in "$SNAPSHOT/"*; do
+ BRANCH=$(basename "$BRANCH_PATH") || exit $?
+ [ -e "$BRANCH_PATH/." ] || continue
+ [ -z "$TARGET" -o "$TARGET" = "$BRANCH" ] || continue
+
+ echo "snapshot='$SNAPSHOT' branch='$BRANCH'" >&2
+
+ # Get the snapshot's ctime
+ DATE=$(perl -e '@a=stat($ARGV[0]) or die "$ARGV[0]: $!";
+ print $a[10];' "$BRANCH_PATH")
+ [ -n "$DATE" ] || exit 3
+
+ TMPIDX=bupindex.$BRANCH.tmp
+ bup index -ux -f "$TMPIDX" "$BRANCH_PATH/" || exit $?
+ bup save --strip --date="$DATE" \
+ -f "$TMPIDX" -n "$BRANCH" \
+ "$BRANCH_PATH/" || exit $?
+ rm "$TMPIDX" || exit $?
+ done
+done
--- /dev/null
+#!/bin/sh
+"""": # -*-python-*-
+bup_python="$(dirname "$0")/bup-python" || exit $?
+exec "$bup_python" "$0" ${1+"$@"}
+"""
+# end of bup preamble
+
+from __future__ import absolute_import, print_function
+from binascii import hexlify
+import sys, stat, time, os, errno, re
+
+from bup import metadata, options, git, index, drecurse, hlinkdb
+from bup.compat import argv_bytes
+from bup.drecurse import recursive_dirlist
+from bup.hashsplit import GIT_MODE_TREE, GIT_MODE_FILE
+from bup.helpers import (add_error, handle_ctrl_c, log, parse_excludes, parse_rx_excludes,
+ progress, qprogress, saved_errors)
+from bup.io import byte_stream, path_msg
+
+
+class IterHelper:
+ def __init__(self, l):
+ self.i = iter(l)
+ self.cur = None
+ self.next()
+
+ def __next__(self):
+ self.cur = next(self.i, None)
+ return self.cur
+
+ next = __next__
+
+def check_index(reader):
+ try:
+ log('check: checking forward iteration...\n')
+ e = None
+ d = {}
+ for e in reader.forward_iter():
+ if e.children_n:
+ if opt.verbose:
+ log('%08x+%-4d %r\n' % (e.children_ofs, e.children_n,
+ path_msg(e.name)))
+ assert(e.children_ofs)
+ assert e.name.endswith(b'/')
+ assert(not d.get(e.children_ofs))
+ d[e.children_ofs] = 1
+ if e.flags & index.IX_HASHVALID:
+ assert(e.sha != index.EMPTY_SHA)
+ assert(e.gitmode)
+ assert not e or bytes(e.name) == b'/' # last entry is *always* /
+ log('check: checking normal iteration...\n')
+ last = None
+ for e in reader:
+ if last:
+ assert(last > e.name)
+ last = e.name
+ except:
+ log('index error! at %r\n' % e)
+ raise
+ log('check: passed.\n')
+
+
+def clear_index(indexfile):
+ indexfiles = [indexfile, indexfile + b'.meta', indexfile + b'.hlink']
+ for indexfile in indexfiles:
+ path = git.repo(indexfile)
+ try:
+ os.remove(path)
+ if opt.verbose:
+ log('clear: removed %s\n' % path_msg(path))
+ except OSError as e:
+ if e.errno != errno.ENOENT:
+ raise
+
+
+def update_index(top, excluded_paths, exclude_rxs, xdev_exceptions, out=None):
+ # tmax and start must be epoch nanoseconds.
+ tmax = (time.time() - 1) * 10**9
+ ri = index.Reader(indexfile)
+ msw = index.MetaStoreWriter(indexfile + b'.meta')
+ wi = index.Writer(indexfile, msw, tmax)
+ rig = IterHelper(ri.iter(name=top))
+ tstart = int(time.time()) * 10**9
+
+ hlinks = hlinkdb.HLinkDB(indexfile + b'.hlink')
+
+ fake_hash = None
+ if opt.fake_valid:
+ def fake_hash(name):
+ return (GIT_MODE_FILE, index.FAKE_SHA)
+
+ total = 0
+ bup_dir = os.path.abspath(git.repo())
+ index_start = time.time()
+ for path, pst in recursive_dirlist([top],
+ xdev=opt.xdev,
+ bup_dir=bup_dir,
+ excluded_paths=excluded_paths,
+ exclude_rxs=exclude_rxs,
+ xdev_exceptions=xdev_exceptions):
+ if opt.verbose>=2 or (opt.verbose==1 and stat.S_ISDIR(pst.st_mode)):
+ out.write(b'%s\n' % path)
+ out.flush()
+ elapsed = time.time() - index_start
+ paths_per_sec = total / elapsed if elapsed else 0
+ qprogress('Indexing: %d (%d paths/s)\r' % (total, paths_per_sec))
+ elif not (total % 128):
+ elapsed = time.time() - index_start
+ paths_per_sec = total / elapsed if elapsed else 0
+ qprogress('Indexing: %d (%d paths/s)\r' % (total, paths_per_sec))
+ total += 1
+
+ while rig.cur and rig.cur.name > path: # deleted paths
+ if rig.cur.exists():
+ rig.cur.set_deleted()
+ rig.cur.repack()
+ if rig.cur.nlink > 1 and not stat.S_ISDIR(rig.cur.mode):
+ hlinks.del_path(rig.cur.name)
+ rig.next()
+
+ if rig.cur and rig.cur.name == path: # paths that already existed
+ need_repack = False
+ if(rig.cur.stale(pst, tstart, check_device=opt.check_device)):
+ try:
+ meta = metadata.from_path(path, statinfo=pst)
+ except (OSError, IOError) as e:
+ add_error(e)
+ rig.next()
+ continue
+ if not stat.S_ISDIR(rig.cur.mode) and rig.cur.nlink > 1:
+ hlinks.del_path(rig.cur.name)
+ if not stat.S_ISDIR(pst.st_mode) and pst.st_nlink > 1:
+ hlinks.add_path(path, pst.st_dev, pst.st_ino)
+ # Clear these so they don't bloat the store -- they're
+ # already in the index (since they vary a lot and they're
+ # fixed length). If you've noticed "tmax", you might
+ # wonder why it's OK to do this, since that code may
+ # adjust (mangle) the index mtime and ctime -- producing
+ # fake values which must not end up in a .bupm. However,
+ # it looks like that shouldn't be possible: (1) When
+ # "save" validates the index entry, it always reads the
+ # metadata from the filesytem. (2) Metadata is only
+ # read/used from the index if hashvalid is true. (3)
+ # "faked" entries will be stale(), and so we'll invalidate
+ # them below.
+ meta.ctime = meta.mtime = meta.atime = 0
+ meta_ofs = msw.store(meta)
+ rig.cur.update_from_stat(pst, meta_ofs)
+ rig.cur.invalidate()
+ need_repack = True
+ if not (rig.cur.flags & index.IX_HASHVALID):
+ if fake_hash:
+ if rig.cur.sha == index.EMPTY_SHA:
+ rig.cur.gitmode, rig.cur.sha = fake_hash(path)
+ rig.cur.flags |= index.IX_HASHVALID
+ need_repack = True
+ if opt.fake_invalid:
+ rig.cur.invalidate()
+ need_repack = True
+ if need_repack:
+ rig.cur.repack()
+ rig.next()
+ else: # new paths
+ try:
+ meta = metadata.from_path(path, statinfo=pst)
+ except (OSError, IOError) as e:
+ add_error(e)
+ continue
+ # See same assignment to 0, above, for rationale.
+ meta.atime = meta.mtime = meta.ctime = 0
+ meta_ofs = msw.store(meta)
+ wi.add(path, pst, meta_ofs, hashgen=fake_hash)
+ if not stat.S_ISDIR(pst.st_mode) and pst.st_nlink > 1:
+ hlinks.add_path(path, pst.st_dev, pst.st_ino)
+
+ elapsed = time.time() - index_start
+ paths_per_sec = total / elapsed if elapsed else 0
+ progress('Indexing: %d, done (%d paths/s).\n' % (total, paths_per_sec))
+
+ hlinks.prepare_save()
+
+ if ri.exists():
+ ri.save()
+ wi.flush()
+ if wi.count:
+ wr = wi.new_reader()
+ if opt.check:
+ log('check: before merging: oldfile\n')
+ check_index(ri)
+ log('check: before merging: newfile\n')
+ check_index(wr)
+ mi = index.Writer(indexfile, msw, tmax)
+
+ for e in index.merge(ri, wr):
+ # FIXME: shouldn't we remove deleted entries eventually? When?
+ mi.add_ixentry(e)
+
+ ri.close()
+ mi.close()
+ wr.close()
+ wi.abort()
+ else:
+ wi.close()
+
+ msw.close()
+ hlinks.commit_save()
+
+
+optspec = """
+bup index <-p|-m|-s|-u|--clear|--check> [options...] <filenames...>
+--
+ Modes:
+p,print print the index entries for the given names (also works with -u)
+m,modified print only added/deleted/modified files (implies -p)
+s,status print each filename with a status char (A/M/D) (implies -p)
+u,update recursively update the index entries for the given file/dir names (default if no mode is specified)
+check carefully check index file integrity
+clear clear the default index
+ Options:
+H,hash print the hash for each object next to its name
+l,long print more information about each file
+no-check-device don't invalidate an entry if the containing device changes
+fake-valid mark all index entries as up-to-date even if they aren't
+fake-invalid mark all index entries as invalid
+f,indexfile= the name of the index file (normally BUP_DIR/bupindex)
+exclude= a path to exclude from the backup (may be repeated)
+exclude-from= skip --exclude paths in file (may be repeated)
+exclude-rx= skip paths matching the unanchored regex (may be repeated)
+exclude-rx-from= skip --exclude-rx patterns in file (may be repeated)
+v,verbose increase log output (can be used more than once)
+x,xdev,one-file-system don't cross filesystem boundaries
+"""
+o = options.Options(optspec)
+(opt, flags, extra) = o.parse(sys.argv[1:])
+
+if not (opt.modified or \
+ opt['print'] or \
+ opt.status or \
+ opt.update or \
+ opt.check or \
+ opt.clear):
+ opt.update = 1
+if (opt.fake_valid or opt.fake_invalid) and not opt.update:
+ o.fatal('--fake-{in,}valid are meaningless without -u')
+if opt.fake_valid and opt.fake_invalid:
+ o.fatal('--fake-valid is incompatible with --fake-invalid')
+if opt.clear and opt.indexfile:
+ o.fatal('cannot clear an external index (via -f)')
+
+# FIXME: remove this once we account for timestamp races, i.e. index;
+# touch new-file; index. It's possible for this to happen quickly
+# enough that new-file ends up with the same timestamp as the first
+# index, and then bup will ignore it.
+tick_start = time.time()
+time.sleep(1 - (tick_start - int(tick_start)))
+
+git.check_repo_or_die()
+
+handle_ctrl_c()
+
+if opt.verbose is None:
+ opt.verbose = 0
+
+if opt.indexfile:
+ indexfile = argv_bytes(opt.indexfile)
+else:
+ indexfile = git.repo(b'bupindex')
+
+if opt.check:
+ log('check: starting initial check.\n')
+ check_index(index.Reader(indexfile))
+
+if opt.clear:
+ log('clear: clearing index.\n')
+ clear_index(indexfile)
+
+sys.stdout.flush()
+out = byte_stream(sys.stdout)
+
+if opt.update:
+ if not extra:
+ o.fatal('update mode (-u) requested but no paths given')
+ extra = [argv_bytes(x) for x in extra]
+ excluded_paths = parse_excludes(flags, o.fatal)
+ exclude_rxs = parse_rx_excludes(flags, o.fatal)
+ xexcept = index.unique_resolved_paths(extra)
+ for rp, path in index.reduce_paths(extra):
+ update_index(rp, excluded_paths, exclude_rxs, xdev_exceptions=xexcept,
+ out=out)
+
+if opt['print'] or opt.status or opt.modified:
+ extra = [argv_bytes(x) for x in extra]
+ for name, ent in index.Reader(indexfile).filter(extra or [b'']):
+ if (opt.modified
+ and (ent.is_valid() or ent.is_deleted() or not ent.mode)):
+ continue
+ line = b''
+ if opt.status:
+ if ent.is_deleted():
+ line += b'D '
+ elif not ent.is_valid():
+ if ent.sha == index.EMPTY_SHA:
+ line += b'A '
+ else:
+ line += b'M '
+ else:
+ line += b' '
+ if opt.hash:
+ line += hexlify(ent.sha) + b' '
+ if opt.long:
+ line += b'%7s %7s ' % (oct(ent.mode), oct(ent.gitmode))
+ out.write(line + (name or b'./') + b'\n')
+
+if opt.check and (opt['print'] or opt.status or opt.modified or opt.update):
+ log('check: starting final check.\n')
+ check_index(index.Reader(indexfile))
+
+if saved_errors:
+ log('WARNING: %d errors encountered.\n' % len(saved_errors))
+ sys.exit(1)
--- /dev/null
+#!/bin/sh
+"""": # -*-python-*-
+bup_python="$(dirname "$0")/bup-python" || exit $?
+exec "$bup_python" "$0" ${1+"$@"}
+"""
+# end of bup preamble
+
+from __future__ import absolute_import
+import sys
+
+from bup import git, options, client
+from bup.helpers import log, saved_errors
+from bup.compat import argv_bytes
+
+
+optspec = """
+[BUP_DIR=...] bup init [-r host:path]
+--
+r,remote= remote repository path
+"""
+o = options.Options(optspec)
+(opt, flags, extra) = o.parse(sys.argv[1:])
+
+if extra:
+ o.fatal("no arguments expected")
+
+
+try:
+ git.init_repo() # local repo
+except git.GitError as e:
+ log("bup: error: could not init repository: %s" % e)
+ sys.exit(1)
+
+if opt.remote:
+ git.check_repo_or_die()
+ cli = client.Client(argv_bytes(opt.remote), create=True)
+ cli.close()
--- /dev/null
+#!/bin/sh
+"""": # -*-python-*-
+bup_python="$(dirname "$0")/bup-python" || exit $?
+exec "$bup_python" "$0" ${1+"$@"}
+"""
+# end of bup preamble
+
+from __future__ import absolute_import
+import sys
+
+from bup import git, options
+from bup.compat import argv_bytes
+from bup.helpers import linereader, log
+from bup.io import byte_stream
+from bup.repo import LocalRepo, RemoteRepo
+
+
+optspec = """
+bup join [-r host:path] [refs or hashes...]
+--
+r,remote= remote repository path
+o= output filename
+"""
+o = options.Options(optspec)
+(opt, flags, extra) = o.parse(sys.argv[1:])
+if opt.remote:
+ opt.remote = argv_bytes(opt.remote)
+
+git.check_repo_or_die()
+
+stdin = byte_stream(sys.stdin)
+
+if not extra:
+ extra = linereader(stdin)
+
+ret = 0
+repo = RemoteRepo(opt.remote) if opt.remote else LocalRepo()
+
+if opt.o:
+ outfile = open(opt.o, 'wb')
+else:
+ sys.stdout.flush()
+ outfile = byte_stream(sys.stdout)
+
+for ref in [argv_bytes(x) for x in extra]:
+ try:
+ for blob in repo.join(ref):
+ outfile.write(blob)
+ except KeyError as e:
+ outfile.flush()
+ log('error: %s\n' % e)
+ ret = 1
+
+sys.exit(ret)
--- /dev/null
+#!/bin/sh
+"""": # -*-python-*-
+bup_python="$(dirname "$0")/bup-python" || exit $?
+exec "$bup_python" "$0" ${1+"$@"}
+"""
+# end of bup preamble
+
+from __future__ import absolute_import, print_function
+from binascii import hexlify, unhexlify
+import sys, os
+
+from bup import git, options
+from bup.compat import argv_bytes
+from bup.helpers import add_error, handle_ctrl_c, log, qprogress, saved_errors
+from bup.io import byte_stream
+
+optspec = """
+bup list-idx [--find=<prefix>] <idxfilenames...>
+--
+find= display only objects that start with <prefix>
+"""
+o = options.Options(optspec)
+(opt, flags, extra) = o.parse(sys.argv[1:])
+
+handle_ctrl_c()
+
+opt.find = argv_bytes(opt.find) if opt.find else b''
+
+if not extra:
+ o.fatal('you must provide at least one filename')
+
+if len(opt.find) > 40:
+ o.fatal('--find parameter must be <= 40 chars long')
+else:
+ if len(opt.find) % 2:
+ s = opt.find + b'0'
+ else:
+ s = opt.find
+ try:
+ bin = unhexlify(s)
+ except TypeError:
+ o.fatal('--find parameter is not a valid hex string')
+
+sys.stdout.flush()
+out = byte_stream(sys.stdout)
+find = opt.find.lower()
+count = 0
+idxfiles = [argv_bytes(x) for x in extra]
+for name in idxfiles:
+ try:
+ ix = git.open_idx(name)
+ except git.GitError as e:
+ add_error('%r: %s' % (name, e))
+ continue
+ if len(opt.find) == 40:
+ if ix.exists(bin):
+ out.write(b'%s %s\n' % (name, find))
+ else:
+ # slow, exhaustive search
+ for _i in ix:
+ i = hexlify(_i)
+ if i.startswith(find):
+ out.write(b'%s %s\n' % (name, i))
+ qprogress('Searching: %d\r' % count)
+ count += 1
+
+if saved_errors:
+ log('WARNING: %d errors encountered while saving.\n' % len(saved_errors))
+ sys.exit(1)
--- /dev/null
+#!/bin/sh
+"""": # -*-python-*-
+bup_python="$(dirname "$0")/bup-python" || exit $?
+exec "$bup_python" "$0" ${1+"$@"}
+"""
+# end of bup preamble
+
+from __future__ import absolute_import
+import sys
+
+from bup import git, ls
+from bup.io import byte_stream
+
+
+git.check_repo_or_die()
+
+sys.stdout.flush()
+out = byte_stream(sys.stdout)
+# Check out lib/bup/ls.py for the opt spec
+rc = ls.via_cmdline(sys.argv[1:], out=out)
+sys.exit(rc)
--- /dev/null
+#!/bin/sh
+"""": # -*-python-*-
+bup_python="$(dirname "$0")/bup-python" || exit $?
+exec "$bup_python" "$0" ${1+"$@"}
+"""
+# end of bup preamble
+
+from __future__ import absolute_import
+import sys, struct, math
+
+from bup import options, git, _helpers
+from bup.helpers import log
+from bup.io import byte_stream
+
+POPULATION_OF_EARTH=6.7e9 # as of September, 2010
+
+optspec = """
+bup margin
+--
+predict Guess object offsets and report the maximum deviation
+ignore-midx Don't use midx files; use only plain pack idx files.
+"""
+o = options.Options(optspec)
+(opt, flags, extra) = o.parse(sys.argv[1:])
+
+if extra:
+ o.fatal("no arguments expected")
+
+git.check_repo_or_die()
+
+mi = git.PackIdxList(git.repo(b'objects/pack'), ignore_midx=opt.ignore_midx)
+
+def do_predict(ix, out):
+ total = len(ix)
+ maxdiff = 0
+ for count,i in enumerate(ix):
+ prefix = struct.unpack('!Q', i[:8])[0]
+ expected = prefix * total // (1 << 64)
+ diff = count - expected
+ maxdiff = max(maxdiff, abs(diff))
+ out.write(b'%d of %d (%.3f%%) '
+ % (maxdiff, len(ix), maxdiff * 100.0 / len(ix)))
+ out.flush()
+ assert(count+1 == len(ix))
+
+sys.stdout.flush()
+out = byte_stream(sys.stdout)
+
+if opt.predict:
+ if opt.ignore_midx:
+ for pack in mi.packs:
+ do_predict(pack, out)
+ else:
+ do_predict(mi, out)
+else:
+ # default mode: find longest matching prefix
+ last = b'\0'*20
+ longmatch = 0
+ for i in mi:
+ if i == last:
+ continue
+ #assert(str(i) >= last)
+ pm = _helpers.bitmatch(last, i)
+ longmatch = max(longmatch, pm)
+ last = i
+ out.write(b'%d\n' % longmatch)
+ log('%d matching prefix bits\n' % longmatch)
+ doublings = math.log(len(mi), 2)
+ bpd = longmatch / doublings
+ log('%.2f bits per doubling\n' % bpd)
+ remain = 160 - longmatch
+ rdoublings = remain / bpd
+ log('%d bits (%.2f doublings) remaining\n' % (remain, rdoublings))
+ larger = 2**rdoublings
+ log('%g times larger is possible\n' % larger)
+ perperson = larger/POPULATION_OF_EARTH
+ log('\nEveryone on earth could have %d data sets like yours, all in one\n'
+ 'repository, and we would expect 1 object collision.\n'
+ % int(perperson))
--- /dev/null
+#!/bin/sh
+"""": # -*-python-*-
+bup_python="$(dirname "$0")/bup-python" || exit $?
+exec "$bup_python" "$0" ${1+"$@"}
+"""
+# end of bup preamble
+
+from __future__ import absolute_import, print_function
+import sys, re, struct, time, resource
+
+from bup import git, bloom, midx, options, _helpers
+from bup.compat import range
+from bup.helpers import handle_ctrl_c
+from bup.io import byte_stream
+
+
+handle_ctrl_c()
+
+
+_linux_warned = 0
+def linux_memstat():
+ global _linux_warned
+ #fields = ['VmSize', 'VmRSS', 'VmData', 'VmStk', 'ms']
+ d = {}
+ try:
+ f = open(b'/proc/self/status', 'rb')
+ except IOError as e:
+ if not _linux_warned:
+ log('Warning: %s\n' % e)
+ _linux_warned = 1
+ return {}
+ for line in f:
+ # Note that on Solaris, this file exists but is binary. If that
+ # happens, this split() might not return two elements. We don't
+ # really need to care about the binary format since this output
+ # isn't used for much and report() can deal with missing entries.
+ t = re.split(br':\s*', line.strip(), 1)
+ if len(t) == 2:
+ k,v = t
+ d[k] = v
+ return d
+
+
+last = last_u = last_s = start = 0
+def report(count, out):
+ global last, last_u, last_s, start
+ headers = ['RSS', 'MajFlt', 'user', 'sys', 'ms']
+ ru = resource.getrusage(resource.RUSAGE_SELF)
+ now = time.time()
+ rss = int(ru.ru_maxrss // 1024)
+ if not rss:
+ rss = linux_memstat().get(b'VmRSS', b'??')
+ fields = [rss,
+ ru.ru_majflt,
+ int((ru.ru_utime - last_u) * 1000),
+ int((ru.ru_stime - last_s) * 1000),
+ int((now - last) * 1000)]
+ fmt = '%9s ' + ('%10s ' * len(fields))
+ if count >= 0:
+ line = fmt % tuple([count] + fields)
+ out.write(line.encode('ascii') + b'\n')
+ else:
+ start = now
+ out.write((fmt % tuple([''] + headers)).encode('ascii') + b'\n')
+ out.flush()
+
+ # don't include time to run report() in usage counts
+ ru = resource.getrusage(resource.RUSAGE_SELF)
+ last_u = ru.ru_utime
+ last_s = ru.ru_stime
+ last = time.time()
+
+
+optspec = """
+bup memtest [-n elements] [-c cycles]
+--
+n,number= number of objects per cycle [10000]
+c,cycles= number of cycles to run [100]
+ignore-midx ignore .midx files, use only .idx files
+existing test with existing objects instead of fake ones
+"""
+o = options.Options(optspec)
+(opt, flags, extra) = o.parse(sys.argv[1:])
+
+if extra:
+ o.fatal('no arguments expected')
+
+git.check_repo_or_die()
+m = git.PackIdxList(git.repo(b'objects/pack'), ignore_midx=opt.ignore_midx)
+
+sys.stdout.flush()
+out = byte_stream(sys.stdout)
+
+report(-1, out)
+_helpers.random_sha()
+report(0, out)
+
+if opt.existing:
+ def foreverit(mi):
+ while 1:
+ for e in mi:
+ yield e
+ objit = iter(foreverit(m))
+
+for c in range(opt.cycles):
+ for n in range(opt.number):
+ if opt.existing:
+ bin = next(objit)
+ assert(m.exists(bin))
+ else:
+ bin = _helpers.random_sha()
+
+ # technically, a randomly generated object id might exist.
+ # but the likelihood of that is the likelihood of finding
+ # a collision in sha-1 by accident, which is so unlikely that
+ # we don't care.
+ assert(not m.exists(bin))
+ report((c+1)*opt.number, out)
+
+if bloom._total_searches:
+ out.write(b'bloom: %d objects searched in %d steps: avg %.3f steps/object\n'
+ % (bloom._total_searches, bloom._total_steps,
+ bloom._total_steps*1.0/bloom._total_searches))
+if midx._total_searches:
+ out.write(b'midx: %d objects searched in %d steps: avg %.3f steps/object\n'
+ % (midx._total_searches, midx._total_steps,
+ midx._total_steps*1.0/midx._total_searches))
+if git._total_searches:
+ out.write(b'idx: %d objects searched in %d steps: avg %.3f steps/object\n'
+ % (git._total_searches, git._total_steps,
+ git._total_steps*1.0/git._total_searches))
+out.write(b'Total time: %.3fs\n' % (time.time() - start))
--- /dev/null
+#!/bin/sh
+"""": # -*-python-*-
+bup_python="$(dirname "$0")/bup-python" || exit $?
+exec "$bup_python" "$0" ${1+"$@"}
+"""
+# end of bup preamble
+
+# Copyright (C) 2010 Rob Browning
+#
+# This code is covered under the terms of the GNU Library General
+# Public License as described in the bup LICENSE file.
+
+# TODO: Add tar-like -C option.
+
+from __future__ import absolute_import
+import sys
+from bup import metadata
+from bup import options
+from bup.compat import argv_bytes
+from bup.io import byte_stream
+from bup.helpers import handle_ctrl_c, log, saved_errors
+
+
+def open_input(name):
+ if not name or name == b'-':
+ return byte_stream(sys.stdin)
+ return open(name, 'rb')
+
+
+def open_output(name):
+ if not name or name == b'-':
+ sys.stdout.flush()
+ return byte_stream(sys.stdout)
+ return open(name, 'wb')
+
+
+optspec = """
+bup meta --create [OPTION ...] <PATH ...>
+bup meta --list [OPTION ...]
+bup meta --extract [OPTION ...]
+bup meta --start-extract [OPTION ...]
+bup meta --finish-extract [OPTION ...]
+bup meta --edit [OPTION ...] <PATH ...>
+--
+c,create write metadata for PATHs to stdout (or --file)
+t,list display metadata
+x,extract perform --start-extract followed by --finish-extract
+start-extract build tree matching metadata provided on standard input (or --file)
+finish-extract finish applying standard input (or --file) metadata to filesystem
+edit alter metadata; write to stdout (or --file)
+f,file= specify source or destination file
+R,recurse recurse into subdirectories
+xdev,one-file-system don't cross filesystem boundaries
+numeric-ids apply numeric IDs (user, group, etc.) rather than names
+symlinks handle symbolic links (default is true)
+paths include paths in metadata (default is true)
+set-uid= set metadata uid (via --edit)
+set-gid= set metadata gid (via --edit)
+set-user= set metadata user (via --edit)
+unset-user remove metadata user (via --edit)
+set-group= set metadata group (via --edit)
+unset-group remove metadata group (via --edit)
+v,verbose increase log output (can be used more than once)
+q,quiet don't show progress meter
+"""
+
+handle_ctrl_c()
+
+o = options.Options(optspec)
+(opt, flags, remainder) = o.parse(['--paths', '--symlinks', '--recurse']
+ + sys.argv[1:])
+
+opt.verbose = opt.verbose or 0
+opt.quiet = opt.quiet or 0
+metadata.verbose = opt.verbose - opt.quiet
+opt.file = argv_bytes(opt.file) if opt.file else None
+
+action_count = sum([bool(x) for x in [opt.create, opt.list, opt.extract,
+ opt.start_extract, opt.finish_extract,
+ opt.edit]])
+if action_count > 1:
+ o.fatal("bup: only one action permitted: --create --list --extract --edit")
+if action_count == 0:
+ o.fatal("bup: no action specified")
+
+if opt.create:
+ if len(remainder) < 1:
+ o.fatal("no paths specified for create")
+ output_file = open_output(opt.file)
+ metadata.save_tree(output_file,
+ [argv_bytes(r) for r in remainder],
+ recurse=opt.recurse,
+ write_paths=opt.paths,
+ save_symlinks=opt.symlinks,
+ xdev=opt.xdev)
+elif opt.list:
+ if len(remainder) > 0:
+ o.fatal("cannot specify paths for --list")
+ src = open_input(opt.file)
+ metadata.display_archive(src, open_output(b'-'))
+elif opt.start_extract:
+ if len(remainder) > 0:
+ o.fatal("cannot specify paths for --start-extract")
+ src = open_input(opt.file)
+ metadata.start_extract(src, create_symlinks=opt.symlinks)
+elif opt.finish_extract:
+ if len(remainder) > 0:
+ o.fatal("cannot specify paths for --finish-extract")
+ src = open_input(opt.file)
+ metadata.finish_extract(src, restore_numeric_ids=opt.numeric_ids)
+elif opt.extract:
+ if len(remainder) > 0:
+ o.fatal("cannot specify paths for --extract")
+ src = open_input(opt.file)
+ metadata.extract(src,
+ restore_numeric_ids=opt.numeric_ids,
+ create_symlinks=opt.symlinks)
+elif opt.edit:
+ if len(remainder) < 1:
+ o.fatal("no paths specified for edit")
+ output_file = open_output(opt.file)
+
+ unset_user = False # True if --unset-user was the last relevant option.
+ unset_group = False # True if --unset-group was the last relevant option.
+ for flag in flags:
+ if flag[0] == '--set-user':
+ unset_user = False
+ elif flag[0] == '--unset-user':
+ unset_user = True
+ elif flag[0] == '--set-group':
+ unset_group = False
+ elif flag[0] == '--unset-group':
+ unset_group = True
+
+ for path in remainder:
+ f = open(argv_bytes(path), 'rb')
+ try:
+ for m in metadata._ArchiveIterator(f):
+ if opt.set_uid is not None:
+ try:
+ m.uid = int(opt.set_uid)
+ except ValueError:
+ o.fatal("uid must be an integer")
+
+ if opt.set_gid is not None:
+ try:
+ m.gid = int(opt.set_gid)
+ except ValueError:
+ o.fatal("gid must be an integer")
+
+ if unset_user:
+ m.user = b''
+ elif opt.set_user is not None:
+ m.user = argv_bytes(opt.set_user)
+
+ if unset_group:
+ m.group = b''
+ elif opt.set_group is not None:
+ m.group = argv_bytes(opt.set_group)
+
+ m.write(output_file)
+ finally:
+ f.close()
+
+
+if saved_errors:
+ log('WARNING: %d errors encountered.\n' % len(saved_errors))
+ sys.exit(1)
+else:
+ sys.exit(0)
--- /dev/null
+#!/bin/sh
+"""": # -*-python-*-
+bup_python="$(dirname "$0")/bup-python" || exit $?
+exec "$bup_python" "$0" ${1+"$@"}
+"""
+# end of bup preamble
+
+from __future__ import absolute_import, print_function
+from binascii import hexlify
+import glob, math, os, resource, struct, sys, tempfile
+
+from bup import options, git, midx, _helpers, xstat
+from bup.compat import argv_bytes, hexstr, range
+from bup.helpers import (Sha1, add_error, atomically_replaced_file, debug1, fdatasync,
+ handle_ctrl_c, log, mmap_readwrite, qprogress,
+ saved_errors, unlink)
+from bup.io import byte_stream, path_msg
+
+
+PAGE_SIZE=4096
+SHA_PER_PAGE=PAGE_SIZE/20.
+
+optspec = """
+bup midx [options...] <idxnames...>
+--
+o,output= output midx filename (default: auto-generated)
+a,auto automatically use all existing .midx/.idx files as input
+f,force merge produce exactly one .midx containing all objects
+p,print print names of generated midx files
+check validate contents of the given midx files (with -a, all midx files)
+max-files= maximum number of idx files to open at once [-1]
+d,dir= directory containing idx/midx files
+"""
+
+merge_into = _helpers.merge_into
+
+
+def _group(l, count):
+ for i in range(0, len(l), count):
+ yield l[i:i+count]
+
+
+def max_files():
+ mf = min(resource.getrlimit(resource.RLIMIT_NOFILE))
+ if mf > 32:
+ mf -= 20 # just a safety margin
+ else:
+ mf -= 6 # minimum safety margin
+ return mf
+
+
+def check_midx(name):
+ nicename = git.repo_rel(name)
+ log('Checking %s.\n' % path_msg(nicename))
+ try:
+ ix = git.open_idx(name)
+ except git.GitError as e:
+ add_error('%s: %s' % (pathmsg(name), e))
+ return
+ for count,subname in enumerate(ix.idxnames):
+ sub = git.open_idx(os.path.join(os.path.dirname(name), subname))
+ for ecount,e in enumerate(sub):
+ if not (ecount % 1234):
+ qprogress(' %d/%d: %s %d/%d\r'
+ % (count, len(ix.idxnames),
+ git.shorten_hash(subname).decode('ascii'),
+ ecount, len(sub)))
+ if not sub.exists(e):
+ add_error("%s: %s: %s missing from idx"
+ % (path_msg(nicename),
+ git.shorten_hash(subname).decode('ascii'),
+ hexstr(e)))
+ if not ix.exists(e):
+ add_error("%s: %s: %s missing from midx"
+ % (path_msg(nicename),
+ git.shorten_hash(subname).decode('ascii'),
+ hexstr(e)))
+ prev = None
+ for ecount,e in enumerate(ix):
+ if not (ecount % 1234):
+ qprogress(' Ordering: %d/%d\r' % (ecount, len(ix)))
+ if e and prev and not e >= prev:
+ add_error('%s: ordering error: %s < %s'
+ % (nicename, hexstr(e), hexstr(prev)))
+ prev = e
+
+
+_first = None
+def _do_midx(outdir, outfilename, infilenames, prefixstr):
+ global _first
+ if not outfilename:
+ assert(outdir)
+ sum = hexlify(Sha1(b'\0'.join(infilenames)).digest())
+ outfilename = b'%s/midx-%s.midx' % (outdir, sum)
+
+ inp = []
+ total = 0
+ allfilenames = []
+ midxs = []
+ try:
+ for name in infilenames:
+ ix = git.open_idx(name)
+ midxs.append(ix)
+ inp.append((
+ ix.map,
+ len(ix),
+ ix.sha_ofs,
+ isinstance(ix, midx.PackMidx) and ix.which_ofs or 0,
+ len(allfilenames),
+ ))
+ for n in ix.idxnames:
+ allfilenames.append(os.path.basename(n))
+ total += len(ix)
+ inp.sort(reverse=True, key=lambda x: x[0][x[2] : x[2] + 20])
+
+ if not _first: _first = outdir
+ dirprefix = (_first != outdir) and git.repo_rel(outdir) + b': ' or b''
+ debug1('midx: %s%screating from %d files (%d objects).\n'
+ % (dirprefix, prefixstr, len(infilenames), total))
+ if (opt.auto and (total < 1024 and len(infilenames) < 3)) \
+ or ((opt.auto or opt.force) and len(infilenames) < 2) \
+ or (opt.force and not total):
+ debug1('midx: nothing to do.\n')
+ return
+
+ pages = int(total/SHA_PER_PAGE) or 1
+ bits = int(math.ceil(math.log(pages, 2)))
+ entries = 2**bits
+ debug1('midx: table size: %d (%d bits)\n' % (entries*4, bits))
+
+ unlink(outfilename)
+ with atomically_replaced_file(outfilename, 'wb') as f:
+ f.write(b'MIDX')
+ f.write(struct.pack('!II', midx.MIDX_VERSION, bits))
+ assert(f.tell() == 12)
+
+ f.truncate(12 + 4*entries + 20*total + 4*total)
+ f.flush()
+ fdatasync(f.fileno())
+
+ fmap = mmap_readwrite(f, close=False)
+ count = merge_into(fmap, bits, total, inp)
+ del fmap # Assume this calls msync() now.
+ f.seek(0, os.SEEK_END)
+ f.write(b'\0'.join(allfilenames))
+ finally:
+ for ix in midxs:
+ if isinstance(ix, midx.PackMidx):
+ ix.close()
+ midxs = None
+ inp = None
+
+
+ # This is just for testing (if you enable this, don't clear inp above)
+ if 0:
+ p = midx.PackMidx(outfilename)
+ assert(len(p.idxnames) == len(infilenames))
+ log(repr(p.idxnames) + '\n')
+ assert(len(p) == total)
+ for pe, e in p, git.idxmerge(inp, final_progress=False):
+ pin = next(pi)
+ assert(i == pin)
+ assert(p.exists(i))
+
+ return total, outfilename
+
+
+def do_midx(outdir, outfilename, infilenames, prefixstr, prout):
+ rv = _do_midx(outdir, outfilename, infilenames, prefixstr)
+ if rv and opt['print']:
+ prout.write(rv[1] + b'\n')
+
+
+def do_midx_dir(path, outfilename, prout):
+ already = {}
+ sizes = {}
+ if opt.force and not opt.auto:
+ midxs = [] # don't use existing midx files
+ else:
+ midxs = glob.glob(b'%s/*.midx' % path)
+ contents = {}
+ for mname in midxs:
+ m = git.open_idx(mname)
+ contents[mname] = [(b'%s/%s' % (path,i)) for i in m.idxnames]
+ sizes[mname] = len(m)
+
+ # sort the biggest+newest midxes first, so that we can eliminate
+ # smaller (or older) redundant ones that come later in the list
+ midxs.sort(key=lambda ix: (-sizes[ix], -xstat.stat(ix).st_mtime))
+
+ for mname in midxs:
+ any = 0
+ for iname in contents[mname]:
+ if not already.get(iname):
+ already[iname] = 1
+ any = 1
+ if not any:
+ debug1('%r is redundant\n' % mname)
+ unlink(mname)
+ already[mname] = 1
+
+ midxs = [k for k in midxs if not already.get(k)]
+ idxs = [k for k in glob.glob(b'%s/*.idx' % path) if not already.get(k)]
+
+ for iname in idxs:
+ i = git.open_idx(iname)
+ sizes[iname] = len(i)
+
+ all = [(sizes[n],n) for n in (midxs + idxs)]
+
+ # FIXME: what are the optimal values? Does this make sense?
+ DESIRED_HWM = opt.force and 1 or 5
+ DESIRED_LWM = opt.force and 1 or 2
+ existed = dict((name,1) for sz,name in all)
+ debug1('midx: %d indexes; want no more than %d.\n'
+ % (len(all), DESIRED_HWM))
+ if len(all) <= DESIRED_HWM:
+ debug1('midx: nothing to do.\n')
+ while len(all) > DESIRED_HWM:
+ all.sort()
+ part1 = [name for sz,name in all[:len(all)-DESIRED_LWM+1]]
+ part2 = all[len(all)-DESIRED_LWM+1:]
+ all = list(do_midx_group(path, outfilename, part1)) + part2
+ if len(all) > DESIRED_HWM:
+ debug1('\nStill too many indexes (%d > %d). Merging again.\n'
+ % (len(all), DESIRED_HWM))
+
+ if opt['print']:
+ for sz,name in all:
+ if not existed.get(name):
+ prout.write(name + b'\n')
+
+
+def do_midx_group(outdir, outfilename, infiles):
+ groups = list(_group(infiles, opt.max_files))
+ gprefix = ''
+ for n,sublist in enumerate(groups):
+ if len(groups) != 1:
+ gprefix = 'Group %d: ' % (n+1)
+ rv = _do_midx(outdir, outfilename, sublist, gprefix)
+ if rv:
+ yield rv
+
+
+handle_ctrl_c()
+
+o = options.Options(optspec)
+(opt, flags, extra) = o.parse(sys.argv[1:])
+opt.dir = argv_bytes(opt.dir) if opt.dir else None
+opt.output = argv_bytes(opt.output) if opt.output else None
+
+if extra and (opt.auto or opt.force):
+ o.fatal("you can't use -f/-a and also provide filenames")
+if opt.check and (not extra and not opt.auto):
+ o.fatal("if using --check, you must provide filenames or -a")
+
+git.check_repo_or_die()
+
+if opt.max_files < 0:
+ opt.max_files = max_files()
+assert(opt.max_files >= 5)
+
+extra = [argv_bytes(x) for x in extra]
+
+if opt.check:
+ # check existing midx files
+ if extra:
+ midxes = extra
+ else:
+ midxes = []
+ paths = opt.dir and [opt.dir] or git.all_packdirs()
+ for path in paths:
+ debug1('midx: scanning %s\n' % path)
+ midxes += glob.glob(os.path.join(path, b'*.midx'))
+ for name in midxes:
+ check_midx(name)
+ if not saved_errors:
+ log('All tests passed.\n')
+else:
+ if extra:
+ sys.stdout.flush()
+ do_midx(git.repo(b'objects/pack'), opt.output, extra, b'',
+ byte_stream(sys.stdout))
+ elif opt.auto or opt.force:
+ sys.stdout.flush()
+ paths = opt.dir and [opt.dir] or git.all_packdirs()
+ for path in paths:
+ debug1('midx: scanning %s\n' % path_msg(path))
+ do_midx_dir(path, opt.output, byte_stream(sys.stdout))
+ else:
+ o.fatal("you must use -f or -a or provide input filenames")
+
+if saved_errors:
+ log('WARNING: %d errors encountered.\n' % len(saved_errors))
+ sys.exit(1)
--- /dev/null
+#!/bin/sh
+"""": # -*-python-*-
+bup_python="$(dirname "$0")/bup-python" || exit $?
+exec "$bup_python" "$0" ${1+"$@"}
+"""
+# end of bup preamble
+
+from __future__ import absolute_import
+import os, sys, subprocess, struct
+
+from bup import options
+from bup.helpers import debug1, debug2, mux
+from bup.io import byte_stream
+
+# Give the subcommand exclusive access to stdin.
+orig_stdin = os.dup(0)
+devnull = os.open(os.devnull, os.O_RDONLY)
+os.dup2(devnull, 0)
+os.close(devnull)
+
+optspec = """
+bup mux command [arguments...]
+--
+"""
+o = options.Options(optspec)
+(opt, flags, extra) = o.parse(sys.argv[1:])
+if len(extra) < 1:
+ o.fatal('command is required')
+
+subcmd = extra
+
+debug2('bup mux: starting %r\n' % (extra,))
+
+outr, outw = os.pipe()
+errr, errw = os.pipe()
+def close_fds():
+ os.close(outr)
+ os.close(errr)
+
+p = subprocess.Popen(subcmd, stdin=orig_stdin, stdout=outw, stderr=errw,
+ close_fds=False, preexec_fn=close_fds)
+os.close(outw)
+os.close(errw)
+sys.stdout.flush()
+out = byte_stream(sys.stdout)
+out.write(b'BUPMUX')
+out.flush()
+mux(p, out.fileno(), outr, errr)
+os.close(outr)
+os.close(errr)
+prv = p.wait()
+
+if prv:
+ debug1('%s exited with code %d\n' % (extra[0], prv))
+
+debug1('bup mux: done\n')
+
+sys.exit(prv)
--- /dev/null
+#!/bin/sh
+"""": # -*-python-*-
+bup_python="$(dirname "$0")/bup-python" || exit $?
+exec "$bup_python" "$0" ${1+"$@"}
+"""
+# end of bup preamble
+
+from __future__ import absolute_import
+import sys, os, struct
+
+from bup import options, helpers, path
+from bup.compat import environ, py_maj
+from bup.io import byte_stream
+
+optspec = """
+bup on--server
+--
+ This command is run automatically by 'bup on'
+"""
+o = options.Options(optspec)
+(opt, flags, extra) = o.parse(sys.argv[1:])
+if extra:
+ o.fatal('no arguments expected')
+
+# get the subcommand's argv.
+# Normally we could just pass this on the command line, but since we'll often
+# be getting called on the other end of an ssh pipe, which tends to mangle
+# argv (by sending it via the shell), this way is much safer.
+
+stdin = byte_stream(sys.stdin)
+buf = stdin.read(4)
+sz = struct.unpack('!I', buf)[0]
+assert(sz > 0)
+assert(sz < 1000000)
+buf = stdin.read(sz)
+assert(len(buf) == sz)
+argv = buf.split(b'\0')
+argv[0] = path.exe()
+argv = [argv[0], b'mux', b'--'] + argv
+
+
+# stdin/stdout are supposedly connected to 'bup server' that the caller
+# started for us (often on the other end of an ssh tunnel), so we don't want
+# to misuse them. Move them out of the way, then replace stdout with
+# a pointer to stderr in case our subcommand wants to do something with it.
+#
+# It might be nice to do the same with stdin, but my experiments showed that
+# ssh seems to make its child's stderr a readable-but-never-reads-anything
+# socket. They really should have used shutdown(SHUT_WR) on the other end
+# of it, but probably didn't. Anyway, it's too messy, so let's just make sure
+# anyone reading from stdin is disappointed.
+#
+# (You can't just leave stdin/stdout "not open" by closing the file
+# descriptors. Then the next file that opens is automatically assigned 0 or 1,
+# and people *trying* to read/write stdin/stdout get screwed.)
+os.dup2(0, 3)
+os.dup2(1, 4)
+os.dup2(2, 1)
+fd = os.open(os.devnull, os.O_RDONLY)
+os.dup2(fd, 0)
+os.close(fd)
+
+environ[b'BUP_SERVER_REVERSE'] = helpers.hostname()
+os.execvp(argv[0], argv)
+sys.exit(99)
--- /dev/null
+#!/bin/sh
+"""": # -*-python-*-
+bup_python="$(dirname "$0")/bup-python" || exit $?
+exec "$bup_python" "$0" ${1+"$@"}
+"""
+# end of bup preamble
+
+from __future__ import absolute_import
+from subprocess import PIPE
+import sys, os, struct, getopt, subprocess, signal
+
+from bup import options, ssh, path
+from bup.compat import argv_bytes
+from bup.helpers import DemuxConn, log
+from bup.io import byte_stream
+
+
+optspec = """
+bup on <hostname> index ...
+bup on <hostname> save ...
+bup on <hostname> split ...
+bup on <hostname> get ...
+"""
+o = options.Options(optspec, optfunc=getopt.getopt)
+(opt, flags, extra) = o.parse(sys.argv[1:])
+if len(extra) < 2:
+ o.fatal('arguments expected')
+
+class SigException(Exception):
+ def __init__(self, signum):
+ self.signum = signum
+ Exception.__init__(self, 'signal %d received' % signum)
+def handler(signum, frame):
+ raise SigException(signum)
+
+signal.signal(signal.SIGTERM, handler)
+signal.signal(signal.SIGINT, handler)
+
+sys.stdout.flush()
+out = byte_stream(sys.stdout)
+
+try:
+ sp = None
+ p = None
+ ret = 99
+
+ hp = argv_bytes(extra[0]).split(b':')
+ if len(hp) == 1:
+ (hostname, port) = (hp[0], None)
+ else:
+ (hostname, port) = hp
+ argv = [argv_bytes(x) for x in extra[1:]]
+ p = ssh.connect(hostname, port, b'on--server', stderr=PIPE)
+
+ try:
+ argvs = b'\0'.join([b'bup'] + argv)
+ p.stdin.write(struct.pack('!I', len(argvs)) + argvs)
+ p.stdin.flush()
+ sp = subprocess.Popen([path.exe(), b'server'],
+ stdin=p.stdout, stdout=p.stdin)
+ p.stdin.close()
+ p.stdout.close()
+ # Demultiplex remote client's stderr (back to stdout/stderr).
+ dmc = DemuxConn(p.stderr.fileno(), open(os.devnull, "wb"))
+ for line in iter(dmc.readline, b''):
+ out.write(line)
+ finally:
+ while 1:
+ # if we get a signal while waiting, we have to keep waiting, just
+ # in case our child doesn't die.
+ try:
+ ret = p.wait()
+ if sp:
+ sp.wait()
+ break
+ except SigException as e:
+ log('\nbup on: %s\n' % e)
+ os.kill(p.pid, e.signum)
+ ret = 84
+except SigException as e:
+ if ret == 0:
+ ret = 99
+ log('\nbup on: %s\n' % e)
+
+sys.exit(ret)
--- /dev/null
+#!/bin/sh
+"""": # -*-python-*-
+bup_python="$(dirname "$0")/bup-python" || exit $?
+exec "$bup_python" "$0" ${1+"$@"}
+"""
+# end of bup preamble
+
+from __future__ import absolute_import, print_function
+from binascii import hexlify, unhexlify
+from collections import defaultdict
+from itertools import groupby
+from sys import stderr
+from time import localtime, strftime, time
+import re, sys
+
+from bup import git, options
+from bup.compat import argv_bytes, int_types
+from bup.gc import bup_gc
+from bup.helpers import die_if_errors, log, partition, period_as_secs
+from bup.io import byte_stream
+from bup.repo import LocalRepo
+from bup.rm import bup_rm
+
+
+def branches(refnames=tuple()):
+ return ((name[11:], hexlify(sha)) for (name,sha)
+ in git.list_refs(patterns=(b'refs/heads/' + n for n in refnames),
+ limit_to_heads=True))
+
+def save_name(branch, utc):
+ return branch + b'/' \
+ + strftime('%Y-%m-%d-%H%M%S', localtime(utc)).encode('ascii')
+
+def classify_saves(saves, period_start):
+ """For each (utc, id) in saves, yield (True, (utc, id)) if the save
+ should be kept and (False, (utc, id)) if the save should be removed.
+ The ids are binary hashes.
+ """
+
+ def retain_newest_in_region(region):
+ for save in region[0:1]:
+ yield True, save
+ for save in region[1:]:
+ yield False, save
+
+ matches, rest = partition(lambda s: s[0] >= period_start['all'], saves)
+ for save in matches:
+ yield True, save
+
+ tm_ranges = ((period_start['dailies'], lambda s: localtime(s[0]).tm_yday),
+ (period_start['monthlies'], lambda s: localtime(s[0]).tm_mon),
+ (period_start['yearlies'], lambda s: localtime(s[0]).tm_year))
+
+ # Break the decreasing utc sorted saves up into the respective
+ # period ranges (dailies, monthlies, ...). Within each range,
+ # group the saves by the period scale (days, months, ...), and
+ # then yield a "keep" action (True, utc) for the newest save in
+ # each group, and a "drop" action (False, utc) for the rest.
+ for pstart, time_region_id in tm_ranges:
+ matches, rest = partition(lambda s: s[0] >= pstart, rest)
+ for region_id, region_saves in groupby(matches, time_region_id):
+ for action in retain_newest_in_region(list(region_saves)):
+ yield action
+
+ # Finally, drop any saves older than the specified periods
+ for save in rest:
+ yield False, save
+
+
+optspec = """
+bup prune-older [options...] [BRANCH...]
+--
+keep-all-for= retain all saves within the PERIOD
+keep-dailies-for= retain the newest save per day within the PERIOD
+keep-monthlies-for= retain the newest save per month within the PERIOD
+keep-yearlies-for= retain the newest save per year within the PERIOD
+wrt= end all periods at this number of seconds since the epoch
+pretend don't prune, just report intended actions to standard output
+gc collect garbage after removals [1]
+gc-threshold= only rewrite a packfile if it's over this percent garbage [10]
+#,compress= set compression level to # (0-9, 9 is highest) [1]
+v,verbose increase log output (can be used more than once)
+unsafe use the command even though it may be DANGEROUS
+"""
+
+o = options.Options(optspec)
+opt, flags, roots = o.parse(sys.argv[1:])
+roots = [argv_bytes(x) for x in roots]
+
+if not opt.unsafe:
+ o.fatal('refusing to run dangerous, experimental command without --unsafe')
+
+now = int(time()) if opt.wrt is None else opt.wrt
+if not isinstance(now, int_types):
+ o.fatal('--wrt value ' + str(now) + ' is not an integer')
+
+period_start = {}
+for period, extent in (('all', opt.keep_all_for),
+ ('dailies', opt.keep_dailies_for),
+ ('monthlies', opt.keep_monthlies_for),
+ ('yearlies', opt.keep_yearlies_for)):
+ if extent:
+ secs = period_as_secs(extent.encode('ascii'))
+ if not secs:
+ o.fatal('%r is not a valid period' % extent)
+ period_start[period] = now - secs
+
+if not period_start:
+ o.fatal('at least one keep argument is required')
+
+period_start = defaultdict(lambda: float('inf'), period_start)
+
+if opt.verbose:
+ epoch_ymd = strftime('%Y-%m-%d-%H%M%S', localtime(0))
+ for kind in ['all', 'dailies', 'monthlies', 'yearlies']:
+ period_utc = period_start[kind]
+ if period_utc != float('inf'):
+ if not (period_utc > float('-inf')):
+ log('keeping all ' + kind)
+ else:
+ try:
+ when = strftime('%Y-%m-%d-%H%M%S', localtime(period_utc))
+ log('keeping ' + kind + ' since ' + when + '\n')
+ except ValueError as ex:
+ if period_utc < 0:
+ log('keeping %s since %d seconds before %s\n'
+ %(kind, abs(period_utc), epoch_ymd))
+ elif period_utc > 0:
+ log('keeping %s since %d seconds after %s\n'
+ %(kind, period_utc, epoch_ymd))
+ else:
+ log('keeping %s since %s\n' % (kind, epoch_ymd))
+
+git.check_repo_or_die()
+
+# This could be more efficient, but for now just build the whole list
+# in memory and let bup_rm() do some redundant work.
+
+def parse_info(f):
+ author_secs = f.readline().strip()
+ return int(author_secs)
+
+sys.stdout.flush()
+out = byte_stream(sys.stdout)
+
+removals = []
+for branch, branch_id in branches(roots):
+ die_if_errors()
+ saves = ((utc, unhexlify(oidx)) for (oidx, utc) in
+ git.rev_list(branch_id, format=b'%at', parse=parse_info))
+ for keep_save, (utc, id) in classify_saves(saves, period_start):
+ assert(keep_save in (False, True))
+ # FIXME: base removals on hashes
+ if opt.pretend:
+ out.write(b'+ ' if keep_save else b'- '
+ + save_name(branch, utc) + b'\n')
+ elif not keep_save:
+ removals.append(save_name(branch, utc))
+
+if not opt.pretend:
+ die_if_errors()
+ repo = LocalRepo()
+ bup_rm(repo, removals, compression=opt.compress, verbosity=opt.verbose)
+ if opt.gc:
+ die_if_errors()
+ bup_gc(threshold=opt.gc_threshold,
+ compression=opt.compress,
+ verbosity=opt.verbose)
+
+die_if_errors()
--- /dev/null
+#!/bin/sh
+
+set -e
+
+top="$(pwd)"
+cmdpath="$0"
+# loop because macos has no recursive resolution
+while test -L "$cmdpath"; do
+ link="$(readlink "$cmdpath")"
+ cd "$(dirname "$cmdpath")"
+ cmdpath="$link"
+done
+script_home="$(cd "$(dirname "$cmdpath")" && pwd -P)"
+cd "$top"
+
+bup_libdir="$script_home/.." # bup_libdir will be adjusted during install
+export PYTHONPATH="$bup_libdir${PYTHONPATH:+:$PYTHONPATH}"
+
+# Force python to use ISO-8859-1 (aka Latin 1), a single-byte
+# encoding, to help avoid any manipulation of data from system APIs
+# (paths, users, groups, command line arguments, etc.)
+
+export PYTHONCOERCECLOCALE=0 # Perhaps not necessary, but shouldn't hurt
+
+# We can't just export LC_CTYPE directly here because the locale might
+# not exist outside python, and then bash (at least) may be cranky.
+
+if [ "${LC_ALL+x}" ]; then
+ unset LC_ALL
+ exec env \
+ BUP_LC_ALL="$LC_ALL" \
+ LC_COLLATE="$LC_ALL" \
+ LC_MONETARY="$LC_ALL" \
+ LC_NUMERIC="$LC_ALL" \
+ LC_TIME="$LC_ALL" \
+ LC_MESSAGES="$LC_ALL" \
+ LC_CTYPE=ISO-8859-1 \
+ @bup_python@ "$@"
+elif [ "${LC_CTYPE+x}" ]; then
+ exec env \
+ BUP_LC_CTYPE="$LC_CTYPE" \
+ LC_CTYPE=ISO-8859-1 \
+ @bup_python@ "$@"
+else
+ exec env \
+ LC_CTYPE=ISO-8859-1 \
+ @bup_python@ "$@"
+fi
--- /dev/null
+#!/bin/sh
+"""": # -*-python-*-
+bup_python="$(dirname "$0")/bup-python" || exit $?
+exec "$bup_python" "$0" ${1+"$@"}
+"""
+# end of bup preamble
+
+from __future__ import absolute_import
+import os, sys
+
+from bup import options, _helpers
+from bup.helpers import atoi, handle_ctrl_c, log, parse_num
+
+
+optspec = """
+bup random [-S seed] <numbytes>
+--
+S,seed= optional random number seed [1]
+f,force print random data to stdout even if it's a tty
+v,verbose print byte counter to stderr
+"""
+o = options.Options(optspec)
+(opt, flags, extra) = o.parse(sys.argv[1:])
+
+if len(extra) != 1:
+ o.fatal("exactly one argument expected")
+
+total = parse_num(extra[0])
+
+handle_ctrl_c()
+
+if opt.force or (not os.isatty(1) and
+ not atoi(os.environ.get('BUP_FORCE_TTY')) & 1):
+ _helpers.write_random(sys.stdout.fileno(), total, opt.seed,
+ opt.verbose and 1 or 0)
+else:
+ log('error: not writing binary data to a terminal. Use -f to force.\n')
+ sys.exit(1)
--- /dev/null
+#!/bin/sh
+"""": # -*-python-*-
+bup_python="$(dirname "$0")/bup-python" || exit $?
+exec "$bup_python" "$0" ${1+"$@"}
+"""
+# end of bup preamble
+
+from __future__ import absolute_import
+from stat import S_ISDIR
+import copy, errno, os, sys, stat, re
+
+from bup import options, git, metadata, vfs
+from bup._helpers import write_sparsely
+from bup.compat import argv_bytes, fsencode, wrap_main
+from bup.helpers import (add_error, chunkyreader, die_if_errors, handle_ctrl_c,
+ log, mkdirp, parse_rx_excludes, progress, qprogress,
+ saved_errors, should_rx_exclude_path, unlink)
+from bup.io import byte_stream
+from bup.repo import LocalRepo, RemoteRepo
+
+
+optspec = """
+bup restore [-r host:path] [-C outdir] </branch/revision/path/to/dir ...>
+--
+r,remote= remote repository path
+C,outdir= change to given outdir before extracting files
+numeric-ids restore numeric IDs (user, group, etc.) rather than names
+exclude-rx= skip paths matching the unanchored regex (may be repeated)
+exclude-rx-from= skip --exclude-rx patterns in file (may be repeated)
+sparse create sparse files
+v,verbose increase log output (can be used more than once)
+map-user= given OLD=NEW, restore OLD user as NEW user
+map-group= given OLD=NEW, restore OLD group as NEW group
+map-uid= given OLD=NEW, restore OLD uid as NEW uid
+map-gid= given OLD=NEW, restore OLD gid as NEW gid
+q,quiet don't show progress meter
+"""
+
+total_restored = 0
+
+# stdout should be flushed after each line, even when not connected to a tty
+stdoutfd = sys.stdout.fileno()
+sys.stdout.flush()
+sys.stdout = os.fdopen(stdoutfd, 'w', 1)
+out = byte_stream(sys.stdout)
+
+def valid_restore_path(path):
+ path = os.path.normpath(path)
+ if path.startswith(b'/'):
+ path = path[1:]
+ if b'/' in path:
+ return True
+
+def parse_owner_mappings(type, options, fatal):
+ """Traverse the options and parse all --map-TYPEs, or call Option.fatal()."""
+ opt_name = '--map-' + type
+ if type in ('uid', 'gid'):
+ value_rx = re.compile(br'^(-?[0-9]+)=(-?[0-9]+)$')
+ else:
+ value_rx = re.compile(br'^([^=]+)=([^=]*)$')
+ owner_map = {}
+ for flag in options:
+ (option, parameter) = flag
+ if option != opt_name:
+ continue
+ parameter = argv_bytes(parameter)
+ match = value_rx.match(parameter)
+ if not match:
+ raise fatal("couldn't parse %r as %s mapping" % (parameter, type))
+ old_id, new_id = match.groups()
+ if type in ('uid', 'gid'):
+ old_id = int(old_id)
+ new_id = int(new_id)
+ owner_map[old_id] = new_id
+ return owner_map
+
+def apply_metadata(meta, name, restore_numeric_ids, owner_map):
+ m = copy.deepcopy(meta)
+ m.user = owner_map['user'].get(m.user, m.user)
+ m.group = owner_map['group'].get(m.group, m.group)
+ m.uid = owner_map['uid'].get(m.uid, m.uid)
+ m.gid = owner_map['gid'].get(m.gid, m.gid)
+ m.apply_to_path(name, restore_numeric_ids = restore_numeric_ids)
+
+def hardlink_compatible(prev_path, prev_item, new_item, top):
+ prev_candidate = top + prev_path
+ if not os.path.exists(prev_candidate):
+ return False
+ prev_meta, new_meta = prev_item.meta, new_item.meta
+ if new_item.oid != prev_item.oid \
+ or new_meta.mtime != prev_meta.mtime \
+ or new_meta.ctime != prev_meta.ctime \
+ or new_meta.mode != prev_meta.mode:
+ return False
+ # FIXME: should we be checking the path on disk, or the recorded metadata?
+ # The exists() above might seem to suggest the former.
+ if not new_meta.same_file(prev_meta):
+ return False
+ return True
+
+def hardlink_if_possible(fullname, item, top, hardlinks):
+ """Find a suitable hardlink target, link to it, and return true,
+ otherwise return false."""
+ # The cwd will be dirname(fullname), and fullname will be
+ # absolute, i.e. /foo/bar, and the caller is expected to handle
+ # restoring the metadata if hardlinking isn't possible.
+
+ # FIXME: we can probably replace the target_vfs_path with the
+ # relevant vfs item
+
+ # hardlinks tracks a list of (restore_path, vfs_path, meta)
+ # triples for each path we've written for a given hardlink_target.
+ # This allows us to handle the case where we restore a set of
+ # hardlinks out of order (with respect to the original save
+ # call(s)) -- i.e. when we don't restore the hardlink_target path
+ # first. This data also allows us to attempt to handle other
+ # situations like hardlink sets that change on disk during a save,
+ # or between index and save.
+
+ target = item.meta.hardlink_target
+ assert(target)
+ assert(fullname.startswith(b'/'))
+ target_versions = hardlinks.get(target)
+ if target_versions:
+ # Check every path in the set that we've written so far for a match.
+ for prev_path, prev_item in target_versions:
+ if hardlink_compatible(prev_path, prev_item, item, top):
+ try:
+ os.link(top + prev_path, top + fullname)
+ return True
+ except OSError as e:
+ if e.errno != errno.EXDEV:
+ raise
+ else:
+ target_versions = []
+ hardlinks[target] = target_versions
+ target_versions.append((fullname, item))
+ return False
+
+def write_file_content(repo, dest_path, vfs_file):
+ with vfs.fopen(repo, vfs_file) as inf:
+ with open(dest_path, 'wb') as outf:
+ for b in chunkyreader(inf):
+ outf.write(b)
+
+def write_file_content_sparsely(repo, dest_path, vfs_file):
+ with vfs.fopen(repo, vfs_file) as inf:
+ outfd = os.open(dest_path, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600)
+ try:
+ trailing_zeros = 0;
+ for b in chunkyreader(inf):
+ trailing_zeros = write_sparsely(outfd, b, 512, trailing_zeros)
+ pos = os.lseek(outfd, trailing_zeros, os.SEEK_END)
+ os.ftruncate(outfd, pos)
+ finally:
+ os.close(outfd)
+
+def restore(repo, parent_path, name, item, top, sparse, numeric_ids, owner_map,
+ exclude_rxs, verbosity, hardlinks):
+ global total_restored
+ mode = vfs.item_mode(item)
+ treeish = S_ISDIR(mode)
+ fullname = parent_path + b'/' + name
+ # Match behavior of index --exclude-rx with respect to paths.
+ if should_rx_exclude_path(fullname + (b'/' if treeish else b''),
+ exclude_rxs):
+ return
+
+ if not treeish:
+ # Do this now so we'll have meta.symlink_target for verbose output
+ item = vfs.augment_item_meta(repo, item, include_size=True)
+ meta = item.meta
+ assert(meta.mode == mode)
+
+ if stat.S_ISDIR(mode):
+ if verbosity >= 1:
+ out.write(b'%s/\n' % fullname)
+ elif stat.S_ISLNK(mode):
+ assert(meta.symlink_target)
+ if verbosity >= 2:
+ out.write(b'%s@ -> %s\n' % (fullname, meta.symlink_target))
+ else:
+ if verbosity >= 2:
+ out.write(fullname + '\n')
+
+ orig_cwd = os.getcwd()
+ try:
+ if treeish:
+ # Assumes contents() returns '.' with the full metadata first
+ sub_items = vfs.contents(repo, item, want_meta=True)
+ dot, item = next(sub_items, None)
+ assert(dot == b'.')
+ item = vfs.augment_item_meta(repo, item, include_size=True)
+ meta = item.meta
+ meta.create_path(name)
+ os.chdir(name)
+ total_restored += 1
+ if verbosity >= 0:
+ qprogress('Restoring: %d\r' % total_restored)
+ for sub_name, sub_item in sub_items:
+ restore(repo, fullname, sub_name, sub_item, top, sparse,
+ numeric_ids, owner_map, exclude_rxs, verbosity,
+ hardlinks)
+ os.chdir(b'..')
+ apply_metadata(meta, name, numeric_ids, owner_map)
+ else:
+ created_hardlink = False
+ if meta.hardlink_target:
+ created_hardlink = hardlink_if_possible(fullname, item, top,
+ hardlinks)
+ if not created_hardlink:
+ meta.create_path(name)
+ if stat.S_ISREG(meta.mode):
+ if sparse:
+ write_file_content_sparsely(repo, name, item)
+ else:
+ write_file_content(repo, name, item)
+ total_restored += 1
+ if verbosity >= 0:
+ qprogress('Restoring: %d\r' % total_restored)
+ if not created_hardlink:
+ apply_metadata(meta, name, numeric_ids, owner_map)
+ finally:
+ os.chdir(orig_cwd)
+
+def main():
+ o = options.Options(optspec)
+ opt, flags, extra = o.parse(sys.argv[1:])
+ verbosity = (opt.verbose or 0) if not opt.quiet else -1
+ if opt.remote:
+ opt.remote = argv_bytes(opt.remote)
+ if opt.outdir:
+ opt.outdir = argv_bytes(opt.outdir)
+
+ git.check_repo_or_die()
+
+ if not extra:
+ o.fatal('must specify at least one filename to restore')
+
+ exclude_rxs = parse_rx_excludes(flags, o.fatal)
+
+ owner_map = {}
+ for map_type in ('user', 'group', 'uid', 'gid'):
+ owner_map[map_type] = parse_owner_mappings(map_type, flags, o.fatal)
+
+ if opt.outdir:
+ mkdirp(opt.outdir)
+ os.chdir(opt.outdir)
+
+ repo = RemoteRepo(opt.remote) if opt.remote else LocalRepo()
+ top = fsencode(os.getcwd())
+ hardlinks = {}
+ for path in [argv_bytes(x) for x in extra]:
+ if not valid_restore_path(path):
+ add_error("path %r doesn't include a branch and revision" % path)
+ continue
+ try:
+ resolved = vfs.resolve(repo, path, want_meta=True, follow=False)
+ except vfs.IOError as e:
+ add_error(e)
+ continue
+ if len(resolved) == 3 and resolved[2][0] == b'latest':
+ # Follow latest symlink to the actual save
+ try:
+ resolved = vfs.resolve(repo, b'latest', parent=resolved[:-1],
+ want_meta=True)
+ except vfs.IOError as e:
+ add_error(e)
+ continue
+ # Rename it back to 'latest'
+ resolved = tuple(elt if i != 2 else (b'latest',) + elt[1:]
+ for i, elt in enumerate(resolved))
+ path_parent, path_name = os.path.split(path)
+ leaf_name, leaf_item = resolved[-1]
+ if not leaf_item:
+ add_error('error: cannot access %r in %r'
+ % ('/'.join(name for name, item in resolved),
+ path))
+ continue
+ if not path_name or path_name == b'.':
+ # Source is /foo/what/ever/ or /foo/what/ever/. -- extract
+ # what/ever/* to the current directory, and if name == '.'
+ # (i.e. /foo/what/ever/.), then also restore what/ever's
+ # metadata to the current directory.
+ treeish = vfs.item_mode(leaf_item)
+ if not treeish:
+ add_error('%r cannot be restored as a directory' % path)
+ else:
+ items = vfs.contents(repo, leaf_item, want_meta=True)
+ dot, leaf_item = next(items, None)
+ assert dot == b'.'
+ for sub_name, sub_item in items:
+ restore(repo, b'', sub_name, sub_item, top,
+ opt.sparse, opt.numeric_ids, owner_map,
+ exclude_rxs, verbosity, hardlinks)
+ if path_name == b'.':
+ leaf_item = vfs.augment_item_meta(repo, leaf_item,
+ include_size=True)
+ apply_metadata(leaf_item.meta, b'.',
+ opt.numeric_ids, owner_map)
+ else:
+ restore(repo, b'', leaf_name, leaf_item, top,
+ opt.sparse, opt.numeric_ids, owner_map,
+ exclude_rxs, verbosity, hardlinks)
+
+ if verbosity >= 0:
+ progress('Restoring: %d, done.\n' % total_restored)
+ die_if_errors()
+
+wrap_main(main)
--- /dev/null
+#!/bin/sh
+"""": # -*-python-*-
+bup_python="$(dirname "$0")/bup-python" || exit $?
+exec "$bup_python" "$0" ${1+"$@"}
+"""
+# end of bup preamble
+
+from __future__ import absolute_import
+import sys
+
+from bup.compat import argv_bytes
+from bup.git import check_repo_or_die
+from bup.options import Options
+from bup.helpers import die_if_errors, handle_ctrl_c, log
+from bup.repo import LocalRepo
+from bup.rm import bup_rm
+
+optspec = """
+bup rm <branch|save...>
+--
+#,compress= set compression level to # (0-9, 9 is highest) [6]
+v,verbose increase verbosity (can be specified multiple times)
+unsafe use the command even though it may be DANGEROUS
+"""
+
+handle_ctrl_c()
+
+o = Options(optspec)
+opt, flags, extra = o.parse(sys.argv[1:])
+
+if not opt.unsafe:
+ o.fatal('refusing to run dangerous, experimental command without --unsafe')
+
+if len(extra) < 1:
+ o.fatal('no paths specified')
+
+check_repo_or_die()
+repo = LocalRepo()
+bup_rm(repo, [argv_bytes(x) for x in extra],
+ compression=opt.compress, verbosity=opt.verbose)
+die_if_errors()
--- /dev/null
+#!/bin/sh
+"""": # -*-python-*-
+bup_python="$(dirname "$0")/bup-python" || exit $?
+exec "$bup_python" "$0" ${1+"$@"}
+"""
+# end of bup preamble
+
+from __future__ import absolute_import, print_function
+from binascii import hexlify
+from errno import EACCES
+from io import BytesIO
+import os, sys, stat, time, math
+
+from bup import hashsplit, git, options, index, client, metadata, hlinkdb
+from bup.compat import argv_bytes, environ
+from bup.hashsplit import GIT_MODE_TREE, GIT_MODE_FILE, GIT_MODE_SYMLINK
+from bup.helpers import (add_error, grafted_path_components, handle_ctrl_c,
+ hostname, istty2, log, parse_date_or_fatal, parse_num,
+ path_components, progress, qprogress, resolve_parent,
+ saved_errors, stripped_path_components,
+ valid_save_name)
+from bup.io import byte_stream, path_msg
+from bup.pwdgrp import userfullname, username
+
+
+optspec = """
+bup save [-tc] [-n name] <filenames...>
+--
+r,remote= hostname:/path/to/repo of remote repository
+t,tree output a tree id
+c,commit output a commit id
+n,name= name of backup set to update (if any)
+d,date= date for the commit (seconds since the epoch)
+v,verbose increase log output (can be used more than once)
+q,quiet don't show progress meter
+smaller= only back up files smaller than n bytes
+bwlimit= maximum bytes/sec to transmit to server
+f,indexfile= the name of the index file (normally BUP_DIR/bupindex)
+strip strips the path to every filename given
+strip-path= path-prefix to be stripped when saving
+graft= a graft point *old_path*=*new_path* (can be used more than once)
+#,compress= set compression level to # (0-9, 9 is highest) [1]
+"""
+o = options.Options(optspec)
+(opt, flags, extra) = o.parse(sys.argv[1:])
+
+if opt.indexfile:
+ opt.indexfile = argv_bytes(opt.indexfile)
+if opt.name:
+ opt.name = argv_bytes(opt.name)
+if opt.remote:
+ opt.remote = argv_bytes(opt.remote)
+if opt.strip_path:
+ opt.strip_path = argv_bytes(opt.strip_path)
+
+git.check_repo_or_die()
+if not (opt.tree or opt.commit or opt.name):
+ o.fatal("use one or more of -t, -c, -n")
+if not extra:
+ o.fatal("no filenames given")
+
+extra = [argv_bytes(x) for x in extra]
+
+opt.progress = (istty2 and not opt.quiet)
+opt.smaller = parse_num(opt.smaller or 0)
+if opt.bwlimit:
+ client.bwlimit = parse_num(opt.bwlimit)
+
+if opt.date:
+ date = parse_date_or_fatal(opt.date, o.fatal)
+else:
+ date = time.time()
+
+if opt.strip and opt.strip_path:
+ o.fatal("--strip is incompatible with --strip-path")
+
+graft_points = []
+if opt.graft:
+ if opt.strip:
+ o.fatal("--strip is incompatible with --graft")
+
+ if opt.strip_path:
+ o.fatal("--strip-path is incompatible with --graft")
+
+ for (option, parameter) in flags:
+ if option == "--graft":
+ parameter = argv_bytes(parameter)
+ splitted_parameter = parameter.split(b'=')
+ if len(splitted_parameter) != 2:
+ o.fatal("a graft point must be of the form old_path=new_path")
+ old_path, new_path = splitted_parameter
+ if not (old_path and new_path):
+ o.fatal("a graft point cannot be empty")
+ graft_points.append((resolve_parent(old_path),
+ resolve_parent(new_path)))
+
+is_reverse = environ.get(b'BUP_SERVER_REVERSE')
+if is_reverse and opt.remote:
+ o.fatal("don't use -r in reverse mode; it's automatic")
+
+name = opt.name
+if name and not valid_save_name(name):
+ o.fatal("'%s' is not a valid branch name" % path_msg(name))
+refname = name and b'refs/heads/%s' % name or None
+if opt.remote or is_reverse:
+ try:
+ cli = client.Client(opt.remote)
+ except client.ClientError as e:
+ log('error: %s' % e)
+ sys.exit(1)
+ oldref = refname and cli.read_ref(refname) or None
+ w = cli.new_packwriter(compression_level=opt.compress)
+else:
+ cli = None
+ oldref = refname and git.read_ref(refname) or None
+ w = git.PackWriter(compression_level=opt.compress)
+
+handle_ctrl_c()
+
+
+# Metadata is stored in a file named .bupm in each directory. The
+# first metadata entry will be the metadata for the current directory.
+# The remaining entries will be for each of the other directory
+# elements, in the order they're listed in the index.
+#
+# Since the git tree elements are sorted according to
+# git.shalist_item_sort_key, the metalist items are accumulated as
+# (sort_key, metadata) tuples, and then sorted when the .bupm file is
+# created. The sort_key should have been computed using the element's
+# mangled name and git mode (after hashsplitting), but the code isn't
+# actually doing that but rather uses the element's real name and mode.
+# This makes things a bit more difficult when reading it back, see
+# vfs.ordered_tree_entries().
+
+# Maintain a stack of information representing the current location in
+# the archive being constructed. The current path is recorded in
+# parts, which will be something like ['', 'home', 'someuser'], and
+# the accumulated content and metadata for of the dirs in parts is
+# stored in parallel stacks in shalists and metalists.
+
+parts = [] # Current archive position (stack of dir names).
+shalists = [] # Hashes for each dir in paths.
+metalists = [] # Metadata for each dir in paths.
+
+
+def _push(part, metadata):
+ # Enter a new archive directory -- make it the current directory.
+ parts.append(part)
+ shalists.append([])
+ metalists.append([(b'', metadata)]) # This dir's metadata (no name).
+
+
+def _pop(force_tree, dir_metadata=None):
+ # Leave the current archive directory and add its tree to its parent.
+ assert(len(parts) >= 1)
+ part = parts.pop()
+ shalist = shalists.pop()
+ metalist = metalists.pop()
+ # FIXME: only test if collision is possible (i.e. given --strip, etc.)?
+ if force_tree:
+ tree = force_tree
+ else:
+ names_seen = set()
+ clean_list = []
+ metaidx = 1 # entry at 0 is for the dir
+ for x in shalist:
+ name = x[1]
+ if name in names_seen:
+ parent_path = b'/'.join(parts) + b'/'
+ add_error('error: ignoring duplicate path %s in %s'
+ % (path_msg(name), path_msg(parent_path)))
+ if not stat.S_ISDIR(x[0]):
+ del metalist[metaidx]
+ else:
+ names_seen.add(name)
+ clean_list.append(x)
+ if not stat.S_ISDIR(x[0]):
+ metaidx += 1
+
+ if dir_metadata: # Override the original metadata pushed for this dir.
+ metalist = [(b'', dir_metadata)] + metalist[1:]
+ sorted_metalist = sorted(metalist, key = lambda x : x[0])
+ metadata = b''.join([m[1].encode() for m in sorted_metalist])
+ metadata_f = BytesIO(metadata)
+ mode, id = hashsplit.split_to_blob_or_tree(w.new_blob, w.new_tree,
+ [metadata_f],
+ keep_boundaries=False)
+ clean_list.append((mode, b'.bupm', id))
+
+ tree = w.new_tree(clean_list)
+ if shalists:
+ shalists[-1].append((GIT_MODE_TREE,
+ git.mangle_name(part,
+ GIT_MODE_TREE, GIT_MODE_TREE),
+ tree))
+ return tree
+
+
+lastremain = None
+def progress_report(n):
+ global count, subcount, lastremain
+ subcount += n
+ cc = count + subcount
+ pct = total and (cc*100.0/total) or 0
+ now = time.time()
+ elapsed = now - tstart
+ kps = elapsed and int(cc/1024./elapsed)
+ kps_frac = 10 ** int(math.log(kps+1, 10) - 1)
+ kps = int(kps/kps_frac)*kps_frac
+ if cc:
+ remain = elapsed*1.0/cc * (total-cc)
+ else:
+ remain = 0.0
+ if (lastremain and (remain > lastremain)
+ and ((remain - lastremain)/lastremain < 0.05)):
+ remain = lastremain
+ else:
+ lastremain = remain
+ hours = int(remain/60/60)
+ mins = int(remain/60 - hours*60)
+ secs = int(remain - hours*60*60 - mins*60)
+ if elapsed < 30:
+ remainstr = ''
+ kpsstr = ''
+ else:
+ kpsstr = '%dk/s' % kps
+ if hours:
+ remainstr = '%dh%dm' % (hours, mins)
+ elif mins:
+ remainstr = '%dm%d' % (mins, secs)
+ else:
+ remainstr = '%ds' % secs
+ qprogress('Saving: %.2f%% (%d/%dk, %d/%d files) %s %s\r'
+ % (pct, cc/1024, total/1024, fcount, ftotal,
+ remainstr, kpsstr))
+
+
+indexfile = opt.indexfile or git.repo(b'bupindex')
+r = index.Reader(indexfile)
+try:
+ msr = index.MetaStoreReader(indexfile + b'.meta')
+except IOError as ex:
+ if ex.errno != EACCES:
+ raise
+ log('error: cannot access %r; have you run bup index?'
+ % path_msg(indexfile))
+ sys.exit(1)
+hlink_db = hlinkdb.HLinkDB(indexfile + b'.hlink')
+
+def already_saved(ent):
+ return ent.is_valid() and w.exists(ent.sha) and ent.sha
+
+def wantrecurse_pre(ent):
+ return not already_saved(ent)
+
+def wantrecurse_during(ent):
+ return not already_saved(ent) or ent.sha_missing()
+
+def find_hardlink_target(hlink_db, ent):
+ if hlink_db and not stat.S_ISDIR(ent.mode) and ent.nlink > 1:
+ link_paths = hlink_db.node_paths(ent.dev, ent.ino)
+ if link_paths:
+ return link_paths[0]
+
+total = ftotal = 0
+if opt.progress:
+ for (transname,ent) in r.filter(extra, wantrecurse=wantrecurse_pre):
+ if not (ftotal % 10024):
+ qprogress('Reading index: %d\r' % ftotal)
+ exists = ent.exists()
+ hashvalid = already_saved(ent)
+ ent.set_sha_missing(not hashvalid)
+ if not opt.smaller or ent.size < opt.smaller:
+ if exists and not hashvalid:
+ total += ent.size
+ ftotal += 1
+ progress('Reading index: %d, done.\n' % ftotal)
+ hashsplit.progress_callback = progress_report
+
+# Root collisions occur when strip or graft options map more than one
+# path to the same directory (paths which originally had separate
+# parents). When that situation is detected, use empty metadata for
+# the parent. Otherwise, use the metadata for the common parent.
+# Collision example: "bup save ... --strip /foo /foo/bar /bar".
+
+# FIXME: Add collision tests, or handle collisions some other way.
+
+# FIXME: Detect/handle strip/graft name collisions (other than root),
+# i.e. if '/foo/bar' and '/bar' both map to '/'.
+
+first_root = None
+root_collision = None
+tstart = time.time()
+count = subcount = fcount = 0
+lastskip_name = None
+lastdir = b''
+for (transname,ent) in r.filter(extra, wantrecurse=wantrecurse_during):
+ (dir, file) = os.path.split(ent.name)
+ exists = (ent.flags & index.IX_EXISTS)
+ hashvalid = already_saved(ent)
+ wasmissing = ent.sha_missing()
+ oldsize = ent.size
+ if opt.verbose:
+ if not exists:
+ status = 'D'
+ elif not hashvalid:
+ if ent.sha == index.EMPTY_SHA:
+ status = 'A'
+ else:
+ status = 'M'
+ else:
+ status = ' '
+ if opt.verbose >= 2:
+ log('%s %-70s\n' % (status, path_msg(ent.name)))
+ elif not stat.S_ISDIR(ent.mode) and lastdir != dir:
+ if not lastdir.startswith(dir):
+ log('%s %-70s\n' % (status, path_msg(os.path.join(dir, b''))))
+ lastdir = dir
+
+ if opt.progress:
+ progress_report(0)
+ fcount += 1
+
+ if not exists:
+ continue
+ if opt.smaller and ent.size >= opt.smaller:
+ if exists and not hashvalid:
+ if opt.verbose:
+ log('skipping large file "%s"\n' % path_msg(ent.name))
+ lastskip_name = ent.name
+ continue
+
+ assert(dir.startswith(b'/'))
+ if opt.strip:
+ dirp = stripped_path_components(dir, extra)
+ elif opt.strip_path:
+ dirp = stripped_path_components(dir, [opt.strip_path])
+ elif graft_points:
+ dirp = grafted_path_components(graft_points, dir)
+ else:
+ dirp = path_components(dir)
+
+ # At this point, dirp contains a representation of the archive
+ # path that looks like [(archive_dir_name, real_fs_path), ...].
+ # So given "bup save ... --strip /foo/bar /foo/bar/baz", dirp
+ # might look like this at some point:
+ # [('', '/foo/bar'), ('baz', '/foo/bar/baz'), ...].
+
+ # This dual representation supports stripping/grafting, where the
+ # archive path may not have a direct correspondence with the
+ # filesystem. The root directory is represented by an initial
+ # component named '', and any component that doesn't have a
+ # corresponding filesystem directory (due to grafting, for
+ # example) will have a real_fs_path of None, i.e. [('', None),
+ # ...].
+
+ if first_root == None:
+ first_root = dirp[0]
+ elif first_root != dirp[0]:
+ root_collision = True
+
+ # If switching to a new sub-tree, finish the current sub-tree.
+ while parts > [x[0] for x in dirp]:
+ _pop(force_tree = None)
+
+ # If switching to a new sub-tree, start a new sub-tree.
+ for path_component in dirp[len(parts):]:
+ dir_name, fs_path = path_component
+ # Not indexed, so just grab the FS metadata or use empty metadata.
+ try:
+ meta = metadata.from_path(fs_path, normalized=True) \
+ if fs_path else metadata.Metadata()
+ except (OSError, IOError) as e:
+ add_error(e)
+ lastskip_name = dir_name
+ meta = metadata.Metadata()
+ _push(dir_name, meta)
+
+ if not file:
+ if len(parts) == 1:
+ continue # We're at the top level -- keep the current root dir
+ # Since there's no filename, this is a subdir -- finish it.
+ oldtree = already_saved(ent) # may be None
+ newtree = _pop(force_tree = oldtree)
+ if not oldtree:
+ if lastskip_name and lastskip_name.startswith(ent.name):
+ ent.invalidate()
+ else:
+ ent.validate(GIT_MODE_TREE, newtree)
+ ent.repack()
+ if exists and wasmissing:
+ count += oldsize
+ continue
+
+ # it's not a directory
+ if hashvalid:
+ id = ent.sha
+ git_name = git.mangle_name(file, ent.mode, ent.gitmode)
+ git_info = (ent.gitmode, git_name, id)
+ shalists[-1].append(git_info)
+ sort_key = git.shalist_item_sort_key((ent.mode, file, id))
+ meta = msr.metadata_at(ent.meta_ofs)
+ meta.hardlink_target = find_hardlink_target(hlink_db, ent)
+ # Restore the times that were cleared to 0 in the metastore.
+ (meta.atime, meta.mtime, meta.ctime) = (ent.atime, ent.mtime, ent.ctime)
+ metalists[-1].append((sort_key, meta))
+ else:
+ id = None
+ if stat.S_ISREG(ent.mode):
+ try:
+ with hashsplit.open_noatime(ent.name) as f:
+ (mode, id) = hashsplit.split_to_blob_or_tree(
+ w.new_blob, w.new_tree, [f],
+ keep_boundaries=False)
+ except (IOError, OSError) as e:
+ add_error('%s: %s' % (ent.name, e))
+ lastskip_name = ent.name
+ elif stat.S_ISDIR(ent.mode):
+ assert(0) # handled above
+ elif stat.S_ISLNK(ent.mode):
+ try:
+ rl = os.readlink(ent.name)
+ except (OSError, IOError) as e:
+ add_error(e)
+ lastskip_name = ent.name
+ else:
+ (mode, id) = (GIT_MODE_SYMLINK, w.new_blob(rl))
+ else:
+ # Everything else should be fully described by its
+ # metadata, so just record an empty blob, so the paths
+ # in the tree and .bupm will match up.
+ (mode, id) = (GIT_MODE_FILE, w.new_blob(b''))
+
+ if id:
+ ent.validate(mode, id)
+ ent.repack()
+ git_name = git.mangle_name(file, ent.mode, ent.gitmode)
+ git_info = (mode, git_name, id)
+ shalists[-1].append(git_info)
+ sort_key = git.shalist_item_sort_key((ent.mode, file, id))
+ hlink = find_hardlink_target(hlink_db, ent)
+ try:
+ meta = metadata.from_path(ent.name, hardlink_target=hlink,
+ normalized=True)
+ except (OSError, IOError) as e:
+ add_error(e)
+ lastskip_name = ent.name
+ meta = metadata.Metadata()
+ metalists[-1].append((sort_key, meta))
+
+ if exists and wasmissing:
+ count += oldsize
+ subcount = 0
+
+
+if opt.progress:
+ pct = total and count*100.0/total or 100
+ progress('Saving: %.2f%% (%d/%dk, %d/%d files), done. \n'
+ % (pct, count/1024, total/1024, fcount, ftotal))
+
+while len(parts) > 1: # _pop() all the parts above the root
+ _pop(force_tree = None)
+assert(len(shalists) == 1)
+assert(len(metalists) == 1)
+
+# Finish the root directory.
+tree = _pop(force_tree = None,
+ # When there's a collision, use empty metadata for the root.
+ dir_metadata = metadata.Metadata() if root_collision else None)
+
+sys.stdout.flush()
+out = byte_stream(sys.stdout)
+
+if opt.tree:
+ out.write(hexlify(tree))
+ out.write(b'\n')
+if opt.commit or name:
+ msg = (b'bup save\n\nGenerated by command:\n%r\n'
+ % [argv_bytes(x) for x in sys.argv])
+ userline = (b'%s <%s@%s>' % (userfullname(), username(), hostname()))
+ commit = w.new_commit(tree, oldref, userline, date, None,
+ userline, date, None, msg)
+ if opt.commit:
+ out.write(hexlify(commit))
+ out.write(b'\n')
+
+msr.close()
+w.close() # must close before we can update the ref
+
+if opt.name:
+ if cli:
+ cli.update_ref(refname, commit, oldref)
+ else:
+ git.update_ref(refname, commit, oldref)
+
+if cli:
+ cli.close()
+
+if saved_errors:
+ log('WARNING: %d errors encountered while saving.\n' % len(saved_errors))
+ sys.exit(1)
--- /dev/null
+#!/bin/sh
+"""": # -*-python-*-
+bup_python="$(dirname "$0")/bup-python" || exit $?
+exec "$bup_python" "$0" ${1+"$@"}
+"""
+# end of bup preamble
+
+from __future__ import absolute_import
+from binascii import hexlify, unhexlify
+import os, sys, struct, subprocess
+
+from bup import options, git, vfs, vint
+from bup.compat import environ, hexstr
+from bup.git import MissingObject
+from bup.helpers import (Conn, debug1, debug2, linereader, lines_until_sentinel,
+ log)
+from bup.io import byte_stream, path_msg
+from bup.repo import LocalRepo
+
+
+suspended_w = None
+dumb_server_mode = False
+repo = None
+
+def do_help(conn, junk):
+ conn.write(b'Commands:\n %s\n' % b'\n '.join(sorted(commands)))
+ conn.ok()
+
+
+def _set_mode():
+ global dumb_server_mode
+ dumb_server_mode = os.path.exists(git.repo(b'bup-dumb-server'))
+ debug1('bup server: serving in %s mode\n'
+ % (dumb_server_mode and 'dumb' or 'smart'))
+
+
+def _init_session(reinit_with_new_repopath=None):
+ global repo
+ if reinit_with_new_repopath is None and git.repodir:
+ if not repo:
+ repo = LocalRepo()
+ return
+ git.check_repo_or_die(reinit_with_new_repopath)
+ if repo:
+ repo.close()
+ repo = LocalRepo()
+ # OK. we now know the path is a proper repository. Record this path in the
+ # environment so that subprocesses inherit it and know where to operate.
+ environ[b'BUP_DIR'] = git.repodir
+ debug1('bup server: bupdir is %s\n' % path_msg(git.repodir))
+ _set_mode()
+
+
+def init_dir(conn, arg):
+ git.init_repo(arg)
+ debug1('bup server: bupdir initialized: %s\n' % path_msg(git.repodir))
+ _init_session(arg)
+ conn.ok()
+
+
+def set_dir(conn, arg):
+ _init_session(arg)
+ conn.ok()
+
+
+def list_indexes(conn, junk):
+ _init_session()
+ suffix = b''
+ if dumb_server_mode:
+ suffix = b' load'
+ for f in os.listdir(git.repo(b'objects/pack')):
+ if f.endswith(b'.idx'):
+ conn.write(b'%s%s\n' % (f, suffix))
+ conn.ok()
+
+
+def send_index(conn, name):
+ _init_session()
+ assert name.find(b'/') < 0
+ assert name.endswith(b'.idx')
+ idx = git.open_idx(git.repo(b'objects/pack/%s' % name))
+ conn.write(struct.pack('!I', len(idx.map)))
+ conn.write(idx.map)
+ conn.ok()
+
+
+def receive_objects_v2(conn, junk):
+ global suspended_w
+ _init_session()
+ suggested = set()
+ if suspended_w:
+ w = suspended_w
+ suspended_w = None
+ else:
+ if dumb_server_mode:
+ w = git.PackWriter(objcache_maker=None)
+ else:
+ w = git.PackWriter()
+ while 1:
+ ns = conn.read(4)
+ if not ns:
+ w.abort()
+ raise Exception('object read: expected length header, got EOF\n')
+ n = struct.unpack('!I', ns)[0]
+ #debug2('expecting %d bytes\n' % n)
+ if not n:
+ debug1('bup server: received %d object%s.\n'
+ % (w.count, w.count!=1 and "s" or ''))
+ fullpath = w.close(run_midx=not dumb_server_mode)
+ if fullpath:
+ (dir, name) = os.path.split(fullpath)
+ conn.write(b'%s.idx\n' % name)
+ conn.ok()
+ return
+ elif n == 0xffffffff:
+ debug2('bup server: receive-objects suspended.\n')
+ suspended_w = w
+ conn.ok()
+ return
+
+ shar = conn.read(20)
+ crcr = struct.unpack('!I', conn.read(4))[0]
+ n -= 20 + 4
+ buf = conn.read(n) # object sizes in bup are reasonably small
+ #debug2('read %d bytes\n' % n)
+ _check(w, n, len(buf), 'object read: expected %d bytes, got %d\n')
+ if not dumb_server_mode:
+ oldpack = w.exists(shar, want_source=True)
+ if oldpack:
+ assert(not oldpack == True)
+ assert(oldpack.endswith(b'.idx'))
+ (dir,name) = os.path.split(oldpack)
+ if not (name in suggested):
+ debug1("bup server: suggesting index %s\n"
+ % git.shorten_hash(name).decode('ascii'))
+ debug1("bup server: because of object %s\n"
+ % hexstr(shar))
+ conn.write(b'index %s\n' % name)
+ suggested.add(name)
+ continue
+ nw, crc = w._raw_write((buf,), sha=shar)
+ _check(w, crcr, crc, 'object read: expected crc %d, got %d\n')
+ # NOTREACHED
+
+
+def _check(w, expected, actual, msg):
+ if expected != actual:
+ w.abort()
+ raise Exception(msg % (expected, actual))
+
+
+def read_ref(conn, refname):
+ _init_session()
+ r = git.read_ref(refname)
+ conn.write(b'%s\n' % hexlify(r) if r else b'')
+ conn.ok()
+
+
+def update_ref(conn, refname):
+ _init_session()
+ newval = conn.readline().strip()
+ oldval = conn.readline().strip()
+ git.update_ref(refname, unhexlify(newval), unhexlify(oldval))
+ conn.ok()
+
+def join(conn, id):
+ _init_session()
+ try:
+ for blob in git.cp().join(id):
+ conn.write(struct.pack('!I', len(blob)))
+ conn.write(blob)
+ except KeyError as e:
+ log('server: error: %s\n' % e)
+ conn.write(b'\0\0\0\0')
+ conn.error(e)
+ else:
+ conn.write(b'\0\0\0\0')
+ conn.ok()
+
+def cat_batch(conn, dummy):
+ _init_session()
+ cat_pipe = git.cp()
+ # For now, avoid potential deadlock by just reading them all
+ for ref in tuple(lines_until_sentinel(conn, b'\n', Exception)):
+ ref = ref[:-1]
+ it = cat_pipe.get(ref)
+ info = next(it)
+ if not info[0]:
+ conn.write(b'missing\n')
+ continue
+ conn.write(b'%s %s %d\n' % info)
+ for buf in it:
+ conn.write(buf)
+ conn.ok()
+
+def refs(conn, args):
+ limit_to_heads, limit_to_tags = args.split()
+ assert limit_to_heads in (b'0', b'1')
+ assert limit_to_tags in (b'0', b'1')
+ limit_to_heads = int(limit_to_heads)
+ limit_to_tags = int(limit_to_tags)
+ _init_session()
+ patterns = tuple(x[:-1] for x in lines_until_sentinel(conn, b'\n', Exception))
+ for name, oid in git.list_refs(patterns=patterns,
+ limit_to_heads=limit_to_heads,
+ limit_to_tags=limit_to_tags):
+ assert b'\n' not in name
+ conn.write(b'%s %s\n' % (hexlify(oid), name))
+ conn.write(b'\n')
+ conn.ok()
+
+def rev_list(conn, _):
+ _init_session()
+ count = conn.readline()
+ if not count:
+ raise Exception('Unexpected EOF while reading rev-list count')
+ assert count == b'\n'
+ count = None
+ fmt = conn.readline()
+ if not fmt:
+ raise Exception('Unexpected EOF while reading rev-list format')
+ fmt = None if fmt == b'\n' else fmt[:-1]
+ refs = tuple(x[:-1] for x in lines_until_sentinel(conn, b'\n', Exception))
+ args = git.rev_list_invocation(refs, format=fmt)
+ p = subprocess.Popen(args, env=git._gitenv(git.repodir),
+ stdout=subprocess.PIPE)
+ while True:
+ out = p.stdout.read(64 * 1024)
+ if not out:
+ break
+ conn.write(out)
+ conn.write(b'\n')
+ rv = p.wait() # not fatal
+ if rv:
+ msg = 'git rev-list returned error %d' % rv
+ conn.error(msg)
+ raise GitError(msg)
+ conn.ok()
+
+def resolve(conn, args):
+ _init_session()
+ (flags,) = args.split()
+ flags = int(flags)
+ want_meta = bool(flags & 1)
+ follow = bool(flags & 2)
+ have_parent = bool(flags & 4)
+ parent = vfs.read_resolution(conn) if have_parent else None
+ path = vint.read_bvec(conn)
+ if not len(path):
+ raise Exception('Empty resolve path')
+ try:
+ res = list(vfs.resolve(repo, path, parent=parent, want_meta=want_meta,
+ follow=follow))
+ except vfs.IOError as ex:
+ res = ex
+ if isinstance(res, vfs.IOError):
+ conn.write(b'\x00') # error
+ vfs.write_ioerror(conn, res)
+ else:
+ conn.write(b'\x01') # success
+ vfs.write_resolution(conn, res)
+ conn.ok()
+
+optspec = """
+bup server
+"""
+o = options.Options(optspec)
+(opt, flags, extra) = o.parse(sys.argv[1:])
+
+if extra:
+ o.fatal('no arguments expected')
+
+debug2('bup server: reading from stdin.\n')
+
+commands = {
+ b'quit': None,
+ b'help': do_help,
+ b'init-dir': init_dir,
+ b'set-dir': set_dir,
+ b'list-indexes': list_indexes,
+ b'send-index': send_index,
+ b'receive-objects-v2': receive_objects_v2,
+ b'read-ref': read_ref,
+ b'update-ref': update_ref,
+ b'join': join,
+ b'cat': join, # apocryphal alias
+ b'cat-batch' : cat_batch,
+ b'refs': refs,
+ b'rev-list': rev_list,
+ b'resolve': resolve
+}
+
+# FIXME: this protocol is totally lame and not at all future-proof.
+# (Especially since we abort completely as soon as *anything* bad happens)
+sys.stdout.flush()
+conn = Conn(byte_stream(sys.stdin), byte_stream(sys.stdout))
+lr = linereader(conn)
+for _line in lr:
+ line = _line.strip()
+ if not line:
+ continue
+ debug1('bup server: command: %r\n' % line)
+ words = line.split(b' ', 1)
+ cmd = words[0]
+ rest = len(words)>1 and words[1] or b''
+ if cmd == b'quit':
+ break
+ else:
+ cmd = commands.get(cmd)
+ if cmd:
+ cmd(conn, rest)
+ else:
+ raise Exception('unknown server command: %r\n' % line)
+
+debug1('bup server: done\n')
--- /dev/null
+#!/bin/sh
+"""": # -*-python-*-
+bup_python="$(dirname "$0")/bup-python" || exit $?
+exec "$bup_python" "$0" ${1+"$@"}
+"""
+# end of bup preamble
+
+from __future__ import absolute_import, division, print_function
+from binascii import hexlify
+import os, sys, time
+
+from bup import hashsplit, git, options, client
+from bup.compat import argv_bytes, environ
+from bup.helpers import (add_error, handle_ctrl_c, hostname, log, parse_num,
+ qprogress, reprogress, saved_errors,
+ valid_save_name,
+ parse_date_or_fatal)
+from bup.io import byte_stream
+from bup.pwdgrp import userfullname, username
+
+
+optspec = """
+bup split [-t] [-c] [-n name] OPTIONS [--git-ids | filenames...]
+bup split -b OPTIONS [--git-ids | filenames...]
+bup split --copy OPTIONS [--git-ids | filenames...]
+bup split --noop [-b|-t] OPTIONS [--git-ids | filenames...]
+--
+ Modes:
+b,blobs output a series of blob ids. Implies --fanout=0.
+t,tree output a tree id
+c,commit output a commit id
+n,name= save the result under the given name
+noop split the input, but throw away the result
+copy split the input, copy it to stdout, don't save to repo
+ Options:
+r,remote= remote repository path
+d,date= date for the commit (seconds since the epoch)
+q,quiet don't print progress messages
+v,verbose increase log output (can be used more than once)
+git-ids read a list of git object ids from stdin and split their contents
+keep-boundaries don't let one chunk span two input files
+bench print benchmark timings to stderr
+max-pack-size= maximum bytes in a single pack
+max-pack-objects= maximum number of objects in a single pack
+fanout= average number of blobs in a single tree
+bwlimit= maximum bytes/sec to transmit to server
+#,compress= set compression level to # (0-9, 9 is highest) [1]
+"""
+handle_ctrl_c()
+
+o = options.Options(optspec)
+(opt, flags, extra) = o.parse(sys.argv[1:])
+if opt.name: opt.name = argv_bytes(opt.name)
+if opt.remote: opt.remote = argv_bytes(opt.remote)
+if opt.verbose is None: opt.verbose = 0
+
+if not (opt.blobs or opt.tree or opt.commit or opt.name or
+ opt.noop or opt.copy):
+ o.fatal("use one or more of -b, -t, -c, -n, --noop, --copy")
+if opt.copy and (opt.blobs or opt.tree):
+ o.fatal('--copy is incompatible with -b, -t')
+if (opt.noop or opt.copy) and (opt.commit or opt.name):
+ o.fatal('--noop and --copy are incompatible with -c, -n')
+if opt.blobs and (opt.tree or opt.commit or opt.name):
+ o.fatal('-b is incompatible with -t, -c, -n')
+if extra and opt.git_ids:
+ o.fatal("don't provide filenames when using --git-ids")
+
+if opt.verbose >= 2:
+ git.verbose = opt.verbose - 1
+ opt.bench = 1
+
+max_pack_size = None
+if opt.max_pack_size:
+ max_pack_size = parse_num(opt.max_pack_size)
+max_pack_objects = None
+if opt.max_pack_objects:
+ max_pack_objects = parse_num(opt.max_pack_objects)
+
+if opt.fanout:
+ hashsplit.fanout = parse_num(opt.fanout)
+if opt.blobs:
+ hashsplit.fanout = 0
+if opt.bwlimit:
+ client.bwlimit = parse_num(opt.bwlimit)
+if opt.date:
+ date = parse_date_or_fatal(opt.date, o.fatal)
+else:
+ date = time.time()
+
+total_bytes = 0
+def prog(filenum, nbytes):
+ global total_bytes
+ total_bytes += nbytes
+ if filenum > 0:
+ qprogress('Splitting: file #%d, %d kbytes\r'
+ % (filenum+1, total_bytes // 1024))
+ else:
+ qprogress('Splitting: %d kbytes\r' % (total_bytes // 1024))
+
+
+is_reverse = environ.get(b'BUP_SERVER_REVERSE')
+if is_reverse and opt.remote:
+ o.fatal("don't use -r in reverse mode; it's automatic")
+start_time = time.time()
+
+if opt.name and not valid_save_name(opt.name):
+ o.fatal("'%r' is not a valid branch name." % opt.name)
+refname = opt.name and b'refs/heads/%s' % opt.name or None
+
+if opt.noop or opt.copy:
+ cli = pack_writer = oldref = None
+elif opt.remote or is_reverse:
+ git.check_repo_or_die()
+ cli = client.Client(opt.remote)
+ oldref = refname and cli.read_ref(refname) or None
+ pack_writer = cli.new_packwriter(compression_level=opt.compress,
+ max_pack_size=max_pack_size,
+ max_pack_objects=max_pack_objects)
+else:
+ git.check_repo_or_die()
+ cli = None
+ oldref = refname and git.read_ref(refname) or None
+ pack_writer = git.PackWriter(compression_level=opt.compress,
+ max_pack_size=max_pack_size,
+ max_pack_objects=max_pack_objects)
+
+input = byte_stream(sys.stdin)
+
+if opt.git_ids:
+ # the input is actually a series of git object ids that we should retrieve
+ # and split.
+ #
+ # This is a bit messy, but basically it converts from a series of
+ # CatPipe.get() iterators into a series of file-type objects.
+ # It would be less ugly if either CatPipe.get() returned a file-like object
+ # (not very efficient), or split_to_shalist() expected an iterator instead
+ # of a file.
+ cp = git.CatPipe()
+ class IterToFile:
+ def __init__(self, it):
+ self.it = iter(it)
+ def read(self, size):
+ v = next(self.it, None)
+ return v or b''
+ def read_ids():
+ while 1:
+ line = input.readline()
+ if not line:
+ break
+ if line:
+ line = line.strip()
+ try:
+ it = cp.get(line.strip())
+ next(it, None) # skip the file info
+ except KeyError as e:
+ add_error('error: %s' % e)
+ continue
+ yield IterToFile(it)
+ files = read_ids()
+else:
+ # the input either comes from a series of files or from stdin.
+ files = extra and (open(argv_bytes(fn), 'rb') for fn in extra) or [input]
+
+if pack_writer:
+ new_blob = pack_writer.new_blob
+ new_tree = pack_writer.new_tree
+elif opt.blobs or opt.tree:
+ # --noop mode
+ new_blob = lambda content: git.calc_hash(b'blob', content)
+ new_tree = lambda shalist: git.calc_hash(b'tree', git.tree_encode(shalist))
+
+sys.stdout.flush()
+out = byte_stream(sys.stdout)
+
+if opt.blobs:
+ shalist = hashsplit.split_to_blobs(new_blob, files,
+ keep_boundaries=opt.keep_boundaries,
+ progress=prog)
+ for (sha, size, level) in shalist:
+ out.write(hexlify(sha) + b'\n')
+ reprogress()
+elif opt.tree or opt.commit or opt.name:
+ if opt.name: # insert dummy_name which may be used as a restore target
+ mode, sha = \
+ hashsplit.split_to_blob_or_tree(new_blob, new_tree, files,
+ keep_boundaries=opt.keep_boundaries,
+ progress=prog)
+ splitfile_name = git.mangle_name(b'data', hashsplit.GIT_MODE_FILE, mode)
+ shalist = [(mode, splitfile_name, sha)]
+ else:
+ shalist = hashsplit.split_to_shalist(
+ new_blob, new_tree, files,
+ keep_boundaries=opt.keep_boundaries, progress=prog)
+ tree = new_tree(shalist)
+else:
+ last = 0
+ it = hashsplit.hashsplit_iter(files,
+ keep_boundaries=opt.keep_boundaries,
+ progress=prog)
+ for (blob, level) in it:
+ hashsplit.total_split += len(blob)
+ if opt.copy:
+ sys.stdout.write(str(blob))
+ megs = hashsplit.total_split // 1024 // 1024
+ if not opt.quiet and last != megs:
+ last = megs
+
+if opt.verbose:
+ log('\n')
+if opt.tree:
+ out.write(hexlify(tree) + b'\n')
+if opt.commit or opt.name:
+ msg = b'bup split\n\nGenerated by command:\n%r\n' % sys.argv
+ ref = opt.name and (b'refs/heads/%s' % opt.name) or None
+ userline = b'%s <%s@%s>' % (userfullname(), username(), hostname())
+ commit = pack_writer.new_commit(tree, oldref, userline, date, None,
+ userline, date, None, msg)
+ if opt.commit:
+ out.write(hexlify(commit) + b'\n')
+
+if pack_writer:
+ pack_writer.close() # must close before we can update the ref
+
+if opt.name:
+ if cli:
+ cli.update_ref(refname, commit, oldref)
+ else:
+ git.update_ref(refname, commit, oldref)
+
+if cli:
+ cli.close()
+
+secs = time.time() - start_time
+size = hashsplit.total_split
+if opt.bench:
+ log('bup: %.2f kbytes in %.2f secs = %.2f kbytes/sec\n'
+ % (size / 1024, secs, size / 1024 / secs))
+
+if saved_errors:
+ log('WARNING: %d errors encountered while saving.\n' % len(saved_errors))
+ sys.exit(1)
--- /dev/null
+#!/bin/sh
+"""": # -*-python-*-
+bup_python="$(dirname "$0")/bup-python" || exit $?
+exec "$bup_python" "$0" ${1+"$@"}
+"""
+# end of bup preamble
+
+from __future__ import absolute_import
+from binascii import hexlify
+import os, sys
+
+from bup import git, options
+from bup.compat import argv_bytes
+from bup.helpers import debug1, handle_ctrl_c, log
+from bup.io import byte_stream, path_msg
+
+# FIXME: review for safe writes.
+
+handle_ctrl_c()
+
+optspec = """
+bup tag
+bup tag [-f] <tag name> <commit>
+bup tag [-f] -d <tag name>
+--
+d,delete= Delete a tag
+f,force Overwrite existing tag, or ignore missing tag when deleting
+"""
+
+o = options.Options(optspec)
+(opt, flags, extra) = o.parse(sys.argv[1:])
+
+git.check_repo_or_die()
+
+tags = [t for sublist in git.tags().values() for t in sublist]
+
+if opt.delete:
+ # git.delete_ref() doesn't complain if a ref doesn't exist. We
+ # could implement this verification but we'd need to read in the
+ # contents of the tag file and pass the hash, and we already know
+ # about the tag's existance via "tags".
+ tag_name = argv_bytes(opt.delete)
+ if not opt.force and tag_name not in tags:
+ log("error: tag '%s' doesn't exist\n" % path_msg(tag_name))
+ sys.exit(1)
+ tag_file = b'refs/tags/%s' % tag_name
+ git.delete_ref(tag_file)
+ sys.exit(0)
+
+if not extra:
+ for t in tags:
+ sys.stdout.flush()
+ out = byte_stream(sys.stdout)
+ out.write(t)
+ out.write(b'\n')
+ sys.exit(0)
+elif len(extra) != 2:
+ o.fatal('expected commit ref and hash')
+
+tag_name, commit = map(argv_bytes, extra[:2])
+if not tag_name:
+ o.fatal("tag name must not be empty.")
+debug1("args: tag name = %s; commit = %s\n"
+ % (path_msg(tag_name), commit.decode('ascii')))
+
+if tag_name in tags and not opt.force:
+ log("bup: error: tag '%s' already exists\n" % path_msg(tag_name))
+ sys.exit(1)
+
+if tag_name.startswith(b'.'):
+ o.fatal("'%s' is not a valid tag name." % path_msg(tag_name))
+
+try:
+ hash = git.rev_parse(commit)
+except git.GitError as e:
+ log("bup: error: %s" % e)
+ sys.exit(2)
+
+if not hash:
+ log("bup: error: commit %s not found.\n" % commit.decode('ascii'))
+ sys.exit(2)
+
+pL = git.PackIdxList(git.repo(b'objects/pack'))
+if not pL.exists(hash):
+ log("bup: error: commit %s not found.\n" % commit.decode('ascii'))
+ sys.exit(2)
+
+tag_file = git.repo(b'refs/tags/' + tag_name)
+try:
+ tag = open(tag_file, 'wb')
+except OSError as e:
+ log("bup: error: could not create tag '%s': %s" % (path_msg(tag_name), e))
+ sys.exit(3)
+with tag as tag:
+ tag.write(hexlify(hash))
+ tag.write(b'\n')
--- /dev/null
+#!/bin/sh
+"""": # -*-python-*-
+bup_python="$(dirname "$0")/bup-python" || exit $?
+exec "$bup_python" "$0" ${1+"$@"}
+"""
+# end of bup preamble
+
+from __future__ import absolute_import
+import sys, time
+from bup import options
+
+optspec = """
+bup tick
+"""
+o = options.Options(optspec)
+(opt, flags, extra) = o.parse(sys.argv[1:])
+
+if extra:
+ o.fatal("no arguments expected")
+
+t = time.time()
+tleft = 1 - (t - int(t))
+time.sleep(tleft)
--- /dev/null
+#!/bin/sh
+"""": # -*-python-*-
+bup_python="$(dirname "$0")/bup-python" || exit $?
+exec "$bup_python" "$0" ${1+"$@"}
+"""
+# end of bup preamble
+
+from __future__ import absolute_import, print_function
+import re, sys
+
+from bup import options
+from bup import version
+
+version_rx = re.compile(r'^[0-9]+\.[0-9]+(\.[0-9]+)?(-[0-9]+-g[0-9abcdef]+)?$')
+
+optspec = """
+bup version [--date|--commit|--tag]
+--
+date display the date this version of bup was created
+commit display the git commit id of this version of bup
+tag display the tag name of this version. If no tag is available, display the commit id
+"""
+o = options.Options(optspec)
+(opt, flags, extra) = o.parse(sys.argv[1:])
+
+
+total = (opt.date or 0) + (opt.commit or 0) + (opt.tag or 0)
+if total > 1:
+ o.fatal('at most one option expected')
+
+
+def version_date():
+ """Format bup's version date string for output."""
+ return version.DATE.split(' ')[0]
+
+
+def version_commit():
+ """Get the commit hash of bup's current version."""
+ return version.COMMIT
+
+
+def version_tag():
+ """Format bup's version tag (the official version number).
+
+ When generated from a commit other than one pointed to with a tag, the
+ returned string will be "unknown-" followed by the first seven positions of
+ the commit hash.
+ """
+ names = version.NAMES.strip()
+ assert(names[0] == '(')
+ assert(names[-1] == ')')
+ names = names[1:-1]
+ l = [n.strip() for n in names.split(',')]
+ for n in l:
+ if n.startswith('tag: ') and version_rx.match(n[5:]):
+ return n[5:]
+ return 'unknown-%s' % version.COMMIT[:7]
+
+
+if opt.date:
+ print(version_date())
+elif opt.commit:
+ print(version_commit())
+else:
+ print(version_tag())
--- /dev/null
+#!/bin/sh
+"""": # -*-python-*-
+bup_python="$(dirname "$0")/bup-python" || exit $?
+exec "$bup_python" "$0" ${1+"$@"}
+"""
+# end of bup preamble
+
+from __future__ import absolute_import, print_function
+from collections import namedtuple
+import mimetypes, os, posixpath, signal, stat, sys, time, urllib, webbrowser
+
+from bup import options, git, vfs
+from bup.helpers import (chunkyreader, debug1, format_filesize, handle_ctrl_c,
+ log, saved_errors)
+from bup.metadata import Metadata
+from bup.path import resource_path
+from bup.repo import LocalRepo
+
+try:
+ from tornado import gen
+ from tornado.httpserver import HTTPServer
+ from tornado.ioloop import IOLoop
+ from tornado.netutil import bind_unix_socket
+ import tornado.web
+except ImportError:
+ log('error: cannot find the python "tornado" module; please install it\n')
+ sys.exit(1)
+
+
+# FIXME: right now the way hidden files are handled causes every
+# directory to be traversed twice.
+
+handle_ctrl_c()
+
+
+def http_date_from_utc_ns(utc_ns):
+ return time.strftime('%a, %d %b %Y %H:%M:%S', time.gmtime(utc_ns / 10**9))
+
+
+def _compute_breadcrumbs(path, show_hidden=False):
+ """Returns a list of breadcrumb objects for a path."""
+ breadcrumbs = []
+ breadcrumbs.append(('[root]', '/'))
+ path_parts = path.split('/')[1:-1]
+ full_path = '/'
+ for part in path_parts:
+ full_path += part + "/"
+ url_append = ""
+ if show_hidden:
+ url_append = '?hidden=1'
+ breadcrumbs.append((part, full_path+url_append))
+ return breadcrumbs
+
+
+def _contains_hidden_files(repo, dir_item):
+ """Return true if the directory contains items with names other than
+ '.' and '..' that begin with '.'
+
+ """
+ for name, item in vfs.contents(repo, dir_item, want_meta=False):
+ if name in ('.', '..'):
+ continue
+ if name.startswith('.'):
+ return True
+ return False
+
+
+def _dir_contents(repo, resolution, show_hidden=False):
+ """Yield the display information for the contents of dir_item."""
+
+ url_query = '?hidden=1' if show_hidden else ''
+
+ def display_info(name, item, resolved_item, display_name=None):
+ # link should be based on fully resolved type to avoid extra
+ # HTTP redirect.
+ if stat.S_ISDIR(vfs.item_mode(resolved_item)):
+ link = urllib.quote(name) + '/'
+ else:
+ link = urllib.quote(name)
+
+ size = vfs.item_size(repo, item)
+ if opt.human_readable:
+ display_size = format_filesize(size)
+ else:
+ display_size = size
+
+ if not display_name:
+ mode = vfs.item_mode(item)
+ if stat.S_ISDIR(mode):
+ display_name = name + '/'
+ elif stat.S_ISLNK(mode):
+ display_name = name + '@'
+ else:
+ display_name = name
+
+ return display_name, link + url_query, display_size
+
+ dir_item = resolution[-1][1]
+ for name, item in vfs.contents(repo, dir_item):
+ if not show_hidden:
+ if (name not in ('.', '..')) and name.startswith('.'):
+ continue
+ if name == '.':
+ yield display_info(name, item, item, '.')
+ parent_item = resolution[-2][1] if len(resolution) > 1 else dir_item
+ yield display_info('..', parent_item, parent_item, '..')
+ continue
+ res = vfs.try_resolve(repo, name, parent=resolution, want_meta=False)
+ res_name, res_item = res[-1]
+ yield display_info(name, item, res_item)
+
+
+class BupRequestHandler(tornado.web.RequestHandler):
+
+ def initialize(self, repo=None):
+ self.repo = repo
+
+ def decode_argument(self, value, name=None):
+ if name == 'path':
+ return value
+ return super(BupRequestHandler, self).decode_argument(value, name)
+
+ def get(self, path):
+ return self._process_request(path)
+
+ def head(self, path):
+ return self._process_request(path)
+
+ def _process_request(self, path):
+ path = urllib.unquote(path)
+ print('Handling request for %s' % path)
+ # Set want_meta because dir metadata won't be fetched, and if
+ # it's not a dir, then we're going to want the metadata.
+ res = vfs.resolve(self.repo, path, want_meta=True)
+ leaf_name, leaf_item = res[-1]
+ if not leaf_item:
+ self.send_error(404)
+ return
+ mode = vfs.item_mode(leaf_item)
+ if stat.S_ISDIR(mode):
+ self._list_directory(path, res)
+ else:
+ self._get_file(self.repo, path, res)
+
+ def _list_directory(self, path, resolution):
+ """Helper to produce a directory listing.
+
+ Return value is either a file object, or None (indicating an
+ error). In either case, the headers are sent.
+ """
+ if not path.endswith('/') and len(path) > 0:
+ print('Redirecting from %s to %s' % (path, path + '/'))
+ return self.redirect(path + '/', permanent=True)
+
+ hidden_arg = self.request.arguments.get('hidden', [0])[-1]
+ try:
+ show_hidden = int(hidden_arg)
+ except ValueError as e:
+ show_hidden = False
+
+ self.render(
+ 'list-directory.html',
+ path=path,
+ breadcrumbs=_compute_breadcrumbs(path, show_hidden),
+ files_hidden=_contains_hidden_files(self.repo, resolution[-1][1]),
+ hidden_shown=show_hidden,
+ dir_contents=_dir_contents(self.repo, resolution,
+ show_hidden=show_hidden))
+
+ @gen.coroutine
+ def _get_file(self, repo, path, resolved):
+ """Process a request on a file.
+
+ Return value is either a file object, or None (indicating an error).
+ In either case, the headers are sent.
+ """
+ file_item = resolved[-1][1]
+ file_item = vfs.augment_item_meta(repo, file_item, include_size=True)
+ meta = file_item.meta
+ ctype = self._guess_type(path)
+ self.set_header("Last-Modified", http_date_from_utc_ns(meta.mtime))
+ self.set_header("Content-Type", ctype)
+
+ self.set_header("Content-Length", str(meta.size))
+ assert len(file_item.oid) == 20
+ self.set_header("Etag", file_item.oid.encode('hex'))
+ if self.request.method != 'HEAD':
+ with vfs.fopen(self.repo, file_item) as f:
+ it = chunkyreader(f)
+ for blob in chunkyreader(f):
+ self.write(blob)
+ raise gen.Return()
+
+ def _guess_type(self, path):
+ """Guess the type of a file.
+
+ Argument is a PATH (a filename).
+
+ Return value is a string of the form type/subtype,
+ usable for a MIME Content-type header.
+
+ The default implementation looks the file's extension
+ up in the table self.extensions_map, using application/octet-stream
+ as a default; however it would be permissible (if
+ slow) to look inside the data to make a better guess.
+ """
+ base, ext = posixpath.splitext(path)
+ if ext in self.extensions_map:
+ return self.extensions_map[ext]
+ ext = ext.lower()
+ if ext in self.extensions_map:
+ return self.extensions_map[ext]
+ else:
+ return self.extensions_map['']
+
+ if not mimetypes.inited:
+ mimetypes.init() # try to read system mime.types
+ extensions_map = mimetypes.types_map.copy()
+ extensions_map.update({
+ '': 'text/plain', # Default
+ '.py': 'text/plain',
+ '.c': 'text/plain',
+ '.h': 'text/plain',
+ })
+
+
+io_loop = None
+
+def handle_sigterm(signum, frame):
+ global io_loop
+ debug1('\nbup-web: signal %d received\n' % signum)
+ log('Shutdown requested\n')
+ if not io_loop:
+ sys.exit(0)
+ io_loop.stop()
+
+
+signal.signal(signal.SIGTERM, handle_sigterm)
+
+UnixAddress = namedtuple('UnixAddress', ['path'])
+InetAddress = namedtuple('InetAddress', ['host', 'port'])
+
+optspec = """
+bup web [[hostname]:port]
+bup web unix://path
+--
+human-readable display human readable file sizes (i.e. 3.9K, 4.7M)
+browser show repository in default browser (incompatible with unix://)
+"""
+o = options.Options(optspec)
+(opt, flags, extra) = o.parse(sys.argv[1:])
+
+if len(extra) > 1:
+ o.fatal("at most one argument expected")
+
+if len(extra) == 0:
+ address = InetAddress(host='127.0.0.1', port=8080)
+else:
+ bind_url = extra[0]
+ if bind_url.startswith('unix://'):
+ address = UnixAddress(path=bind_url[len('unix://'):])
+ else:
+ addr_parts = extra[0].split(':', 1)
+ if len(addr_parts) == 1:
+ host = '127.0.0.1'
+ port = addr_parts[0]
+ else:
+ host, port = addr_parts
+ try:
+ port = int(port)
+ except (TypeError, ValueError) as ex:
+ o.fatal('port must be an integer, not %r' % port)
+ address = InetAddress(host=host, port=port)
+
+git.check_repo_or_die()
+
+settings = dict(
+ debug = 1,
+ template_path = resource_path('web'),
+ static_path = resource_path('web/static')
+)
+
+# Disable buffering on stdout, for debug messages
+sys.stdout = os.fdopen(sys.stdout.fileno(), 'w', 0)
+
+application = tornado.web.Application([
+ (r"(?P<path>/.*)", BupRequestHandler, dict(repo=LocalRepo())),
+], **settings)
+
+http_server = HTTPServer(application)
+io_loop_pending = IOLoop.instance()
+
+if isinstance(address, InetAddress):
+ http_server.listen(address.port, address.host)
+ try:
+ sock = http_server._socket # tornado < 2.0
+ except AttributeError as e:
+ sock = http_server._sockets.values()[0]
+ print('Serving HTTP on %s:%d...' % sock.getsockname())
+ if opt.browser:
+ browser_addr = 'http://' + address[0] + ':' + str(address[1])
+ io_loop_pending.add_callback(lambda : webbrowser.open(browser_addr))
+elif isinstance(address, UnixAddress):
+ unix_socket = bind_unix_socket(address.path)
+ http_server.add_socket(unix_socket)
+ print('Serving HTTP on filesystem socket %r' % address.path)
+else:
+ log('error: unexpected address %r', address)
+ sys.exit(1)
+
+io_loop = io_loop_pending
+io_loop.start()
+
+if saved_errors:
+ log('WARNING: %d errors encountered while saving.\n' % len(saved_errors))
+ sys.exit(1)
--- /dev/null
+#!/bin/sh
+"""": # -*-python-*-
+bup_python="$(dirname "$0")/bup-python" || exit $?
+exec "$bup_python" "$0" ${1+"$@"}
+"""
+# end of bup preamble
+
+# Copyright (C) 2010 Rob Browning
+#
+# This code is covered under the terms of the GNU Library General
+# Public License as described in the bup LICENSE file.
+
+from __future__ import absolute_import, print_function
+import sys, stat, errno
+
+from bup import metadata, options, xstat
+from bup.compat import argv_bytes
+from bup.helpers import add_error, handle_ctrl_c, parse_timestamp, saved_errors, \
+ add_error, log
+from bup.io import byte_stream
+
+
+def parse_timestamp_arg(field, value):
+ res = str(value) # Undo autoconversion.
+ try:
+ res = parse_timestamp(res)
+ except ValueError as ex:
+ if ex.args:
+ o.fatal('unable to parse %s resolution "%s" (%s)'
+ % (field, value, ex))
+ else:
+ o.fatal('unable to parse %s resolution "%s"' % (field, value))
+
+ if res != 1 and res % 10:
+ o.fatal('%s resolution "%s" must be a power of 10' % (field, value))
+ return res
+
+
+optspec = """
+bup xstat pathinfo [OPTION ...] <PATH ...>
+--
+v,verbose increase log output (can be used more than once)
+q,quiet don't show progress meter
+exclude-fields= exclude comma-separated fields
+include-fields= include comma-separated fields (definitive if first)
+atime-resolution= limit s, ms, us, ns, 10ns (value must be a power of 10) [ns]
+mtime-resolution= limit s, ms, us, ns, 10ns (value must be a power of 10) [ns]
+ctime-resolution= limit s, ms, us, ns, 10ns (value must be a power of 10) [ns]
+"""
+
+target_filename = b''
+active_fields = metadata.all_fields
+
+handle_ctrl_c()
+
+o = options.Options(optspec)
+(opt, flags, remainder) = o.parse(sys.argv[1:])
+
+atime_resolution = parse_timestamp_arg('atime', opt.atime_resolution)
+mtime_resolution = parse_timestamp_arg('mtime', opt.mtime_resolution)
+ctime_resolution = parse_timestamp_arg('ctime', opt.ctime_resolution)
+
+treat_include_fields_as_definitive = True
+for flag, value in flags:
+ if flag == '--exclude-fields':
+ exclude_fields = frozenset(value.split(','))
+ for f in exclude_fields:
+ if not f in metadata.all_fields:
+ o.fatal(f + ' is not a valid field name')
+ active_fields = active_fields - exclude_fields
+ treat_include_fields_as_definitive = False
+ elif flag == '--include-fields':
+ include_fields = frozenset(value.split(','))
+ for f in include_fields:
+ if not f in metadata.all_fields:
+ o.fatal(f + ' is not a valid field name')
+ if treat_include_fields_as_definitive:
+ active_fields = include_fields
+ treat_include_fields_as_definitive = False
+ else:
+ active_fields = active_fields | include_fields
+
+opt.verbose = opt.verbose or 0
+opt.quiet = opt.quiet or 0
+metadata.verbose = opt.verbose - opt.quiet
+
+sys.stdout.flush()
+out = byte_stream(sys.stdout)
+
+first_path = True
+for path in remainder:
+ path = argv_bytes(path)
+ try:
+ m = metadata.from_path(path, archive_path = path)
+ except (OSError,IOError) as e:
+ if e.errno == errno.ENOENT:
+ add_error(e)
+ continue
+ else:
+ raise
+ if metadata.verbose >= 0:
+ if not first_path:
+ out.write(b'\n')
+ if atime_resolution != 1:
+ m.atime = (m.atime / atime_resolution) * atime_resolution
+ if mtime_resolution != 1:
+ m.mtime = (m.mtime / mtime_resolution) * mtime_resolution
+ if ctime_resolution != 1:
+ m.ctime = (m.ctime / ctime_resolution) * ctime_resolution
+ out.write(metadata.detailed_bytes(m, active_fields))
+ out.write(b'\n')
+ first_path = False
+
+if saved_errors:
+ log('WARNING: %d errors encountered.\n' % len(saved_errors))
+ sys.exit(1)
+else:
+ sys.exit(0)
WVPASS bup init
WVPASS cd "$tmpdir"
WVPASS mkdir rdiff-backup
-WVPASS rdiff-backup "$top/cmd" rdiff-backup
+WVPASS rdiff-backup "$top/lib/cmd" rdiff-backup
WVPASS bup tick
WVPASS rdiff-backup "$top/Documentation" rdiff-backup
WVPASS bup import-rdiff-backup rdiff-backup import-rdiff-backup
projects / bup.git / commitdiff