"bup save -n webserver --strip-path=/root/chroots" would
be saved as */webserver/etc*
+--graft=*old_path*=*new_path*
+: a graft point *old_path*=*new_path* (can be used more than
+ once).
+
+ A directory */root/chroot/a/etc* saved with
+ "bup save -n chroots --graft /root/chroot/a/etc=/chroots/a"
+ would be saved as */chroots/a/etc*
# EXAMPLE
-
+
$ bup index -ux /etc
Indexing: 1981, done.
-
+
$ bup save -r myserver: -n my-pc-backup --bwlimit=50k /etc
Reading index: 1981, done.
- Saving: 100.00% (998/998k, 1981/1981 files), done.
-
+ Saving: 100.00% (998/998k, 1981/1981 files), done.
+
+
+
+ $ ls /home/joe/chroots/httpd
+ bin var
+
+ $ bup index -ux /home/joe/chroots/httpd
+ Indexing: 1337, done.
+
+ $ bup save --strip -n joes-httpd-chroot /home/joe/chroots/httpd
+ Reading index: 1337, done.
+ Saving: 100.00% (998/998k, 1337/1337 files), done.
+
+ $ bup ls joes-httpd-chroot/latest/
+ bin/
+ var/
+
+
+ $ bup save --strip-prefix=/home/joe/chroots -n joes-chroots \
+ /home/joe/chroots/httpd
+ Reading index: 1337, done.
+ Saving: 100.00% (998/998k, 1337/1337 files), done.
+
+ $ bup ls joes-chroots/latest/
+ httpd/
+
+
+ $ bup save --graft /home/joe/chroots/httpd=/http-chroot \
+ -n joe
+ /home/joe/chroots/httpd
+ Reading index: 1337, done.
+ Saving: 100.00% (998/998k, 1337/1337 files), done.
+
+ $ bup ls joe/latest/
+ http-chroot/
+
# SEE ALSO
There is normally no reason to run `bup server` yourself.
+# MODES
+
+smart
+: In this mode, the server checks each incoming object
+ against the idx files in its repository. If any object
+ already exists, it tells the client about the idx file
+ it was found in, allowing the client to download that
+ idx and avoid sending duplicate data.
+
+dumb
+: In this mode, the server will not check its local index
+ before writing an object. To avoid writing duplicate
+ objects, the server will tell the client to download all
+ of its .idx files at the start of the session. This
+ mode is useful on low powered server hardware (ie
+ router/slow NAS).
+
+# FILES
+
+$BUP_DIR/bup-dumb-server
+: Activate dumb server mode, as discussed above.
+
# SEE ALSO
`bup-save`(1), `bup-split`(1)
ret = 99
try:
- hostname = extra[0]
+ hp = extra[0].split(':')
+ if len(hp) == 1:
+ (hostname, port) = (hp[0], None)
+ else:
+ (hostname, port) = hp
+
argv = extra[1:]
- p = ssh.connect(hostname, 'on--server')
+ p = ssh.connect(hostname, port, 'on--server')
argvs = '\0'.join(['bup'] + argv)
p.stdin.write(struct.pack('!I', len(argvs)) + argvs)
f,indexfile= the name of the index file (normally BUP_DIR/bupindex)
strip strips the path to every filename given
strip-path= path-prefix to be stripped when saving
+graft= a graft point *old_path*=*new_path* (can be used morethan once)
"""
o = options.Options('bup save', optspec)
(opt, flags, extra) = o.parse(sys.argv[1:])
if opt.strip and opt.strip_path:
o.fatal("--strip is incompatible with --strip-path")
+graft_points = []
+if opt.graft:
+ if opt.strip:
+ o.fatal("--strip is incompatible with --graft")
+
+ if opt.strip_path:
+ o.fatal("--strip-path is incompatible with --graft")
+
+ for (option, parameter) in flags:
+ if option == "--graft":
+ splitted_parameter = parameter.split('=')
+ if len(splitted_parameter) != 2:
+ o.fatal("a graft point must be of the form old_path=new_path")
+ graft_points.append((realpath(splitted_parameter[0]),
+ realpath(splitted_parameter[1])))
+
is_reverse = os.environ.get('BUP_SERVER_REVERSE')
if is_reverse and opt.remote:
o.fatal("don't use -r in reverse mode; it's automatic")
dirp = stripped_base_path.split('/')
elif opt.strip_path:
dirp = strip_path(opt.strip_path, dir).split('/')
+ elif graft_points:
+ grafted = graft_path(graft_points, dir)
+ dirp = grafted.split('/')
else:
dirp = dir.split('/')
while parts > dirp:
#!/usr/bin/env python
-import sys, struct
+import os, sys, struct
from bup import options, git
from bup.helpers import *
suspended_w = None
+dumb_server_mode = False
+
+def _set_mode():
+ global dumb_server_mode
+ dumb_server_mode = os.path.exists(git.repo('bup-dumb-server'))
+ debug1('bup server: serving in %s mode\n'
+ % (dumb_server_mode and 'dumb' or 'smart'))
def init_dir(conn, arg):
git.init_repo(arg)
debug1('bup server: bupdir initialized: %r\n' % git.repodir)
+ _set_mode()
conn.ok()
def set_dir(conn, arg):
git.check_repo_or_die(arg)
debug1('bup server: bupdir is %r\n' % git.repodir)
+ _set_mode()
conn.ok()
def list_indexes(conn, junk):
git.check_repo_or_die()
+ suffix = ''
+ if dumb_server_mode:
+ suffix = ' load'
for f in os.listdir(git.repo('objects/pack')):
if f.endswith('.idx'):
- conn.write('%s\n' % f)
+ conn.write('%s%s\n' % (f, suffix))
conn.ok()
if not n:
debug1('bup server: received %d object%s.\n'
% (w.count, w.count!=1 and "s" or ''))
- fullpath = w.close()
+ fullpath = w.close(run_midx=not dumb_server_mode)
if fullpath:
(dir, name) = os.path.split(fullpath)
conn.write('%s.idx\n' % name)
w.abort()
raise Exception('object read: expected %d bytes, got %d\n'
% (n, len(buf)))
- oldpack = w.exists(shar)
+ if dumb_server_mode:
+ oldpack = None
+ else:
+ oldpack = w.exists(shar)
# FIXME: we only suggest a single index per cycle, because the client
# is currently too dumb to download more than one per cycle anyway.
# Actually we should fix the client, but this is a minor optimization
bwcount = len(sub) # might be less than 4096
bwtime = next
return (bwcount, bwtime)
-
+
+
+def parse_remote(remote):
+ protocol = r'([a-z]+)://'
+ host = r'(?P<sb>\[)?((?(sb)[0-9a-f:]+|[^:/]+))(?(sb)\])'
+ port = r'(?::(\d+))?'
+ path = r'(/.*)?'
+ url_match = re.match(
+ '%s(?:%s%s)?%s' % (protocol, host, port, path), remote, re.I)
+ if url_match:
+ assert(url_match.group(1) in ('ssh', 'bup', 'file'))
+ return url_match.group(1,3,4,5)
+ else:
+ rs = remote.split(':', 1)
+ if len(rs) == 1 or rs[0] in ('', '-'):
+ return 'file', None, None, rs[-1]
+ else:
+ return 'ssh', rs[0], None, rs[1]
+
class Client:
def __init__(self, remote, create=False):
- self._busy = self.conn = self.p = self.pout = self.pin = None
+ self._busy = self.conn = None
+ self.sock = self.p = self.pout = self.pin = None
is_reverse = os.environ.get('BUP_SERVER_REVERSE')
if is_reverse:
assert(not remote)
remote = '%s:' % is_reverse
- rs = remote.split(':', 1)
- if len(rs) == 1:
- (host, dir) = (None, remote)
- else:
- (host, dir) = rs
- (self.host, self.dir) = (host, dir)
+ (self.protocol, self.host, self.port, self.dir) = parse_remote(remote)
self.cachedir = git.repo('index-cache/%s'
% re.sub(r'[^@\w]', '_',
- "%s:%s" % (host, dir)))
+ "%s:%s" % (self.host, self.dir)))
if is_reverse:
self.pout = os.fdopen(3, 'rb')
self.pin = os.fdopen(4, 'wb')
else:
- try:
- self.p = ssh.connect(host, 'server')
- self.pout = self.p.stdout
- self.pin = self.p.stdin
- except OSError, e:
- raise ClientError, 'connect: %s' % e, sys.exc_info()[2]
+ if self.protocol in ('ssh', 'file'):
+ try:
+ # FIXME: ssh and file shouldn't use the same module
+ self.p = ssh.connect(self.host, self.port, 'server')
+ self.pout = self.p.stdout
+ self.pin = self.p.stdin
+ except OSError, e:
+ raise ClientError, 'connect: %s' % e, sys.exc_info()[2]
+ elif self.protocol == 'bup':
+ self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+ self.sock.connect((self.host, self.port or 1982))
+ self.pout = self.sock.makefile('rb')
+ self.pin = self.sock.makefile('wb')
self.conn = Conn(self.pout, self.pin)
- if dir:
- dir = re.sub(r'[\r\n]', ' ', dir)
+ if self.dir:
+ self.dir = re.sub(r'[\r\n]', ' ', self.dir)
if create:
- self.conn.write('init-dir %s\n' % dir)
+ self.conn.write('init-dir %s\n' % self.dir)
else:
- self.conn.write('set-dir %s\n' % dir)
+ self.conn.write('set-dir %s\n' % self.dir)
self.check_ok()
- self.sync_indexes_del()
+ self.sync_indexes()
def __del__(self):
try:
while self.pout.read(65536):
pass
self.pout.close()
+ if self.sock:
+ self.sock.close()
if self.p:
self.p.wait()
rv = self.p.wait()
if rv:
raise ClientError('server tunnel returned exit code %d' % rv)
self.conn = None
- self.p = self.pin = self.pout = None
+ self.sock = self.p = self.pin = self.pout = None
def check_ok(self):
if self.p:
def _not_busy(self):
self._busy = None
- def sync_indexes_del(self):
+ def sync_indexes(self):
self.check_busy()
conn = self.conn
+ mkdirp(self.cachedir)
+ # All cached idxs are extra until proven otherwise
+ extra = set()
+ for f in os.listdir(self.cachedir):
+ debug1('%s\n' % f)
+ if f.endswith('.idx'):
+ extra.add(f)
+ needed = set()
conn.write('list-indexes\n')
- packdir = git.repo('objects/pack')
- all = {}
- needed = {}
for line in linereader(conn):
if not line:
break
- all[line] = 1
assert(line.find('/') < 0)
- if not os.path.exists(os.path.join(self.cachedir, line)):
- needed[line] = 1
- self.check_ok()
+ parts = line.split(' ')
+ idx = parts[0]
+ if len(parts) == 2 and parts[1] == 'load' and idx not in extra:
+ # If the server requests that we load an idx and we don't
+ # already have a copy of it, it is needed
+ needed.add(idx)
+ # Any idx that the server has heard of is proven not extra
+ extra.discard(idx)
- mkdirp(self.cachedir)
- for f in os.listdir(self.cachedir):
- if f.endswith('.idx') and not f in all:
- debug1('client: pruning old index: %r\n' % f)
- os.unlink(os.path.join(self.cachedir, f))
+ self.check_ok()
+ debug1('client: removing extra indexes: %s\n' % extra)
+ for idx in extra:
+ os.unlink(os.path.join(self.cachedir, idx))
+ debug1('client: server requested load of: %s\n' % needed)
+ for idx in needed:
+ self.sync_index(idx)
def sync_index(self, name):
#debug1('requesting %r\n' % name)
f.close()
os.unlink(self.filename + '.pack')
- def _end(self):
+ def _end(self, run_midx=True):
f = self.file
if not f: return None
self.file = None
os.rename(self.filename + '.pack', nameprefix + '.pack')
os.rename(self.filename + '.idx', nameprefix + '.idx')
- auto_midx(repo('objects/pack'))
+ if run_midx:
+ auto_midx(repo('objects/pack'))
return nameprefix
- def close(self):
+ def close(self, run_midx=True):
"""Close the pack file and move it to its definitive path."""
- return self._end()
+ return self._end(run_midx=run_midx)
def _write_pack_idx_v2(self, file, idx, packbin):
sum = Sha1()
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
close_fds = True,
+ bufsize = 4096,
preexec_fn = _gitenv)
def _fast_get(self, id):
assert(not id.startswith('-'))
self.inprogress = id
self.p.stdin.write('%s\n' % id)
+ self.p.stdin.flush()
hdr = self.p.stdout.readline()
if hdr.endswith(' missing\n'):
self.inprogress = None
"""Helper functions and classes for bup."""
+
import sys, os, pwd, subprocess, errno, socket, select, mmap, stat, re
from bup import _version
def strip_base_path(path, base_paths):
"""Strips the base path from a given path.
+
Determines the base path for the given string and the strips it
using strip_path().
Iterates over all base_paths from long to short, to prevent that
return strip_path(bp, normalized_path)
return path
+def graft_path(graft_points, path):
+ normalized_path = os.path.realpath(path)
+ for graft_point in graft_points:
+ old_prefix, new_prefix = graft_point
+ if normalized_path.startswith(old_prefix):
+ return re.sub(r'^' + old_prefix, new_prefix, normalized_path)
+ return normalized_path
+
# hashlib is only available in python 2.5 or higher, but the 'sha' module
# produces a DeprecationWarning in python 2.6 or higher. We want to support
from bup import helpers
-def connect(rhost, subcmd):
+def connect(rhost, port, subcmd):
"""Connect to 'rhost' and execute the bup subcommand 'subcmd' on it."""
assert(not re.search(r'[^\w-]', subcmd))
main_exe = os.environ.get('BUP_MAIN_EXE') or sys.argv[0]
cmd = r"""
sh -c PATH=%s:'$PATH BUP_DEBUG=%s BUP_FORCE_TTY=%s bup %s'
""" % (escapedir, buglvl, force_tty, subcmd)
- argv = ['ssh', rhost, '--', cmd.strip()]
+ argv = ['ssh']
+ if port:
+ argv.extend(('-p', port))
+ argv.extend((rhost, '--', cmd.strip()))
#helpers.log('argv is: %r\n' % argv)
def setup():
# runs in the child process
rw.new_blob(s1)
+@wvtest
+def test_dumb_client_server():
+ os.environ['BUP_MAIN_EXE'] = '../../../bup'
+ os.environ['BUP_DIR'] = bupdir = 'buptest_tclient.tmp'
+ subprocess.call(['rm', '-rf', bupdir])
+ git.init_repo(bupdir)
+ os.mknod(git.repo('bup-dumb-server'))
+
+ lw = git.PackWriter()
+ lw.new_blob(s1)
+ lw.close()
+
+ c = client.Client(bupdir, create=True)
+ rw = c.new_packwriter()
+ WVPASSEQ(len(os.listdir(c.cachedir)), 1)
+ rw.new_blob(s1)
+ WVPASSEQ(len(os.listdir(c.cachedir)), 1)
+ rw.new_blob(s2)
+ rw.close()
+ WVPASSEQ(len(os.listdir(c.cachedir)), 2)
+
+
@wvtest
def test_midx_refreshing():
os.environ['BUP_MAIN_EXE'] = bupmain = '../../../bup'
WVPASSEQ(len(pi.packs), 2)
pi.refresh(skip_midx=False)
WVPASSEQ(len(pi.packs), 1)
+
+@wvtest
+def test_remote_parsing():
+ tests = (
+ (':/bup', ('file', None, None, '/bup')),
+ ('file:///bup', ('file', None, None, '/bup')),
+ ('192.168.1.1:/bup', ('ssh', '192.168.1.1', None, '/bup')),
+ ('ssh://192.168.1.1:2222/bup', ('ssh', '192.168.1.1', '2222', '/bup')),
+ ('ssh://[ff:fe::1]:2222/bup', ('ssh', 'ff:fe::1', '2222', '/bup')),
+ ('bup://foo.com:1950', ('bup', 'foo.com', '1950', None)),
+ ('bup://foo.com:1950/bup', ('bup', 'foo.com', '1950', '/bup')),
+ ('bup://[ff:fe::1]/bup', ('bup', 'ff:fe::1', None, '/bup')),
+ )
+ for remote, values in tests:
+ WVPASSEQ(client.parse_remote(remote), values)
+ try:
+ client.parse_remote('http://asdf.com/bup')
+ WVFAIL()
+ except AssertionError:
+ WVPASS()
r = w._write_pack_idx_v2(f, idx, pack_bin)
f.seek(0)
i = git.PackIdxV2(name, f)
- WVPASS(i.find_offset(obj_bin)==0xfffffffff)
- WVPASS(i.find_offset(obj2_bin)==0xffffffffff)
- WVPASS(i.find_offset(obj3_bin)==0xff)
+ WVPASSEQ(i.find_offset(obj_bin), 0xfffffffff)
+ WVPASSEQ(i.find_offset(obj2_bin), 0xffffffffff)
+ WVPASSEQ(i.find_offset(obj3_bin), 0xff)
f.close()
os.remove(name)
WVPASSEQ(result, "/a")
+@wvtest
+def test_graft_path():
+ middle_matching_old_path = "/user"
+ non_matching_old_path = "/usr"
+ matching_old_path = "/home"
+ matching_full_path = "/home/user"
+ new_path = "/opt"
+
+ all_graft_points = [(middle_matching_old_path, new_path),
+ (non_matching_old_path, new_path),
+ (matching_old_path, new_path)]
+
+ path = "/home/user/"
+
+ WVPASSEQ(graft_path([(middle_matching_old_path, new_path)], path),
+ "/home/user")
+ WVPASSEQ(graft_path([(non_matching_old_path, new_path)], path),
+ "/home/user")
+ WVPASSEQ(graft_path([(matching_old_path, new_path)], path), "/opt/user")
+ WVPASSEQ(graft_path(all_graft_points, path), "/opt/user")
+ WVPASSEQ(graft_path([(matching_full_path, new_path)], path),
+ "/opt")
d/
f"
+WVSTART "graft_points"
+D=graft-points.tmp
+rm -rf $D
+mkdir $D
+export BUP_DIR="$D/.bup"
+WVPASS bup init
+touch $D/a
+WVPASS bup random 128k >$D/b
+mkdir $D/d $D/d/e
+WVPASS bup random 512 >$D/f
+WVPASS bup index -ux $D
+bup save --graft $TOP/$D=/grafted -n graft-point-absolute $D
+WVPASSEQ "$(bup ls graft-point-absolute/latest/grafted/)" "a
+b
+d/
+f"
+bup save --graft $D=grafted -n graft-point-relative $D
+WVPASSEQ "$(bup ls graft-point-relative/latest/$TOP/grafted/)" "a
+b
+d/
+f"
+
WVSTART "indexfile"
D=indexfile.tmp
INDEXFILE=tmpindexfile.tmp
projects / bup.git / commitdiff