3 # https://sourceware.org/bugzilla/show_bug.cgi?id=26034
4 export "BUP_ARGV_0"="$0"
7 export "BUP_ARGV_${arg_i}"="$arg"
11 # Here to end of preamble replaced during install
12 bup_python="$(dirname "$0")/../../config/bin/python" || exit $?
13 exec "$bup_python" "$0"
17 from __future__ import absolute_import, print_function
18 from binascii import hexlify
19 from errno import EACCES
20 from io import BytesIO
21 import math, os, stat, sys, time
23 sys.path[:0] = [os.path.dirname(os.path.realpath(__file__)) + '/..']
25 from bup import compat, hashsplit, git, options, index, client, metadata
26 from bup import hlinkdb
27 from bup.compat import argv_bytes, environ
28 from bup.hashsplit import GIT_MODE_TREE, GIT_MODE_FILE, GIT_MODE_SYMLINK
29 from bup.helpers import (add_error, grafted_path_components, handle_ctrl_c,
30 hostname, istty2, log, parse_date_or_fatal, parse_num,
31 path_components, progress, qprogress, resolve_parent,
32 saved_errors, stripped_path_components,
34 from bup.io import byte_stream, path_msg
35 from bup.pwdgrp import userfullname, username
39 bup save [-tc] [-n name] <filenames...>
41 r,remote= hostname:/path/to/repo of remote repository
42 t,tree output a tree id
43 c,commit output a commit id
44 n,name= name of backup set to update (if any)
45 d,date= date for the commit (seconds since the epoch)
46 v,verbose increase log output (can be used more than once)
47 q,quiet don't show progress meter
48 smaller= only back up files smaller than n bytes
49 bwlimit= maximum bytes/sec to transmit to server
50 f,indexfile= the name of the index file (normally BUP_DIR/bupindex)
51 strip strips the path to every filename given
52 strip-path= path-prefix to be stripped when saving
53 graft= a graft point *old_path*=*new_path* (can be used more than once)
54 #,compress= set compression level to # (0-9, 9 is highest) [1]
56 o = options.Options(optspec)
57 opt, flags, extra = o.parse(compat.argv[1:])
60 opt.indexfile = argv_bytes(opt.indexfile)
62 opt.name = argv_bytes(opt.name)
64 opt.remote = argv_bytes(opt.remote)
66 opt.strip_path = argv_bytes(opt.strip_path)
68 git.check_repo_or_die()
69 if not (opt.tree or opt.commit or opt.name):
70 o.fatal("use one or more of -t, -c, -n")
72 o.fatal("no filenames given")
74 extra = [argv_bytes(x) for x in extra]
76 opt.progress = (istty2 and not opt.quiet)
77 opt.smaller = parse_num(opt.smaller or 0)
79 client.bwlimit = parse_num(opt.bwlimit)
82 date = parse_date_or_fatal(opt.date, o.fatal)
86 if opt.strip and opt.strip_path:
87 o.fatal("--strip is incompatible with --strip-path")
92 o.fatal("--strip is incompatible with --graft")
95 o.fatal("--strip-path is incompatible with --graft")
97 for (option, parameter) in flags:
98 if option == "--graft":
99 parameter = argv_bytes(parameter)
100 splitted_parameter = parameter.split(b'=')
101 if len(splitted_parameter) != 2:
102 o.fatal("a graft point must be of the form old_path=new_path")
103 old_path, new_path = splitted_parameter
104 if not (old_path and new_path):
105 o.fatal("a graft point cannot be empty")
106 graft_points.append((resolve_parent(old_path),
107 resolve_parent(new_path)))
109 is_reverse = environ.get(b'BUP_SERVER_REVERSE')
110 if is_reverse and opt.remote:
111 o.fatal("don't use -r in reverse mode; it's automatic")
114 if name and not valid_save_name(name):
115 o.fatal("'%s' is not a valid branch name" % path_msg(name))
116 refname = name and b'refs/heads/%s' % name or None
117 if opt.remote or is_reverse:
119 cli = client.Client(opt.remote)
120 except client.ClientError as e:
123 oldref = refname and cli.read_ref(refname) or None
124 w = cli.new_packwriter(compression_level=opt.compress)
127 oldref = refname and git.read_ref(refname) or None
128 w = git.PackWriter(compression_level=opt.compress)
133 # Metadata is stored in a file named .bupm in each directory. The
134 # first metadata entry will be the metadata for the current directory.
135 # The remaining entries will be for each of the other directory
136 # elements, in the order they're listed in the index.
138 # Since the git tree elements are sorted according to
139 # git.shalist_item_sort_key, the metalist items are accumulated as
140 # (sort_key, metadata) tuples, and then sorted when the .bupm file is
141 # created. The sort_key should have been computed using the element's
142 # mangled name and git mode (after hashsplitting), but the code isn't
143 # actually doing that but rather uses the element's real name and mode.
144 # This makes things a bit more difficult when reading it back, see
145 # vfs.ordered_tree_entries().
147 # Maintain a stack of information representing the current location in
148 # the archive being constructed. The current path is recorded in
149 # parts, which will be something like ['', 'home', 'someuser'], and
150 # the accumulated content and metadata for of the dirs in parts is
151 # stored in parallel stacks in shalists and metalists.
153 parts = [] # Current archive position (stack of dir names).
154 shalists = [] # Hashes for each dir in paths.
155 metalists = [] # Metadata for each dir in paths.
158 def _push(part, metadata):
159 # Enter a new archive directory -- make it the current directory.
162 metalists.append([(b'', metadata)]) # This dir's metadata (no name).
165 def _pop(force_tree, dir_metadata=None):
166 # Leave the current archive directory and add its tree to its parent.
167 assert(len(parts) >= 1)
169 shalist = shalists.pop()
170 metalist = metalists.pop()
171 # FIXME: only test if collision is possible (i.e. given --strip, etc.)?
177 metaidx = 1 # entry at 0 is for the dir
180 if name in names_seen:
181 parent_path = b'/'.join(parts) + b'/'
182 add_error('error: ignoring duplicate path %s in %s'
183 % (path_msg(name), path_msg(parent_path)))
184 if not stat.S_ISDIR(x[0]):
185 del metalist[metaidx]
189 if not stat.S_ISDIR(x[0]):
192 if dir_metadata: # Override the original metadata pushed for this dir.
193 metalist = [(b'', dir_metadata)] + metalist[1:]
194 sorted_metalist = sorted(metalist, key = lambda x : x[0])
195 metadata = b''.join([m[1].encode() for m in sorted_metalist])
196 metadata_f = BytesIO(metadata)
197 mode, id = hashsplit.split_to_blob_or_tree(w.new_blob, w.new_tree,
199 keep_boundaries=False)
200 clean_list.append((mode, b'.bupm', id))
202 tree = w.new_tree(clean_list)
204 shalists[-1].append((GIT_MODE_TREE,
205 git.mangle_name(part,
206 GIT_MODE_TREE, GIT_MODE_TREE),
212 def progress_report(n):
213 global count, subcount, lastremain
215 cc = count + subcount
216 pct = total and (cc*100.0/total) or 0
218 elapsed = now - tstart
219 kps = elapsed and int(cc/1024./elapsed)
220 kps_frac = 10 ** int(math.log(kps+1, 10) - 1)
221 kps = int(kps/kps_frac)*kps_frac
223 remain = elapsed*1.0/cc * (total-cc)
226 if (lastremain and (remain > lastremain)
227 and ((remain - lastremain)/lastremain < 0.05)):
231 hours = int(remain/60/60)
232 mins = int(remain/60 - hours*60)
233 secs = int(remain - hours*60*60 - mins*60)
238 kpsstr = '%dk/s' % kps
240 remainstr = '%dh%dm' % (hours, mins)
242 remainstr = '%dm%d' % (mins, secs)
244 remainstr = '%ds' % secs
245 qprogress('Saving: %.2f%% (%d/%dk, %d/%d files) %s %s\r'
246 % (pct, cc/1024, total/1024, fcount, ftotal,
250 indexfile = opt.indexfile or git.repo(b'bupindex')
251 r = index.Reader(indexfile)
253 msr = index.MetaStoreReader(indexfile + b'.meta')
254 except IOError as ex:
255 if ex.errno != EACCES:
257 log('error: cannot access %r; have you run bup index?'
258 % path_msg(indexfile))
260 hlink_db = hlinkdb.HLinkDB(indexfile + b'.hlink')
262 def already_saved(ent):
263 return ent.is_valid() and w.exists(ent.sha) and ent.sha
265 def wantrecurse_pre(ent):
266 return not already_saved(ent)
268 def wantrecurse_during(ent):
269 return not already_saved(ent) or ent.sha_missing()
271 def find_hardlink_target(hlink_db, ent):
272 if hlink_db and not stat.S_ISDIR(ent.mode) and ent.nlink > 1:
273 link_paths = hlink_db.node_paths(ent.dev, ent.ino)
279 for (transname,ent) in r.filter(extra, wantrecurse=wantrecurse_pre):
280 if not (ftotal % 10024):
281 qprogress('Reading index: %d\r' % ftotal)
282 exists = ent.exists()
283 hashvalid = already_saved(ent)
284 ent.set_sha_missing(not hashvalid)
285 if not opt.smaller or ent.size < opt.smaller:
286 if exists and not hashvalid:
289 progress('Reading index: %d, done.\n' % ftotal)
290 hashsplit.progress_callback = progress_report
292 # Root collisions occur when strip or graft options map more than one
293 # path to the same directory (paths which originally had separate
294 # parents). When that situation is detected, use empty metadata for
295 # the parent. Otherwise, use the metadata for the common parent.
296 # Collision example: "bup save ... --strip /foo /foo/bar /bar".
298 # FIXME: Add collision tests, or handle collisions some other way.
300 # FIXME: Detect/handle strip/graft name collisions (other than root),
301 # i.e. if '/foo/bar' and '/bar' both map to '/'.
304 root_collision = None
306 count = subcount = fcount = 0
309 for (transname,ent) in r.filter(extra, wantrecurse=wantrecurse_during):
310 (dir, file) = os.path.split(ent.name)
311 exists = (ent.flags & index.IX_EXISTS)
312 hashvalid = already_saved(ent)
313 wasmissing = ent.sha_missing()
319 if ent.sha == index.EMPTY_SHA:
326 log('%s %-70s\n' % (status, path_msg(ent.name)))
327 elif not stat.S_ISDIR(ent.mode) and lastdir != dir:
328 if not lastdir.startswith(dir):
329 log('%s %-70s\n' % (status, path_msg(os.path.join(dir, b''))))
338 if opt.smaller and ent.size >= opt.smaller:
339 if exists and not hashvalid:
341 log('skipping large file "%s"\n' % path_msg(ent.name))
342 lastskip_name = ent.name
345 assert(dir.startswith(b'/'))
347 dirp = stripped_path_components(dir, extra)
349 dirp = stripped_path_components(dir, [opt.strip_path])
351 dirp = grafted_path_components(graft_points, dir)
353 dirp = path_components(dir)
355 # At this point, dirp contains a representation of the archive
356 # path that looks like [(archive_dir_name, real_fs_path), ...].
357 # So given "bup save ... --strip /foo/bar /foo/bar/baz", dirp
358 # might look like this at some point:
359 # [('', '/foo/bar'), ('baz', '/foo/bar/baz'), ...].
361 # This dual representation supports stripping/grafting, where the
362 # archive path may not have a direct correspondence with the
363 # filesystem. The root directory is represented by an initial
364 # component named '', and any component that doesn't have a
365 # corresponding filesystem directory (due to grafting, for
366 # example) will have a real_fs_path of None, i.e. [('', None),
369 if first_root == None:
371 elif first_root != dirp[0]:
372 root_collision = True
374 # If switching to a new sub-tree, finish the current sub-tree.
375 while parts > [x[0] for x in dirp]:
376 _pop(force_tree = None)
378 # If switching to a new sub-tree, start a new sub-tree.
379 for path_component in dirp[len(parts):]:
380 dir_name, fs_path = path_component
381 # Not indexed, so just grab the FS metadata or use empty metadata.
383 meta = metadata.from_path(fs_path, normalized=True) \
384 if fs_path else metadata.Metadata()
385 except (OSError, IOError) as e:
387 lastskip_name = dir_name
388 meta = metadata.Metadata()
389 _push(dir_name, meta)
393 continue # We're at the top level -- keep the current root dir
394 # Since there's no filename, this is a subdir -- finish it.
395 oldtree = already_saved(ent) # may be None
396 newtree = _pop(force_tree = oldtree)
398 if lastskip_name and lastskip_name.startswith(ent.name):
401 ent.validate(GIT_MODE_TREE, newtree)
403 if exists and wasmissing:
407 # it's not a directory
410 git_name = git.mangle_name(file, ent.mode, ent.gitmode)
411 git_info = (ent.gitmode, git_name, id)
412 shalists[-1].append(git_info)
413 sort_key = git.shalist_item_sort_key((ent.mode, file, id))
414 meta = msr.metadata_at(ent.meta_ofs)
415 meta.hardlink_target = find_hardlink_target(hlink_db, ent)
416 # Restore the times that were cleared to 0 in the metastore.
417 (meta.atime, meta.mtime, meta.ctime) = (ent.atime, ent.mtime, ent.ctime)
418 metalists[-1].append((sort_key, meta))
421 if stat.S_ISREG(ent.mode):
423 with hashsplit.open_noatime(ent.name) as f:
424 (mode, id) = hashsplit.split_to_blob_or_tree(
425 w.new_blob, w.new_tree, [f],
426 keep_boundaries=False)
427 except (IOError, OSError) as e:
428 add_error('%s: %s' % (ent.name, e))
429 lastskip_name = ent.name
430 elif stat.S_ISDIR(ent.mode):
431 assert(0) # handled above
432 elif stat.S_ISLNK(ent.mode):
434 rl = os.readlink(ent.name)
435 except (OSError, IOError) as e:
437 lastskip_name = ent.name
439 (mode, id) = (GIT_MODE_SYMLINK, w.new_blob(rl))
441 # Everything else should be fully described by its
442 # metadata, so just record an empty blob, so the paths
443 # in the tree and .bupm will match up.
444 (mode, id) = (GIT_MODE_FILE, w.new_blob(b''))
447 ent.validate(mode, id)
449 git_name = git.mangle_name(file, ent.mode, ent.gitmode)
450 git_info = (mode, git_name, id)
451 shalists[-1].append(git_info)
452 sort_key = git.shalist_item_sort_key((ent.mode, file, id))
453 hlink = find_hardlink_target(hlink_db, ent)
455 meta = metadata.from_path(ent.name, hardlink_target=hlink,
457 except (OSError, IOError) as e:
459 lastskip_name = ent.name
460 meta = metadata.Metadata()
461 metalists[-1].append((sort_key, meta))
463 if exists and wasmissing:
469 pct = total and count*100.0/total or 100
470 progress('Saving: %.2f%% (%d/%dk, %d/%d files), done. \n'
471 % (pct, count/1024, total/1024, fcount, ftotal))
473 while len(parts) > 1: # _pop() all the parts above the root
474 _pop(force_tree = None)
475 assert(len(shalists) == 1)
476 assert(len(metalists) == 1)
478 # Finish the root directory.
479 tree = _pop(force_tree = None,
480 # When there's a collision, use empty metadata for the root.
481 dir_metadata = metadata.Metadata() if root_collision else None)
484 out = byte_stream(sys.stdout)
487 out.write(hexlify(tree))
489 if opt.commit or name:
490 msg = (b'bup save\n\nGenerated by command:\n%r\n'
491 % [argv_bytes(x) for x in sys.argv])
492 userline = (b'%s <%s@%s>' % (userfullname(), username(), hostname()))
493 commit = w.new_commit(tree, oldref, userline, date, None,
494 userline, date, None, msg)
496 out.write(hexlify(commit))
500 w.close() # must close before we can update the ref
504 cli.update_ref(refname, commit, oldref)
506 git.update_ref(refname, commit, oldref)
512 log('WARNING: %d errors encountered while saving.\n' % len(saved_errors))