2 from __future__ import absolute_import, print_function
3 from binascii import hexlify
4 from errno import EACCES
6 import math, os, stat, sys, time
8 from bup import compat, hashsplit, git, options, index, client, metadata
9 from bup import hlinkdb
10 from bup.compat import argv_bytes, environ, nullcontext
11 from bup.hashsplit import GIT_MODE_TREE, GIT_MODE_FILE, GIT_MODE_SYMLINK
12 from bup.helpers import (add_error, grafted_path_components, handle_ctrl_c,
13 hostname, istty2, log, parse_date_or_fatal, parse_num,
14 path_components, progress, qprogress, resolve_parent,
15 saved_errors, stripped_path_components,
17 from bup.io import byte_stream, path_msg
18 from bup.pwdgrp import userfullname, username
19 from bup.tree import StackDir
23 bup save [-tc] [-n name] <filenames...>
25 r,remote= hostname:/path/to/repo of remote repository
26 t,tree output a tree id
27 c,commit output a commit id
28 n,name= name of backup set to update (if any)
29 d,date= date for the commit (seconds since the epoch)
30 v,verbose increase log output (can be used more than once)
31 q,quiet don't show progress meter
32 smaller= only back up files smaller than n bytes
33 bwlimit= maximum bytes/sec to transmit to server
34 f,indexfile= the name of the index file (normally BUP_DIR/bupindex)
35 strip strips the path to every filename given
36 strip-path= path-prefix to be stripped when saving
37 graft= a graft point *old_path*=*new_path* (can be used more than once)
38 #,compress= set compression level to # (0-9, 9 is highest) [1]
44 after_nondir_metadata_stat = None
46 def before_saving_regular_file(name):
50 def opts_from_cmdline(argv):
51 o = options.Options(optspec)
52 opt, flags, extra = o.parse_bytes(argv[1:])
55 opt.indexfile = argv_bytes(opt.indexfile)
57 opt.name = argv_bytes(opt.name)
59 opt.remote = argv_bytes(opt.remote)
61 opt.strip_path = argv_bytes(opt.strip_path)
62 if not (opt.tree or opt.commit or opt.name):
63 o.fatal("use one or more of -t, -c, -n")
65 o.fatal("no filenames given")
67 opt.date = parse_date_or_fatal(opt.date, o.fatal)
69 opt.date = time.time()
71 opt.progress = (istty2 and not opt.quiet)
72 opt.smaller = parse_num(opt.smaller or 0)
75 opt.bwlimit = parse_num(opt.bwlimit)
77 if opt.strip and opt.strip_path:
78 o.fatal("--strip is incompatible with --strip-path")
80 opt.sources = [argv_bytes(x) for x in extra]
85 o.fatal("--strip is incompatible with --graft")
88 o.fatal("--strip-path is incompatible with --graft")
90 for (option, parameter) in flags:
91 if option == "--graft":
92 parameter = argv_bytes(parameter)
93 splitted_parameter = parameter.split(b'=')
94 if len(splitted_parameter) != 2:
95 o.fatal("a graft point must be of the form old_path=new_path")
96 old_path, new_path = splitted_parameter
97 if not (old_path and new_path):
98 o.fatal("a graft point cannot be empty")
99 grafts.append((resolve_parent(old_path),
100 resolve_parent(new_path)))
103 opt.is_reverse = environ.get(b'BUP_SERVER_REVERSE')
104 if opt.is_reverse and opt.remote:
105 o.fatal("don't use -r in reverse mode; it's automatic")
107 if opt.name and not valid_save_name(opt.name):
108 o.fatal("'%s' is not a valid branch name" % path_msg(opt.name))
112 def save_tree(opt, reader, hlink_db, msr, w):
113 # Metadata is stored in a file named .bupm in each directory. The
114 # first metadata entry will be the metadata for the current directory.
115 # The remaining entries will be for each of the other directory
116 # elements, in the order they're listed in the index.
118 # Since the git tree elements are sorted according to
119 # git.shalist_item_sort_key, the metalist items are accumulated as
120 # (sort_key, metadata) tuples, and then sorted when the .bupm file is
121 # created. The sort_key should have been computed using the element's
122 # mangled name and git mode (after hashsplitting), but the code isn't
123 # actually doing that but rather uses the element's real name and mode.
124 # This makes things a bit more difficult when reading it back, see
125 # vfs.ordered_tree_entries().
127 # Maintain a stack of information representing the current location in
128 # the archive being constructed. The current path is recorded in
129 # parts, which will be something like
130 # [StackDir(name=''), StackDir(name='home'), StackDir(name='someuser')],
131 # and the accumulated content and metadata for files in the dirs is stored
132 # in the .items member of the StackDir.
136 def _push(part, metadata):
137 # Enter a new archive directory -- make it the current directory.
138 item = StackDir(part, metadata)
142 def _pop(force_tree=None, dir_metadata=None):
143 # Leave the current archive directory and add its tree to its parent.
145 # FIXME: only test if collision is possible (i.e. given --strip, etc.)?
153 if name in names_seen:
154 parent_path = b'/'.join(x.name for x in stack) + b'/'
155 add_error('error: ignoring duplicate path %s in %s'
156 % (path_msg(name), path_msg(parent_path)))
161 # if set, overrides the original metadata pushed for this dir.
162 if dir_metadata is None:
163 dir_metadata = item.meta
164 metalist = [(b'', dir_metadata)]
165 metalist += [(git.shalist_item_sort_key((entry.mode, entry.name, None)),
167 for entry in clean_list if entry.mode != GIT_MODE_TREE]
168 metalist.sort(key = lambda x: x[0])
169 metadata = BytesIO(b''.join(m[1].encode() for m in metalist))
170 mode, id = hashsplit.split_to_blob_or_tree(w.new_blob, w.new_tree,
172 keep_boundaries=False)
173 shalist = [(mode, b'.bupm', id)]
174 shalist += [(entry.gitmode,
175 git.mangle_name(entry.name, entry.mode, entry.gitmode),
177 for entry in clean_list]
179 tree = w.new_tree(shalist)
181 stack[-1].append(item.name, GIT_MODE_TREE, GIT_MODE_TREE, tree, None)
185 # Hack around lack of nonlocal vars in python 2
187 _nonlocal['count'] = 0
188 _nonlocal['subcount'] = 0
189 _nonlocal['lastremain'] = None
191 def progress_report(n):
192 _nonlocal['subcount'] += n
193 cc = _nonlocal['count'] + _nonlocal['subcount']
194 pct = total and (cc*100.0/total) or 0
196 elapsed = now - tstart
197 kps = elapsed and int(cc/1024./elapsed)
198 kps_frac = 10 ** int(math.log(kps+1, 10) - 1)
199 kps = int(kps/kps_frac)*kps_frac
201 remain = elapsed*1.0/cc * (total-cc)
204 if (_nonlocal['lastremain'] and (remain > _nonlocal['lastremain'])
205 and ((remain - _nonlocal['lastremain'])/_nonlocal['lastremain'] < 0.05)):
206 remain = _nonlocal['lastremain']
208 _nonlocal['lastremain'] = remain
209 hours = int(remain/60/60)
210 mins = int(remain/60 - hours*60)
211 secs = int(remain - hours*60*60 - mins*60)
216 kpsstr = '%dk/s' % kps
218 remainstr = '%dh%dm' % (hours, mins)
220 remainstr = '%dm%d' % (mins, secs)
222 remainstr = '%ds' % secs
223 qprogress('Saving: %.2f%% (%d/%dk, %d/%d files) %s %s\r'
224 % (pct, cc/1024, total/1024, fcount, ftotal,
228 def already_saved(ent):
229 return ent.is_valid() and w.exists(ent.sha) and ent.sha
231 def wantrecurse_pre(ent):
232 return not already_saved(ent)
234 def wantrecurse_during(ent):
235 return not already_saved(ent) or ent.sha_missing()
237 def find_hardlink_target(hlink_db, ent):
238 if hlink_db and not stat.S_ISDIR(ent.mode) and ent.nlink > 1:
239 link_paths = hlink_db.node_paths(ent.dev, ent.ino)
246 for transname, ent in reader.filter(opt.sources,
247 wantrecurse=wantrecurse_pre):
248 if not (ftotal % 10024):
249 qprogress('Reading index: %d\r' % ftotal)
250 exists = ent.exists()
251 hashvalid = already_saved(ent)
252 ent.set_sha_missing(not hashvalid)
253 if not opt.smaller or ent.size < opt.smaller:
254 if exists and not hashvalid:
257 progress('Reading index: %d, done.\n' % ftotal)
258 hashsplit.progress_callback = progress_report
260 # Root collisions occur when strip or graft options map more than one
261 # path to the same directory (paths which originally had separate
262 # parents). When that situation is detected, use empty metadata for
263 # the parent. Otherwise, use the metadata for the common parent.
264 # Collision example: "bup save ... --strip /foo /foo/bar /bar".
266 # FIXME: Add collision tests, or handle collisions some other way.
268 # FIXME: Detect/handle strip/graft name collisions (other than root),
269 # i.e. if '/foo/bar' and '/bar' both map to '/'.
272 root_collision = None
277 for transname, ent in reader.filter(opt.sources,
278 wantrecurse=wantrecurse_during):
279 (dir, file) = os.path.split(ent.name)
280 exists = (ent.flags & index.IX_EXISTS)
281 hashvalid = already_saved(ent)
282 wasmissing = ent.sha_missing()
288 if ent.sha == index.EMPTY_SHA:
295 log('%s %-70s\n' % (status, path_msg(ent.name)))
296 elif not stat.S_ISDIR(ent.mode) and lastdir != dir:
297 if not lastdir.startswith(dir):
298 log('%s %-70s\n' % (status, path_msg(os.path.join(dir, b''))))
307 if opt.smaller and ent.size >= opt.smaller:
308 if exists and not hashvalid:
310 log('skipping large file "%s"\n' % path_msg(ent.name))
311 lastskip_name = ent.name
314 assert(dir.startswith(b'/'))
316 dirp = stripped_path_components(dir, opt.sources)
318 dirp = stripped_path_components(dir, [opt.strip_path])
320 dirp = grafted_path_components(opt.grafts, dir)
322 dirp = path_components(dir)
324 # At this point, dirp contains a representation of the archive
325 # path that looks like [(archive_dir_name, real_fs_path), ...].
326 # So given "bup save ... --strip /foo/bar /foo/bar/baz", dirp
327 # might look like this at some point:
328 # [('', '/foo/bar'), ('baz', '/foo/bar/baz'), ...].
330 # This dual representation supports stripping/grafting, where the
331 # archive path may not have a direct correspondence with the
332 # filesystem. The root directory is represented by an initial
333 # component named '', and any component that doesn't have a
334 # corresponding filesystem directory (due to grafting, for
335 # example) will have a real_fs_path of None, i.e. [('', None),
338 if first_root == None:
340 elif first_root != dirp[0]:
341 root_collision = True
343 # If switching to a new sub-tree, finish the current sub-tree.
344 while [x.name for x in stack] > [x[0] for x in dirp]:
347 # If switching to a new sub-tree, start a new sub-tree.
348 for path_component in dirp[len(stack):]:
349 dir_name, fs_path = path_component
350 # Not indexed, so just grab the FS metadata or use empty metadata.
352 meta = metadata.from_path(fs_path, normalized=True) \
353 if fs_path else metadata.Metadata()
354 except (OSError, IOError) as e:
356 lastskip_name = dir_name
357 meta = metadata.Metadata()
358 _push(dir_name, meta)
362 continue # We're at the top level -- keep the current root dir
363 # Since there's no filename, this is a subdir -- finish it.
364 oldtree = already_saved(ent) # may be None
365 newtree = _pop(force_tree = oldtree)
367 if lastskip_name and lastskip_name.startswith(ent.name):
370 ent.validate(GIT_MODE_TREE, newtree)
372 if exists and wasmissing:
373 _nonlocal['count'] += oldsize
376 # it's not a directory
378 meta = msr.metadata_at(ent.meta_ofs)
379 meta.hardlink_target = find_hardlink_target(hlink_db, ent)
380 # Restore the times that were cleared to 0 in the metastore.
381 (meta.atime, meta.mtime, meta.ctime) = (ent.atime, ent.mtime, ent.ctime)
382 stack[-1].append(file, ent.mode, ent.gitmode, ent.sha, meta)
385 hlink = find_hardlink_target(hlink_db, ent)
387 meta = metadata.from_path(ent.name, hardlink_target=hlink,
389 after_stat=after_nondir_metadata_stat)
390 except (OSError, IOError) as e:
392 lastskip_name = ent.name
394 if stat.S_IFMT(ent.mode) != stat.S_IFMT(meta.mode):
395 # The mode changed since we indexed the file, this is bad.
396 # This can cause two issues:
397 # 1) We e.g. think the file is a regular file, but now it's
398 # something else (a device, socket, FIFO or symlink, etc.)
399 # and _read_ from it when we shouldn't.
400 # 2) We then record it as valid, but don't update the index
401 # metadata, and on a subsequent save it has 'hashvalid'
402 # but is recorded as the file type from the index, when
403 # the content is something else ...
404 # Avoid all of these consistency issues by just skipping such
405 # things - it really ought to not happen anyway.
406 add_error("%s: mode changed since indexing, skipping." % path_msg(ent.name))
407 lastskip_name = ent.name
409 if stat.S_ISREG(ent.mode):
411 # If the file changes while we're reading it, then our reading
412 # may stop at some point, but the stat() above may have gotten
413 # a different size already. Recalculate the meta size so that
414 # the repository records the accurate size in the metadata, even
415 # if the other stat() data might be slightly older than the file
416 # content (which we can't fix, this is inherently racy, but we
417 # can prevent the size mismatch.)
420 meta.size += len(data)
421 return w.new_blob(data)
422 before_saving_regular_file(ent.name)
423 with hashsplit.open_noatime(ent.name) as f:
424 (mode, id) = hashsplit.split_to_blob_or_tree(
425 new_blob, w.new_tree, [f],
426 keep_boundaries=False)
427 except (IOError, OSError) as e:
428 add_error('%s: %s' % (ent.name, e))
429 lastskip_name = ent.name
430 elif stat.S_ISDIR(ent.mode):
431 assert(0) # handled above
432 elif stat.S_ISLNK(ent.mode):
433 mode, id = (GIT_MODE_SYMLINK, w.new_blob(meta.symlink_target))
435 # Everything else should be fully described by its
436 # metadata, so just record an empty blob, so the paths
437 # in the tree and .bupm will match up.
438 (mode, id) = (GIT_MODE_FILE, w.new_blob(b''))
441 ent.validate(mode, id)
443 stack[-1].append(file, ent.mode, ent.gitmode, id, meta)
445 if exists and wasmissing:
446 _nonlocal['count'] += oldsize
447 _nonlocal['subcount'] = 0
451 pct = total and _nonlocal['count']*100.0/total or 100
452 progress('Saving: %.2f%% (%d/%dk, %d/%d files), done. \n'
453 % (pct, _nonlocal['count']/1024, total/1024, fcount, ftotal))
455 while len(stack) > 1: # _pop() all the parts above the root
458 # Finish the root directory.
459 # When there's a collision, use empty metadata for the root.
460 tree = _pop(dir_metadata = metadata.Metadata() if root_collision else None)
465 def commit_tree(tree, parent, date, argv, writer):
466 if compat.py_maj > 2:
467 # Strip b prefix from python 3 bytes reprs to preserve previous format
468 msgcmd = b'[%s]' % b', '.join([repr(argv_bytes(x))[1:].encode('ascii')
472 msg = b'bup save\n\nGenerated by command:\n%s\n' % msgcmd
473 userline = (b'%s <%s@%s>' % (userfullname(), username(), hostname()))
474 return writer.new_commit(tree, parent, userline, date, None,
475 userline, date, None, msg)
480 opt = opts_from_cmdline(argv)
481 client.bwlimit = opt.bwlimit
482 git.check_repo_or_die()
484 remote_dest = opt.remote or opt.is_reverse
490 cli = repo = client.Client(opt.remote)
491 except client.ClientError as e:
495 # cli creation must be last nontrivial command in each if clause above
498 w = git.PackWriter(compression_level=opt.compress)
500 w = cli.new_packwriter(compression_level=opt.compress)
504 out = byte_stream(sys.stdout)
507 refname = b'refs/heads/%s' % opt.name
508 parent = repo.read_ref(refname)
510 refname = parent = None
512 indexfile = opt.indexfile or git.repo(b'bupindex')
514 msr = index.MetaStoreReader(indexfile + b'.meta')
515 except IOError as ex:
516 if ex.errno != EACCES:
518 log('error: cannot access %r; have you run bup index?'
519 % path_msg(indexfile))
522 hlinkdb.HLinkDB(indexfile + b'.hlink') as hlink_db, \
523 index.Reader(indexfile) as reader:
524 tree = save_tree(opt, reader, hlink_db, msr, w)
526 out.write(hexlify(tree))
528 if opt.commit or opt.name:
529 commit = commit_tree(tree, parent, opt.date, argv, w)
531 out.write(hexlify(commit))
534 # packwriter must be closed before we can update the ref
536 repo.update_ref(refname, commit, parent)
539 log('WARNING: %d errors encountered while saving.\n' % len(saved_errors))