3 bup_python="$(dirname "$0")/bup-python" || exit $?
4 exec "$bup_python" "$0" ${1+"$@"}
8 from __future__ import absolute_import, print_function
9 from binascii import hexlify
10 import sys, stat, time, os, errno, re
12 from bup import metadata, options, git, index, drecurse, hlinkdb
13 from bup.compat import argv_bytes
14 from bup.drecurse import recursive_dirlist
15 from bup.hashsplit import GIT_MODE_TREE, GIT_MODE_FILE
16 from bup.helpers import (add_error, handle_ctrl_c, log, parse_excludes, parse_rx_excludes,
17 progress, qprogress, saved_errors)
18 from bup.io import byte_stream, path_msg
22 def __init__(self, l):
28 self.cur = next(self.i, None)
33 def check_index(reader):
35 log('check: checking forward iteration...\n')
38 for e in reader.forward_iter():
41 log('%08x+%-4d %r\n' % (e.children_ofs, e.children_n,
43 assert(e.children_ofs)
44 assert e.name.endswith(b'/')
45 assert(not d.get(e.children_ofs))
47 if e.flags & index.IX_HASHVALID:
48 assert(e.sha != index.EMPTY_SHA)
50 assert not e or bytes(e.name) == b'/' # last entry is *always* /
51 log('check: checking normal iteration...\n')
58 log('index error! at %r\n' % e)
60 log('check: passed.\n')
63 def clear_index(indexfile):
64 indexfiles = [indexfile, indexfile + b'.meta', indexfile + b'.hlink']
65 for indexfile in indexfiles:
66 path = git.repo(indexfile)
70 log('clear: removed %s\n' % path_msg(path))
72 if e.errno != errno.ENOENT:
76 def update_index(top, excluded_paths, exclude_rxs, xdev_exceptions, out=None):
77 # tmax and start must be epoch nanoseconds.
78 tmax = (time.time() - 1) * 10**9
79 ri = index.Reader(indexfile)
80 msw = index.MetaStoreWriter(indexfile + b'.meta')
81 wi = index.Writer(indexfile, msw, tmax)
82 rig = IterHelper(ri.iter(name=top))
83 tstart = int(time.time()) * 10**9
85 hlinks = hlinkdb.HLinkDB(indexfile + b'.hlink')
90 return (GIT_MODE_FILE, index.FAKE_SHA)
93 bup_dir = os.path.abspath(git.repo())
94 index_start = time.time()
95 for path, pst in recursive_dirlist([top],
98 excluded_paths=excluded_paths,
99 exclude_rxs=exclude_rxs,
100 xdev_exceptions=xdev_exceptions):
101 if opt.verbose>=2 or (opt.verbose==1 and stat.S_ISDIR(pst.st_mode)):
102 out.write(b'%s\n' % path)
104 elapsed = time.time() - index_start
105 paths_per_sec = total / elapsed if elapsed else 0
106 qprogress('Indexing: %d (%d paths/s)\r' % (total, paths_per_sec))
107 elif not (total % 128):
108 elapsed = time.time() - index_start
109 paths_per_sec = total / elapsed if elapsed else 0
110 qprogress('Indexing: %d (%d paths/s)\r' % (total, paths_per_sec))
113 while rig.cur and rig.cur.name > path: # deleted paths
115 rig.cur.set_deleted()
117 if rig.cur.nlink > 1 and not stat.S_ISDIR(rig.cur.mode):
118 hlinks.del_path(rig.cur.name)
121 if rig.cur and rig.cur.name == path: # paths that already existed
123 if(rig.cur.stale(pst, tstart, check_device=opt.check_device)):
125 meta = metadata.from_path(path, statinfo=pst)
126 except (OSError, IOError) as e:
130 if not stat.S_ISDIR(rig.cur.mode) and rig.cur.nlink > 1:
131 hlinks.del_path(rig.cur.name)
132 if not stat.S_ISDIR(pst.st_mode) and pst.st_nlink > 1:
133 hlinks.add_path(path, pst.st_dev, pst.st_ino)
134 # Clear these so they don't bloat the store -- they're
135 # already in the index (since they vary a lot and they're
136 # fixed length). If you've noticed "tmax", you might
137 # wonder why it's OK to do this, since that code may
138 # adjust (mangle) the index mtime and ctime -- producing
139 # fake values which must not end up in a .bupm. However,
140 # it looks like that shouldn't be possible: (1) When
141 # "save" validates the index entry, it always reads the
142 # metadata from the filesytem. (2) Metadata is only
143 # read/used from the index if hashvalid is true. (3)
144 # "faked" entries will be stale(), and so we'll invalidate
146 meta.ctime = meta.mtime = meta.atime = 0
147 meta_ofs = msw.store(meta)
148 rig.cur.update_from_stat(pst, meta_ofs)
151 if not (rig.cur.flags & index.IX_HASHVALID):
153 if rig.cur.sha == index.EMPTY_SHA:
154 rig.cur.gitmode, rig.cur.sha = fake_hash(path)
155 rig.cur.flags |= index.IX_HASHVALID
165 meta = metadata.from_path(path, statinfo=pst)
166 except (OSError, IOError) as e:
169 # See same assignment to 0, above, for rationale.
170 meta.atime = meta.mtime = meta.ctime = 0
171 meta_ofs = msw.store(meta)
172 wi.add(path, pst, meta_ofs, hashgen=fake_hash)
173 if not stat.S_ISDIR(pst.st_mode) and pst.st_nlink > 1:
174 hlinks.add_path(path, pst.st_dev, pst.st_ino)
176 elapsed = time.time() - index_start
177 paths_per_sec = total / elapsed if elapsed else 0
178 progress('Indexing: %d, done (%d paths/s).\n' % (total, paths_per_sec))
180 hlinks.prepare_save()
188 log('check: before merging: oldfile\n')
190 log('check: before merging: newfile\n')
192 mi = index.Writer(indexfile, msw, tmax)
194 for e in index.merge(ri, wr):
195 # FIXME: shouldn't we remove deleted entries eventually? When?
210 bup index <-p|-m|-s|-u|--clear|--check> [options...] <filenames...>
213 p,print print the index entries for the given names (also works with -u)
214 m,modified print only added/deleted/modified files (implies -p)
215 s,status print each filename with a status char (A/M/D) (implies -p)
216 u,update recursively update the index entries for the given file/dir names (default if no mode is specified)
217 check carefully check index file integrity
218 clear clear the default index
220 H,hash print the hash for each object next to its name
221 l,long print more information about each file
222 no-check-device don't invalidate an entry if the containing device changes
223 fake-valid mark all index entries as up-to-date even if they aren't
224 fake-invalid mark all index entries as invalid
225 f,indexfile= the name of the index file (normally BUP_DIR/bupindex)
226 exclude= a path to exclude from the backup (may be repeated)
227 exclude-from= skip --exclude paths in file (may be repeated)
228 exclude-rx= skip paths matching the unanchored regex (may be repeated)
229 exclude-rx-from= skip --exclude-rx patterns in file (may be repeated)
230 v,verbose increase log output (can be used more than once)
231 x,xdev,one-file-system don't cross filesystem boundaries
233 o = options.Options(optspec)
234 (opt, flags, extra) = o.parse(sys.argv[1:])
236 if not (opt.modified or \
243 if (opt.fake_valid or opt.fake_invalid) and not opt.update:
244 o.fatal('--fake-{in,}valid are meaningless without -u')
245 if opt.fake_valid and opt.fake_invalid:
246 o.fatal('--fake-valid is incompatible with --fake-invalid')
247 if opt.clear and opt.indexfile:
248 o.fatal('cannot clear an external index (via -f)')
250 # FIXME: remove this once we account for timestamp races, i.e. index;
251 # touch new-file; index. It's possible for this to happen quickly
252 # enough that new-file ends up with the same timestamp as the first
253 # index, and then bup will ignore it.
254 tick_start = time.time()
255 time.sleep(1 - (tick_start - int(tick_start)))
257 git.check_repo_or_die()
261 if opt.verbose is None:
265 indexfile = argv_bytes(opt.indexfile)
267 indexfile = git.repo(b'bupindex')
270 log('check: starting initial check.\n')
271 check_index(index.Reader(indexfile))
274 log('clear: clearing index.\n')
275 clear_index(indexfile)
278 out = byte_stream(sys.stdout)
282 o.fatal('update mode (-u) requested but no paths given')
283 extra = [argv_bytes(x) for x in extra]
284 excluded_paths = parse_excludes(flags, o.fatal)
285 exclude_rxs = parse_rx_excludes(flags, o.fatal)
286 xexcept = index.unique_resolved_paths(extra)
287 for rp, path in index.reduce_paths(extra):
288 update_index(rp, excluded_paths, exclude_rxs, xdev_exceptions=xexcept,
291 if opt['print'] or opt.status or opt.modified:
292 extra = [argv_bytes(x) for x in extra]
293 for name, ent in index.Reader(indexfile).filter(extra or [b'']):
295 and (ent.is_valid() or ent.is_deleted() or not ent.mode)):
301 elif not ent.is_valid():
302 if ent.sha == index.EMPTY_SHA:
309 line += hexlify(ent.sha) + b' '
311 line += b'%7s %7s ' % (oct(ent.mode), oct(ent.gitmode))
312 out.write(line + (name or b'./') + b'\n')
314 if opt.check and (opt['print'] or opt.status or opt.modified or opt.update):
315 log('check: starting final check.\n')
316 check_index(index.Reader(indexfile))
319 log('WARNING: %d errors encountered.\n' % len(saved_errors))