3 # https://sourceware.org/bugzilla/show_bug.cgi?id=26034
4 export "BUP_ARGV_0"="$0"
7 export "BUP_ARGV_${arg_i}"="$arg"
11 # Here to end of preamble replaced during install
12 bup_python="$(dirname "$0")/../../config/bin/python" || exit $?
13 exec "$bup_python" "$0"
17 from __future__ import absolute_import, print_function
18 from binascii import hexlify
19 import errno, os, re, stat, sys, time
21 sys.path[:0] = [os.path.dirname(os.path.realpath(__file__)) + '/..']
23 from bup import compat, metadata, options, git, index, drecurse, hlinkdb
24 from bup.compat import argv_bytes
25 from bup.drecurse import recursive_dirlist
26 from bup.hashsplit import GIT_MODE_TREE, GIT_MODE_FILE
27 from bup.helpers import (add_error, handle_ctrl_c, log, parse_excludes, parse_rx_excludes,
28 progress, qprogress, saved_errors)
29 from bup.io import byte_stream, path_msg
33 def __init__(self, l):
39 self.cur = next(self.i, None)
44 def check_index(reader):
46 log('check: checking forward iteration...\n')
49 for e in reader.forward_iter():
52 log('%08x+%-4d %r\n' % (e.children_ofs, e.children_n,
54 assert(e.children_ofs)
55 assert e.name.endswith(b'/')
56 assert(not d.get(e.children_ofs))
58 if e.flags & index.IX_HASHVALID:
59 assert(e.sha != index.EMPTY_SHA)
61 assert not e or bytes(e.name) == b'/' # last entry is *always* /
62 log('check: checking normal iteration...\n')
69 log('index error! at %r\n' % e)
71 log('check: passed.\n')
74 def clear_index(indexfile):
75 indexfiles = [indexfile, indexfile + b'.meta', indexfile + b'.hlink']
76 for indexfile in indexfiles:
77 path = git.repo(indexfile)
81 log('clear: removed %s\n' % path_msg(path))
83 if e.errno != errno.ENOENT:
87 def update_index(top, excluded_paths, exclude_rxs, xdev_exceptions, out=None):
88 # tmax and start must be epoch nanoseconds.
89 tmax = (time.time() - 1) * 10**9
90 ri = index.Reader(indexfile)
91 msw = index.MetaStoreWriter(indexfile + b'.meta')
92 wi = index.Writer(indexfile, msw, tmax)
93 rig = IterHelper(ri.iter(name=top))
94 tstart = int(time.time()) * 10**9
96 hlinks = hlinkdb.HLinkDB(indexfile + b'.hlink')
101 return (GIT_MODE_FILE, index.FAKE_SHA)
104 bup_dir = os.path.abspath(git.repo())
105 index_start = time.time()
106 for path, pst in recursive_dirlist([top],
109 excluded_paths=excluded_paths,
110 exclude_rxs=exclude_rxs,
111 xdev_exceptions=xdev_exceptions):
112 if opt.verbose>=2 or (opt.verbose==1 and stat.S_ISDIR(pst.st_mode)):
113 out.write(b'%s\n' % path)
115 elapsed = time.time() - index_start
116 paths_per_sec = total / elapsed if elapsed else 0
117 qprogress('Indexing: %d (%d paths/s)\r' % (total, paths_per_sec))
118 elif not (total % 128):
119 elapsed = time.time() - index_start
120 paths_per_sec = total / elapsed if elapsed else 0
121 qprogress('Indexing: %d (%d paths/s)\r' % (total, paths_per_sec))
124 while rig.cur and rig.cur.name > path: # deleted paths
126 rig.cur.set_deleted()
128 if rig.cur.nlink > 1 and not stat.S_ISDIR(rig.cur.mode):
129 hlinks.del_path(rig.cur.name)
132 if rig.cur and rig.cur.name == path: # paths that already existed
134 if(rig.cur.stale(pst, tstart, check_device=opt.check_device)):
136 meta = metadata.from_path(path, statinfo=pst)
137 except (OSError, IOError) as e:
141 if not stat.S_ISDIR(rig.cur.mode) and rig.cur.nlink > 1:
142 hlinks.del_path(rig.cur.name)
143 if not stat.S_ISDIR(pst.st_mode) and pst.st_nlink > 1:
144 hlinks.add_path(path, pst.st_dev, pst.st_ino)
145 # Clear these so they don't bloat the store -- they're
146 # already in the index (since they vary a lot and they're
147 # fixed length). If you've noticed "tmax", you might
148 # wonder why it's OK to do this, since that code may
149 # adjust (mangle) the index mtime and ctime -- producing
150 # fake values which must not end up in a .bupm. However,
151 # it looks like that shouldn't be possible: (1) When
152 # "save" validates the index entry, it always reads the
153 # metadata from the filesytem. (2) Metadata is only
154 # read/used from the index if hashvalid is true. (3)
155 # "faked" entries will be stale(), and so we'll invalidate
157 meta.ctime = meta.mtime = meta.atime = 0
158 meta_ofs = msw.store(meta)
159 rig.cur.update_from_stat(pst, meta_ofs)
162 if not (rig.cur.flags & index.IX_HASHVALID):
164 if rig.cur.sha == index.EMPTY_SHA:
165 rig.cur.gitmode, rig.cur.sha = fake_hash(path)
166 rig.cur.flags |= index.IX_HASHVALID
176 meta = metadata.from_path(path, statinfo=pst)
177 except (OSError, IOError) as e:
180 # See same assignment to 0, above, for rationale.
181 meta.atime = meta.mtime = meta.ctime = 0
182 meta_ofs = msw.store(meta)
183 wi.add(path, pst, meta_ofs, hashgen=fake_hash)
184 if not stat.S_ISDIR(pst.st_mode) and pst.st_nlink > 1:
185 hlinks.add_path(path, pst.st_dev, pst.st_ino)
187 elapsed = time.time() - index_start
188 paths_per_sec = total / elapsed if elapsed else 0
189 progress('Indexing: %d, done (%d paths/s).\n' % (total, paths_per_sec))
191 hlinks.prepare_save()
199 log('check: before merging: oldfile\n')
201 log('check: before merging: newfile\n')
203 mi = index.Writer(indexfile, msw, tmax)
205 for e in index.merge(ri, wr):
206 # FIXME: shouldn't we remove deleted entries eventually? When?
221 bup index <-p|-m|-s|-u|--clear|--check> [options...] <filenames...>
224 p,print print the index entries for the given names (also works with -u)
225 m,modified print only added/deleted/modified files (implies -p)
226 s,status print each filename with a status char (A/M/D) (implies -p)
227 u,update recursively update the index entries for the given file/dir names (default if no mode is specified)
228 check carefully check index file integrity
229 clear clear the default index
231 H,hash print the hash for each object next to its name
232 l,long print more information about each file
233 no-check-device don't invalidate an entry if the containing device changes
234 fake-valid mark all index entries as up-to-date even if they aren't
235 fake-invalid mark all index entries as invalid
236 f,indexfile= the name of the index file (normally BUP_DIR/bupindex)
237 exclude= a path to exclude from the backup (may be repeated)
238 exclude-from= skip --exclude paths in file (may be repeated)
239 exclude-rx= skip paths matching the unanchored regex (may be repeated)
240 exclude-rx-from= skip --exclude-rx patterns in file (may be repeated)
241 v,verbose increase log output (can be used more than once)
242 x,xdev,one-file-system don't cross filesystem boundaries
244 o = options.Options(optspec)
245 opt, flags, extra = o.parse(compat.argv[1:])
247 if not (opt.modified or \
254 if (opt.fake_valid or opt.fake_invalid) and not opt.update:
255 o.fatal('--fake-{in,}valid are meaningless without -u')
256 if opt.fake_valid and opt.fake_invalid:
257 o.fatal('--fake-valid is incompatible with --fake-invalid')
258 if opt.clear and opt.indexfile:
259 o.fatal('cannot clear an external index (via -f)')
261 # FIXME: remove this once we account for timestamp races, i.e. index;
262 # touch new-file; index. It's possible for this to happen quickly
263 # enough that new-file ends up with the same timestamp as the first
264 # index, and then bup will ignore it.
265 tick_start = time.time()
266 time.sleep(1 - (tick_start - int(tick_start)))
268 git.check_repo_or_die()
272 if opt.verbose is None:
276 indexfile = argv_bytes(opt.indexfile)
278 indexfile = git.repo(b'bupindex')
281 log('check: starting initial check.\n')
282 check_index(index.Reader(indexfile))
285 log('clear: clearing index.\n')
286 clear_index(indexfile)
289 out = byte_stream(sys.stdout)
293 o.fatal('update mode (-u) requested but no paths given')
294 extra = [argv_bytes(x) for x in extra]
295 excluded_paths = parse_excludes(flags, o.fatal)
296 exclude_rxs = parse_rx_excludes(flags, o.fatal)
297 xexcept = index.unique_resolved_paths(extra)
298 for rp, path in index.reduce_paths(extra):
299 update_index(rp, excluded_paths, exclude_rxs, xdev_exceptions=xexcept,
302 if opt['print'] or opt.status or opt.modified:
303 extra = [argv_bytes(x) for x in extra]
304 for name, ent in index.Reader(indexfile).filter(extra or [b'']):
306 and (ent.is_valid() or ent.is_deleted() or not ent.mode)):
312 elif not ent.is_valid():
313 if ent.sha == index.EMPTY_SHA:
320 line += hexlify(ent.sha) + b' '
322 line += b'%7s %7s ' % (oct(ent.mode).encode('ascii'),
323 oct(ent.gitmode).encode('ascii'))
324 out.write(line + (name or b'./') + b'\n')
326 if opt.check and (opt['print'] or opt.status or opt.modified or opt.update):
327 log('check: starting final check.\n')
328 check_index(index.Reader(indexfile))
331 log('WARNING: %d errors encountered.\n' % len(saved_errors))