3 bup_python="$(dirname "$0")/bup-python" || exit $?
4 exec "$bup_python" "$0" ${1+"$@"}
8 import sys, stat, time, os, errno, re
10 from bup import metadata, options, git, index, drecurse, hlinkdb
11 from bup.drecurse import recursive_dirlist
12 from bup.hashsplit import GIT_MODE_TREE, GIT_MODE_FILE
13 from bup.helpers import (add_error, handle_ctrl_c, log, parse_excludes, parse_rx_excludes,
14 progress, qprogress, saved_errors)
18 def __init__(self, l):
25 self.cur = self.i.next()
31 def check_index(reader):
33 log('check: checking forward iteration...\n')
36 for e in reader.forward_iter():
39 log('%08x+%-4d %r\n' % (e.children_ofs, e.children_n,
41 assert(e.children_ofs)
42 assert(e.name.endswith('/'))
43 assert(not d.get(e.children_ofs))
45 if e.flags & index.IX_HASHVALID:
46 assert(e.sha != index.EMPTY_SHA)
48 assert(not e or e.name == '/') # last entry is *always* /
49 log('check: checking normal iteration...\n')
56 log('index error! at %r\n' % e)
58 log('check: passed.\n')
61 def clear_index(indexfile):
62 indexfiles = [indexfile, indexfile + '.meta', indexfile + '.hlink']
63 for indexfile in indexfiles:
64 path = git.repo(indexfile)
68 log('clear: removed %s\n' % path)
70 if e.errno != errno.ENOENT:
74 def update_index(top, excluded_paths, exclude_rxs, xdev_exceptions):
75 # tmax and start must be epoch nanoseconds.
76 tmax = (time.time() - 1) * 10**9
77 ri = index.Reader(indexfile)
78 msw = index.MetaStoreWriter(indexfile + '.meta')
79 wi = index.Writer(indexfile, msw, tmax)
80 rig = IterHelper(ri.iter(name=top))
81 tstart = int(time.time()) * 10**9
83 hlinks = hlinkdb.HLinkDB(indexfile + '.hlink')
88 return (GIT_MODE_FILE, index.FAKE_SHA)
91 bup_dir = os.path.abspath(git.repo())
92 index_start = time.time()
93 for path, pst in recursive_dirlist([top],
96 excluded_paths=excluded_paths,
97 exclude_rxs=exclude_rxs,
98 xdev_exceptions=xdev_exceptions):
99 if opt.verbose>=2 or (opt.verbose==1 and stat.S_ISDIR(pst.st_mode)):
100 sys.stdout.write('%s\n' % path)
102 elapsed = time.time() - index_start
103 paths_per_sec = total / elapsed if elapsed else 0
104 qprogress('Indexing: %d (%d paths/s)\r' % (total, paths_per_sec))
105 elif not (total % 128):
106 elapsed = time.time() - index_start
107 paths_per_sec = total / elapsed if elapsed else 0
108 qprogress('Indexing: %d (%d paths/s)\r' % (total, paths_per_sec))
111 while rig.cur and rig.cur.name > path: # deleted paths
113 rig.cur.set_deleted()
115 if rig.cur.nlink > 1 and not stat.S_ISDIR(rig.cur.mode):
116 hlinks.del_path(rig.cur.name)
119 if rig.cur and rig.cur.name == path: # paths that already existed
121 if(rig.cur.stale(pst, tstart, check_device=opt.check_device)):
123 meta = metadata.from_path(path, statinfo=pst)
124 except (OSError, IOError) as e:
128 if not stat.S_ISDIR(rig.cur.mode) and rig.cur.nlink > 1:
129 hlinks.del_path(rig.cur.name)
130 if not stat.S_ISDIR(pst.st_mode) and pst.st_nlink > 1:
131 hlinks.add_path(path, pst.st_dev, pst.st_ino)
132 # Clear these so they don't bloat the store -- they're
133 # already in the index (since they vary a lot and they're
134 # fixed length). If you've noticed "tmax", you might
135 # wonder why it's OK to do this, since that code may
136 # adjust (mangle) the index mtime and ctime -- producing
137 # fake values which must not end up in a .bupm. However,
138 # it looks like that shouldn't be possible: (1) When
139 # "save" validates the index entry, it always reads the
140 # metadata from the filesytem. (2) Metadata is only
141 # read/used from the index if hashvalid is true. (3)
142 # "faked" entries will be stale(), and so we'll invalidate
144 meta.ctime = meta.mtime = meta.atime = 0
145 meta_ofs = msw.store(meta)
146 rig.cur.update_from_stat(pst, meta_ofs)
149 if not (rig.cur.flags & index.IX_HASHVALID):
151 rig.cur.gitmode, rig.cur.sha = fake_hash(path)
152 rig.cur.flags |= index.IX_HASHVALID
162 meta = metadata.from_path(path, statinfo=pst)
163 except (OSError, IOError) as e:
166 # See same assignment to 0, above, for rationale.
167 meta.atime = meta.mtime = meta.ctime = 0
168 meta_ofs = msw.store(meta)
169 wi.add(path, pst, meta_ofs, hashgen=fake_hash)
170 if not stat.S_ISDIR(pst.st_mode) and pst.st_nlink > 1:
171 hlinks.add_path(path, pst.st_dev, pst.st_ino)
173 elapsed = time.time() - index_start
174 paths_per_sec = total / elapsed if elapsed else 0
175 progress('Indexing: %d, done (%d paths/s).\n' % (total, paths_per_sec))
177 hlinks.prepare_save()
185 log('check: before merging: oldfile\n')
187 log('check: before merging: newfile\n')
189 mi = index.Writer(indexfile, msw, tmax)
191 for e in index.merge(ri, wr):
192 # FIXME: shouldn't we remove deleted entries eventually? When?
207 bup index <-p|-m|-s|-u|--clear|--check> [options...] <filenames...>
210 p,print print the index entries for the given names (also works with -u)
211 m,modified print only added/deleted/modified files (implies -p)
212 s,status print each filename with a status char (A/M/D) (implies -p)
213 u,update recursively update the index entries for the given file/dir names (default if no mode is specified)
214 check carefully check index file integrity
215 clear clear the default index
217 H,hash print the hash for each object next to its name
218 l,long print more information about each file
219 no-check-device don't invalidate an entry if the containing device changes
220 fake-valid mark all index entries as up-to-date even if they aren't
221 fake-invalid mark all index entries as invalid
222 f,indexfile= the name of the index file (normally BUP_DIR/bupindex)
223 exclude= a path to exclude from the backup (may be repeated)
224 exclude-from= skip --exclude paths in file (may be repeated)
225 exclude-rx= skip paths matching the unanchored regex (may be repeated)
226 exclude-rx-from= skip --exclude-rx patterns in file (may be repeated)
227 v,verbose increase log output (can be used more than once)
228 x,xdev,one-file-system don't cross filesystem boundaries
230 o = options.Options(optspec)
231 (opt, flags, extra) = o.parse(sys.argv[1:])
233 if not (opt.modified or \
240 if (opt.fake_valid or opt.fake_invalid) and not opt.update:
241 o.fatal('--fake-{in,}valid are meaningless without -u')
242 if opt.fake_valid and opt.fake_invalid:
243 o.fatal('--fake-valid is incompatible with --fake-invalid')
244 if opt.clear and opt.indexfile:
245 o.fatal('cannot clear an external index (via -f)')
247 # FIXME: remove this once we account for timestamp races, i.e. index;
248 # touch new-file; index. It's possible for this to happen quickly
249 # enough that new-file ends up with the same timestamp as the first
250 # index, and then bup will ignore it.
251 tick_start = time.time()
252 time.sleep(1 - (tick_start - int(tick_start)))
254 git.check_repo_or_die()
255 indexfile = opt.indexfile or git.repo('bupindex')
260 log('check: starting initial check.\n')
261 check_index(index.Reader(indexfile))
264 log('clear: clearing index.\n')
265 clear_index(indexfile)
269 o.fatal('update mode (-u) requested but no paths given')
270 excluded_paths = parse_excludes(flags, o.fatal)
271 exclude_rxs = parse_rx_excludes(flags, o.fatal)
272 xexcept = index.unique_resolved_paths(extra)
273 for rp, path in index.reduce_paths(extra):
274 update_index(rp, excluded_paths, exclude_rxs, xdev_exceptions=xexcept)
276 if opt['print'] or opt.status or opt.modified:
277 for (name, ent) in index.Reader(indexfile).filter(extra or ['']):
279 and (ent.is_valid() or ent.is_deleted() or not ent.mode)):
285 elif not ent.is_valid():
286 if ent.sha == index.EMPTY_SHA:
293 line += ent.sha.encode('hex') + ' '
295 line += "%7s %7s " % (oct(ent.mode), oct(ent.gitmode))
296 print line + (name or './')
298 if opt.check and (opt['print'] or opt.status or opt.modified or opt.update):
299 log('check: starting final check.\n')
300 check_index(index.Reader(indexfile))
303 log('WARNING: %d errors encountered.\n' % len(saved_errors))