3 bup_python="$(dirname "$0")/bup-python" || exit $?
4 exec "$bup_python" "$0" ${1+"$@"}
8 import sys, stat, time, os, errno, re
10 from bup import metadata, options, git, index, drecurse, hlinkdb
11 from bup.drecurse import recursive_dirlist
12 from bup.hashsplit import GIT_MODE_TREE, GIT_MODE_FILE
13 from bup.helpers import (add_error, handle_ctrl_c, log, parse_excludes, parse_rx_excludes,
14 progress, qprogress, saved_errors)
18 def __init__(self, l):
24 self.cur = next(self.i, None)
28 def check_index(reader):
30 log('check: checking forward iteration...\n')
33 for e in reader.forward_iter():
36 log('%08x+%-4d %r\n' % (e.children_ofs, e.children_n,
38 assert(e.children_ofs)
39 assert(e.name.endswith('/'))
40 assert(not d.get(e.children_ofs))
42 if e.flags & index.IX_HASHVALID:
43 assert(e.sha != index.EMPTY_SHA)
45 assert(not e or e.name == '/') # last entry is *always* /
46 log('check: checking normal iteration...\n')
53 log('index error! at %r\n' % e)
55 log('check: passed.\n')
58 def clear_index(indexfile):
59 indexfiles = [indexfile, indexfile + '.meta', indexfile + '.hlink']
60 for indexfile in indexfiles:
61 path = git.repo(indexfile)
65 log('clear: removed %s\n' % path)
67 if e.errno != errno.ENOENT:
71 def update_index(top, excluded_paths, exclude_rxs, xdev_exceptions):
72 # tmax and start must be epoch nanoseconds.
73 tmax = (time.time() - 1) * 10**9
74 ri = index.Reader(indexfile)
75 msw = index.MetaStoreWriter(indexfile + '.meta')
76 wi = index.Writer(indexfile, msw, tmax)
77 rig = IterHelper(ri.iter(name=top))
78 tstart = int(time.time()) * 10**9
80 hlinks = hlinkdb.HLinkDB(indexfile + '.hlink')
85 return (GIT_MODE_FILE, index.FAKE_SHA)
88 bup_dir = os.path.abspath(git.repo())
89 index_start = time.time()
90 for path, pst in recursive_dirlist([top],
93 excluded_paths=excluded_paths,
94 exclude_rxs=exclude_rxs,
95 xdev_exceptions=xdev_exceptions):
96 if opt.verbose>=2 or (opt.verbose==1 and stat.S_ISDIR(pst.st_mode)):
97 sys.stdout.write('%s\n' % path)
99 elapsed = time.time() - index_start
100 paths_per_sec = total / elapsed if elapsed else 0
101 qprogress('Indexing: %d (%d paths/s)\r' % (total, paths_per_sec))
102 elif not (total % 128):
103 elapsed = time.time() - index_start
104 paths_per_sec = total / elapsed if elapsed else 0
105 qprogress('Indexing: %d (%d paths/s)\r' % (total, paths_per_sec))
108 while rig.cur and rig.cur.name > path: # deleted paths
110 rig.cur.set_deleted()
112 if rig.cur.nlink > 1 and not stat.S_ISDIR(rig.cur.mode):
113 hlinks.del_path(rig.cur.name)
116 if rig.cur and rig.cur.name == path: # paths that already existed
118 if(rig.cur.stale(pst, tstart, check_device=opt.check_device)):
120 meta = metadata.from_path(path, statinfo=pst)
121 except (OSError, IOError) as e:
125 if not stat.S_ISDIR(rig.cur.mode) and rig.cur.nlink > 1:
126 hlinks.del_path(rig.cur.name)
127 if not stat.S_ISDIR(pst.st_mode) and pst.st_nlink > 1:
128 hlinks.add_path(path, pst.st_dev, pst.st_ino)
129 # Clear these so they don't bloat the store -- they're
130 # already in the index (since they vary a lot and they're
131 # fixed length). If you've noticed "tmax", you might
132 # wonder why it's OK to do this, since that code may
133 # adjust (mangle) the index mtime and ctime -- producing
134 # fake values which must not end up in a .bupm. However,
135 # it looks like that shouldn't be possible: (1) When
136 # "save" validates the index entry, it always reads the
137 # metadata from the filesytem. (2) Metadata is only
138 # read/used from the index if hashvalid is true. (3)
139 # "faked" entries will be stale(), and so we'll invalidate
141 meta.ctime = meta.mtime = meta.atime = 0
142 meta_ofs = msw.store(meta)
143 rig.cur.update_from_stat(pst, meta_ofs)
146 if not (rig.cur.flags & index.IX_HASHVALID):
148 rig.cur.gitmode, rig.cur.sha = fake_hash(path)
149 rig.cur.flags |= index.IX_HASHVALID
159 meta = metadata.from_path(path, statinfo=pst)
160 except (OSError, IOError) as e:
163 # See same assignment to 0, above, for rationale.
164 meta.atime = meta.mtime = meta.ctime = 0
165 meta_ofs = msw.store(meta)
166 wi.add(path, pst, meta_ofs, hashgen=fake_hash)
167 if not stat.S_ISDIR(pst.st_mode) and pst.st_nlink > 1:
168 hlinks.add_path(path, pst.st_dev, pst.st_ino)
170 elapsed = time.time() - index_start
171 paths_per_sec = total / elapsed if elapsed else 0
172 progress('Indexing: %d, done (%d paths/s).\n' % (total, paths_per_sec))
174 hlinks.prepare_save()
182 log('check: before merging: oldfile\n')
184 log('check: before merging: newfile\n')
186 mi = index.Writer(indexfile, msw, tmax)
188 for e in index.merge(ri, wr):
189 # FIXME: shouldn't we remove deleted entries eventually? When?
204 bup index <-p|-m|-s|-u|--clear|--check> [options...] <filenames...>
207 p,print print the index entries for the given names (also works with -u)
208 m,modified print only added/deleted/modified files (implies -p)
209 s,status print each filename with a status char (A/M/D) (implies -p)
210 u,update recursively update the index entries for the given file/dir names (default if no mode is specified)
211 check carefully check index file integrity
212 clear clear the default index
214 H,hash print the hash for each object next to its name
215 l,long print more information about each file
216 no-check-device don't invalidate an entry if the containing device changes
217 fake-valid mark all index entries as up-to-date even if they aren't
218 fake-invalid mark all index entries as invalid
219 f,indexfile= the name of the index file (normally BUP_DIR/bupindex)
220 exclude= a path to exclude from the backup (may be repeated)
221 exclude-from= skip --exclude paths in file (may be repeated)
222 exclude-rx= skip paths matching the unanchored regex (may be repeated)
223 exclude-rx-from= skip --exclude-rx patterns in file (may be repeated)
224 v,verbose increase log output (can be used more than once)
225 x,xdev,one-file-system don't cross filesystem boundaries
227 o = options.Options(optspec)
228 (opt, flags, extra) = o.parse(sys.argv[1:])
230 if not (opt.modified or \
237 if (opt.fake_valid or opt.fake_invalid) and not opt.update:
238 o.fatal('--fake-{in,}valid are meaningless without -u')
239 if opt.fake_valid and opt.fake_invalid:
240 o.fatal('--fake-valid is incompatible with --fake-invalid')
241 if opt.clear and opt.indexfile:
242 o.fatal('cannot clear an external index (via -f)')
244 # FIXME: remove this once we account for timestamp races, i.e. index;
245 # touch new-file; index. It's possible for this to happen quickly
246 # enough that new-file ends up with the same timestamp as the first
247 # index, and then bup will ignore it.
248 tick_start = time.time()
249 time.sleep(1 - (tick_start - int(tick_start)))
251 git.check_repo_or_die()
252 indexfile = opt.indexfile or git.repo('bupindex')
257 log('check: starting initial check.\n')
258 check_index(index.Reader(indexfile))
261 log('clear: clearing index.\n')
262 clear_index(indexfile)
266 o.fatal('update mode (-u) requested but no paths given')
267 excluded_paths = parse_excludes(flags, o.fatal)
268 exclude_rxs = parse_rx_excludes(flags, o.fatal)
269 xexcept = index.unique_resolved_paths(extra)
270 for rp, path in index.reduce_paths(extra):
271 update_index(rp, excluded_paths, exclude_rxs, xdev_exceptions=xexcept)
273 if opt['print'] or opt.status or opt.modified:
274 for (name, ent) in index.Reader(indexfile).filter(extra or ['']):
276 and (ent.is_valid() or ent.is_deleted() or not ent.mode)):
282 elif not ent.is_valid():
283 if ent.sha == index.EMPTY_SHA:
290 line += ent.sha.encode('hex') + ' '
292 line += "%7s %7s " % (oct(ent.mode), oct(ent.gitmode))
293 print line + (name or './')
295 if opt.check and (opt['print'] or opt.status or opt.modified or opt.update):
296 log('check: starting final check.\n')
297 check_index(index.Reader(indexfile))
300 log('WARNING: %d errors encountered.\n' % len(saved_errors))