3 bup_python="$(dirname "$0")/bup-python" || exit $?
4 exec "$bup_python" "$0" ${1+"$@"}
8 import sys, stat, time, os, errno, re
10 from bup import metadata, options, git, index, drecurse, hlinkdb
11 from bup.drecurse import recursive_dirlist
12 from bup.hashsplit import GIT_MODE_TREE, GIT_MODE_FILE
13 from bup.helpers import (add_error, handle_ctrl_c, log, parse_excludes, parse_rx_excludes,
14 progress, qprogress, saved_errors)
18 def __init__(self, l):
25 self.cur = self.i.next()
31 def check_index(reader):
33 log('check: checking forward iteration...\n')
36 for e in reader.forward_iter():
39 log('%08x+%-4d %r\n' % (e.children_ofs, e.children_n,
41 assert(e.children_ofs)
42 assert(e.name.endswith('/'))
43 assert(not d.get(e.children_ofs))
45 if e.flags & index.IX_HASHVALID:
46 assert(e.sha != index.EMPTY_SHA)
48 assert(not e or e.name == '/') # last entry is *always* /
49 log('check: checking normal iteration...\n')
56 log('index error! at %r\n' % e)
58 log('check: passed.\n')
61 def clear_index(indexfile):
62 indexfiles = [indexfile, indexfile + '.meta', indexfile + '.hlink']
63 for indexfile in indexfiles:
64 path = git.repo(indexfile)
68 log('clear: removed %s\n' % path)
70 if e.errno != errno.ENOENT:
74 def update_index(top, excluded_paths, exclude_rxs, xdev_exceptions):
75 # tmax and start must be epoch nanoseconds.
76 tmax = (time.time() - 1) * 10**9
77 ri = index.Reader(indexfile)
78 msw = index.MetaStoreWriter(indexfile + '.meta')
79 wi = index.Writer(indexfile, msw, tmax)
80 rig = IterHelper(ri.iter(name=top))
81 tstart = int(time.time()) * 10**9
83 hlinks = hlinkdb.HLinkDB(indexfile + '.hlink')
88 return (GIT_MODE_FILE, index.FAKE_SHA)
91 bup_dir = os.path.abspath(git.repo())
92 index_start = time.time()
93 for path, pst in recursive_dirlist([top],
96 excluded_paths=excluded_paths,
97 exclude_rxs=exclude_rxs,
98 xdev_exceptions=xdev_exceptions):
99 if opt.verbose>=2 or (opt.verbose==1 and stat.S_ISDIR(pst.st_mode)):
100 sys.stdout.write('%s\n' % path)
102 elapsed = time.time() - index_start
103 paths_per_sec = total / elapsed if elapsed else 0
104 qprogress('Indexing: %d (%d paths/s)\r' % (total, paths_per_sec))
105 elif not (total % 128):
106 elapsed = time.time() - index_start
107 paths_per_sec = total / elapsed if elapsed else 0
108 qprogress('Indexing: %d (%d paths/s)\r' % (total, paths_per_sec))
110 while rig.cur and rig.cur.name > path: # deleted paths
112 rig.cur.set_deleted()
114 if rig.cur.nlink > 1 and not stat.S_ISDIR(rig.cur.mode):
115 hlinks.del_path(rig.cur.name)
117 if rig.cur and rig.cur.name == path: # paths that already existed
119 meta = metadata.from_path(path, statinfo=pst)
120 except (OSError, IOError) as e:
124 if not stat.S_ISDIR(rig.cur.mode) and rig.cur.nlink > 1:
125 hlinks.del_path(rig.cur.name)
126 if not stat.S_ISDIR(pst.st_mode) and pst.st_nlink > 1:
127 hlinks.add_path(path, pst.st_dev, pst.st_ino)
128 # Clear these so they don't bloat the store -- they're
129 # already in the index (since they vary a lot and they're
130 # fixed length). If you've noticed "tmax", you might
131 # wonder why it's OK to do this, since that code may
132 # adjust (mangle) the index mtime and ctime -- producing
133 # fake values which must not end up in a .bupm. However,
134 # it looks like that shouldn't be possible: (1) When
135 # "save" validates the index entry, it always reads the
136 # metadata from the filesytem. (2) Metadata is only
137 # read/used from the index if hashvalid is true. (3) index
138 # always invalidates "faked" entries, because "old != new"
140 meta.ctime = meta.mtime = meta.atime = 0
141 meta_ofs = msw.store(meta)
142 rig.cur.from_stat(pst, meta_ofs, tstart,
143 check_device=opt.check_device)
144 if not (rig.cur.flags & index.IX_HASHVALID):
146 (rig.cur.gitmode, rig.cur.sha) = hashgen(path)
147 rig.cur.flags |= index.IX_HASHVALID
154 meta = metadata.from_path(path, statinfo=pst)
155 except (OSError, IOError) as e:
158 # See same assignment to 0, above, for rationale.
159 meta.atime = meta.mtime = meta.ctime = 0
160 meta_ofs = msw.store(meta)
161 wi.add(path, pst, meta_ofs, hashgen = hashgen)
162 if not stat.S_ISDIR(pst.st_mode) and pst.st_nlink > 1:
163 hlinks.add_path(path, pst.st_dev, pst.st_ino)
165 elapsed = time.time() - index_start
166 paths_per_sec = total / elapsed if elapsed else 0
167 progress('Indexing: %d, done (%d paths/s).\n' % (total, paths_per_sec))
169 hlinks.prepare_save()
177 log('check: before merging: oldfile\n')
179 log('check: before merging: newfile\n')
181 mi = index.Writer(indexfile, msw, tmax)
183 for e in index.merge(ri, wr):
184 # FIXME: shouldn't we remove deleted entries eventually? When?
199 bup index <-p|-m|-s|-u|--clear|--check> [options...] <filenames...>
202 p,print print the index entries for the given names (also works with -u)
203 m,modified print only added/deleted/modified files (implies -p)
204 s,status print each filename with a status char (A/M/D) (implies -p)
205 u,update recursively update the index entries for the given file/dir names (default if no mode is specified)
206 check carefully check index file integrity
207 clear clear the default index
209 H,hash print the hash for each object next to its name
210 l,long print more information about each file
211 no-check-device don't invalidate an entry if the containing device changes
212 fake-valid mark all index entries as up-to-date even if they aren't
213 fake-invalid mark all index entries as invalid
214 f,indexfile= the name of the index file (normally BUP_DIR/bupindex)
215 exclude= a path to exclude from the backup (may be repeated)
216 exclude-from= skip --exclude paths in file (may be repeated)
217 exclude-rx= skip paths matching the unanchored regex (may be repeated)
218 exclude-rx-from= skip --exclude-rx patterns in file (may be repeated)
219 v,verbose increase log output (can be used more than once)
220 x,xdev,one-file-system don't cross filesystem boundaries
222 o = options.Options(optspec)
223 (opt, flags, extra) = o.parse(sys.argv[1:])
225 if not (opt.modified or \
232 if (opt.fake_valid or opt.fake_invalid) and not opt.update:
233 o.fatal('--fake-{in,}valid are meaningless without -u')
234 if opt.fake_valid and opt.fake_invalid:
235 o.fatal('--fake-valid is incompatible with --fake-invalid')
236 if opt.clear and opt.indexfile:
237 o.fatal('cannot clear an external index (via -f)')
239 # FIXME: remove this once we account for timestamp races, i.e. index;
240 # touch new-file; index. It's possible for this to happen quickly
241 # enough that new-file ends up with the same timestamp as the first
242 # index, and then bup will ignore it.
243 tick_start = time.time()
244 time.sleep(1 - (tick_start - int(tick_start)))
246 git.check_repo_or_die()
247 indexfile = opt.indexfile or git.repo('bupindex')
252 log('check: starting initial check.\n')
253 check_index(index.Reader(indexfile))
256 log('clear: clearing index.\n')
257 clear_index(indexfile)
261 o.fatal('update mode (-u) requested but no paths given')
262 excluded_paths = parse_excludes(flags, o.fatal)
263 exclude_rxs = parse_rx_excludes(flags, o.fatal)
264 xexcept = index.unique_resolved_paths(extra)
265 for rp, path in index.reduce_paths(extra):
266 update_index(rp, excluded_paths, exclude_rxs, xdev_exceptions=xexcept)
268 if opt['print'] or opt.status or opt.modified:
269 for (name, ent) in index.Reader(indexfile).filter(extra or ['']):
271 and (ent.is_valid() or ent.is_deleted() or not ent.mode)):
277 elif not ent.is_valid():
278 if ent.sha == index.EMPTY_SHA:
285 line += ent.sha.encode('hex') + ' '
287 line += "%7s %7s " % (oct(ent.mode), oct(ent.gitmode))
288 print line + (name or './')
290 if opt.check and (opt['print'] or opt.status or opt.modified or opt.update):
291 log('check: starting final check.\n')
292 check_index(index.Reader(indexfile))
295 log('WARNING: %d errors encountered.\n' % len(saved_errors))