3 import sys, stat, time, os, errno
4 from bup import metadata, options, git, index, drecurse, hlinkdb
5 from bup.helpers import *
6 from bup.hashsplit import GIT_MODE_TREE, GIT_MODE_FILE
16 self.cur = self.i.next()
22 def check_index(reader):
24 log('check: checking forward iteration...\n')
27 for e in reader.forward_iter():
30 log('%08x+%-4d %r\n' % (e.children_ofs, e.children_n,
32 assert(e.children_ofs)
33 assert(e.name.endswith('/'))
34 assert(not d.get(e.children_ofs))
36 if e.flags & index.IX_HASHVALID:
37 assert(e.sha != index.EMPTY_SHA)
39 assert(not e or e.name == '/') # last entry is *always* /
40 log('check: checking normal iteration...\n')
47 log('index error! at %r\n' % e)
49 log('check: passed.\n')
52 def clear_index(indexfile):
53 indexfiles = [indexfile, indexfile + '.meta', indexfile + '.hlink']
54 for indexfile in indexfiles:
55 path = git.repo(indexfile)
59 log('clear: removed %s\n' % path)
61 if e.errno != errno.ENOENT:
65 def update_index(top, excluded_paths):
66 # tmax and start must be epoch nanoseconds.
67 tmax = (time.time() - 1) * 10**9
68 ri = index.Reader(indexfile)
69 msw = index.MetaStoreWriter(indexfile + '.meta')
70 wi = index.Writer(indexfile, msw, tmax)
71 rig = IterHelper(ri.iter(name=top))
72 tstart = int(time.time()) * 10**9
74 hlinks = hlinkdb.HLinkDB(indexfile + '.hlink')
79 return (GIT_MODE_FILE, index.FAKE_SHA)
82 bup_dir = os.path.abspath(git.repo())
83 for (path,pst) in drecurse.recursive_dirlist([top], xdev=opt.xdev,
85 excluded_paths=excluded_paths):
86 if opt.verbose>=2 or (opt.verbose==1 and stat.S_ISDIR(pst.st_mode)):
87 sys.stdout.write('%s\n' % path)
89 qprogress('Indexing: %d\r' % total)
90 elif not (total % 128):
91 qprogress('Indexing: %d\r' % total)
93 while rig.cur and rig.cur.name > path: # deleted paths
97 if rig.cur.nlink > 1 and not stat.S_ISDIR(rig.cur.mode):
98 hlinks.del_path(rig.cur.name)
100 if rig.cur and rig.cur.name == path: # paths that already existed
101 if not stat.S_ISDIR(rig.cur.mode) and rig.cur.nlink > 1:
102 hlinks.del_path(rig.cur.name)
103 if not stat.S_ISDIR(pst.st_mode) and pst.st_nlink > 1:
104 hlinks.add_path(path, pst.st_dev, pst.st_ino)
105 meta = metadata.from_path(path, statinfo=pst)
106 # Clear these so they don't bloat the store -- they're
107 # already in the index (since they vary a lot and they're
108 # fixed length). If you've noticed "tmax", you might
109 # wonder why it's OK to do this, since that code may
110 # adjust (mangle) the index mtime and ctime -- producing
111 # fake values which must not end up in a .bupm. However,
112 # it looks like that shouldn't be possible: (1) When
113 # "save" validates the index entry, it always reads the
114 # metadata from the filesytem. (2) Metadata is only
115 # read/used from the index if hashvalid is true. (3) index
116 # always invalidates "faked" entries, because "old != new"
118 meta.ctime = meta.mtime = meta.atime = 0
119 meta_ofs = msw.store(meta)
120 rig.cur.from_stat(pst, meta_ofs, tstart)
121 if not (rig.cur.flags & index.IX_HASHVALID):
123 (rig.cur.gitmode, rig.cur.sha) = hashgen(path)
124 rig.cur.flags |= index.IX_HASHVALID
130 meta = metadata.from_path(path, statinfo=pst)
131 # See same assignment to 0, above, for rationale.
132 meta.atime = meta.mtime = meta.ctime = 0
133 meta_ofs = msw.store(meta)
134 wi.add(path, pst, meta_ofs, hashgen = hashgen)
135 if not stat.S_ISDIR(pst.st_mode) and pst.st_nlink > 1:
136 hlinks.add_path(path, pst.st_dev, pst.st_ino)
138 progress('Indexing: %d, done.\n' % total)
140 hlinks.prepare_save()
148 log('check: before merging: oldfile\n')
150 log('check: before merging: newfile\n')
152 mi = index.Writer(indexfile, msw, tmax)
154 for e in index.merge(ri, wr):
155 # FIXME: shouldn't we remove deleted entries eventually? When?
170 bup index <-p|m|s|u> [options...] <filenames...>
173 p,print print the index entries for the given names (also works with -u)
174 m,modified print only added/deleted/modified files (implies -p)
175 s,status print each filename with a status char (A/M/D) (implies -p)
176 u,update recursively update the index entries for the given file/dir names (default if no mode is specified)
177 check carefully check index file integrity
178 clear clear the index
180 H,hash print the hash for each object next to its name
181 l,long print more information about each file
182 fake-valid mark all index entries as up-to-date even if they aren't
183 fake-invalid mark all index entries as invalid
184 f,indexfile= the name of the index file (normally BUP_DIR/bupindex)
185 exclude= a path to exclude from the backup (can be used more than once)
186 exclude-from= a file that contains exclude paths (can be used more than once)
187 v,verbose increase log output (can be used more than once)
188 x,xdev,one-file-system don't cross filesystem boundaries
190 o = options.Options(optspec)
191 (opt, flags, extra) = o.parse(sys.argv[1:])
193 if not (opt.modified or \
200 if (opt.fake_valid or opt.fake_invalid) and not opt.update:
201 o.fatal('--fake-{in,}valid are meaningless without -u')
202 if opt.fake_valid and opt.fake_invalid:
203 o.fatal('--fake-valid is incompatible with --fake-invalid')
205 # FIXME: remove this once we account for timestamp races, i.e. index;
206 # touch new-file; index. It's possible for this to happen quickly
207 # enough that new-file ends up with the same timestamp as the first
208 # index, and then bup will ignore it.
209 tick_start = time.time()
210 time.sleep(1 - (tick_start - int(tick_start)))
212 git.check_repo_or_die()
213 indexfile = opt.indexfile or git.repo('bupindex')
218 log('check: starting initial check.\n')
219 check_index(index.Reader(indexfile))
222 log('clear: clearing index.\n')
223 clear_index(indexfile)
225 excluded_paths = parse_excludes(flags, o.fatal)
226 paths = index.reduce_paths(extra)
230 o.fatal('update mode (-u) requested but no paths given')
231 for (rp,path) in paths:
232 update_index(rp, excluded_paths)
234 if opt['print'] or opt.status or opt.modified:
235 for (name, ent) in index.Reader(indexfile).filter(extra or ['']):
237 and (ent.is_valid() or ent.is_deleted() or not ent.mode)):
243 elif not ent.is_valid():
244 if ent.sha == index.EMPTY_SHA:
251 line += ent.sha.encode('hex') + ' '
253 line += "%7s %7s " % (oct(ent.mode), oct(ent.gitmode))
254 print line + (name or './')
256 if opt.check and (opt['print'] or opt.status or opt.modified or opt.update):
257 log('check: starting final check.\n')
258 check_index(index.Reader(indexfile))
261 log('WARNING: %d errors encountered.\n' % len(saved_errors))