3 import sys, stat, time, os
4 from bup import metadata, options, git, index, drecurse, hlinkdb
5 from bup.helpers import *
6 from bup.hashsplit import GIT_MODE_TREE, GIT_MODE_FILE
16 self.cur = self.i.next()
22 def check_index(reader):
24 log('check: checking forward iteration...\n')
27 for e in reader.forward_iter():
30 log('%08x+%-4d %r\n' % (e.children_ofs, e.children_n,
32 assert(e.children_ofs)
33 assert(e.name.endswith('/'))
34 assert(not d.get(e.children_ofs))
36 if e.flags & index.IX_HASHVALID:
37 assert(e.sha != index.EMPTY_SHA)
39 assert(not e or e.name == '/') # last entry is *always* /
40 log('check: checking normal iteration...\n')
47 log('index error! at %r\n' % e)
49 log('check: passed.\n')
52 def update_index(top, excluded_paths):
53 # tmax and start must be epoch nanoseconds.
54 tmax = (time.time() - 1) * 10**9
55 ri = index.Reader(indexfile)
56 msw = index.MetaStoreWriter(indexfile + '.meta')
57 wi = index.Writer(indexfile, msw, tmax)
58 rig = IterHelper(ri.iter(name=top))
59 tstart = int(time.time()) * 10**9
61 hlinks = hlinkdb.HLinkDB(indexfile + '.hlink')
66 return (GIT_MODE_FILE, index.FAKE_SHA)
69 bup_dir = os.path.abspath(git.repo())
70 for (path,pst) in drecurse.recursive_dirlist([top], xdev=opt.xdev,
72 excluded_paths=excluded_paths):
73 if opt.verbose>=2 or (opt.verbose==1 and stat.S_ISDIR(pst.st_mode)):
74 sys.stdout.write('%s\n' % path)
76 qprogress('Indexing: %d\r' % total)
77 elif not (total % 128):
78 qprogress('Indexing: %d\r' % total)
80 while rig.cur and rig.cur.name > path: # deleted paths
84 if rig.cur.nlink > 1 and not stat.S_ISDIR(rig.cur.mode):
85 hlinks.del_path(rig.cur.name)
87 if rig.cur and rig.cur.name == path: # paths that already existed
88 if not stat.S_ISDIR(rig.cur.mode) and rig.cur.nlink > 1:
89 hlinks.del_path(rig.cur.name)
90 if not stat.S_ISDIR(pst.st_mode) and pst.st_nlink > 1:
91 hlinks.add_path(path, pst.st_dev, pst.st_ino)
92 meta = metadata.from_path(path, statinfo=pst)
93 # Clear these so they don't bloat the store -- they're
94 # already in the index (since they vary a lot and they're
95 # fixed length). If you've noticed "tmax", you might
96 # wonder why it's OK to do this, since that code may
97 # adjust (mangle) the index mtime and ctime -- producing
98 # fake values which must not end up in a .bupm. However,
99 # it looks like that shouldn't be possible: (1) When
100 # "save" validates the index entry, it always reads the
101 # metadata from the filesytem. (2) Metadata is only
102 # read/used from the index if hashvalid is true. (3) index
103 # always invalidates "faked" entries, because "old != new"
105 meta.ctime = meta.mtime = meta.atime = 0
106 meta_ofs = msw.store(meta)
107 rig.cur.from_stat(pst, meta_ofs, tstart)
108 if not (rig.cur.flags & index.IX_HASHVALID):
110 (rig.cur.gitmode, rig.cur.sha) = hashgen(path)
111 rig.cur.flags |= index.IX_HASHVALID
117 meta = metadata.from_path(path, statinfo=pst)
118 # See same assignment to 0, above, for rationale.
119 meta.atime = meta.mtime = meta.ctime = 0
120 meta_ofs = msw.store(meta)
121 wi.add(path, pst, meta_ofs, hashgen = hashgen)
122 if not stat.S_ISDIR(pst.st_mode) and pst.st_nlink > 1:
123 hlinks.add_path(path, pst.st_dev, pst.st_ino)
125 progress('Indexing: %d, done.\n' % total)
127 hlinks.prepare_save()
135 log('check: before merging: oldfile\n')
137 log('check: before merging: newfile\n')
139 mi = index.Writer(indexfile, msw, tmax)
141 for e in index.merge(ri, wr):
142 # FIXME: shouldn't we remove deleted entries eventually? When?
157 bup index <-p|m|s|u> [options...] <filenames...>
160 p,print print the index entries for the given names (also works with -u)
161 m,modified print only added/deleted/modified files (implies -p)
162 s,status print each filename with a status char (A/M/D) (implies -p)
163 u,update recursively update the index entries for the given file/dir names (default if no mode is specified)
164 check carefully check index file integrity
166 H,hash print the hash for each object next to its name
167 l,long print more information about each file
168 fake-valid mark all index entries as up-to-date even if they aren't
169 fake-invalid mark all index entries as invalid
170 f,indexfile= the name of the index file (normally BUP_DIR/bupindex)
171 exclude= a path to exclude from the backup (can be used more than once)
172 exclude-from= a file that contains exclude paths (can be used more than once)
173 v,verbose increase log output (can be used more than once)
174 x,xdev,one-file-system don't cross filesystem boundaries
176 o = options.Options(optspec)
177 (opt, flags, extra) = o.parse(sys.argv[1:])
179 if not (opt.modified or opt['print'] or opt.status or opt.update or opt.check):
181 if (opt.fake_valid or opt.fake_invalid) and not opt.update:
182 o.fatal('--fake-{in,}valid are meaningless without -u')
183 if opt.fake_valid and opt.fake_invalid:
184 o.fatal('--fake-valid is incompatible with --fake-invalid')
186 # FIXME: remove this once we account for timestamp races, i.e. index;
187 # touch new-file; index. It's possible for this to happen quickly
188 # enough that new-file ends up with the same timestamp as the first
189 # index, and then bup will ignore it.
190 tick_start = time.time()
191 time.sleep(1 - (tick_start - int(tick_start)))
193 git.check_repo_or_die()
194 indexfile = opt.indexfile or git.repo('bupindex')
199 log('check: starting initial check.\n')
200 check_index(index.Reader(indexfile))
202 excluded_paths = drecurse.parse_excludes(flags)
204 paths = index.reduce_paths(extra)
208 o.fatal('update mode (-u) requested but no paths given')
209 for (rp,path) in paths:
210 update_index(rp, excluded_paths)
212 if opt['print'] or opt.status or opt.modified:
213 for (name, ent) in index.Reader(indexfile).filter(extra or ['']):
215 and (ent.is_valid() or ent.is_deleted() or not ent.mode)):
221 elif not ent.is_valid():
222 if ent.sha == index.EMPTY_SHA:
229 line += ent.sha.encode('hex') + ' '
231 line += "%7s %7s " % (oct(ent.mode), oct(ent.gitmode))
232 print line + (name or './')
234 if opt.check and (opt['print'] or opt.status or opt.modified or opt.update):
235 log('check: starting final check.\n')
236 check_index(index.Reader(indexfile))
239 log('WARNING: %d errors encountered.\n' % len(saved_errors))