X-Git-Url: https://arthur.barton.de/gitweb/?a=blobdiff_plain;f=lib%2Fbup%2Fdrecurse.py;h=84fbce608281f5c2a1cc0c1ffed8e5300cc68689;hb=ed52e7041e90514d6f4cfabd08639b8fa43530d0;hp=2dbe50c7934420f5e11d7b525d63f3f1626749d0;hpb=2a191b01e6082564f188d1517241d19e4a6c5c7b;p=bup.git diff --git a/lib/bup/drecurse.py b/lib/bup/drecurse.py index 2dbe50c..84fbce6 100644 --- a/lib/bup/drecurse.py +++ b/lib/bup/drecurse.py @@ -1,7 +1,21 @@ + +from __future__ import absolute_import import stat, os -from bup.helpers import * + +from bup.compat import str_type +from bup.helpers \ + import (add_error, + debug1, + finalized, + resolve_parent, + should_rx_exclude_path) +from bup.io import path_msg import bup.xstat as xstat +# the use of fchdir() and lstat() is for two reasons: +# - help out the kernel by not making it repeatedly look up the absolute path +# - avoid race conditions caused by doing listdir() on a changing symlink + try: O_LARGEFILE = os.O_LARGEFILE except AttributeError: @@ -12,129 +26,106 @@ except AttributeError: O_NOFOLLOW = 0 -# the use of fchdir() and lstat() is for two reasons: -# - help out the kernel by not making it repeatedly look up the absolute path -# - avoid race conditions caused by doing listdir() on a changing symlink -class OsFile: - def __init__(self, path): - self.fd = None - self.fd = os.open(path, os.O_RDONLY|O_LARGEFILE|O_NOFOLLOW|os.O_NDELAY) - - def __del__(self): - if self.fd: - fd = self.fd - self.fd = None - os.close(fd) - - def fchdir(self): - os.fchdir(self.fd) - - def stat(self): - return xstat.fstat(self.fd) +def finalized_fd(path): + fd = os.open(path, os.O_RDONLY|O_LARGEFILE|O_NOFOLLOW|os.O_NDELAY) + return finalized(fd, lambda x: os.close(x)) -_IFMT = stat.S_IFMT(0xffffffff) # avoid function call in inner loop def _dirlist(): l = [] - for n in os.listdir('.'): + for n in os.listdir(b'.'): try: st = xstat.lstat(n) - except OSError, e: - add_error(Exception('%s: %s' % (realpath(n), str(e)))) + except OSError as e: + add_error(Exception('%s: %s' % (resolve_parent(n), str(e)))) continue - if (st.st_mode & _IFMT) == stat.S_IFDIR: - n += '/' + if stat.S_ISDIR(st.st_mode): + n += b'/' l.append((n,st)) l.sort(reverse=True) return l - -def _recursive_dirlist(prepend, xdev, bup_dir=None, excluded_paths=None): +def _recursive_dirlist(prepend, xdev, bup_dir=None, + excluded_paths=None, + exclude_rxs=None, + xdev_exceptions=frozenset()): for (name,pst) in _dirlist(): - if name.endswith('/'): - if xdev != None and pst.st_dev != xdev: - log('Skipping %r: different filesystem.\n' % (prepend+name)) + path = prepend + name + if excluded_paths: + if os.path.normpath(path) in excluded_paths: + debug1('Skipping %r: excluded.\n' % path_msg(path)) continue + if exclude_rxs and should_rx_exclude_path(path, exclude_rxs): + continue + if name.endswith(b'/'): if bup_dir != None: - if os.path.normpath(prepend+name) == bup_dir: - log('Skipping BUP_DIR.\n') - continue - if excluded_paths: - if os.path.normpath(prepend+name) in excluded_paths: - log('Skipping %r: excluded.\n' % (prepend+name)) + if os.path.normpath(path) == bup_dir: + debug1('Skipping BUP_DIR.\n') continue - try: - OsFile(name).fchdir() - except OSError, e: - add_error('%s: %s' % (prepend, e)) + if xdev != None and pst.st_dev != xdev \ + and path not in xdev_exceptions: + debug1('Skipping contents of %r: different filesystem.\n' + % path_msg(path)) else: - for i in _recursive_dirlist(prepend=prepend+name, xdev=xdev, - bup_dir=bup_dir, - excluded_paths=excluded_paths): - yield i - os.chdir('..') - yield (prepend + name, pst) - - -def recursive_dirlist(paths, xdev, bup_dir=None, excluded_paths=None): - startdir = OsFile('.') - try: - assert(type(paths) != type('')) - for path in paths: - try: - pst = xstat.lstat(path) - if stat.S_ISLNK(pst.st_mode): - yield (path, pst) - continue - except OSError, e: - add_error('recursive_dirlist: %s' % e) - continue - try: - pfile = OsFile(path) - except OSError, e: - add_error(e) - continue - pst = pfile.stat() - if xdev: - xdev = pst.st_dev - else: - xdev = None - if stat.S_ISDIR(pst.st_mode): - pfile.fchdir() - prepend = os.path.join(path, '') - for i in _recursive_dirlist(prepend=prepend, xdev=xdev, - bup_dir=bup_dir, - excluded_paths=excluded_paths): - yield i - startdir.fchdir() - else: - prepend = path - yield (prepend,pst) - except: + try: + with finalized_fd(name) as fd: + os.fchdir(fd) + except OSError as e: + add_error('%s: %s' % (prepend, e)) + else: + for i in _recursive_dirlist(prepend=prepend+name, xdev=xdev, + bup_dir=bup_dir, + excluded_paths=excluded_paths, + exclude_rxs=exclude_rxs, + xdev_exceptions=xdev_exceptions): + yield i + os.chdir(b'..') + yield (path, pst) + + +def recursive_dirlist(paths, xdev, bup_dir=None, + excluded_paths=None, + exclude_rxs=None, + xdev_exceptions=frozenset()): + with finalized_fd(b'.') as startdir: try: - startdir.fchdir() + assert not isinstance(paths, str_type) + for path in paths: + try: + pst = xstat.lstat(path) + if stat.S_ISLNK(pst.st_mode): + yield (path, pst) + continue + except OSError as e: + add_error('recursive_dirlist: %s' % e) + continue + try: + opened_pfile = finalized_fd(path) + except OSError as e: + add_error(e) + continue + with opened_pfile as pfile: + pst = xstat.fstat(pfile) + if xdev: + xdev = pst.st_dev + else: + xdev = None + if stat.S_ISDIR(pst.st_mode): + os.fchdir(pfile) + prepend = os.path.join(path, b'') + for i in _recursive_dirlist(prepend=prepend, xdev=xdev, + bup_dir=bup_dir, + excluded_paths=excluded_paths, + exclude_rxs=exclude_rxs, + xdev_exceptions=xdev_exceptions): + yield i + os.fchdir(startdir) + else: + prepend = path + yield (prepend,pst) except: - pass - raise - -def parse_excludes(flags): - excluded_paths = [] - - for flag in flags: - (option, parameter) = flag - if option == '--exclude': - excluded_paths.append(realpath(parameter)) - - if option == '--exclude-from': try: - try: - f = open(realpath(parameter)) - for exclude_path in f.readlines(): - excluded_paths.append(realpath(exclude_path.strip())) - except Error, e: - log("warning: couldn't read %s" % parameter) - finally: - f.close() - - return excluded_paths - + os.fchdir(startdir) + except: + pass + raise