1 """Helper functions and classes for bup."""
2 import sys, os, pwd, subprocess, errno, socket, select, mmap, stat, re
3 from bup import _version
5 # This function should really be in helpers, not in bup.options. But we
6 # want options.py to be standalone so people can include it in other projects.
7 from bup.options import _tty_width
12 """Convert the string 's' to an integer. Return 0 if s is not a number."""
20 """Convert the string 's' to a float. Return 0 if s is not a number."""
22 return float(s or '0')
27 buglvl = atoi(os.environ.get('BUP_DEBUG', 0))
30 # Write (blockingly) to sockets that may or may not be in blocking mode.
31 # We need this because our stderr is sometimes eaten by subprocesses
32 # (probably ssh) that sometimes make it nonblocking, if only temporarily,
33 # leading to race conditions. Ick. We'll do it the hard way.
34 def _hard_write(fd, buf):
36 (r,w,x) = select.select([], [fd], [], None)
38 raise IOError('select(fd) returned without being writable')
40 sz = os.write(fd, buf)
42 if e.errno != errno.EAGAIN:
48 """Print a log message to stderr."""
50 _hard_write(sys.stderr.fileno(), s)
63 def mkdirp(d, mode=None):
64 """Recursively create directories on path 'd'.
66 Unlike os.makedirs(), it doesn't raise an exception if the last element of
67 the path already exists.
75 if e.errno == errno.EEXIST:
82 """Get the next item from an iterator, None if we reached the end."""
90 """Delete a file at path 'f' if it currently exists.
92 Unlike os.unlink(), does not throw an exception if the file didn't already
98 if e.errno == errno.ENOENT:
99 pass # it doesn't exist, that's what you asked for
103 """Run a subprocess and return its output."""
104 p = subprocess.Popen(argv, stdout=subprocess.PIPE)
111 """Get the absolute path of a file.
113 Behaves like os.path.realpath, but doesn't follow a symlink for the last
114 element. (ie. if 'p' itself is a symlink, this one won't follow it, but it
115 will follow symlinks in p's directory)
121 if st and stat.S_ISLNK(st.st_mode):
122 (dir, name) = os.path.split(p)
123 dir = os.path.realpath(dir)
124 out = os.path.join(dir, name)
126 out = os.path.realpath(p)
127 #log('realpathing:%r,%r\n' % (p, out))
133 """Get the user's login name."""
138 _username = pwd.getpwuid(uid)[0]
140 _username = 'user%d' % uid
146 """Get the user's full name."""
148 if not _userfullname:
151 _userfullname = pwd.getpwuid(uid)[4].split(',')[0]
153 _userfullname = 'user%d' % uid
159 """Get the FQDN of this machine."""
162 _hostname = socket.getfqdn()
166 _resource_path = None
167 def resource_path(subdir=''):
168 global _resource_path
169 if not _resource_path:
170 _resource_path = os.environ.get('BUP_RESOURCE_PATH') or '.'
171 return os.path.join(_resource_path, subdir)
173 class NotOk(Exception):
177 """A helper class for bup's client-server protocol."""
178 def __init__(self, inp, outp):
182 def read(self, size):
183 """Read 'size' bytes from input stream."""
185 return self.inp.read(size)
188 """Read from input stream until a newline is found."""
190 return self.inp.readline()
192 def write(self, data):
193 """Write 'data' to output stream."""
194 #log('%d writing: %d bytes\n' % (os.getpid(), len(data)))
195 self.outp.write(data)
198 """Return true if input stream is readable."""
199 [rl, wl, xl] = select.select([self.inp.fileno()], [], [], 0)
201 assert(rl[0] == self.inp.fileno())
207 """Indicate end of output from last sent command."""
211 """Indicate server error to the client."""
212 s = re.sub(r'\s+', ' ', str(s))
213 self.write('\nerror %s\n' % s)
215 def _check_ok(self, onempty):
218 for rl in linereader(self.inp):
219 #log('%d got line: %r\n' % (os.getpid(), rl))
220 if not rl: # empty line
224 elif rl.startswith('error '):
225 #log('client: error: %s\n' % rl[6:])
229 raise Exception('server exited unexpectedly; see errors above')
231 def drain_and_check_ok(self):
232 """Remove all data for the current command from input stream."""
235 return self._check_ok(onempty)
238 """Verify that server action completed successfully."""
240 raise Exception('expected "ok", got %r' % rl)
241 return self._check_ok(onempty)
245 """Generate a list of input lines from 'f' without terminating newlines."""
253 def chunkyreader(f, count = None):
254 """Generate a list of chunks of data read from 'f'.
256 If count is None, read until EOF is reached.
258 If count is a positive integer, read 'count' bytes from 'f'. If EOF is
259 reached while reading, raise IOError.
263 b = f.read(min(count, 65536))
265 raise IOError('EOF with %d bytes remaining' % count)
276 """Append "/" to 's' if it doesn't aleady end in "/"."""
277 if s and not s.endswith('/'):
283 def _mmap_do(f, sz, flags, prot):
285 st = os.fstat(f.fileno())
288 # trying to open a zero-length map gives an error, but an empty
289 # string has all the same behaviour of a zero-length map, ie. it has
292 map = mmap.mmap(f.fileno(), sz, flags, prot)
293 f.close() # map will persist beyond file close
297 def mmap_read(f, sz = 0):
298 """Create a read-only memory mapped region on file 'f'.
300 If sz is 0, the region will cover the entire file.
302 return _mmap_do(f, sz, mmap.MAP_PRIVATE, mmap.PROT_READ)
305 def mmap_readwrite(f, sz = 0):
306 """Create a read-write memory mapped region on file 'f'.
308 If sz is 0, the region will cover the entire file.
310 return _mmap_do(f, sz, mmap.MAP_SHARED, mmap.PROT_READ|mmap.PROT_WRITE)
314 """Parse data size information into a float number.
316 Here are some examples of conversions:
317 199.2k means 203981 bytes
318 1GB means 1073741824 bytes
319 2.1 tb means 2199023255552 bytes
321 g = re.match(r'([-+\d.e]+)\s*(\w*)', str(s))
323 raise ValueError("can't parse %r as a number" % s)
324 (val, unit) = g.groups()
327 if unit in ['t', 'tb']:
328 mult = 1024*1024*1024*1024
329 elif unit in ['g', 'gb']:
330 mult = 1024*1024*1024
331 elif unit in ['m', 'mb']:
333 elif unit in ['k', 'kb']:
335 elif unit in ['', 'b']:
338 raise ValueError("invalid unit %r in number %r" % (unit, s))
343 """Count the number of elements in an iterator. (consumes the iterator)"""
344 return reduce(lambda x,y: x+1, l)
349 """Append an error message to the list of saved errors.
351 Once processing is able to stop and output the errors, the saved errors are
352 accessible in the module variable helpers.saved_errors.
354 saved_errors.append(e)
357 istty = os.isatty(2) or atoi(os.environ.get('BUP_FORCE_TTY'))
359 """Calls log(s) if stderr is a TTY. Does nothing otherwise."""
365 """Replace the default exception handler for KeyboardInterrupt (Ctrl-C).
367 The new exception handler will make sure that bup will exit without an ugly
368 stacktrace when Ctrl-C is hit.
370 oldhook = sys.excepthook
371 def newhook(exctype, value, traceback):
372 if exctype == KeyboardInterrupt:
373 log('Interrupted.\n')
375 return oldhook(exctype, value, traceback)
376 sys.excepthook = newhook
379 def columnate(l, prefix):
380 """Format elements of 'l' in columns with 'prefix' leading each line.
382 The number of columns is determined automatically based on the string
388 clen = max(len(s) for s in l)
389 ncols = (tty_width() - len(prefix)) / (clen + 2)
394 while len(l) % ncols:
397 for s in range(0, len(l), rows):
398 cols.append(l[s:s+rows])
400 for row in zip(*cols):
401 out += prefix + ''.join(('%-*s' % (clen+2, s)) for s in row) + '\n'
404 def parse_date_or_fatal(str, fatal):
405 """Parses the given date or calls Option.fatal().
406 For now we expect a string that contains a float."""
409 except ValueError, e:
410 raise fatal('invalid date format (should be a float): %r' % e)
414 def strip_path(prefix, path):
415 """Strips a given prefix from a path.
417 First both paths are normalized.
419 Raises an Exception if no prefix is given.
422 raise Exception('no path given')
424 normalized_prefix = os.path.realpath(prefix)
425 debug2("normalized_prefix: %s\n" % normalized_prefix)
426 normalized_path = os.path.realpath(path)
427 debug2("normalized_path: %s\n" % normalized_path)
428 if normalized_path.startswith(normalized_prefix):
429 return normalized_path[len(normalized_prefix):]
433 def strip_base_path(path, base_paths):
434 """Strips the base path from a given path.
436 Determines the base path for the given string and the strips it
438 Iterates over all base_paths from long to short, to prevent that
439 a too short base_path is removed.
441 normalized_path = os.path.realpath(path)
442 sorted_base_paths = sorted(base_paths, key=len, reverse=True)
443 for bp in sorted_base_paths:
444 if normalized_path.startswith(os.path.realpath(bp)):
445 return strip_path(bp, normalized_path)
449 # hashlib is only available in python 2.5 or higher, but the 'sha' module
450 # produces a DeprecationWarning in python 2.6 or higher. We want to support
451 # python 2.4 and above without any stupid warnings, so let's try using hashlib
452 # first, and downgrade if it fails.
463 """Format bup's version date string for output."""
464 return _version.DATE.split(' ')[0]
466 def version_commit():
467 """Get the commit hash of bup's current version."""
468 return _version.COMMIT
471 """Format bup's version tag (the official version number).
473 When generated from a commit other than one pointed to with a tag, the
474 returned string will be "unknown-" followed by the first seven positions of
477 names = _version.NAMES.strip()
478 assert(names[0] == '(')
479 assert(names[-1] == ')')
481 l = [n.strip() for n in names.split(',')]
483 if n.startswith('tag: bup-'):
485 return 'unknown-%s' % _version.COMMIT[:7]