1 """Helper functions and classes for bup."""
2 import sys, os, pwd, subprocess, errno, socket, select, mmap, stat, re
3 from bup import _version
5 # This function should really be in helpers, not in bup.options. But we
6 # want options.py to be standalone so people can include it in other projects.
7 from bup.options import _tty_width
12 """Convert the string 's' to an integer. Return 0 if s is not a number."""
20 """Convert the string 's' to a float. Return 0 if s is not a number."""
22 return float(s or '0')
27 buglvl = atoi(os.environ.get('BUP_DEBUG', 0))
30 # Write (blockingly) to sockets that may or may not be in blocking mode.
31 # We need this because our stderr is sometimes eaten by subprocesses
32 # (probably ssh) that sometimes make it nonblocking, if only temporarily,
33 # leading to race conditions. Ick. We'll do it the hard way.
34 def _hard_write(fd, buf):
36 (r,w,x) = select.select([], [fd], [], None)
38 raise IOError('select(fd) returned without being writable')
40 sz = os.write(fd, buf)
42 if e.errno != errno.EAGAIN:
48 """Print a log message to stderr."""
50 _hard_write(sys.stderr.fileno(), s)
63 def mkdirp(d, mode=None):
64 """Recursively create directories on path 'd'.
66 Unlike os.makedirs(), it doesn't raise an exception if the last element of
67 the path already exists.
75 if e.errno == errno.EEXIST:
82 """Get the next item from an iterator, None if we reached the end."""
90 """Delete a file at path 'f' if it currently exists.
92 Unlike os.unlink(), does not throw an exception if the file didn't already
98 if e.errno == errno.ENOENT:
99 pass # it doesn't exist, that's what you asked for
103 """Run a subprocess and return its output."""
104 p = subprocess.Popen(argv, stdout=subprocess.PIPE)
111 """Get the absolute path of a file.
113 Behaves like os.path.realpath, but doesn't follow a symlink for the last
114 element. (ie. if 'p' itself is a symlink, this one won't follow it, but it
115 will follow symlinks in p's directory)
121 if st and stat.S_ISLNK(st.st_mode):
122 (dir, name) = os.path.split(p)
123 dir = os.path.realpath(dir)
124 out = os.path.join(dir, name)
126 out = os.path.realpath(p)
127 #log('realpathing:%r,%r\n' % (p, out))
133 """Get the user's login name."""
138 _username = pwd.getpwuid(uid)[0]
140 _username = 'user%d' % uid
146 """Get the user's full name."""
148 if not _userfullname:
151 _userfullname = pwd.getpwuid(uid)[4].split(',')[0]
153 _userfullname = 'user%d' % uid
159 """Get the FQDN of this machine."""
162 _hostname = socket.getfqdn()
166 _resource_path = None
167 def resource_path(subdir=''):
168 global _resource_path
169 if not _resource_path:
170 _resource_path = os.environ.get('BUP_RESOURCE_PATH') or '.'
171 return os.path.join(_resource_path, subdir)
173 class NotOk(Exception):
177 """A helper class for bup's client-server protocol."""
178 def __init__(self, inp, outp):
182 def read(self, size):
183 """Read 'size' bytes from input stream."""
185 return self.inp.read(size)
188 """Read from input stream until a newline is found."""
190 return self.inp.readline()
192 def write(self, data):
193 """Write 'data' to output stream."""
194 #log('%d writing: %d bytes\n' % (os.getpid(), len(data)))
195 self.outp.write(data)
198 """Return true if input stream is readable."""
199 [rl, wl, xl] = select.select([self.inp.fileno()], [], [], 0)
201 assert(rl[0] == self.inp.fileno())
207 """Indicate end of output from last sent command."""
211 """Indicate server error to the client."""
212 s = re.sub(r'\s+', ' ', str(s))
213 self.write('\nerror %s\n' % s)
215 def _check_ok(self, onempty):
218 for rl in linereader(self.inp):
219 #log('%d got line: %r\n' % (os.getpid(), rl))
220 if not rl: # empty line
224 elif rl.startswith('error '):
225 #log('client: error: %s\n' % rl[6:])
229 raise Exception('server exited unexpectedly; see errors above')
231 def drain_and_check_ok(self):
232 """Remove all data for the current command from input stream."""
235 return self._check_ok(onempty)
238 """Verify that server action completed successfully."""
240 raise Exception('expected "ok", got %r' % rl)
241 return self._check_ok(onempty)
245 """Generate a list of input lines from 'f' without terminating newlines."""
253 def chunkyreader(f, count = None):
254 """Generate a list of chunks of data read from 'f'.
256 If count is None, read until EOF is reached.
258 If count is a positive integer, read 'count' bytes from 'f'. If EOF is
259 reached while reading, raise IOError.
263 b = f.read(min(count, 65536))
265 raise IOError('EOF with %d bytes remaining' % count)
276 """Append "/" to 's' if it doesn't aleady end in "/"."""
277 if s and not s.endswith('/'):
283 def _mmap_do(f, sz, flags, prot):
285 st = os.fstat(f.fileno())
287 map = mmap.mmap(f.fileno(), sz, flags, prot)
288 f.close() # map will persist beyond file close
292 def mmap_read(f, sz = 0):
293 """Create a read-only memory mapped region on file 'f'.
295 If sz is 0, the region will cover the entire file.
297 return _mmap_do(f, sz, mmap.MAP_PRIVATE, mmap.PROT_READ)
300 def mmap_readwrite(f, sz = 0):
301 """Create a read-write memory mapped region on file 'f'.
303 If sz is 0, the region will cover the entire file.
305 return _mmap_do(f, sz, mmap.MAP_SHARED, mmap.PROT_READ|mmap.PROT_WRITE)
309 """Parse data size information into a float number.
311 Here are some examples of conversions:
312 199.2k means 203981 bytes
313 1GB means 1073741824 bytes
314 2.1 tb means 2199023255552 bytes
316 g = re.match(r'([-+\d.e]+)\s*(\w*)', str(s))
318 raise ValueError("can't parse %r as a number" % s)
319 (val, unit) = g.groups()
322 if unit in ['t', 'tb']:
323 mult = 1024*1024*1024*1024
324 elif unit in ['g', 'gb']:
325 mult = 1024*1024*1024
326 elif unit in ['m', 'mb']:
328 elif unit in ['k', 'kb']:
330 elif unit in ['', 'b']:
333 raise ValueError("invalid unit %r in number %r" % (unit, s))
338 """Count the number of elements in an iterator. (consumes the iterator)"""
339 return reduce(lambda x,y: x+1, l)
344 """Append an error message to the list of saved errors.
346 Once processing is able to stop and output the errors, the saved errors are
347 accessible in the module variable helpers.saved_errors.
349 saved_errors.append(e)
352 istty = os.isatty(2) or atoi(os.environ.get('BUP_FORCE_TTY'))
354 """Calls log(s) if stderr is a TTY. Does nothing otherwise."""
360 """Replace the default exception handler for KeyboardInterrupt (Ctrl-C).
362 The new exception handler will make sure that bup will exit without an ugly
363 stacktrace when Ctrl-C is hit.
365 oldhook = sys.excepthook
366 def newhook(exctype, value, traceback):
367 if exctype == KeyboardInterrupt:
368 log('Interrupted.\n')
370 return oldhook(exctype, value, traceback)
371 sys.excepthook = newhook
374 def columnate(l, prefix):
375 """Format elements of 'l' in columns with 'prefix' leading each line.
377 The number of columns is determined automatically based on the string
383 clen = max(len(s) for s in l)
384 ncols = (tty_width() - len(prefix)) / (clen + 2)
389 while len(l) % ncols:
392 for s in range(0, len(l), rows):
393 cols.append(l[s:s+rows])
395 for row in zip(*cols):
396 out += prefix + ''.join(('%-*s' % (clen+2, s)) for s in row) + '\n'
399 def parse_date_or_fatal(str, fatal):
400 """Parses the given date or calls Option.fatal().
401 For now we expect a string that contains a float."""
404 except ValueError, e:
405 raise fatal('invalid date format (should be a float): %r' % e)
410 # hashlib is only available in python 2.5 or higher, but the 'sha' module
411 # produces a DeprecationWarning in python 2.6 or higher. We want to support
412 # python 2.4 and above without any stupid warnings, so let's try using hashlib
413 # first, and downgrade if it fails.
424 """Format bup's version date string for output."""
425 return _version.DATE.split(' ')[0]
427 def version_commit():
428 """Get the commit hash of bup's current version."""
429 return _version.COMMIT
432 """Format bup's version tag (the official version number).
434 When generated from a commit other than one pointed to with a tag, the
435 returned string will be "unknown-" followed by the first seven positions of
438 names = _version.NAMES.strip()
439 assert(names[0] == '(')
440 assert(names[-1] == ')')
442 l = [n.strip() for n in names.split(',')]
444 if n.startswith('tag: bup-'):
446 return 'unknown-%s' % _version.COMMIT[:7]