From 5dfc5f8eb00873664d0a76e8680d77097e1a7cf3 Mon Sep 17 00:00:00 2001 From: Rob Browning Date: Tue, 17 Dec 2013 20:19:44 -0600 Subject: [PATCH] Don't fail tests when the timestamp read resolution is higher than write. Previously, if bup was able to read path timestamps at a higher resolution than it could write them, tests would fail. This situation can occur (for example) when the stat() resolution is 1ns, but either the underlying filesystem's isn't, or bup wasn't able to find utimensat at build time. To fix this, compute the maximum resolution of the test filesystem (via "t/ns-timestamp-resolutions HERE"), and then limit the "bup xstat" timestamp resolution in the tests to match via a new --mtime-resolution argument. For completeness, also add --atime-resolution and --ctime-resolution arguments. Thanks to Tim Riemenschneider for reporting the problem. Signed-off-by: Rob Browning --- cmd/xstat-cmd.py | 33 ++++++++++++++++++++++++- lib/bup/helpers.py | 20 ++++++++++++++++ t/ns-timestamp-resolutions | 49 ++++++++++++++++++++++++++++++++++++++ t/test-meta.sh | 13 +++++++++- 4 files changed, 113 insertions(+), 2 deletions(-) create mode 100755 t/ns-timestamp-resolutions diff --git a/cmd/xstat-cmd.py b/cmd/xstat-cmd.py index 11b0b58..85071d2 100755 --- a/cmd/xstat-cmd.py +++ b/cmd/xstat-cmd.py @@ -5,7 +5,25 @@ # Public License as described in the bup LICENSE file. import sys, stat, errno from bup import metadata, options, xstat -from bup.helpers import handle_ctrl_c, saved_errors, add_error, log +from bup.helpers import handle_ctrl_c, parse_timestamp, saved_errors, \ + add_error, log + + +def parse_timestamp_arg(field, value): + res = str(value) # Undo autoconversion. + try: + res = parse_timestamp(res) + except ValueError, ex: + if ex.args: + o.fatal('unable to parse %s resolution "%s" (%s)' + % (field, value, ex)) + else: + o.fatal('unable to parse %s resolution "%s"' % (field, value)) + + if res != 1 and res % 10: + o.fatal('%s resolution "%s" must be a power of 10' % (field, value)) + return res + optspec = """ bup xstat pathinfo [OPTION ...] @@ -14,6 +32,9 @@ v,verbose increase log output (can be used more than once) q,quiet don't show progress meter exclude-fields= exclude comma-separated fields include-fields= include comma-separated fields (definitive if first) +atime-resolution= limit s, ms, us, ns, 10ns (value must be a power of 10) [ns] +mtime-resolution= limit s, ms, us, ns, 10ns (value must be a power of 10) [ns] +ctime-resolution= limit s, ms, us, ns, 10ns (value must be a power of 10) [ns] """ target_filename = '' @@ -24,6 +45,10 @@ handle_ctrl_c() o = options.Options(optspec) (opt, flags, remainder) = o.parse(sys.argv[1:]) +atime_resolution = parse_timestamp_arg('atime', opt.atime_resolution) +mtime_resolution = parse_timestamp_arg('mtime', opt.mtime_resolution) +ctime_resolution = parse_timestamp_arg('ctime', opt.ctime_resolution) + treat_include_fields_as_definitive = True for flag, value in flags: if flag == '--exclude-fields': @@ -61,6 +86,12 @@ for path in remainder: if metadata.verbose >= 0: if not first_path: print + if atime_resolution != 1: + m.atime = (m.atime / atime_resolution) * atime_resolution + if mtime_resolution != 1: + m.mtime = (m.mtime / mtime_resolution) * mtime_resolution + if ctime_resolution != 1: + m.ctime = (m.ctime / ctime_resolution) * ctime_resolution print metadata.detailed_str(m, active_fields) first_path = False diff --git a/lib/bup/helpers.py b/lib/bup/helpers.py index c090bc0..45fcbb5 100644 --- a/lib/bup/helpers.py +++ b/lib/bup/helpers.py @@ -625,6 +625,26 @@ def mmap_readwrite_private(f, sz = 0, close=True): close) +def parse_timestamp(epoch_str): + """Return the number of nanoseconds since the epoch that are described +by epoch_str (100ms, 100ns, ...); when epoch_str cannot be parsed, +throw a ValueError that may contain additional information.""" + ns_per = {'s' : 1000000000, + 'ms' : 1000000, + 'us' : 1000, + 'ns' : 1} + match = re.match(r'^((?:[-+]?[0-9]+)?)(s|ms|us|ns)$', epoch_str) + if not match: + if re.match(r'^([-+]?[0-9]+)$', epoch_str): + raise ValueError('must include units, i.e. 100ns, 100ms, ...') + raise ValueError() + (n, units) = match.group(1, 2) + if not n: + n = 1 + n = int(n) + return n * ns_per[units] + + def parse_num(s): """Parse data size information into a float number. diff --git a/t/ns-timestamp-resolutions b/t/ns-timestamp-resolutions new file mode 100755 index 0000000..d1bb785 --- /dev/null +++ b/t/ns-timestamp-resolutions @@ -0,0 +1,49 @@ +#!/usr/bin/env python + +import os, sys + +argv = sys.argv +exe = os.path.realpath(argv[0]) +exepath = os.path.split(exe)[0] or '.' +exeprefix = os.path.split(os.path.abspath(exepath))[0] + +# fix the PYTHONPATH to include our lib dir +libpath = os.path.join(exepath, '..', 'lib') +sys.path[:0] = [libpath] +os.environ['PYTHONPATH'] = libpath + ':' + os.environ.get('PYTHONPATH', '') + +import bup.xstat as xstat +from bup.helpers import handle_ctrl_c, saved_errors +from bup import metadata, options + +optspec = """ +ns-timestamp-resolutions TEST_FILE_NAME +-- +""" + +handle_ctrl_c() + +o = options.Options(optspec) +(opt, flags, extra) = o.parse(sys.argv[1:]) + +if len(extra) != 1: + o.fatal('must specify a test file name') + +target = extra[0] + +open(target, 'w').close() +xstat.utime(target, (123456789, 123456789)) +meta = metadata.from_path(target) + +def ns_resolution(x): + n = 1; + while n < 10**9 and x % 10 == 0: + x /= 10 + n *= 10 + return n + +print ns_resolution(meta.atime), ns_resolution(meta.mtime) + +if saved_errors: + log('warning: %d errors encountered\n' % len(saved_errors)) + sys.exit(1) diff --git a/t/test-meta.sh b/t/test-meta.sh index b247e7a..4f8c0c4 100755 --- a/t/test-meta.sh +++ b/t/test-meta.sh @@ -7,6 +7,15 @@ set -o pipefail TOP="$(WVPASS pwd)" || exit $? export BUP_DIR="$TOP/buptest.tmp" +WVPASS force-delete "$TOP/bupmeta.tmp" +timestamp_resolutions="$(t/ns-timestamp-resolutions "$TOP/bupmeta.tmp")" \ + || exit $? +WVPASS rm "$TOP/bupmeta.tmp" +atime_resolution="$(echo $timestamp_resolutions | WVPASS cut -d' ' -f 1)" \ + || exit $? +mtime_resolution="$(echo $timestamp_resolutions | WVPASS cut -d' ' -f 2)" \ + || exit $? + bup() { "$TOP/bup" "$@" @@ -34,7 +43,9 @@ genstat() export PATH="$TOP:$PATH" # pick up bup # Skip atime (test elsewhere) to avoid the observer effect. WVPASS find . | WVPASS sort \ - | WVPASS xargs bup xstat --exclude-fields ctime,atime,size + | WVPASS xargs bup xstat \ + --mtime-resolution "$mtime_resolution"ns \ + --exclude-fields ctime,atime,size ) } -- 2.39.2