From 82a633b54577a876f0333082e4f69b3740de2c22 Mon Sep 17 00:00:00 2001 From: Avery Pennarun Date: Mon, 7 Jun 2010 19:02:23 -0400 Subject: [PATCH] cmd/{save,split}: add a --bwlimit option. This allows you to limit how much upstream bandwidth 'bup save' and 'bup split' will use. Specify it as a number of bytes/second, or with the 'k' or 'M' or (lucky you!) 'G' suffixes for larger values. Signed-off-by: Avery Pennarun --- cmd/save-cmd.py | 3 +++ cmd/split-cmd.py | 5 ++++- lib/bup/client.py | 33 +++++++++++++++++++++++++++++++-- 3 files changed, 38 insertions(+), 3 deletions(-) diff --git a/cmd/save-cmd.py b/cmd/save-cmd.py index e840e55..59b9cb0 100755 --- a/cmd/save-cmd.py +++ b/cmd/save-cmd.py @@ -14,6 +14,7 @@ n,name= name of backup set to update (if any) v,verbose increase log output (can be used more than once) q,quiet don't show progress meter smaller= only back up files smaller than n bytes +bwlimit= maximum bytes/sec to transmit to server """ o = options.Options('bup save', optspec) (opt, flags, extra) = o.parse(sys.argv[1:]) @@ -26,6 +27,8 @@ if not extra: opt.progress = (istty and not opt.quiet) opt.smaller = parse_num(opt.smaller or 0) +if opt.bwlimit: + client.bwlimit = parse_num(opt.bwlimit) is_reverse = os.environ.get('BUP_SERVER_REVERSE') if is_reverse and opt.remote: diff --git a/cmd/split-cmd.py b/cmd/split-cmd.py index ff73c3f..b065892 100755 --- a/cmd/split-cmd.py +++ b/cmd/split-cmd.py @@ -20,7 +20,8 @@ copy just copy input to output, hashsplitting along the way bench print benchmark timings to stderr max-pack-size= maximum bytes in a single pack max-pack-objects= maximum number of objects in a single pack -fanout= maximum number of blobs in a single tree +fanout= maximum number of blobs in a single tree +bwlimit= maximum bytes/sec to transmit to server """ o = options.Options('bup split', optspec) (opt, flags, extra) = o.parse(sys.argv[1:]) @@ -44,6 +45,8 @@ if opt.fanout: hashsplit.fanout = parse_num(opt.fanout) if opt.blobs: hashsplit.fanout = 0 +if opt.bwlimit: + client.bwlimit = parse_num(opt.bwlimit) is_reverse = os.environ.get('BUP_SERVER_REVERSE') if is_reverse and opt.remote: diff --git a/lib/bup/client.py b/lib/bup/client.py index 32e0e0d..e471cf5 100644 --- a/lib/bup/client.py +++ b/lib/bup/client.py @@ -1,12 +1,37 @@ -import re, struct, errno, select +import re, struct, errno, select, time from bup import git, ssh from bup.helpers import * +bwlimit = None + class ClientError(Exception): pass +def _raw_write_bwlimit(f, buf, bwcount, bwtime): + if not bwlimit: + f.write(buf) + return (len(buf), time.time()) + else: + # We want to write in reasonably large blocks, but not so large that + # they're likely to overflow a router's queue. So our bwlimit timing + # has to be pretty granular. Also, if it takes too long from one + # transmit to the next, we can't just make up for lost time to bring + # the average back up to bwlimit - that will risk overflowing the + # outbound queue, which defeats the purpose. So if we fall behind + # by more than one block delay, we shouldn't ever try to catch up. + for i in xrange(0,len(buf),4096): + now = time.time() + next = max(now, bwtime + 1.0*bwcount/bwlimit) + time.sleep(next-now) + sub = buf[i:i+4096] + f.write(sub) + bwcount = len(sub) # might be less than 4096 + bwtime = next + return (bwcount, bwtime) + + class Client: def __init__(self, remote, create=False): self._busy = self.conn = self.p = self.pout = self.pin = None @@ -208,6 +233,8 @@ class PackWriter_Remote(git.PackWriter): self.onclose = onclose self.ensure_busy = ensure_busy self._packopen = False + self._bwcount = 0 + self._bwtime = time.time() def _open(self): if not self._packopen: @@ -251,7 +278,9 @@ class PackWriter_Remote(git.PackWriter): self.ensure_busy() data = ''.join(datalist) assert(len(data)) - self.file.write(struct.pack('!I', len(data)) + data) + outbuf = struct.pack('!I', len(data)) + data + (self._bwcount, self._bwtime) = \ + _raw_write_bwlimit(self.file, outbuf, self._bwcount, self._bwtime) self.outbytes += len(data) self.count += 1 -- 2.39.2