Remove max_pack_size and max_pack_objects globals

Instead, have the packwriters take optional arguments with the same
names and purposes.

Signed-off-by: Rob Browning <rlb@defaultvalue.org>
Tested-by: Rob Browning <rlb@defaultvalue.org>

diff --git a/cmd/split-cmd.py b/cmd/split-cmd.py
index 637bbd9af6ca593a19eef6b5d17c2ed8001ca46d..5f52e7086ce5c7172ffc06761354115ef17f9f33 100755 (executable)
--- a/cmd/split-cmd.py
+++ b/cmd/split-cmd.py
@@ -58,10 +58,14 @@ if extra and opt.git_ids:
 if opt.verbose >= 2:
     git.verbose = opt.verbose - 1
     opt.bench = 1
+
+max_pack_size = None
 if opt.max_pack_size:
-    git.max_pack_size = parse_num(opt.max_pack_size)
+    max_pack_size = parse_num(opt.max_pack_size)
+max_pack_objects = None
 if opt.max_pack_objects:
-    git.max_pack_objects = parse_num(opt.max_pack_objects)
+    max_pack_objects = parse_num(opt.max_pack_objects)
+
 if opt.fanout:
     hashsplit.fanout = parse_num(opt.fanout)
 if opt.blobs:
@@ -97,11 +101,15 @@ if opt.noop or opt.copy:
 elif opt.remote or is_reverse:
     cli = client.Client(opt.remote)
     oldref = refname and cli.read_ref(refname) or None
-    pack_writer = cli.new_packwriter(compression_level=opt.compress)
+    pack_writer = cli.new_packwriter(compression_level=opt.compress,
+                                     max_pack_size=max_pack_size,
+                                     max_pack_objects=max_pack_objects)
 else:
     cli = None
     oldref = refname and git.read_ref(refname) or None
-    pack_writer = git.PackWriter(compression_level=opt.compress)
+    pack_writer = git.PackWriter(compression_level=opt.compress,
+                                 max_pack_size=max_pack_size,
+                                 max_pack_objects=max_pack_objects)
 
 if opt.git_ids:
     # the input is actually a series of git object ids that we should retrieve

diff --git a/lib/bup/client.py b/lib/bup/client.py
index ff5d75264ae1d521a8c36ed7dae26ba154643b23..aa894599f9d63065b1087ad886b60e4db1bd02d6 100644 (file)
--- a/lib/bup/client.py
+++ b/lib/bup/client.py
@@ -238,7 +238,8 @@ class Client:
             self.conn.write('%s\n' % ob)
         return idx
 
-    def new_packwriter(self, compression_level = 1):
+    def new_packwriter(self, compression_level=1,
+                       max_pack_size=None, max_pack_objects=None):
         self.check_busy()
         def _set_busy():
             self._busy = 'receive-objects-v2'
@@ -249,7 +250,9 @@ class Client:
                                  onopen = _set_busy,
                                  onclose = self._not_busy,
                                  ensure_busy = self.ensure_busy,
-                                 compression_level = compression_level)
+                                 compression_level=compression_level,
+                                 max_pack_size=max_pack_size,
+                                 max_pack_objects=max_pack_objects)
 
     def read_ref(self, refname):
         self.check_busy()
@@ -287,10 +290,14 @@ class PackWriter_Remote(git.PackWriter):
     def __init__(self, conn, objcache_maker, suggest_packs,
                  onopen, onclose,
                  ensure_busy,
-                 compression_level=1):
+                 compression_level=1,
+                 max_pack_size=None,
+                 max_pack_objects=None):
         git.PackWriter.__init__(self,
                                 objcache_maker=objcache_maker,
-                                compression_level=compression_level)
+                                compression_level=compression_level,
+                                max_pack_size=max_pack_size,
+                                max_pack_objects=max_pack_objects)
         self.file = conn
         self.filename = 'remote socket'
         self.suggest_packs = suggest_packs

diff --git a/lib/bup/git.py b/lib/bup/git.py
index 8dd92361f964c367d9ab74c0e2523368ff31f564..a0b42ffaf3997eced2b5745b776ef3da58354b77 100644 (file)
--- a/lib/bup/git.py
+++ b/lib/bup/git.py
@@ -16,10 +16,6 @@ from bup.helpers import (Sha1, add_error, chunkyreader, debug1, debug2,
                          unlink, username, userfullname,
                          utc_offset_str)
 
-
-max_pack_size = 1000*1000*1000  # larger packs will slow down pruning
-max_pack_objects = 200*1000  # cache memory usage is about 83 bytes per object
-
 verbose = 0
 ignore_midx = 0
 repodir = None  # The default repository, once initialized
@@ -596,7 +592,9 @@ def _make_objcache():
 class PackWriter:
     """Writes Git objects inside a pack file."""
     def __init__(self, objcache_maker=_make_objcache, compression_level=1,
-                 run_midx=True, on_pack_finish=None):
+                 run_midx=True, on_pack_finish=None,
+                 max_pack_size=None, max_pack_objects=None):
+        self.repo_dir = repo()
         self.file = None
         self.parentfd = None
         self.count = 0
@@ -608,13 +606,19 @@ class PackWriter:
         self.compression_level = compression_level
         self.run_midx=run_midx
         self.on_pack_finish = on_pack_finish
+        # larger packs will slow down pruning
+        self.max_pack_size = max_pack_size if max_pack_size \
+                             else 1000 * 1000 * 1000
+        # cache memory usage is about 83 bytes per object
+        self.max_pack_objects = max_pack_objects if max_pack_objects \
+                                else max(1, self.max_pack_size // 5000)
 
     def __del__(self):
         self.close()
 
     def _open(self):
         if not self.file:
-            objdir = dir=repo('objects')
+            objdir = dir = os.path.join(self.repo_dir, 'objects')
             fd, name = tempfile.mkstemp(suffix='.pack', dir=objdir)
             try:
                 self.file = os.fdopen(fd, 'w+b')
@@ -666,7 +670,8 @@ class PackWriter:
         size, crc = self._raw_write(_encode_packobj(type, content,
                                                     self.compression_level),
                                     sha=sha)
-        if self.outbytes >= max_pack_size or self.count >= max_pack_objects:
+        if self.outbytes >= self.max_pack_size \
+           or self.count >= self.max_pack_objects:
             self.breakpoint()
         return sha
 
@@ -778,8 +783,8 @@ class PackWriter:
             f.close()
 
         obj_list_sha = self._write_pack_idx_v2(self.filename + '.idx', idx, packbin)
-
-        nameprefix = repo('objects/pack/pack-%s' % obj_list_sha)
+        nameprefix = os.path.join(self.repo_dir,
+                                  'objects/pack/pack-' +  obj_list_sha)
         if os.path.exists(self.filename + '.map'):
             os.unlink(self.filename + '.map')
         os.rename(self.filename + '.pack', nameprefix + '.pack')
@@ -790,7 +795,7 @@ class PackWriter:
             os.close(self.parentfd)
 
         if run_midx:
-            auto_midx(repo('objects/pack'))
+            auto_midx(os.path.join(self.repo_dir, 'objects/pack'))
 
         if self.on_pack_finish:
             self.on_pack_finish(nameprefix)