From 8d47ea88a71a945cef155724b57af190c503e6d3 Mon Sep 17 00:00:00 2001
From: Mathias Hansen <me@codemonkey.io>
Date: Mon, 5 Oct 2026 10:47:42 +0200
Subject: [PATCH] Raise zip entry cap to 8 GiB and make it configurable

au/vic/statewide ships a 3.2 GB parcels zip whose VMPROP.gdb/a00000014.gdbtable
declares 2,540,059,567 bytes, over the 2 GiB per-entry cap, so the source
failed. The cap now defaults to 8 GiB and BATCH_MACHINE_MAX_ZIP_ENTRY_BYTES
overrides it.

Claude-Session: https://claude.ai/code/session_01VHQ8tPQxvwtMYJfu4EGeos
---
 openaddr/conform.py       | 16 ++++++++++++----
 openaddr/tests/conform.py | 31 +++++++++++++++++++++++++++++++
 2 files changed, 43 insertions(+), 4 deletions(-)

diff --git a/openaddr/conform.py b/openaddr/conform.py
index 993834e7..58b889ac 100644
--- a/openaddr/conform.py
+++ b/openaddr/conform.py
@@ -212,10 +212,17 @@ class ZipDecompressTask(DecompressionTask):
     # any per-entry file type filtering applies. Cap the declared
     # (uncompressed) size of any single entry and how many nested zips deep
     # we'll recurse, so worst case is bounded and the job fails loudly
-    # instead of exhausting disk.
-    MAX_ZIP_ENTRY_BYTES = 2 * 1024 ** 3  # 2GB
+    # instead of exhausting disk. The per-entry cap defaults to 8GiB and can
+    # be overridden with the BATCH_MACHINE_MAX_ZIP_ENTRY_BYTES env var.
+    MAX_ZIP_ENTRY_BYTES = 8 * 1024 ** 3  # 8GiB
     MAX_NESTED_ZIP_DEPTH = 10
 
+    def _max_entry_bytes(self):
+        override = os.environ.get('BATCH_MACHINE_MAX_ZIP_ENTRY_BYTES')
+        if override:
+            return int(override)
+        return self.MAX_ZIP_ENTRY_BYTES
+
     def decompress(self, source_paths, workdir, filenames):
         output_files = []
         expand_path = os.path.join(workdir, UNZIPPED_DIRNAME)
@@ -283,6 +290,7 @@ def _extract_zip(self, source_path, expand_path, filenames):
             further nested zips.
         '''
         found = set()
+        max_entry_bytes = self._max_entry_bytes()
         with ZipFile(source_path, 'r') as z:
             for zinfo in z.infolist():
                 name = zinfo.filename
@@ -290,10 +298,10 @@ def _extract_zip(self, source_path, expand_path, filenames):
                 # Check the declared (uncompressed) size from the zip's
                 # central directory before extracting anything - a bomb's
                 # compressed size can be tiny, but its declared size isn't.
-                if zinfo.file_size > self.MAX_ZIP_ENTRY_BYTES:
+                if zinfo.file_size > max_entry_bytes:
                     raise DecompressionError(
                         "Refusing to extract {} - declared size {} bytes exceeds {} byte limit, possible zip bomb"
-                        .format(name, zinfo.file_size, self.MAX_ZIP_ENTRY_BYTES)
+                        .format(name, zinfo.file_size, max_entry_bytes)
                     )
 
                 # Nested zip files are always extracted regardless of the
diff --git a/openaddr/tests/conform.py b/openaddr/tests/conform.py
index fc943488..90a40dd7 100644
--- a/openaddr/tests/conform.py
+++ b/openaddr/tests/conform.py
@@ -2928,6 +2928,37 @@ def test_oversized_entry_raises_decompression_error(self):
             with self.assertRaises(DecompressionError):
                 task.decompress([outer_zip_path], self.workdir, [])
 
+    def test_entry_under_limit_extracts(self):
+        outer_zip_path = self._make_outer_zip()
+
+        task = ZipDecompressTask()
+        with mock.patch.object(ZipDecompressTask, 'MAX_ZIP_ENTRY_BYTES', 10 * 1024 * 1024):
+            output_files = task.decompress([outer_zip_path], self.workdir, [])
+
+        self.assertIn('readme.txt', {os.path.basename(path) for path in output_files})
+
+    def test_default_entry_limit_admits_entries_over_two_gib(self):
+        self.assertEqual(ZipDecompressTask.MAX_ZIP_ENTRY_BYTES, 8 * 1024 ** 3)
+        self.assertGreater(ZipDecompressTask.MAX_ZIP_ENTRY_BYTES, 2540059567)
+
+    def test_entry_limit_env_override_refuses_oversized_entry(self):
+        outer_zip_path = self._make_outer_zip()
+
+        task = ZipDecompressTask()
+        with mock.patch.dict(os.environ, {'BATCH_MACHINE_MAX_ZIP_ENTRY_BYTES': '10'}):
+            with self.assertRaises(DecompressionError):
+                task.decompress([outer_zip_path], self.workdir, [])
+
+    def test_entry_limit_env_override_raises_cap(self):
+        outer_zip_path = self._make_outer_zip()
+
+        task = ZipDecompressTask()
+        with mock.patch.object(ZipDecompressTask, 'MAX_ZIP_ENTRY_BYTES', 10):
+            with mock.patch.dict(os.environ, {'BATCH_MACHINE_MAX_ZIP_ENTRY_BYTES': str(10 * 1024 * 1024)}):
+                output_files = task.decompress([outer_zip_path], self.workdir, [])
+
+        self.assertIn('readme.txt', {os.path.basename(path) for path in output_files})
+
     def test_deeply_nested_zips_raise_decompression_error(self):
         ''' A chain of nested zips deeper than the configured limit should
             be refused, as a zip bomb guard against chained amplification.