From 7e0ad817512a8d74fb8135ecea59d80f57355106 Mon Sep 17 00:00:00 2001 From: Mathias Hansen Date: Mon, 5 Oct 2026 12:48:55 +0200 Subject: [PATCH] Raise zip entry cap to 16 GiB gb/countrywide ships AddressBase Core as a single CSV whose zip entry declares 9,043,535,569 bytes, over the 8 GiB per-entry cap, so every run of the source fails. The default cap is now 16 GiB. Claude-Session: https://claude.ai/code/session_01M9dZ4UWYaMwr9e9cwwKpyS --- openaddr/conform.py | 4 ++-- openaddr/tests/conform.py | 5 +++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/openaddr/conform.py b/openaddr/conform.py index 58b889ac..92707df7 100644 --- a/openaddr/conform.py +++ b/openaddr/conform.py @@ -212,9 +212,9 @@ class ZipDecompressTask(DecompressionTask): # any per-entry file type filtering applies. Cap the declared # (uncompressed) size of any single entry and how many nested zips deep # we'll recurse, so worst case is bounded and the job fails loudly - # instead of exhausting disk. The per-entry cap defaults to 8GiB and can + # instead of exhausting disk. The per-entry cap defaults to 16GiB and can # be overridden with the BATCH_MACHINE_MAX_ZIP_ENTRY_BYTES env var. - MAX_ZIP_ENTRY_BYTES = 8 * 1024 ** 3 # 8GiB + MAX_ZIP_ENTRY_BYTES = 16 * 1024 ** 3 # 16GiB MAX_NESTED_ZIP_DEPTH = 10 def _max_entry_bytes(self): diff --git a/openaddr/tests/conform.py b/openaddr/tests/conform.py index 90a40dd7..7b22955d 100644 --- a/openaddr/tests/conform.py +++ b/openaddr/tests/conform.py @@ -2937,9 +2937,10 @@ def test_entry_under_limit_extracts(self): self.assertIn('readme.txt', {os.path.basename(path) for path in output_files}) - def test_default_entry_limit_admits_entries_over_two_gib(self): - self.assertEqual(ZipDecompressTask.MAX_ZIP_ENTRY_BYTES, 8 * 1024 ** 3) + def test_default_entry_limit_admits_large_known_entries(self): + self.assertEqual(ZipDecompressTask.MAX_ZIP_ENTRY_BYTES, 16 * 1024 ** 3) self.assertGreater(ZipDecompressTask.MAX_ZIP_ENTRY_BYTES, 2540059567) + self.assertGreater(ZipDecompressTask.MAX_ZIP_ENTRY_BYTES, 9043535569) def test_entry_limit_env_override_refuses_oversized_entry(self): outer_zip_path = self._make_outer_zip()