diff --git a/README.md b/README.md index de62ef3..6b518f0 100644 --- a/README.md +++ b/README.md @@ -44,12 +44,22 @@ rejects an output path that aliases an input PDF. **The first field is not an exact original PDF byte span.** pypdf's extraction helpers may discard or retain some delimiter whitespace. The second field is **not measured decoded pixels**, not an extracted PNG/JPEG file size, and not -allocated image memory. Named/custom color spaces are explicitly unknown rather -than guessed. DCT/JPX payloads are never treated as decoded rasters. +allocated image memory. Device and Indexed color spaces can be resolved directly +or by name in the current page/Form `/Resources` dictionary. Indexed images use +one component per sample, regardless of the palette's base space. Other custom +spaces (including ICCBased, Separation and DeviceN), missing names, and cyclic +aliases remain explicitly unknown. Lookup is bounded to 32 steps and never +reads image pixels or decodes profiles. See PDF 32000-1, sections 7.8.3 and 8.9.7 +([Adobe reference](https://opensource.adobe.com/dc-acrobat-sdk-docs/pdfstandards/PDF32000_2008.pdf)). +DCT/JPX payloads are never treated as decoded rasters. Forms are traversed through actual `Do` instructions, including nested Forms. An indirect Form reused three times contributes one definition and three -invocations. Cycles, excessive nesting/invocations, page limits and image limits +invocations. A Form's own resources take precedence over inherited page +resources. If reuse under different inherited resources produces different +estimates, the definition's estimate becomes unknown with reason +`context_dependent_color_space`; invocation counting still includes every use. +Cycles, excessive nesting/invocations, page limits and image limits produce explicit partial results. Uninvoked resource entries are not scanned. Top-level content is keyed per page; a content stream shared between pages is therefore measured per page. This distinction is recorded here for corpus users. @@ -107,8 +117,17 @@ bytes. Two DCT payloads of 3,056 and 2,079 bytes have estimated packed sample si of 56,394 and 15,183 bytes respectively. Encoded and estimated sample limits therefore answer different questions. The estimates remain metadata-derived. -One document contributes 13,591 definitions and dominates pooled percentiles; -1,684 sample estimates are unknown. The JPX control contains an XObject and does +The historical report was produced by version 0.1.0, with 1,684 unresolved +sample estimates. Version 0.2.0 resolves all 1,684 through named Indexed color +spaces in `plan-excerpt.pdf`. The comparison in +[`research/resource-color-spaces.json`](research/resource-color-spaces.json) +uses the same eight PDF hashes, statuses and 15,453 definitions; the encoded +payload distribution is unchanged. Reproduce it with `--max-images 20000` +(the default 10,000 limit truncates the largest file). No additional independent +sample is claimed. + +One document contributes 13,591 definitions and dominates pooled percentiles. +The JPX control contains an XObject and does not establish inline-JPX coverage. This convenience sample does not justify a universal hard limit or estimate ecosystem prevalence. No upstream review or acceptance is claimed. @@ -123,7 +142,9 @@ python -m unittest discover -s tests -v Synthetic PDF fixtures test parser-backed payload measurements, filter arrays, long-form keys, false-positive text strings, bit-packed rows, image masks, repeated -Forms, recursion, unknown color spaces, corruption, encryption, scan limits, +Forms, recursion, indirect named color spaces, Form resource precedence, +context-dependent reuse, cyclic aliases, unknown color spaces, corruption, +encryption, scan limits, timeouts and the Windows-compatible spawn-based CLI. ## Contribution and provenance diff --git a/pdf_inline_audit.py b/pdf_inline_audit.py index cb3be26..4399e67 100644 --- a/pdf_inline_audit.py +++ b/pdf_inline_audit.py @@ -17,7 +17,7 @@ from pypdf import PdfReader from pypdf.generic import ContentStream, DictionaryObject -VERSION = "0.1.0" +VERSION = "0.2.0" @dataclass(frozen=True) @@ -33,7 +33,37 @@ class ScanLimit(Exception): """A caller-selected limit prevents complete measurement.""" -def sample_estimate(settings: dict) -> tuple[int | None, str | None]: +def color_components(color: Any, resources: dict | None = None) -> int | None: + """Resolve known device/Indexed spaces, without decoding profiles or pixels.""" + spaces = None + seen = set() + for _ in range(32): + if hasattr(color, "get_object"): + color = color.get_object() + if isinstance(color, list): + return 1 if color and str(color[0]) in ("/I", "/Indexed") else None + if not isinstance(color, str): + return None + channels = { + "/G": 1, "/DeviceGray": 1, "/RGB": 3, "/DeviceRGB": 3, + "/CMYK": 4, "/DeviceCMYK": 4, + }.get(color) + if channels is not None: + return channels + if color in seen: + return None + if spaces is None: + spaces = (resources or {}).get("/ColorSpace", {}) + if hasattr(spaces, "get_object"): + spaces = spaces.get_object() + if not isinstance(spaces, dict): + return None + seen.add(color) + color = spaces.get(color) + return None + + +def sample_estimate(settings: dict, resources: dict | None = None) -> tuple[int | None, str | None]: """Packed samples including row padding, not a measured decoded size.""" width = settings.get("/W", settings.get("/Width")) height = settings.get("/H", settings.get("/Height")) @@ -44,12 +74,7 @@ def sample_estimate(settings: dict) -> tuple[int | None, str | None]: if mask and bits != 1: return None, "contradictory_image_mask_bits" color = settings.get("/CS", settings.get("/ColorSpace")) - channels = 1 if mask else { - "/G": 1, "/DeviceGray": 1, "/RGB": 3, "/DeviceRGB": 3, - "/CMYK": 4, "/DeviceCMYK": 4, - }.get(str(color)) - if isinstance(color, list) and color and str(color[0]) in ("/I", "/Indexed"): - channels = 1 + channels = 1 if mask else color_components(color, resources) if any(isinstance(v, bool) or not isinstance(v, int) or v <= 0 for v in (width, height, bits)): return None, "missing_or_invalid_dimensions_or_bits" @@ -107,13 +132,17 @@ def walk(stream: Any, resources: Any, label: str, page: int, for index, (operands, operator) in enumerate(operations[label]): if operator == b"INLINE IMAGE": key = (label, index) + settings = operands["settings"] + estimated, unknown = sample_estimate(settings, resources) if key in definitions: - definitions[key]["invocations"] += 1 + previous = definitions[key] + previous["invocations"] += 1 + if (previous["estimated_packed_sample_bytes"], previous["estimate_unknown_reason"]) != (estimated, unknown): + previous["estimated_packed_sample_bytes"] = None + previous["estimate_unknown_reason"] = "context_dependent_color_space" continue if len(definitions) >= limits.max_images: raise ScanLimit("image_count_limit") - settings = operands["settings"] - estimated, unknown = sample_estimate(settings) item = { "stream": label, "operation_index": index, "first_page": page, "invocations": 1, diff --git a/pyproject.toml b/pyproject.toml index 51b1384..4f87f7c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "pdf-inline-audit" -version = "0.1.0" +version = "0.2.0" description = "Reproducible corpus statistics for PDF inline image payloads" readme = "README.md" requires-python = ">=3.10" diff --git a/research/resource-color-spaces.json b/research/resource-color-spaces.json new file mode 100644 index 0000000..02ab1bb --- /dev/null +++ b/research/resource-color-spaces.json @@ -0,0 +1,163 @@ +{ + "date": "2026-10-21", + "tool_version": "0.2.0", + "pypdf_version": "6.21.1", + "python_version": "3.13.5", + "baseline_report": "inline-regressions.json", + "corpus_provenance": "Same eight public upstream fixtures and SHA256 values as baseline_report; no additional independent corpus or redistributed PDFs.", + "reproduction": { + "command": "pdf-inline-audit inline-fixtures --max-images 20000 --output resource-report.json", + "limits": { + "max_file_bytes": 52428800, + "max_pages": 2000, + "max_images": 20000, + "max_form_depth": 32, + "max_form_invocations": 5000 + }, + "timeout_seconds_per_file": 20, + "exit_status": 1, + "exit_status_reason": "The same deliberately malformed DCT fixture remains partial; seven files complete." + }, + "before": { + "file_status_counts": { + "ok": 7, + "partial": 1 + }, + "unique_inline_definitions": 15453, + "inline_invocations": 15453, + "parsed_encoded_payload_bytes": { + "count": 15453, + "minimum": 1, + "maximum": 3996, + "p50_nearest_rank": 272, + "p95_nearest_rank": 2604, + "greater_than_4096": 0, + "greater_than_8192": 0 + }, + "estimated_packed_sample_bytes": { + "count": 13769, + "minimum": 1, + "maximum": 56394, + "p50_nearest_rank": 216, + "p95_nearest_rank": 2516, + "greater_than_4096": 2, + "greater_than_8192": 2 + }, + "unknown_sample_estimates": 1684 + }, + "after": { + "file_status_counts": { + "ok": 7, + "partial": 1 + }, + "unique_inline_definitions": 15453, + "inline_invocations": 15453, + "parsed_encoded_payload_bytes": { + "count": 15453, + "minimum": 1, + "maximum": 3996, + "p50_nearest_rank": 272, + "p95_nearest_rank": 2604, + "greater_than_4096": 0, + "greater_than_8192": 0 + }, + "estimated_packed_sample_bytes": { + "count": 15453, + "minimum": 1, + "maximum": 56394, + "p50_nearest_rank": 272, + "p95_nearest_rank": 2608, + "greater_than_4096": 2, + "greater_than_8192": 2 + }, + "unknown_sample_estimates": 0 + }, + "comparison": { + "file_hashes_statuses_and_definition_counts_unchanged": true, + "encoded_payload_distribution_unchanged": true, + "newly_resolved_estimates": 1684, + "explanation": "Named Indexed color spaces in plan-excerpt.pdf now resolve through the current page resource dictionary. One sample component is used per index, with row byte padding. No image pixels or ICC profiles are decoded." + }, + "per_file": [ + { + "source": "inline-fixtures/bi-text.pdf", + "sha256": "edec7c0cb675c14d68c2ee15f7fa05a3508d797e68421669e20525acfb8620d3", + "status": "ok", + "inline_definitions": 1, + "previous_unknown_estimates": 0, + "unknown_estimates": 0, + "errors": [] + }, + { + "source": "inline-fixtures/bug1065245.pdf", + "sha256": "00beff9834bc18913a93f0c7bc0572187845c665e29a1ff3aa0bc3b8121feee5", + "status": "ok", + "inline_definitions": 3, + "previous_unknown_estimates": 0, + "unknown_estimates": 0, + "errors": [] + }, + { + "source": "inline-fixtures/inline-dct-early-eof.pdf", + "sha256": "cd0bbc59418c682021fad122aec4d6de463077883702849f826775b3d07b7dd5", + "status": "partial", + "inline_definitions": 0, + "previous_unknown_estimates": 0, + "unknown_estimates": 0, + "errors": [ + "page:1:PdfReadError:Unexpected end of stream." + ] + }, + { + "source": "inline-fixtures/jpx-no-spacecode.pdf", + "sha256": "73cdad27df4070317e1de3548fb9cc326807e790067f5148b07724589d412148", + "status": "ok", + "inline_definitions": 0, + "previous_unknown_estimates": 0, + "unknown_estimates": 0, + "errors": [] + }, + { + "source": "inline-fixtures/lebo102.pdf", + "sha256": "9fbb0bee282e9a6736da00a970516fccb405f06939fce37e2923f9e9b2074844", + "status": "ok", + "inline_definitions": 13591, + "previous_unknown_estimates": 0, + "unknown_estimates": 0, + "errors": [] + }, + { + "source": "inline-fixtures/pdfjs-issue14256.pdf", + "sha256": "6f93d05677c6ceea8d95acd032c19c0e6c88cdb917f73639aa931cd2395d88f9", + "status": "ok", + "inline_definitions": 8, + "previous_unknown_estimates": 0, + "unknown_estimates": 0, + "errors": [] + }, + { + "source": "inline-fixtures/plan-excerpt.pdf", + "sha256": "79d9ffcb529bbb6c1fba336ebfa963c4fe77b13ec652f9d9902bd55e1033453a", + "status": "ok", + "inline_definitions": 1849, + "previous_unknown_estimates": 1684, + "unknown_estimates": 0, + "errors": [] + }, + { + "source": "inline-fixtures/tt-inline.pdf", + "sha256": "ad7046301abfe41a95bbb91169c3d9d6197f413172b7f4447ec0e3ca0c9b6257", + "status": "ok", + "inline_definitions": 1, + "previous_unknown_estimates": 0, + "unknown_estimates": 0, + "errors": [] + } + ], + "limitations": [ + "Metadata-derived estimates, not measured decoded storage or allocated memory.", + "Same deliberately selected regression corpus; not a new independent sample or ecosystem prevalence estimate.", + "One file dominates pooled statistics. No inline JPX coverage or universal hard size limit follows.", + "Unsupported color spaces remain unknown. Reused Forms with conflicting inherited component counts remain unknown." + ] +} diff --git a/tests/test_audit.py b/tests/test_audit.py index 8cd2f7c..9d0431d 100644 --- a/tests/test_audit.py +++ b/tests/test_audit.py @@ -1,26 +1,36 @@ import json from pathlib import Path from pypdf import PdfWriter -from pypdf.generic import ArrayObject, BooleanObject, DecodedStreamObject, DictionaryObject, NameObject, NumberObject +from pypdf.generic import ArrayObject, BooleanObject, ByteStringObject, DecodedStreamObject, DictionaryObject, NameObject, NumberObject from pdf_inline_audit import Limits, distribution, isolated_scan, main, sample_estimate, scan_pdf -def make_pdf(path, streams, form_data=None, recursive=False): +def make_pdf(path, streams, form_data=None, recursive=False, color_spaces=None, form_color_spaces=None): writer = PdfWriter() + + def color_resources(spaces): + table = DictionaryObject({NameObject(name): writer._add_object(value) + for name, value in spaces.items()}) + return DictionaryObject({NameObject('/ColorSpace'): writer._add_object(table)}) + form_ref = None if form_data is not None: form = DecodedStreamObject() form.set_data(form_data) form.update({NameObject('/Subtype'): NameObject('/Form'), NameObject('/BBox'): ArrayObject([NumberObject(v) for v in (0, 0, 10, 10)])}) form_ref = writer._add_object(form) + if form_color_spaces is not None: + form[NameObject('/Resources')] = color_resources(form_color_spaces) if recursive: form[NameObject('/Resources')] = DictionaryObject({NameObject('/XObject'): DictionaryObject({NameObject('/Fm'): form_ref})}) - for data in streams: + for index, data in enumerate(streams): page = writer.add_blank_page(100, 100) content = DecodedStreamObject() content.set_data(data) page[NameObject('/Contents')] = writer._add_object(content) if form_ref is not None: page[NameObject('/Resources')] = DictionaryObject({NameObject('/XObject'): DictionaryObject({NameObject('/Fm'): form_ref})}) + if color_spaces is not None: + page.setdefault(NameObject('/Resources'), DictionaryObject()).update(color_resources(color_spaces[index])) writer.write(path) return path @@ -142,6 +152,45 @@ def test_unresolved_colorspace_is_unknown_not_guessed(self): assert estimate is None assert reason == 'unresolved_or_unsupported_color_space' + def test_named_indexed_colorspace_resolves_indirect_page_resources(self): + indexed = ArrayObject([NameObject('/Indexed'), NameObject('/DeviceRGB'), + NumberObject(1), ByteStringObject(b'\x00\x00\x00\xff\xff\xff')]) + path = make_pdf(self.tmp_path / 'named-indexed.pdf', + [inline(b'00000000>', b'/W 9 /H 2 /BPC 1 /CS /Palette /F /AHx')], + color_spaces=[{'/Palette': indexed}]) + result = scan_pdf(path) + assert result['status'] == 'ok' + assert result['images'][0]['estimated_packed_sample_bytes'] == 4 + + def test_form_colorspace_overrides_page_resource_name(self): + path = make_pdf(self.tmp_path / 'form-colors.pdf', [b'/Fm Do'], + inline(b'000102030405060708090a0b>', b'/W 2 /H 2 /BPC 8 /CS /Custom /F /AHx'), + color_spaces=[{'/Custom': NameObject('/DeviceGray')}], + form_color_spaces={'/Custom': NameObject('/DeviceRGB')}) + result = scan_pdf(path) + assert result['status'] == 'ok' + assert result['images'][0]['estimated_packed_sample_bytes'] == 12 + + def test_reused_form_with_conflicting_inherited_colorspaces_is_unknown(self): + path = make_pdf(self.tmp_path / 'context-dependent.pdf', [b'/Fm Do', b'/Fm Do'], + inline(b'000102>', b'/W 1 /H 1 /BPC 8 /CS /Custom /F /AHx'), + color_spaces=[{'/Custom': NameObject('/DeviceGray')}, + {'/Custom': NameObject('/DeviceRGB')}]) + result = scan_pdf(path) + assert result['status'] == 'ok' + assert len(result['images']) == 1 + image = result['images'][0] + assert image['invocations'] == 2 + assert image['estimated_packed_sample_bytes'] is None + assert image['estimate_unknown_reason'] == 'context_dependent_color_space' + + def test_cyclic_colorspace_aliases_remain_unknown(self): + estimate, reason = sample_estimate( + {'/W': 2, '/H': 3, '/BPC': 8, '/CS': '/A'}, + {'/ColorSpace': {'/A': '/B', '/B': '/A'}}) + assert estimate is None + assert reason == 'unresolved_or_unsupported_color_space' + def test_indexed_colorspace_has_one_component(self): tmp_path = self.tmp_path assert sample_estimate({'/W': 9, '/H': 2, '/BPC': 1, '/CS': ['/I', '/RGB']}) == (4, None)