From 6701a751147b69f4ea960729760d52abd36e999f Mon Sep 17 00:00:00 2001 From: EricPaque <60603143+Radisio@users.noreply.github.com> Date: Tue, 11 Aug 2026 10:10:13 +0200 Subject: [PATCH] Add pdf form, docx and xlsx handler + tests --- CLAUDE.md | 27 ++++- README.md | 52 ++++++++- adapters/renderers/_ooxml.py | 51 +++++++++ adapters/renderers/docx.py | 102 +++++++++++++++++ adapters/renderers/pdf_form.py | 172 ++++++++++++++++++++++++++++ adapters/renderers/registry.py | 14 +++ adapters/renderers/xlsx.py | 147 +++++++++++++++++------- domain/models.py | 91 ++++++++++++++- requirements/worker.txt | 12 ++ tests/conftest.py | 187 +++++++++++++++++++++++++++++++ tests/renderers/test_docx.py | 96 ++++++++++++++++ tests/renderers/test_pdf_form.py | 107 ++++++++++++++++++ tests/renderers/test_registry.py | 46 ++++++++ tests/renderers/test_xlsx.py | 122 +++++++++++++++++++- 14 files changed, 1169 insertions(+), 57 deletions(-) create mode 100644 adapters/renderers/_ooxml.py create mode 100644 adapters/renderers/docx.py create mode 100644 adapters/renderers/pdf_form.py create mode 100644 tests/renderers/test_docx.py create mode 100644 tests/renderers/test_pdf_form.py create mode 100644 tests/renderers/test_registry.py diff --git a/CLAUDE.md b/CLAUDE.md index 9811a17..691f4c6 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -38,7 +38,8 @@ adapters/ # concrete port impls — the only place frameworks live validator_jsonschema.py # Draft 2020-12 gate nats_transport.py # publish results / DLQ render_executor.py # Inline (tests) + ProcessPool (prod) - renderers/ # jinja_html_pdf (WeasyPrint), xlsx (openpyxl), registry, _paths (traversal guard) + renderers/ # jinja_html_pdf (WeasyPrint), xlsx (openpyxl), docx (docxtpl), + # pdf_form (pypdf), registry, _paths (traversal guard), _ooxml (zip repack) core/ # infra cloned/trimmed from simulation-key config.py queue/{init.py} storage/client.py logging.py tracing.py metrics.py worker/ @@ -67,10 +68,26 @@ of truth for both `error.permanent` and the ack/nak decision. so `./logo.png`/`./invoice.css`/fonts resolve. Templates receive `data` + `locale`; `StrictUndefined` makes a missing field a hard error (no silent blanks). - `xlsx` → openpyxl: workbook **defined names** matching top-level `data` keys, - plus `data["cells"]` (`"A1"` / `"Sheet!A1"` → value, applied last). -- Manifest has two **optional** fields beyond the spec: `entrypoint` (default - `template.html` / `template.xlsx`) and `output_basename` (default = last - dot-segment of `id`, e.g. `billing.invoice` → `invoice.pdf`). + manifest-declared repeating **blocks**, then `data["cells"]` (`"A1"` / + `"Sheet!A1"` → value, applied last). +- `docx` → docxtpl (Jinja inside a real `.docx`). Pure Python — unlike + WeasyPrint it imports and tests fine on a bare host. `autoescape=True` is + mandatory: a `.docx` body is XML, so an unescaped `&` corrupts the package. +- `pdf-form` → pypdf: fills the authority's own AcroForm. `manifest.fields` maps + a data key to the (auto-generated) field name; checkbox export states are read + from the PDF, not configured. +- Manifest optional fields beyond the spec: `entrypoint` (default + `template.html` / `.xlsx` / `.docx` / `.pdf`), `output_basename` (default = + last dot-segment of `id`), `blocks` (xlsx), `fields` (pdf-form). +- **Blocks write in place, never `insert_rows`** — openpyxl does not fix up + formulas, merged ranges, defined names, data validations or print areas on a + shift, and regulator forms are full of all of them. Capacity is + `required_fields.properties..maxItems`; overflow is a **permanent** + `VALIDATION_ERROR` (a retry of the same list can never fit). Unused rows are + hidden, not deleted. +- **Adding an engine is guarded at both ends**: `domain/models.py` raises at + import if `_DEFAULT_ENTRYPOINT` misses an `Engine`, and + `tests/renderers/test_registry.py` asserts every `Engine` has a renderer. ## Reliability (JetStream) diff --git a/README.md b/README.md index dcdfb2f..95b5109 100644 --- a/README.md +++ b/README.md @@ -47,7 +47,8 @@ adapters/ # concrete implementations of the ports validator_jsonschema.py Draft 2020-12 validation gate nats_transport.py publish results / DLQ render_executor.py inline + process-pool execution - renderers/ jinja-html→PDF (WeasyPrint), xlsx (openpyxl), registry + renderers/ jinja-html→PDF (WeasyPrint), xlsx (openpyxl), + docx (docxtpl), pdf-form (pypdf), registry, _ooxml core/ # reused infra (settings, logging, tracing, metrics, queue, storage client) worker/ # main.py entrypoint + dispatcher (subscription, ack/nak/DLQ) ``` @@ -106,20 +107,59 @@ Error codes: `VALIDATION_ERROR`, `TEMPLATE_NOT_FOUND`, `UNSUPPORTED_FORMAT` } ``` -- `engine` — `jinja-html` (PDF/HTML via Jinja2 + WeasyPrint) or `xlsx` (openpyxl). -- `entrypoint` *(optional)* — entry file; defaults `template.html` / `template.xlsx`. +- `engine` — one of: + + | engine | produces | library | use for | + |---|---|---|---| + | `jinja-html` | `pdf`, `html` | Jinja2 + WeasyPrint | documents we author | + | `xlsx` | `xlsx` | openpyxl | spreadsheet forms | + | `docx` | `docx` | docxtpl | documents whose mandated format is Word | + | `pdf-form` | `pdf` | pypdf | filling an authority's own fillable PDF | + +- `entrypoint` *(optional)* — entry file; defaults `template.html` / + `template.xlsx` / `template.docx` / `template.pdf`. - `output_basename` *(optional)* — artifact base name; defaults to the last dot-segment of `id` (`billing.invoice` → `invoice.pdf`). - `required_fields` — JSON Schema; an empty schema validates everything. +- `blocks` *(xlsx only)* — repeating regions, see below. +- `fields` *(pdf-form only)* — data key → AcroForm field name. **jinja-html templates** receive the request `data` as `data` and the locale as `locale` (e.g. `{{ data.invoice_number }}`). Relative assets (`./invoice.css`, `./logo.png`, fonts) resolve against the template directory. Missing fields are a hard error (no silent blanks). See `tests/fixtures/templates/billing_invoice/`. -**xlsx templates** are filled from `data` two ways: workbook **defined names** -matching top-level `data` keys, and an explicit `data.cells` map of -`"A1"` / `"Sheet!A1"` → value (applied last). +**docx templates** work the same way — `{{ data.x }}` written inside Word. + +**xlsx templates** are filled from `data` three ways: workbook **defined names** +matching top-level `data` keys, **repeating blocks**, and an explicit +`data.cells` map of `"A1"` / `"Sheet!A1"` → value (applied last). + +A block turns a variable-length list into worksheet rows. Layout is declared by +the *manifest*, never by the caller — the anchor cell is a fact about the +workbook: + +```json +"blocks": [{ + "source": "participants", + "anchor": "Membres et actionnaires!A4", + "columns": ["categorie", "nom", "rue", "code_postal"], + "hide_unused_rows": true +}] +``` + +`data.participants` is then a list of dicts and row *i* column *j* is +`rows[i][columns[j]]`. Rows are written **in place** into a band the template +already provisions — never inserted, because openpyxl does not fix up formulas, +merged ranges, defined names, data validations or print areas when rows shift. +Capacity comes from `required_fields.properties..maxItems` (or an +explicit `max_rows`), and overflowing it is a **permanent** `VALIDATION_ERROR` +raised before any render. Unused rows are hidden, not deleted. + +**pdf-form templates** are the authority's own fillable PDF; only field values +change. Official forms auto-name their fields, so the manifest maps them: +`"fields": {"community_name": "Champ de texte 68"}`. Checkbox export states are +read from the PDF itself, so `/Oui`, `/Yes` and `/1` all work without config. ## Reliability (JetStream) diff --git a/adapters/renderers/_ooxml.py b/adapters/renderers/_ooxml.py new file mode 100644 index 0000000..e697952 --- /dev/null +++ b/adapters/renderers/_ooxml.py @@ -0,0 +1,51 @@ +"""Deterministic repacking for OOXML (Office Open XML) documents. + +``.xlsx`` and ``.docx`` are both OPC packages: a zip whose members include +``docProps/core.xml`` carrying ````/````. +Both writers we use stamp wall-clock time — openpyxl rewrites ``core.xml``'s +modified date inside ``save()``, and both libraries let ``zipfile`` stamp each +member with ``time.localtime()``. Neither is a property of (template, data), so +neither may reach the bytes we hash. + +Shared by the xlsx and docx renderers so the two cannot drift. +""" + +from __future__ import annotations + +import io +import re +import zipfile + +# Fixed (not wall-clock) so normalised packages are reproducible. +_FIXED_ZIP_DATE = (1980, 1, 1, 0, 0, 0) +_FIXED_ISO = b"2000-01-01T00:00:00Z" +_CREATED_RE = re.compile(rb"(]*>)[^<]*()") +_MODIFIED_RE = re.compile(rb"(]*>)[^<]*()") + +_CORE_PROPS = "docProps/core.xml" +# OPC requires the content-types stream to be the first part in the package. +# Sorted order happens to put it first today ('[' is 0x5B, below every other +# part's initial letter), but relying on that is a latent corruption bug — so +# emit it explicitly first instead. +_CONTENT_TYPES = "[Content_Types].xml" + + +def normalise_ooxml_bytes(raw: bytes) -> bytes: + """Repack an OOXML package so identical input yields identical bytes.""" + with zipfile.ZipFile(io.BytesIO(raw)) as src: + members = {name: src.read(name) for name in src.namelist()} + + ordered = [name for name in (_CONTENT_TYPES,) if name in members] + ordered += sorted(name for name in members if name != _CONTENT_TYPES) + + out = io.BytesIO() + with zipfile.ZipFile(out, "w", zipfile.ZIP_DEFLATED) as dst: + for name in ordered: + data = members[name] + if name == _CORE_PROPS: + data = _CREATED_RE.sub(rb"\g<1>" + _FIXED_ISO + rb"\g<2>", data) + data = _MODIFIED_RE.sub(rb"\g<1>" + _FIXED_ISO + rb"\g<2>", data) + info = zipfile.ZipInfo(name, date_time=_FIXED_ZIP_DATE) + info.compress_type = zipfile.ZIP_DEFLATED + dst.writestr(info, data) + return out.getvalue() diff --git a/adapters/renderers/docx.py b/adapters/renderers/docx.py new file mode 100644 index 0000000..58c1e6d --- /dev/null +++ b/adapters/renderers/docx.py @@ -0,0 +1,102 @@ +"""Renderer for the ``docx`` engine: Jinja inside a real ``.docx`` via docxtpl. + +For documents whose mandated format is Word — the CWaPE standard DSO agreements, +for instance — where a PDF would not be accepted. Authoring matches the +``jinja-html`` bundles: the same ``{{ data.x }}`` expressions, but written inside +Word rather than in HTML. + +``StrictUndefined`` mirrors the jinja-html renderer: a missing field is a hard +error, never a silent blank, which is the right default for an official document. + +Determinism: docxtpl introduces nothing wall-clock of its own, but python-docx +lets ``zipfile`` stamp members with the current time, so the package is repacked +through ``_ooxml.normalise_ooxml_bytes`` — the same pass the xlsx renderer uses, +since both are OPC packages. + +Unlike WeasyPrint, docxtpl is pure Python (its only compiled dependency is lxml, +whose wheels bundle libxml2/libxslt), so this renderer imports and its tests run +on a bare developer host. It is still imported lazily, matching the convention. +""" + +from __future__ import annotations + +import datetime +import io +from collections.abc import Mapping +from typing import Any + +from adapters.renderers._ooxml import normalise_ooxml_bytes +from adapters.renderers._paths import resolve_template_file +from domain.errors import DocGenError, RenderError +from domain.models import OutputFormat, RenderedArtifact +from domain.ports import TemplateBundle + +_FIXED_TIMESTAMP = datetime.datetime(2000, 1, 1) +_NORMALISED_AUTHOR = "document-generation" + + +class DocxRenderer: + """Stateless (hence picklable for the process pool) docx renderer.""" + + def render( + self, + bundle: TemplateBundle, + data: Mapping[str, Any], + fmt: OutputFormat, + *, + locale: str | None, + ) -> RenderedArtifact: + if fmt is not OutputFormat.DOCX: + raise RenderError(f"docx engine cannot produce {fmt.value!r}") + try: + content = self._render(bundle, dict(data), locale) + except DocGenError: + raise + except Exception as exc: # any docxtpl/python-docx failure → transient + raise RenderError(f"docx render failed: {exc}") from exc + + return RenderedArtifact( + format=fmt, + filename=bundle.manifest.filename_for(fmt), + content=content, + ) + + @classmethod + def _render(cls, bundle: TemplateBundle, data: dict[str, Any], locale: str | None) -> bytes: + # Lazy import, and the Jinja Environment is built here rather than held + # on the instance: an Environment is not reliably picklable and this + # renderer crosses the process-pool boundary. + from docxtpl import DocxTemplate # type: ignore[import-untyped] + from jinja2 import Environment, StrictUndefined + + entry = resolve_template_file(bundle.root, bundle.manifest.resolve_entrypoint()) + template = DocxTemplate(str(entry)) + # autoescape is mandatory here, not a lint appeasement: a .docx body is + # XML, so an unescaped "&" or "<" in a value (think "Smith & Co") + # produces a corrupt package Word refuses to open. + # + # finalize maps None to an empty string. Jinja's default is to print the + # literal "None", which in a contract reads as a filled-in value and is + # far worse than a visible blank. StrictUndefined still catches a field + # the caller forgot entirely — the two guard different mistakes. + environment = Environment( + undefined=StrictUndefined, + autoescape=True, + finalize=lambda value: "" if value is None else value, + ) + template.render({"data": data, "locale": locale}, environment) + cls._normalise_properties(template.docx) + + buffer = io.BytesIO() + template.save(buffer) + return normalise_ooxml_bytes(buffer.getvalue()) + + @staticmethod + def _normalise_properties(document: Any) -> None: + """Clear author/revision metadata so the template author does not leak.""" + props = document.core_properties + props.created = _FIXED_TIMESTAMP + props.modified = _FIXED_TIMESTAMP + props.last_modified_by = _NORMALISED_AUTHOR + props.author = _NORMALISED_AUTHOR + props.revision = 1 diff --git a/adapters/renderers/pdf_form.py b/adapters/renderers/pdf_form.py new file mode 100644 index 0000000..831d978 --- /dev/null +++ b/adapters/renderers/pdf_form.py @@ -0,0 +1,172 @@ +"""Renderer for the ``pdf-form`` engine: fill an existing AcroForm PDF. + +Unlike ``jinja-html``, which *authors* a PDF, this engine takes the authority's +own fillable form and sets its field values. Everything else — layout, legal +text, logos, page count — stays exactly as the regulator published it, which is +the whole point when the filed artifact must be the official document rather +than a reproduction of it. + +Official forms name their fields automatically (``Champ de texte 68``, +``Case à cocher 81``), so the manifest carries the translation: +``fields: {"": ""}``. That map is a fact about the +template, so it lives in the bundle. + +A value may also map to a **list** of field names, for the comb boxes these forms +use to spell a code out one character per box (an 18-digit EAN across 16 boxes, +say). The string is then spread left-aligned across them and any leftover box is +cleared. + +Checkbox fields are detected from the PDF's own ``/FT`` and export states — a +truthy value selects the non-``/Off`` state, a falsy one selects ``/Off`` — so a +form using ``/Oui``, ``/Yes`` or ``/1`` needs no manifest change. + +A field the manifest does not map is left untouched, and the artifact stays a +fillable AcroForm — so anything we cannot derive remains editable by hand in the +user's PDF reader rather than being lost. + +Determinism: pypdf derives the trailer ``/ID`` from content rather than +randomness, and ``/Info`` dates are inherited from the source document (which is +itself fixed), so the output is a pure function of (template, data). The +``/ID`` is pinned defensively anyway, and ``tests/renderers/test_pdf_form.py`` +asserts byte-equality so a dependency upgrade cannot regress it silently. + +pypdf is imported lazily, matching the WeasyPrint convention — though unlike +WeasyPrint it is pure Python and needs no native libraries. +""" + +from __future__ import annotations + +import hashlib +import io +from collections.abc import Mapping +from typing import Any + +from adapters.renderers._paths import resolve_template_file +from domain.errors import DocGenError, RenderError, TemplateNotFoundError +from domain.models import OutputFormat, RenderedArtifact +from domain.ports import TemplateBundle + +_FIELD_TYPE = "/FT" +_BUTTON = "/Btn" +_STATES = "/_States_" +_OFF = "/Off" + + +class PdfFormRenderer: + """Stateless (hence picklable for the process pool) AcroForm filler.""" + + def render( + self, + bundle: TemplateBundle, + data: Mapping[str, Any], + fmt: OutputFormat, + *, + locale: str | None, + ) -> RenderedArtifact: + if fmt is not OutputFormat.PDF: + raise RenderError(f"pdf-form engine cannot produce {fmt.value!r}") + try: + content = self._render(bundle, dict(data)) + except DocGenError: + raise + except Exception as exc: # any pypdf failure → transient render error + raise RenderError(f"pdf-form render failed: {exc}") from exc + + return RenderedArtifact( + format=fmt, + filename=bundle.manifest.filename_for(fmt), + content=content, + ) + + @classmethod + def _render(cls, bundle: TemplateBundle, data: dict[str, Any]) -> bytes: + # Lazy import: keeps the process-pool import cost and the test import + # graph unchanged, matching the jinja-html renderer's convention. + from pypdf import PdfReader, PdfWriter + + manifest = bundle.manifest + entry = resolve_template_file(bundle.root, manifest.resolve_entrypoint()) + reader = PdfReader(str(entry)) + declared = reader.get_fields() or {} + if not declared: + raise TemplateNotFoundError( + f"template {manifest.id!r} declares engine 'pdf-form' but " + f"{manifest.resolve_entrypoint()!r} has no AcroForm fields" + ) + + values = cls._resolve_values(manifest.fields, declared, data) + + writer = PdfWriter(clone_from=reader) + # Without this, viewers that do not generate appearance streams + # themselves render a filled field as blank. + writer.set_need_appearances_writer(True) + if values: + for page in writer.pages: + writer.update_page_form_field_values(page, values) + + cls._pin_document_id(writer, manifest.id, manifest.version) + buffer = io.BytesIO() + writer.write(buffer) + return buffer.getvalue() + + @staticmethod + def _resolve_values( + mapping: Mapping[str, str | list[str]], + declared: Mapping[str, Any], + data: Mapping[str, Any], + ) -> dict[str, Any]: + """Translate ``data`` keys to AcroForm field names, coercing per field type. + + A mapped field that the PDF does not declare is a template-authoring + error and permanent — the same class as a bad entrypoint, so it raises + the same ``TemplateNotFoundError``. + """ + values: dict[str, Any] = {} + for key, target in mapping.items(): + names = [target] if isinstance(target, str) else list(target) + for field_name in names: + if field_name not in declared: + raise TemplateNotFoundError( + f"manifest maps {key!r} to AcroForm field {field_name!r}, " + f"which this PDF does not declare" + ) + if key not in data: + continue # unsupplied optional field — leave the form's own value + if isinstance(target, str): + values[target] = _coerce(declared[target], data[key]) + else: + # Comb boxes: one character per field, left-aligned, the rest + # cleared so a shorter value never leaves stale digits behind. + characters = "" if data[key] is None else str(data[key]) + for position, field_name in enumerate(names): + values[field_name] = characters[position] if position < len(characters) else "" + return values + + @staticmethod + def _pin_document_id(writer: Any, template_id: str, version: str) -> None: + """Pin the trailer /ID so reproducibility cannot depend on pypdf internals.""" + from pypdf.generic import ArrayObject, ByteStringObject + + digest = hashlib.sha256(f"{template_id}:{version}".encode()).digest()[:16] + if hasattr(writer, "_ID"): + writer._ID = ArrayObject([ByteStringObject(digest), ByteStringObject(digest)]) + + +def _coerce(field: Mapping[str, Any], value: Any) -> Any: + """Coerce a data value to what the AcroForm field expects.""" + if field.get(_FIELD_TYPE) == _BUTTON: + states = [s for s in (field.get(_STATES) or []) if s != _OFF] + on_state = states[0] if states else "/Yes" + return on_state if _is_checked(value) else _OFF + if value is None: + return "" + if isinstance(value, bool): + return "Oui" if value else "Non" + return str(value) + + +def _is_checked(value: Any) -> bool: + """Interpret a checkbox value, tolerating the string forms JSON tends to carry.""" + if isinstance(value, str): + return value.strip().lower() in {"true", "yes", "oui", "on", "1", "x"} + return bool(value) diff --git a/adapters/renderers/registry.py b/adapters/renderers/registry.py index f4194aa..16fad4e 100644 --- a/adapters/renderers/registry.py +++ b/adapters/renderers/registry.py @@ -4,11 +4,17 @@ pair means the format is unsupported for that engine (→ ``UNSUPPORTED_FORMAT``). The renderer instances are stateless, so they are safely reused across requests and picklable for the process pool. + +Note the deliberate absences: ``pdf-form`` produces only ``pdf`` and ``docx`` +only ``docx``. Converting either to another format would need LibreOffice, so a +request for one fails permanently and clearly rather than half-working. """ from __future__ import annotations +from adapters.renderers.docx import DocxRenderer from adapters.renderers.jinja_html_pdf import JinjaHtmlRenderer +from adapters.renderers.pdf_form import PdfFormRenderer from adapters.renderers.xlsx import XlsxRenderer from domain.models import Engine, OutputFormat from domain.ports import Renderer @@ -18,11 +24,19 @@ class DefaultRendererRegistry: def __init__(self) -> None: jinja = JinjaHtmlRenderer() xlsx = XlsxRenderer() + docx = DocxRenderer() + pdf_form = PdfFormRenderer() self._renderers: dict[tuple[Engine, OutputFormat], Renderer] = { (Engine.JINJA_HTML, OutputFormat.PDF): jinja, (Engine.JINJA_HTML, OutputFormat.HTML): jinja, (Engine.XLSX, OutputFormat.XLSX): xlsx, + (Engine.DOCX, OutputFormat.DOCX): docx, + (Engine.PDF_FORM, OutputFormat.PDF): pdf_form, } def get(self, engine: Engine, fmt: OutputFormat) -> Renderer | None: return self._renderers.get((engine, fmt)) + + def engines(self) -> set[Engine]: + """Every engine with at least one renderer. Used by the parity test.""" + return {engine for engine, _fmt in self._renderers} diff --git a/adapters/renderers/xlsx.py b/adapters/renderers/xlsx.py index e3c6596..9df934b 100644 --- a/adapters/renderers/xlsx.py +++ b/adapters/renderers/xlsx.py @@ -1,34 +1,43 @@ """Renderer for the ``xlsx`` engine: openpyxl over a template workbook. -Loads the template workbook and writes ``data`` into it via two generic +Loads the template workbook and writes ``data`` into it via three generic mechanisms (no domain knowledge): * **Defined names** — for every workbook defined name that matches a top-level ``data`` key, the named cell(s) are set to that value. +* **Repeating blocks** — the *manifest* declares where a variable-length list + lands (``Manifest.blocks``); ``data[block.source]`` supplies the rows. Layout + belongs to the template, never to the caller. * **Explicit cells** — ``data["cells"]`` maps ``"A1"`` (active sheet) or ``"Sheet!A1"`` to a value, applied last so it wins. +Blocks write **in place** into a band the template already provisions and never +call ``insert_rows``: openpyxl does not fix up formulas, merged ranges, defined +names, conditional formatting, data validations, print areas or row heights when +rows shift, and official forms are full of all of those. Unused rows are hidden +rather than deleted, for the same reason. + Determinism: the output bytes are a pure function of (template, data). openpyxl rewrites ``docProps/core.xml``'s ```` to ``now()`` inside ``save()`` and stamps each zip member with the current time, so the workbook is -repacked deterministically (fixed member dates + fixed core timestamps) before -hashing — identical input yields identical bytes and sha256. +repacked deterministically before hashing (see ``_ooxml``). """ from __future__ import annotations import io -import re -import zipfile -from collections.abc import Mapping +from collections.abc import Iterable, Mapping from datetime import datetime from typing import Any from openpyxl import load_workbook +from openpyxl.cell.cell import MergedCell +from openpyxl.utils import coordinate_to_tuple +from adapters.renderers._ooxml import normalise_ooxml_bytes from adapters.renderers._paths import resolve_template_file -from domain.errors import DocGenError, RenderError -from domain.models import OutputFormat, RenderedArtifact +from domain.errors import DocGenError, RenderError, SchemaValidationError +from domain.models import BlockSpec, Manifest, OutputFormat, RenderedArtifact from domain.ports import TemplateBundle # Fixed (not wall-clock) so normalised workbooks are reproducible. Naive on @@ -36,12 +45,6 @@ _FIXED_TIMESTAMP = datetime(2000, 1, 1) _NORMALISED_AUTHOR = "document-generation" -# Deterministic zip repack: a fixed member date and forced core timestamps. -_FIXED_ZIP_DATE = (1980, 1, 1, 0, 0, 0) -_FIXED_ISO = b"2000-01-01T00:00:00Z" -_CREATED_RE = re.compile(rb"(]*>)[^<]*()") -_MODIFIED_RE = re.compile(rb"(]*>)[^<]*()") - class XlsxRenderer: """Stateless (hence picklable for the process pool) xlsx renderer.""" @@ -71,19 +74,24 @@ def render( @classmethod def _render(cls, bundle: TemplateBundle, data: dict[str, Any]) -> bytes: - entry = resolve_template_file(bundle.root, bundle.manifest.resolve_entrypoint()) + manifest = bundle.manifest + entry = resolve_template_file(bundle.root, manifest.resolve_entrypoint()) workbook = load_workbook(entry) - cls._apply_defined_names(workbook, data) + # A block source is a list; a defined name pointing at one would make + # openpyxl raise on "cannot convert". Blocks own those keys. + cls._apply_defined_names(workbook, data, skip={b.source for b in manifest.blocks}) + cls._apply_blocks(workbook, manifest, data) cls._apply_cells(workbook, data.get("cells", {})) cls._normalise_properties(workbook) buffer = io.BytesIO() workbook.save(buffer) - return _normalise_xlsx_bytes(buffer.getvalue()) + return normalise_ooxml_bytes(buffer.getvalue()) @staticmethod - def _apply_defined_names(workbook: Any, data: dict[str, Any]) -> None: + def _apply_defined_names(workbook: Any, data: dict[str, Any], skip: Iterable[str] = ()) -> None: + skipped = set(skip) for name, defined in workbook.defined_names.items(): - if name not in data: + if name not in data or name in skipped: continue for sheet_name, coordinate in defined.destinations: # destinations yields absolute refs ("$B$1"); only single cells @@ -93,6 +101,83 @@ def _apply_defined_names(workbook: Any, data: dict[str, Any]) -> None: continue workbook[sheet_name][coord] = data[name] + # -- repeating blocks --------------------------------------------------- + + @classmethod + def _apply_blocks(cls, workbook: Any, manifest: Manifest, data: dict[str, Any]) -> None: + for block in manifest.blocks: + rows = data.get(block.source) or [] + if not isinstance(rows, list): + raise SchemaValidationError( + f"block source {block.source!r} must be a list, got {type(rows).__name__}" + ) + capacity = manifest.block_capacity(block) + if len(rows) > capacity: + # Permanent: no retry of the same over-long list can ever fit. + raise SchemaValidationError( + f"{len(rows)} rows supplied for block {block.source!r} but the " + f"template reserves only {capacity}" + ) + sheet, first_row, first_col = cls._resolve_anchor(workbook, block) + for offset, row in enumerate(rows): + for column_offset, key in enumerate(block.columns): + cls._set_cell( + sheet, + first_row + offset, + first_col + column_offset, + cls._normalise_value(row.get(key)), + ) + if block.hide_unused_rows: + for row_index in range(first_row + len(rows), first_row + capacity): + sheet.row_dimensions[row_index].hidden = True + + @staticmethod + def _resolve_anchor(workbook: Any, block: BlockSpec) -> tuple[Any, int, int]: + """Resolve a block's anchor to (worksheet, row, column) 1-based indices.""" + if block.anchor_name is not None: + defined = workbook.defined_names.get(block.anchor_name) + if defined is None: + raise SchemaValidationError( + f"block {block.source!r} anchors on defined name " + f"{block.anchor_name!r}, which the workbook does not declare" + ) + sheet_name, coordinate = next(iter(defined.destinations)) + reference = f"{sheet_name}!{coordinate}" + else: + reference = block.anchor or "" + + sheet_name, separator, coordinate = reference.rpartition("!") + sheet = workbook[sheet_name] if separator else workbook.active + row, column = coordinate_to_tuple(coordinate.replace("$", "")) + return sheet, row, column + + @staticmethod + def _set_cell(sheet: Any, row: int, column: int, value: Any) -> None: + """Write one cell, redirecting a merged cell to its range's top-left. + + Writing to a ``MergedCell`` raises (it is a read-only view); the value + belongs on the anchor cell of the range that covers it. + """ + cell = sheet.cell(row=row, column=column) + if isinstance(cell, MergedCell): + for merged in sheet.merged_cells.ranges: + if (row, column) in merged.cells: + sheet.cell(row=merged.min_row, column=merged.min_col).value = value + return + return # merged view with no range (corrupt template) — skip silently + cell.value = value + + @staticmethod + def _normalise_value(value: Any) -> Any: + """Canonicalise 'absent' to None in exactly one place. + + ``None`` clears a cell; ``""`` writes an inline-string element. Different + XML, different sha256 — so the two must never both reach a cell. + """ + return None if value == "" else value + + # -- explicit cells ----------------------------------------------------- + @staticmethod def _apply_cells(workbook: Any, cells: Any) -> None: for ref, value in dict(cells).items(): @@ -109,27 +194,3 @@ def _normalise_properties(workbook: Any) -> None: props.modified = _FIXED_TIMESTAMP props.creator = _NORMALISED_AUTHOR props.lastModifiedBy = _NORMALISED_AUTHOR - - -def _normalise_xlsx_bytes(raw: bytes) -> bytes: - """Repack the saved workbook so identical input yields identical bytes. - - openpyxl's ``save()`` overwrites ```` with ``now()`` and the - zip members carry the current time. Rebuild the archive with sorted members, - a fixed member date, and fixed created/modified timestamps. - """ - with zipfile.ZipFile(io.BytesIO(raw)) as src: - names = sorted(src.namelist()) - members = {name: src.read(name) for name in names} - - out = io.BytesIO() - with zipfile.ZipFile(out, "w", zipfile.ZIP_DEFLATED) as dst: - for name in names: - data = members[name] - if name == "docProps/core.xml": - data = _CREATED_RE.sub(rb"\g<1>" + _FIXED_ISO + rb"\g<2>", data) - data = _MODIFIED_RE.sub(rb"\g<1>" + _FIXED_ISO + rb"\g<2>", data) - info = zipfile.ZipInfo(name, date_time=_FIXED_ZIP_DATE) - info.compress_type = zipfile.ZIP_DEFLATED - dst.writestr(info, data) - return out.getvalue() diff --git a/domain/models.py b/domain/models.py index b33f5a8..dd0cc71 100644 --- a/domain/models.py +++ b/domain/models.py @@ -14,7 +14,7 @@ from enum import StrEnum from typing import Any -from pydantic import BaseModel, ConfigDict, Field +from pydantic import BaseModel, ConfigDict, Field, model_validator # --------------------------------------------------------------------------- # Content types per output format. Kept here (domain) because the format set is @@ -23,6 +23,7 @@ _CONTENT_TYPES: dict[str, str] = { "pdf": "application/pdf", "xlsx": "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", + "docx": "application/vnd.openxmlformats-officedocument.wordprocessingml.document", "html": "text/html; charset=utf-8", } @@ -30,6 +31,7 @@ class OutputFormat(StrEnum): PDF = "pdf" XLSX = "xlsx" + DOCX = "docx" HTML = "html" @property @@ -50,6 +52,11 @@ class Engine(StrEnum): JINJA_HTML = "jinja-html" XLSX = "xlsx" + DOCX = "docx" + # Fills an existing fillable (AcroForm) PDF rather than authoring one. Used + # for official regulator forms, where the filed document must BE the + # authority's own file with only its field values set. + PDF_FORM = "pdf-form" class GenerationStatus(StrEnum): @@ -61,8 +68,22 @@ class GenerationStatus(StrEnum): _DEFAULT_ENTRYPOINT: dict[Engine, str] = { Engine.JINJA_HTML: "template.html", Engine.XLSX: "template.xlsx", + Engine.DOCX: "template.docx", + Engine.PDF_FORM: "template.pdf", } +# A missing entry would raise a bare KeyError from resolve_entrypoint() — and a +# KeyError is not a DocGenError, so the orchestrator would not catch it and the +# dispatcher would misclassify a template-config bug as a transient failure, +# burning every retry before the DLQ. Fail at import instead, in every +# environment including tests. Not an `assert`: `python -O` strips those, and +# this guard is most needed in exactly the optimised container image. +if set(_DEFAULT_ENTRYPOINT) != set(Engine): + raise RuntimeError( + "every Engine needs a default entrypoint; missing " + f"{sorted(set(Engine) - set(_DEFAULT_ENTRYPOINT))}" + ) + # --------------------------------------------------------------------------- # Request @@ -153,6 +174,41 @@ def to_json_bytes(self) -> bytes: # --------------------------------------------------------------------------- # Manifest (ships with every template) # --------------------------------------------------------------------------- +class BlockSpec(BaseModel): + """A repeating region of a spreadsheet template, declared by the template. + + Layout (which sheet, which cell, which column order, how many rows fit) is a + fact about the workbook, so it lives here — in the bundle — and never in the + caller's ``data``. A caller supplies only the list: ``data[source]`` is a + list of dicts, and row *i* column *j* is ``rows[i][columns[j]]``. + """ + + model_config = ConfigDict(extra="ignore") + + source: str = Field(min_length=1) + columns: list[str] = Field(min_length=1) + # Exactly one of the two. ``anchor`` is the primary form because real + # regulator workbooks rarely carry usable defined names (and where they do, + # they are data-validation list sources, not layout markers). + anchor: str | None = None # "Sheet name!A4" or "A4" (active sheet) + anchor_name: str | None = None # a workbook defined name + # Capacity of the pre-provisioned band. ``None`` derives it from the + # ``maxItems`` of this source in ``required_fields`` (single source of truth). + max_rows: int | None = Field(default=None, ge=1) + # Blank out-of-use rows in a bordered/banded template look like empty table + # rows; hiding them is deterministic and preserves every style and range, + # unlike delete_rows(). + hide_unused_rows: bool = True + + @model_validator(mode="after") + def _exactly_one_anchor(self) -> BlockSpec: + if bool(self.anchor) == bool(self.anchor_name): + raise ValueError( + f"block {self.source!r} must declare exactly one of " f"'anchor' or 'anchor_name'" + ) + return self + + class Manifest(BaseModel): # ``extra="ignore"`` so a manifest authored against a future, richer schema # still loads here. @@ -168,6 +224,39 @@ class Manifest(BaseModel): entrypoint: str | None = None output_basename: str | None = None + # engine="xlsx": repeating regions this workbook provisions. + blocks: list[BlockSpec] = Field(default_factory=list) + # engine="pdf-form": semantic data key → AcroForm field name. Official forms + # name their fields "Champ de texte 68", so the bundle owns the translation. + # A list of names is a comb: the value is spelled one character per box. + fields: dict[str, str | list[str]] = Field(default_factory=dict) + + @model_validator(mode="after") + def _blocks_have_resolvable_capacity(self) -> Manifest: + """Resolve every block's capacity now, so a render can never fail on it. + + ``template_store._load_manifest`` turns a ValueError here into a + permanent ``TemplateNotFoundError`` at fetch time — which is what a + template-authoring mistake deserves. + """ + for block in self.blocks: + self.block_capacity(block) + return self + + def block_capacity(self, block: BlockSpec) -> int: + """Rows the template reserves for ``block`` — explicit, or from the schema.""" + if block.max_rows is not None: + return block.max_rows + properties = self.required_fields.get("properties") + schema = properties.get(block.source) if isinstance(properties, dict) else None + max_items = schema.get("maxItems") if isinstance(schema, dict) else None + if isinstance(max_items, int) and max_items >= 1: + return max_items + raise ValueError( + f"block {block.source!r} has no capacity: set 'max_rows' on the block, or " + f"'maxItems' on required_fields.properties.{block.source}" + ) + def resolve_entrypoint(self) -> str: return self.entrypoint or _DEFAULT_ENTRYPOINT[self.engine] diff --git a/requirements/worker.txt b/requirements/worker.txt index ba1fa77..fc0e033 100644 --- a/requirements/worker.txt +++ b/requirements/worker.txt @@ -11,3 +11,15 @@ weasyprint==63.1 # XLSX templating. openpyxl==3.1.5 + +# DOCX templating — Jinja inside a real .docx. Pure Python apart from lxml, +# whose wheels bundle libxml2/libxslt statically, so unlike WeasyPrint there is +# nothing to install at the OS level and these renderers work on a bare host. +# lxml is pinned explicitly rather than floated: it is the only compiled +# dependency in the tree and a source-build fallback is a nasty CI failure. +docxtpl==0.20.2 +python-docx==1.2.0 +lxml==6.1.1 + +# AcroForm filling for official regulator PDFs (engine "pdf-form"). Pure Python. +pypdf==5.1.0 diff --git a/tests/conftest.py b/tests/conftest.py index 243a270..08a2edd 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -24,6 +24,7 @@ from domain.errors import DocGenError from domain.models import ( + BlockSpec, Engine, GenerationRequest, Manifest, @@ -103,6 +104,192 @@ def xlsx_bundle(tmp_path: Path) -> TemplateBundle: return TemplateBundle(manifest=manifest, root=directory) +@pytest.fixture +def xlsx_block_bundle(tmp_path: Path) -> TemplateBundle: + """A workbook with a header row and a 4-row participant band at A4. + + Mirrors the real CWaPE annexes: a titled sheet, a header row, then a + pre-provisioned band the renderer writes into without shifting anything. + """ + directory = tmp_path / "xlsx_block_template" + directory.mkdir() + + workbook = Workbook() + sheet = workbook.active + sheet.title = "Participants" + sheet["A1"] = "PARTICIPANTS" + sheet["A3"], sheet["B3"], sheet["C3"] = "Nom", "EAN", "Localite" + workbook.save(directory / "template.xlsx") + + manifest = Manifest( + id="admin.participants", + version="1", + engine=Engine.XLSX, + supported_formats=[OutputFormat.XLSX], + required_fields={ + "type": "object", + "properties": { + "participants": {"type": "array", "maxItems": 4}, + }, + }, + entrypoint="template.xlsx", + output_basename="participants", + blocks=[ + BlockSpec( + source="participants", + anchor="Participants!A4", + columns=["nom", "ean", "localite"], + ) + ], + ) + (directory / "manifest.json").write_text(manifest.model_dump_json()) + return TemplateBundle(manifest=manifest, root=directory) + + +@pytest.fixture +def docx_bundle(tmp_path: Path) -> TemplateBundle: + """A .docx carrying docxtpl Jinja placeholders, built at test time.""" + from docx import Document + + directory = tmp_path / "docx_template" + directory.mkdir() + + document = Document() + document.add_heading("Convention", level=1) + document.add_paragraph("Communauté : {{ data.community_name }}") + document.add_paragraph("Représentant : {{ data.representative }}") + document.save(str(directory / "template.docx")) + + manifest = Manifest( + id="admin.agreement", + version="1", + engine=Engine.DOCX, + supported_formats=[OutputFormat.DOCX], + required_fields={}, + entrypoint="template.docx", + output_basename="agreement", + ) + (directory / "manifest.json").write_text(manifest.model_dump_json()) + return TemplateBundle(manifest=manifest, root=directory) + + +@pytest.fixture +def pdf_form_bundle(tmp_path: Path) -> TemplateBundle: + """A two-field AcroForm PDF (one text field, one checkbox) built at test time. + + Built rather than committed so the fixture carries no third-party document + and the field names are meaningful in assertions. + """ + directory = tmp_path / "pdf_form_template" + directory.mkdir() + _write_acroform_pdf(directory / "template.pdf") + + manifest = Manifest( + id="admin.declaration", + version="1", + engine=Engine.PDF_FORM, + supported_formats=[OutputFormat.PDF], + required_fields={}, + entrypoint="template.pdf", + output_basename="declaration", + fields={"community_name": "Champ de texte 1", "agreed": "Case a cocher 1"}, + ) + (directory / "manifest.json").write_text(manifest.model_dump_json()) + return TemplateBundle(manifest=manifest, root=directory) + + +def _write_acroform_pdf(path: Path) -> None: + """Write a minimal one-page PDF with a text field and a checkbox. + + A well-formed AcroForm carries a ``/DR`` (default resources, with a font) and + a ``/DA`` (default appearance) — pypdf needs both to regenerate a field's + appearance stream. Real regulator forms have them; a fixture without them + fails in a way that looks like a renderer bug, so they are set here. + """ + from pypdf import PdfWriter + from pypdf.generic import ( + ArrayObject, + BooleanObject, + DictionaryObject, + NameObject, + NumberObject, + TextStringObject, + ) + + writer = PdfWriter() + page = writer.add_blank_page(width=595, height=842) + + helvetica = writer._add_object( + DictionaryObject( + { + NameObject("/Type"): NameObject("/Font"), + NameObject("/Subtype"): NameObject("/Type1"), + NameObject("/BaseFont"): NameObject("/Helvetica"), + NameObject("/Encoding"): NameObject("/WinAnsiEncoding"), + } + ) + ) + default_resources = DictionaryObject( + {NameObject("/Font"): DictionaryObject({NameObject("/Helv"): helvetica})} + ) + default_appearance = TextStringObject("/Helv 0 Tf 0 g") + + def _field(name: str, rect: list[int], extra: dict[str, Any]) -> Any: + field = DictionaryObject( + { + NameObject("/Type"): NameObject("/Annot"), + NameObject("/Subtype"): NameObject("/Widget"), + NameObject("/T"): TextStringObject(name), + NameObject("/Rect"): ArrayObject([NumberObject(n) for n in rect]), + NameObject("/F"): NumberObject(4), + } + ) + field.update({NameObject(k): v for k, v in extra.items()}) + return writer._add_object(field) + + text_ref = _field( + "Champ de texte 1", + [50, 700, 300, 720], + { + "/FT": NameObject("/Tx"), + "/V": TextStringObject(""), + "/DA": default_appearance, + }, + ) + checkbox_ref = _field( + "Case a cocher 1", + [50, 650, 70, 670], + { + "/FT": NameObject("/Btn"), + "/V": NameObject("/Off"), + "/AS": NameObject("/Off"), + "/AP": DictionaryObject( + { + NameObject("/N"): DictionaryObject( + { + NameObject("/Oui"): writer._add_object(DictionaryObject()), + NameObject("/Off"): writer._add_object(DictionaryObject()), + } + ) + } + ), + }, + ) + + annots = ArrayObject([text_ref, checkbox_ref]) + page[NameObject("/Annots")] = annots + writer._root_object[NameObject("/AcroForm")] = DictionaryObject( + { + NameObject("/Fields"): annots, + NameObject("/NeedAppearances"): BooleanObject(True), + NameObject("/DR"): default_resources, + NameObject("/DA"): default_appearance, + } + ) + with open(path, "wb") as handle: + writer.write(handle) + + # --------------------------------------------------------------------------- # In-memory fakes (duck-typed against domain.ports) # --------------------------------------------------------------------------- diff --git a/tests/renderers/test_docx.py b/tests/renderers/test_docx.py new file mode 100644 index 0000000..7dc11c3 --- /dev/null +++ b/tests/renderers/test_docx.py @@ -0,0 +1,96 @@ +"""DOCX renderer tests. + +Unlike the WeasyPrint PDF test, these run everywhere: docxtpl is pure Python +apart from lxml, which ships self-contained wheels. +""" + +from __future__ import annotations + +import dataclasses +import io + +import pytest + +from adapters.renderers.docx import DocxRenderer +from domain.errors import RenderError, TemplateNotFoundError +from domain.models import OutputFormat + +_DATA = {"community_name": "Communauté ACME", "representative": "Alice Dupont"} + + +def _render(bundle, data=None, fmt=OutputFormat.DOCX): + return DocxRenderer().render(bundle, _DATA if data is None else data, fmt, locale="fr-BE") + + +def _text(content: bytes) -> str: + from docx import Document + + return "\n".join(p.text for p in Document(io.BytesIO(content)).paragraphs) + + +def test_render_substitutes_placeholders(docx_bundle): + artifact = _render(docx_bundle) + + assert artifact.format is OutputFormat.DOCX + assert artifact.filename == "agreement.docx" + assert ( + artifact.content_type + == "application/vnd.openxmlformats-officedocument.wordprocessingml.document" + ) + body = _text(artifact.content) + assert "Communauté ACME" in body + assert "Alice Dupont" in body + assert "{{" not in body + + +def test_xml_special_characters_survive(docx_bundle): + """A .docx body is XML: an unescaped '&' produces a package Word refuses.""" + artifact = _render( + docx_bundle, + data={"community_name": "Smith & Co ", "representative": 'A "B"'}, + ) + body = _text(artifact.content) # parses the package — corrupt XML raises here + assert "Smith & Co " in body + assert 'A "B"' in body + + +def test_missing_field_is_a_hard_error(docx_bundle): + """StrictUndefined: an official document must never render a silent blank.""" + with pytest.raises(RenderError): + _render(docx_bundle, data={"community_name": "Only one"}) + + +def test_a_null_value_renders_blank_not_the_word_none(docx_bundle): + """In a contract, a literal "None" reads as a filled-in value.""" + artifact = _render(docx_bundle, data={"community_name": "ACME", "representative": None}) + body = _text(artifact.content) + assert "None" not in body + assert "ACME" in body + + +def test_render_bytes_are_deterministic(docx_bundle): + # python-docx lets zipfile stamp members with time.localtime(); the OOXML + # repack is what keeps the sha256 a function of (template, data). + assert _render(docx_bundle).content == _render(docx_bundle).content + + +def test_author_metadata_is_normalised(docx_bundle): + from docx import Document + + props = Document(io.BytesIO(_render(docx_bundle).content)).core_properties + assert props.author == "document-generation" + assert props.last_modified_by == "document-generation" + + +def test_entrypoint_traversal_is_rejected(docx_bundle): + evil = dataclasses.replace( + docx_bundle, + manifest=docx_bundle.manifest.model_copy(update={"entrypoint": "../escape.docx"}), + ) + with pytest.raises(TemplateNotFoundError): + DocxRenderer().render(evil, _DATA, OutputFormat.DOCX, locale=None) + + +def test_wrong_format_raises_render_error(docx_bundle): + with pytest.raises(RenderError): + _render(docx_bundle, fmt=OutputFormat.PDF) diff --git a/tests/renderers/test_pdf_form.py b/tests/renderers/test_pdf_form.py new file mode 100644 index 0000000..3c922b8 --- /dev/null +++ b/tests/renderers/test_pdf_form.py @@ -0,0 +1,107 @@ +"""pdf-form renderer tests: filling an official AcroForm rather than authoring one.""" + +from __future__ import annotations + +import dataclasses +import io + +import pytest +from pypdf import PdfReader + +from adapters.renderers.pdf_form import PdfFormRenderer +from domain.errors import RenderError, TemplateNotFoundError +from domain.models import OutputFormat + +_DATA = {"community_name": "Communauté ACME", "agreed": True} + + +def _render(bundle, data=None, fmt=OutputFormat.PDF): + return PdfFormRenderer().render(bundle, _DATA if data is None else data, fmt, locale=None) + + +def _fields(content: bytes) -> dict: + return PdfReader(io.BytesIO(content)).get_fields() or {} + + +def test_render_fills_text_and_checkbox(pdf_form_bundle): + artifact = _render(pdf_form_bundle) + + assert artifact.format is OutputFormat.PDF + assert artifact.filename == "declaration.pdf" + fields = _fields(artifact.content) + assert fields["Champ de texte 1"]["/V"] == "Communauté ACME" + # The export state comes from the PDF itself, not from the manifest. + assert fields["Case a cocher 1"]["/V"] == "/Oui" + + +def test_falsy_checkbox_selects_off(pdf_form_bundle): + artifact = _render(pdf_form_bundle, {"community_name": "X", "agreed": False}) + assert _fields(artifact.content)["Case a cocher 1"]["/V"] == "/Off" + + +@pytest.mark.parametrize("raw", ["oui", "TRUE", "x", "1", "on"]) +def test_string_checkbox_values_are_understood(pdf_form_bundle, raw): + """JSON payloads routinely carry booleans as strings; a wrong read is silent.""" + artifact = _render(pdf_form_bundle, {"community_name": "X", "agreed": raw}) + assert _fields(artifact.content)["Case a cocher 1"]["/V"] == "/Oui" + + +def test_unsupplied_field_is_left_alone(pdf_form_bundle): + """An absent key must not blank the form's own default.""" + artifact = _render(pdf_form_bundle, {"community_name": "Only this"}) + assert _fields(artifact.content)["Champ de texte 1"]["/V"] == "Only this" + + +def test_need_appearances_is_set(pdf_form_bundle): + """Without it, Chrome/Preview render a filled field as blank.""" + reader = PdfReader(io.BytesIO(_render(pdf_form_bundle).content)) + # pypdf returns a BooleanObject, not the True singleton. + assert bool(reader.trailer["/Root"]["/AcroForm"]["/NeedAppearances"]) is True + + +def test_render_bytes_are_deterministic(pdf_form_bundle): + # The artifact sha256 is the document's identity in document_version, so a + # wall-clock date or a random /ID leaking into the bytes would be a defect. + assert _render(pdf_form_bundle).content == _render(pdf_form_bundle).content + + +def test_unknown_acroform_field_is_permanent(pdf_form_bundle): + """A manifest naming a field the PDF lacks is a template bug, not a retry.""" + broken = dataclasses.replace( + pdf_form_bundle, + manifest=pdf_form_bundle.manifest.model_copy( + update={"fields": {"community_name": "No Such Field"}} + ), + ) + with pytest.raises(TemplateNotFoundError): + PdfFormRenderer().render(broken, _DATA, OutputFormat.PDF, locale=None) + + +def test_pdf_without_acroform_is_permanent(pdf_form_bundle, tmp_path): + from pypdf import PdfWriter + + writer = PdfWriter() + writer.add_blank_page(width=595, height=842) + with open(pdf_form_bundle.root / "flat.pdf", "wb") as handle: + writer.write(handle) + + flat = dataclasses.replace( + pdf_form_bundle, + manifest=pdf_form_bundle.manifest.model_copy(update={"entrypoint": "flat.pdf"}), + ) + with pytest.raises(TemplateNotFoundError): + PdfFormRenderer().render(flat, {}, OutputFormat.PDF, locale=None) + + +def test_entrypoint_traversal_is_rejected(pdf_form_bundle): + evil = dataclasses.replace( + pdf_form_bundle, + manifest=pdf_form_bundle.manifest.model_copy(update={"entrypoint": "../escape.pdf"}), + ) + with pytest.raises(TemplateNotFoundError): + PdfFormRenderer().render(evil, {}, OutputFormat.PDF, locale=None) + + +def test_wrong_format_raises_render_error(pdf_form_bundle): + with pytest.raises(RenderError): + _render(pdf_form_bundle, fmt=OutputFormat.XLSX) diff --git a/tests/renderers/test_registry.py b/tests/renderers/test_registry.py new file mode 100644 index 0000000..95dae49 --- /dev/null +++ b/tests/renderers/test_registry.py @@ -0,0 +1,46 @@ +"""Registry parity: adding an engine must not be possible to half-do. + +Together with the module-level assert in ``domain.models`` (every Engine has a +default entrypoint), this makes "add an engine" a fully-guarded change: forget +either half and the suite fails immediately, rather than a template failing at +render time and being misclassified as a transient error. +""" + +from __future__ import annotations + +import pytest + +from adapters.renderers.registry import DefaultRendererRegistry +from domain.models import Engine, OutputFormat + + +def test_every_engine_has_at_least_one_renderer(): + assert DefaultRendererRegistry().engines() == set(Engine) + + +@pytest.mark.parametrize( + ("engine", "fmt"), + [ + (Engine.JINJA_HTML, OutputFormat.PDF), + (Engine.JINJA_HTML, OutputFormat.HTML), + (Engine.XLSX, OutputFormat.XLSX), + (Engine.DOCX, OutputFormat.DOCX), + (Engine.PDF_FORM, OutputFormat.PDF), + ], +) +def test_supported_pairs_resolve(engine, fmt): + assert DefaultRendererRegistry().get(engine, fmt) is not None + + +@pytest.mark.parametrize( + ("engine", "fmt"), + [ + # Converting either to another format would need LibreOffice; failing + # permanently via UNSUPPORTED_FORMAT is the intended behaviour. + (Engine.PDF_FORM, OutputFormat.HTML), + (Engine.DOCX, OutputFormat.PDF), + (Engine.XLSX, OutputFormat.PDF), + ], +) +def test_deliberately_unsupported_pairs_return_none(engine, fmt): + assert DefaultRendererRegistry().get(engine, fmt) is None diff --git a/tests/renderers/test_xlsx.py b/tests/renderers/test_xlsx.py index d76b474..41f3730 100644 --- a/tests/renderers/test_xlsx.py +++ b/tests/renderers/test_xlsx.py @@ -9,8 +9,8 @@ from openpyxl import load_workbook from adapters.renderers.xlsx import XlsxRenderer -from domain.errors import RenderError, TemplateNotFoundError -from domain.models import OutputFormat +from domain.errors import RenderError, SchemaValidationError, TemplateNotFoundError +from domain.models import BlockSpec, Manifest, OutputFormat _DATA = { "company": "ACME Corp", # written via the workbook's "company" defined name @@ -67,3 +67,121 @@ def test_wrong_format_raises_render_error(xlsx_bundle): def test_missing_sheet_raises_render_error(xlsx_bundle): with pytest.raises(RenderError): _render(xlsx_bundle, data={"cells": {"NoSuchSheet!A1": 1}}) + + +# --------------------------------------------------------------------------- +# Repeating blocks — the variable-length participant/installation lists that +# official annexes are made of. Layout is manifest-owned; data supplies rows. +# --------------------------------------------------------------------------- + +_ROWS = [ + {"nom": "Alice Dupont", "ean": "541448000000000001", "localite": "Namur"}, + {"nom": "ACME SRL", "ean": "541448000000000002", "localite": "Liège"}, +] + + +def _participants(content: bytes): + return load_workbook(io.BytesIO(content))["Participants"] + + +def test_block_writes_one_row_per_item(xlsx_block_bundle): + artifact = XlsxRenderer().render( + xlsx_block_bundle, {"participants": _ROWS}, OutputFormat.XLSX, locale=None + ) + + sheet = _participants(artifact.content) + assert (sheet["A4"].value, sheet["B4"].value, sheet["C4"].value) == ( + "Alice Dupont", + "541448000000000001", + "Namur", + ) + assert sheet["A5"].value == "ACME SRL" + assert sheet["A3"].value == "Nom" # header untouched + + +def test_block_hides_unused_rows_instead_of_deleting_them(xlsx_block_bundle): + """Deleting would break merged ranges, formulas and print areas below.""" + artifact = XlsxRenderer().render( + xlsx_block_bundle, {"participants": _ROWS}, OutputFormat.XLSX, locale=None + ) + + sheet = _participants(artifact.content) + assert sheet.row_dimensions[6].hidden is True # capacity is 4, rows 4-5 used + assert sheet.row_dimensions[7].hidden is True + assert sheet.row_dimensions[4].hidden is False + + +def test_block_overflow_is_permanent(xlsx_block_bundle): + """maxItems is the template's capacity; no retry of the same list can fit.""" + with pytest.raises(SchemaValidationError, match="reserves only 4"): + XlsxRenderer().render( + xlsx_block_bundle, {"participants": _ROWS * 3}, OutputFormat.XLSX, locale=None + ) + + +def test_block_empty_list_renders_cleanly(xlsx_block_bundle): + artifact = XlsxRenderer().render( + xlsx_block_bundle, {"participants": []}, OutputFormat.XLSX, locale=None + ) + assert _participants(artifact.content)["A4"].value is None + + +def test_block_missing_column_key_clears_the_cell(xlsx_block_bundle): + """None and "" must not both reach a cell — they produce different bytes.""" + artifact = XlsxRenderer().render( + xlsx_block_bundle, + {"participants": [{"nom": "Solo", "localite": ""}]}, + OutputFormat.XLSX, + locale=None, + ) + sheet = _participants(artifact.content) + assert sheet["B4"].value is None # key absent + assert sheet["C4"].value is None # empty string canonicalised to None + + +def test_block_bytes_are_deterministic(xlsx_block_bundle): + render = lambda: XlsxRenderer().render( # noqa: E731 + xlsx_block_bundle, {"participants": _ROWS}, OutputFormat.XLSX, locale=None + ) + assert render().content == render().content + + +def test_block_source_is_not_written_through_a_defined_name(xlsx_block_bundle): + """A defined name colliding with a block source would make openpyxl raise. + + Real CWaPE workbooks name their data-validation list sources, so a collision + is a live possibility rather than a hypothetical. + """ + from openpyxl import load_workbook as _load + from openpyxl.workbook.defined_name import DefinedName + + workbook = _load(xlsx_block_bundle.root / "template.xlsx") + workbook.defined_names.add(DefinedName("participants", attr_text="Participants!$E$1")) + workbook.save(xlsx_block_bundle.root / "template.xlsx") + + artifact = XlsxRenderer().render( + xlsx_block_bundle, {"participants": _ROWS}, OutputFormat.XLSX, locale=None + ) + sheet = _participants(artifact.content) + assert sheet["E1"].value is None # the list was NOT written into the named cell + assert sheet["A4"].value == "Alice Dupont" + + +def test_block_without_resolvable_capacity_is_rejected_at_manifest_load(): + """Fails when the bundle is fetched, not mid-render.""" + with pytest.raises(ValueError, match="no capacity"): + Manifest( + id="x.y", + version="1", + engine="xlsx", + supported_formats=["xlsx"], + required_fields={}, + blocks=[BlockSpec(source="rows", anchor="A1", columns=["a"])], + ) + + +def test_block_needs_exactly_one_anchor(): + with pytest.raises(ValueError, match="exactly one"): + BlockSpec(source="rows", columns=["a"]) + with pytest.raises(ValueError, match="exactly one"): + BlockSpec(source="rows", columns=["a"], anchor="A1", anchor_name="rows_start")