diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 622f8d34..58754c3a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -30,7 +30,12 @@ jobs: sudo apt-get install -y libegl1 libxkbcommon-x11-0 libdbus-1-3 libxcb-icccm4 libxcb-image0 libxcb-keysyms1 libxcb-randr0 libxcb-render-util0 libxcb-xinerama0 libxcb-xfixes0 x11-utils libgl1 - name: Install Dependencies - run: uv sync --group dev + run: >- + uv sync --locked + --group dev + --group coolscan-roll + --group fauxice + --group fauxice-hybrid - name: Run Linting (Ruff) run: uv run ruff check . diff --git a/NOTICE.md b/NOTICE.md index a3b1a9c1..6973197c 100644 --- a/NOTICE.md +++ b/NOTICE.md @@ -3,3 +3,12 @@ concepts (continuous defect score, score-weighted multiscale reconstruction, original-floor writer rule, interior-radius routing with feathered compositing) from digital-fauxice, Copyright (c) 2026 Rohan Pandula, MIT License. https://github.com/rohanpandula/digital-fauxice + +The binary tables and portable-oracle validation receipt under +negpy/assets/portable_cms, the fixed output table under +negpy/assets/portable_builder, and the 492-byte output ICC profile embedded in +negpy/services/roll/nikon_icc.py are vendor-derived interoperability data +recovered from the pinned Nikon LS-5000 color path. They are data, not a +redistribution of CML4.dll, LS5000.md3, Nikon Scan, or Wine. Nikon and COOLSCAN +are trademarks of Nikon Corporation; this project is not affiliated with or +endorsed by Nikon. diff --git a/build.py b/build.py index 3cf4a25e..eef8d565 100644 --- a/build.py +++ b/build.py @@ -1,8 +1,15 @@ +import ctypes import glob +import importlib +import importlib.util +import inspect import os import platform +import plistlib +import re import shutil import subprocess +from pathlib import Path import PyInstaller.__main__ @@ -24,6 +31,152 @@ is_macos = system == "Darwin" is_linux = system == "Linux" +COOLSCAN_FEATURE_ENV = "NEGPY_ENABLE_COOLSCAN" +COOLSCAN_LIBUSB_ENV = "NEGPY_LIBUSB_DYLIB" +MACOS_CODESIGN_IDENTITY_ENV = "NEGPY_CODESIGN_IDENTITY" +MACOS_BUNDLE_IDENTIFIER = "org.negpy.NegPy" +MACOS_ENTITLEMENTS_FILE = str(Path(__file__).resolve().parent / "macos" / "NegPy.entitlements") +MACOS_RUNTIME_ENTITLEMENTS = { + "com.apple.security.cs.allow-unsigned-executable-memory": True, +} +MACOS_CS_RUNTIME_FLAG = 0x10000 +COOLSCAN_LIBUSB_DESTINATION = "coolscanpy/_native" +COOLSCAN_LIBUSB_FILENAME = "libusb-1.0.dylib" +CAPTURE_HELPER_FLAG = "--ls5000-capture-helper" +LIVE_ACCEPTANCE_FLAG = "--ls5000-live-acceptance" +PACKAGING_SMOKE_FLAG = "--negpy-packaging-smoke" +PACKAGING_SMOKE_OK = "NegPy frozen scanner packaging smoke passed" + + +def _coolscan_support_enabled() -> bool: + """Enable the optional frozen scanner seam when its build group is present.""" + + configured = os.environ.get(COOLSCAN_FEATURE_ENV) + available = importlib.util.find_spec("coolscanpy") is not None + if configured is None: + enabled = available + elif configured.strip().lower() in {"1", "true", "yes", "on"}: + enabled = True + elif configured.strip().lower() in {"0", "false", "no", "off"}: + enabled = False + else: + raise RuntimeError(f"{COOLSCAN_FEATURE_ENV} must be one of 1/0, true/false, yes/no, or on/off") + + if enabled and not available: + raise RuntimeError("Coolscan support was requested but coolscanpy is unavailable; sync the coolscan-roll build group first") + if enabled and importlib.util.find_spec("portable_digital_ice") is None: + raise RuntimeError("Coolscan support was requested but portable_digital_ice is unavailable; sync the fauxice build group first") + return enabled + + +def resolve_macos_libusb_dylib() -> str: + """Resolve one absolute, architecture-compatible libusb 1.0 dylib.""" + + override = os.environ.get(COOLSCAN_LIBUSB_ENV) + if override: + if not os.path.isabs(override): + raise RuntimeError(f"{COOLSCAN_LIBUSB_ENV} must be an absolute path") + candidates = (override,) + elif platform.machine() == "arm64": + candidates = ( + "/opt/homebrew/opt/libusb/lib/libusb-1.0.dylib", + "/opt/local/lib/libusb-1.0.dylib", + "/usr/local/opt/libusb/lib/libusb-1.0.dylib", + ) + else: + candidates = ( + "/usr/local/opt/libusb/lib/libusb-1.0.dylib", + "/opt/local/lib/libusb-1.0.dylib", + "/opt/homebrew/opt/libusb/lib/libusb-1.0.dylib", + ) + + rejected: list[str] = [] + for candidate in candidates: + # Preserve the stable alias instead of resolving Homebrew's symlink to + # libusb-1.0.0.dylib: PyInstaller uses the source basename at the + # requested destination, and the frozen resolver requires this exact + # package-owned filename. + path = Path(os.path.abspath(candidate)) + if not path.is_file(): + rejected.append(f"{candidate} (missing)") + continue + if path.name != COOLSCAN_LIBUSB_FILENAME: + rejected.append(f"{path} (filename must be {COOLSCAN_LIBUSB_FILENAME})") + continue + architecture = subprocess.run( + ["/usr/bin/lipo", str(path), "-verify_arch", platform.machine()], + check=False, + capture_output=True, + text=True, + ) + if architecture.returncode != 0: + rejected.append(f"{path} (wrong architecture)") + continue + return str(path) + + details = "; ".join(rejected) + raise RuntimeError( + "Coolscan support requires an architecture-compatible libusb 1.0 dylib. " + f"Install it with Homebrew or set {COOLSCAN_LIBUSB_ENV} to an absolute path. " + f"Checked: {details}" + ) + + +COOLSCAN_SUPPORT_ENABLED = _coolscan_support_enabled() +MACOS_LIBUSB_SOURCE = resolve_macos_libusb_dylib() if is_macos and COOLSCAN_SUPPORT_ENABLED else None +MACOS_CODESIGN_IDENTITY = os.environ.get(MACOS_CODESIGN_IDENTITY_ENV, "-").strip() +if is_macos and not MACOS_CODESIGN_IDENTITY: + raise RuntimeError(f"{MACOS_CODESIGN_IDENTITY_ENV} must not be empty") + +_REQUIRED_COOLSCAN_API = { + "coolscanpy": ( + "Device", + "DigitalIceAcquisition", + "DigitalIceAcquisitionEvidence", + "build_digital_ice_acquisition_evidence", + ), + "coolscanpy.protocol.ls5000_single_pass.density": ( + "NikonDensityEvidence", + "NikonExactBuilderEvidence", + "build_nikon_density_evidence", + "build_nikon_exact_builder_evidence", + ), + "coolscanpy.protocol.ls5000_single_pass.bundle": ("verify_capture_bundle",), +} + + +def preflight_coolscan_runtime() -> str: + """Reject a stale or unsealed Coolscan checkout before PyInstaller runs.""" + + loaded: dict[str, object] = {} + missing: list[str] = [] + for module_name, symbols in _REQUIRED_COOLSCAN_API.items(): + module = importlib.import_module(module_name) + loaded[module_name] = module + missing.extend(f"{module_name}.{symbol}" for symbol in symbols if not callable(getattr(module, symbol, None))) + if missing: + raise RuntimeError("Coolscan build API is stale or incomplete: " + ", ".join(missing)) + + device_type = getattr(loaded["coolscanpy"], "Device") + roll_parameters = inspect.signature(device_type.roll).parameters + attempts_root = roll_parameters.get("attempts_root") + if attempts_root is None or attempts_root.kind is not inspect.Parameter.KEYWORD_ONLY or attempts_root.default is not None: + raise RuntimeError( + "Coolscan build API is stale or incomplete: coolscanpy.Device.roll must expose optional keyword-only attempts_root" + ) + + bundle_module = loaded["coolscanpy.protocol.ls5000_single_pass.bundle"] + verify_capture_bundle = getattr(bundle_module, "verify_capture_bundle") + bundle_sha256 = verify_capture_bundle(require_python_sources=True) + if ( + not isinstance(bundle_sha256, str) + or len(bundle_sha256) != 64 + or any(character not in "0123456789abcdef" for character in bundle_sha256) + ): + raise RuntimeError("Coolscan capture bundle returned an invalid SHA-256 identity") + return bundle_sha256 + + # Basic PyInstaller arguments params = [ ENTRY_POINT, @@ -32,6 +185,9 @@ "--windowed", # GUI app, no console "--clean", "--noconfirm", + *([f"--osx-bundle-identifier={MACOS_BUNDLE_IDENTIFIER}"] if is_macos else []), + *([f"--osx-entitlements-file={MACOS_ENTITLEMENTS_FILE}"] if is_macos else []), + *([f"--codesign-identity={MACOS_CODESIGN_IDENTITY}"] if is_macos and MACOS_CODESIGN_IDENTITY != "-" else []), # Hidden imports "--hidden-import=rawpy", "--hidden-import=cv2", @@ -47,6 +203,11 @@ "--hidden-import=jinja2", "--hidden-import=PyQt6", "--hidden-import=qtawesome", + # The frozen app re-enters itself with this worker flag. Keep the worker + # explicit even though --collect-all also finds it, so that entry contract + # cannot disappear in a future package layout change. + *(["--hidden-import=coolscanpy.protocol.ls5000_single_pass.worker"] if COOLSCAN_SUPPORT_ENABLED else []), + *(["--hidden-import=negpy.services.roll.live_acceptance"] if COOLSCAN_SUPPORT_ENABLED else []), # Scanner support: bundle the python-sane C extension but NOT libsane.so.1. # libsane.so.1 must come from the host so SANE can find its backend plugins # in /usr/lib/sane/. See libs_to_remove in package_linux(). @@ -63,6 +224,9 @@ "--collect-all=rawpy", "--collect-all=imageio", "--collect-all=imagecodecs", + *(["--collect-all=coolscanpy"] if COOLSCAN_SUPPORT_ENABLED else []), + "--collect-all=portable_digital_ice", + *([f"--add-binary={MACOS_LIBUSB_SOURCE}:{COOLSCAN_LIBUSB_DESTINATION}"] if MACOS_LIBUSB_SOURCE is not None else []), # Data files "--add-data=negpy/features/exposure/shaders:negpy/features/exposure/shaders", "--add-data=negpy/features/geometry/shaders:negpy/features/geometry/shaders", @@ -71,6 +235,9 @@ "--add-data=negpy/features/lab/shaders:negpy/features/lab/shaders", "--add-data=negpy/features/finish/shaders:negpy/features/finish/shaders", "--add-data=negpy/desktop/view/styles:negpy/desktop/view/styles", + "--add-data=negpy/assets/portable_builder:negpy/assets/portable_builder", + "--add-data=negpy/assets/portable_cms:negpy/assets/portable_cms", + "--add-data=negpy/services/roll/portable_oracle_evaluator.py:negpy/services/roll", "--add-data=icc:icc", "--add-data=media:media", "--add-data=crosstalk:crosstalk", @@ -348,12 +515,218 @@ def package_macos(): shutil.rmtree(temp_dmg_dir) +def verify_macos_runtime_entitlements(app_path: str) -> dict[str, bool]: + """Fail closed unless the signed host has the one required JIT exception.""" + + app_path = os.path.abspath(app_path) + completed = subprocess.run( + [ + "/usr/bin/codesign", + "--display", + "--entitlements", + "-", + "--xml", + app_path, + ], + check=True, + capture_output=True, + ) + try: + embedded = plistlib.loads(completed.stdout) + except (plistlib.InvalidFileException, ValueError, TypeError) as error: + raise RuntimeError("signed NegPy app has unreadable hardened-runtime entitlements") from error + if embedded != MACOS_RUNTIME_ENTITLEMENTS: + raise RuntimeError( + f"signed NegPy app has unexpected hardened-runtime entitlements: expected {MACOS_RUNTIME_ENTITLEMENTS!r}, got {embedded!r}" + ) + return embedded + + +def verify_macos_hardened_runtime(app_path: str) -> int: + """Fail closed unless the host CodeDirectory enables hardened runtime.""" + + app_path = os.path.abspath(app_path) + completed = subprocess.run( + ["/usr/bin/codesign", "--display", "--verbose=4", app_path], + check=True, + capture_output=True, + text=True, + ) + details = f"{completed.stdout}\n{completed.stderr}" + match = re.search(r"\bflags=0x([0-9a-fA-F]+)\b", details) + flags = int(match.group(1), 16) if match is not None else 0 + if flags & MACOS_CS_RUNTIME_FLAG == 0: + raise RuntimeError("signed NegPy app is missing the hardened-runtime CodeDirectory flag") + return flags + + +def sign_macos_scanner_runtime(app_path: str) -> str: + """Sign the bundled libusb dylib, reseal the app, and verify both.""" + + app_path = os.path.abspath(app_path) + + frameworks_libusb = os.path.join( + app_path, + "Contents", + "Frameworks", + COOLSCAN_LIBUSB_DESTINATION, + COOLSCAN_LIBUSB_FILENAME, + ) + resources_libusb = os.path.join( + app_path, + "Contents", + "Resources", + COOLSCAN_LIBUSB_DESTINATION, + COOLSCAN_LIBUSB_FILENAME, + ) + bundled_libusb = next( + (path for path in (frameworks_libusb, resources_libusb) if os.path.isfile(path)), + None, + ) + if bundled_libusb is None: + raise RuntimeError( + f"PyInstaller did not place the required libusb dylib at {COOLSCAN_LIBUSB_DESTINATION}/{COOLSCAN_LIBUSB_FILENAME}" + ) + + identity = os.environ.get(MACOS_CODESIGN_IDENTITY_ENV, "-").strip() + if not identity: + raise RuntimeError(f"{MACOS_CODESIGN_IDENTITY_ENV} must not be empty") + timestamp_option = "--timestamp=none" if identity == "-" else "--timestamp" + codesign = "/usr/bin/codesign" + + subprocess.run( + [codesign, "--force", "--sign", identity, timestamp_option, bundled_libusb], + check=True, + ) + subprocess.run( + [ + codesign, + "--force", + "--sign", + identity, + timestamp_option, + # PyInstaller applies this same policy to the initial bundle + # signature. Reapply it explicitly here because changing the + # bundled libusb invalidates the outer seal; relying on ambient + # entitlement preservation made Developer ID builds fragile. + "--entitlements", + MACOS_ENTITLEMENTS_FILE, + "--preserve-metadata=requirements,flags,runtime", + app_path, + ], + check=True, + ) + subprocess.run( + [codesign, "--verify", "--strict", "--verbose=2", bundled_libusb], + check=True, + ) + subprocess.run( + [codesign, "--verify", "--deep", "--strict", "--verbose=2", app_path], + check=True, + ) + verify_macos_runtime_entitlements(app_path) + if identity != "-": + verify_macos_hardened_runtime(app_path) + return bundled_libusb + + +def smoke_macos_scanner_runtime(app_path: str) -> dict[str, str]: + """Exercise the signed frozen seams without opening Qt or enumerating USB.""" + + app_path = os.path.abspath(app_path) + + executable = os.path.join(app_path, "Contents", "MacOS", APP_NAME) + bundled_libusb = os.path.join( + app_path, + "Contents", + "Frameworks", + COOLSCAN_LIBUSB_DESTINATION, + COOLSCAN_LIBUSB_FILENAME, + ) + if not os.path.isfile(executable) or not os.access(executable, os.X_OK): + raise RuntimeError(f"frozen NegPy executable is missing or not executable: {executable}") + if not os.path.isfile(bundled_libusb): + raise RuntimeError(f"frozen libusb is missing: {bundled_libusb}") + + # Loading the library and resolving symbols does not initialize a libusb + # context and therefore cannot enumerate or claim any USB device. + libusb = ctypes.CDLL(bundled_libusb) + for symbol in ("libusb_init", "libusb_exit"): + if not hasattr(libusb, symbol): + raise RuntimeError(f"frozen libusb is missing required symbol {symbol}") + + environment = os.environ.copy() + environment.pop("PYTHONHOME", None) + environment.pop("PYTHONPATH", None) + for variable in ( + "DYLD_FALLBACK_FRAMEWORK_PATH", + "DYLD_FALLBACK_LIBRARY_PATH", + "DYLD_FRAMEWORK_PATH", + "DYLD_INSERT_LIBRARIES", + "DYLD_LIBRARY_PATH", + ): + environment.pop(variable, None) + + def run_offline(arguments: list[str], *, label: str) -> str: + if LIVE_ACCEPTANCE_FLAG in arguments: + raise RuntimeError(f"{label} attempted to enable live acceptance") + if "--live" in arguments: + raise RuntimeError(f"{label} attempted to enable live scanner access") + completed = subprocess.run( + [executable, *arguments], + cwd=os.path.dirname(app_path), + env=environment, + check=False, + capture_output=True, + text=True, + timeout=120, + ) + if completed.returncode != 0: + raise RuntimeError( + f"{label} failed with exit code {completed.returncode}\nstdout:\n{completed.stdout}\nstderr:\n{completed.stderr}" + ) + return completed.stdout + + packaging_output = run_offline([PACKAGING_SMOKE_FLAG], label="frozen import/assets smoke") + if PACKAGING_SMOKE_OK not in packaging_output: + raise RuntimeError("frozen import/assets smoke did not emit its success receipt") + + helper_output = run_offline( + [ + CAPTURE_HELPER_FLAG, + "--frame", + "18", + "--boundary-offset-rows", + "0", + "--confirm-full-capture", + ], + label="frozen LS-5000 helper dry-run", + ) + for expected in ("619458560 bytes", "dry run only; scanner was not accessed"): + if expected not in helper_output: + raise RuntimeError(f"frozen LS-5000 helper dry-run did not emit {expected!r}") + + return { + "executable": executable, + "libusb": bundled_libusb, + "packaging_output": packaging_output, + "helper_output": helper_output, + } + + def build(): print(f"Building {APP_NAME} for {system}...") + if COOLSCAN_SUPPORT_ENABLED: + bundle_sha256 = preflight_coolscan_runtime() + print(f"Verified Coolscan capture bundle: {bundle_sha256}") print("PyInstaller parameters:", params) PyInstaller.__main__.run(params) + if is_macos and COOLSCAN_SUPPORT_ENABLED: + sign_macos_scanner_runtime(os.path.join("dist", f"{APP_NAME}.app")) + smoke_macos_scanner_runtime(os.path.join("dist", f"{APP_NAME}.app")) + print("Build complete.") if os.path.exists("dist"): print(f"Contents of dist: {os.listdir('dist')}") diff --git a/desktop.py b/desktop.py index 77cfeedb..d4716488 100644 --- a/desktop.py +++ b/desktop.py @@ -1,7 +1,6 @@ import sys import io import faulthandler -from negpy.desktop.main import main def init_streams(): @@ -26,4 +25,11 @@ def flush(self): faulthandler.enable() except Exception: pass - main() + # Frozen scanner helpers and the offline packaging smoke must not import + # the Qt desktop. Keep this dispatch before the GUI module import. + from negpy.desktop.frozen_entry import dispatch_frozen_auxiliary + + if not dispatch_frozen_auxiliary(sys.argv): + from negpy.desktop.main import main + + main() diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 53f52b5c..9bf86bf1 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -7,6 +7,10 @@ ## 0.41.0 - New: **Stitch multi-shot scans** — select overlapping shots of one frame (e.g. a 6×6 scanned in two halves) on the contact sheet and pick **Stitch selected frames**. Alignment, exposure matching and blending happen on the linear scan data before conversion, so the result develops like a single raw. No new file is written: the composite edits and exports like any frame, and **Unstitch** restores the parts. IR dust data is kept when all parts have it. +- New: **Direct Nikon LS-5000 whole-roll scanning** — NegPy can drive a converted SA-21/SA-30 feeder through CoolscanPy, preview and approve the roll, batch-capture 16-bit RGBI, retain scanner evidence, and publish unrepaired, Digital ICE/Hybrid-repaired, and byte-verified Nikon-colour positive tiers. Hybrid synthesis is configurable and fail-closed at a maximum of 10%, with a per-frame recommendation explaining when it is useful. +- New: **Roll-scanner controls that match the workflow** — the contact sheet is shown in the central viewer, with a raw-negative/auto-toned-positive preview toggle, automatic crop and orientation support, Safe Stop, and a one-click **Eject Roll** action that invalidates the old registration after a confirmed eject. +- Fix: **Roll previews no longer look washed out** — preview exposure now ignores isolated hot pixels and the positive view tones optical density per channel. This changes display thumbnails only; saved Nikon-exact TIFF pixels and profiles are untouched. +- Fix: **A timed-out roll eject no longer reads like a definite failure** — the button stays latched against a second transport action and explains that the eject may already have been dispatched, so the operator checks the film physically instead of retrying. - New: **Manage Database…** — dialog to inspect and clear stored data; Clear Saved Edits or Reset Everything, both guarded. @linkmodo - Change: **Copy/paste and Apply-to-roll are now per-setting** — instead of a handful of broad section checkboxes, pasting settings and **Apply settings…** open a picker that lists exactly the settings you changed on the source frame, grouped in collapsible sections (Tone, Colour, Lab, Toning, Finish, Crop, Process, Retouch, Metadata, Export) with each value shown. Tick only what you want, hit Apply. Paste now pops the same picker (Ctrl+V) rather than replacing everything at once; per-frame things like dust spots, heal strokes and crop bounds are never overwritten. - New: **Contact sheet labels and colours** — per-frame labels plus background/label colour, saved as templates. @jboneng diff --git a/docs/COOLSCANPY_ROLL_SCANNING.md b/docs/COOLSCANPY_ROLL_SCANNING.md new file mode 100644 index 00000000..7749e0dd --- /dev/null +++ b/docs/COOLSCANPY_ROLL_SCANNING.md @@ -0,0 +1,535 @@ +# Roll scanning with coolscanpy + +This page describes NegPy's optional integration with +[coolscanpy](https://github.com/rohanpandula/coolscanpy), a standalone +Python library for whole-roll scanning on a Nikon Coolscan LS-5000 with a +roll feeder. coolscanpy talks to the scanner directly over USB and does not +need SANE for the accepted color-negative path. NegPy consumes coolscanpy as +an optional dependency that the rest of the application never requires. + +## What it adds + +The plain Scan panel in NegPy scans one frame at a time through SANE. A +roll feeder changes the shape of the problem. You preview the whole roll +in one transport pass, check or nudge each frame's spacing, and then run a +batch fine scan across a list of slots without reloading the film. That +whole-roll workflow is what coolscanpy implements: preview, spacing +correction, approval of any frame the transport could not place +automatically, and batch scanning with per-frame receipts. This +integration exposes all of it inside NegPy, including a desktop panel. + +Color-negative roll scanning is the accepted path. Conventional silver +black-and-white fine scanning remains explicitly refused by this pinned +release rather than silently taking a different capture route; the current +sidebar opens rolls as color negative. + +## Install + +Nothing here is required to run NegPy. The feature is entirely absent +until coolscanpy is importable, and every entry point checks for that +before doing anything else. + +If you use uv, sync the new dependency group the same way you already +sync the `scanner` group for python-sane or the `camera` group for +gphoto2: + +``` +uv sync --group coolscan-roll +``` + +That group pins the live-validated immutable commit +`970d18e4b092737a633bd20141d3eaa62b25dcc6`, built on the fixed-size +Nikon frame-table repair and the merged streaming +finalization from [PR #1](https://github.com/rohanpandula/coolscanpy/pull/1). +The pin includes exact USB-topology ownership, six-strip leading-edge +handling, Nikon density/exact-builder evidence, and Digital ICE acquisition +evidence, retained caller-owned attempt evidence, terminal-tail-safe +short-strip mapping, and complete continuation-frame meter-layout receipts +(required for multi-frame Nikon-exact publication and for NegPy's +six-frame capture-evidence validation). Its complete Coolscan suite passed before the pin moved, and +`build.py` independently rejects a stale API or capture-bundle hash before +packaging. The pin can move to a tagged release once those APIs are published. + +The local macOS build may use the default ad-hoc code-signing identity (`-`) +for on-machine integrity and testing. That is not publisher identity and does +not make the DMG a notarized release. A distributable build must use a real +Developer ID Application identity through `NEGPY_CODESIGN_IDENTITY`, pass the +same offline frozen smoke, and then complete Apple's notarization and stapling +workflow. `build.py` currently signs and verifies the app bundle but does not +claim to perform that final notarization step. + +The hardened Developer ID build carries one runtime exception: +`com.apple.security.cs.allow-unsigned-executable-memory`. Numba's llvmlite +runtime allocates anonymous read/write pages and later changes them to +read/execute; LLVM 20 does not use `MAP_JIT` for that path, so Apple's +`allow-jit` entitlement is not applicable. The build intentionally does not +disable library validation, executable-page protection, or DYLD protections. +PyInstaller applies the entitlement to the initial bundle seal, the libusb +post-processing step reapplies it explicitly, and the build reads the signed +entitlement and hardened-runtime flag back before running the offline smoke. +That smoke explicitly executes llvmlite's anonymous RW-to-RX allocation check, +so a signed build fails before release if the JIT exception is absent. + +If you manage your own environment instead, install the package directly: + +``` +pip install "coolscanpy @ git+https://github.com/rohanpandula/coolscanpy.git@970d18e4b092737a633bd20141d3eaa62b25dcc6" +``` + +The accepted color-negative roll workflow has no SANE dependency. +coolscanpy's separate plain single-frame `scan()` path may require SANE; see +the coolscanpy README for that setup. + +## Hardware support + +coolscanpy, and by extension this integration, has been validated against +one specific setup: a Nikon Super Coolscan 5000 ED (LS-5000) running +firmware 1.03, with an SA-21 roll feeder wired for SA-30 compatibility. +Live hardware acceptance completed on 2026-07-23: USB enumeration, device +ownership, a 36-frame preview with manual-review flagging, and 36 fine scans +at 4,000 dpi/16-bit RGBI all completed against that pin. Every frame recorded +2,980 reads and 619,458,560 capture bytes. The retained journals and scanner +evidence are banked under +`reverse_engineering/results/NEGPY-LS5000-FULL-ROLL-LIVE-20260723-A/evidence/`. + +Every other Coolscan model and every roll feeder other than the SA-21/SA-30 +combination above is untested. The transport protocol is not assumed to be +LS-5000-only where it does not have to be, but nothing beyond that one +combination has scanned real film. Windows has never run the roll engine +against hardware. + +## Durable live-acceptance evidence + +The six-frame command-line acceptance requires `--attempts-root`. Point it at +an existing, empty, non-symlink directory owned by this one run. It must be +different from `--output-dir`, and the run receipt must live outside both +directories. NegPy passes that directory to +`coolscanpy.Device.roll(..., attempts_root=...)`; unlike Coolscan's default +temporary workspace, its worker journals and durable acquisition evidence +survive a successful close and remain available for audit. Fine-scan +`capture.bin` streams are verified scratch: Coolscan hashes and finalizes each +one, then intentionally deletes it after the frame has been consumed. + +The final run receipt records these files under `capture_evidence`. NegPy +hashes a complete path/size/SHA-256 inventory, caps its canonical manifest at +128 KiB, and binds it to exactly one completed slots-1-through-6 batch. A +successful binding requires the hashed batch job, the session journal's +`batch_job_sha256`, all six frame journals, every fine-capture hash and byte +count recorded before scratch deletion, all six +durable meter sidecars and parent acknowledgements, the first-frame preview, +transport table, and frame map, exact USB topology, and the sealed +plan/engine/bundle identities. Extra attempt trees, incomplete or inconsistent +frame evidence, mixed sessions, and post-hash mutations are fatal. If evidence +changes during success or failure finalization, the receipt is downgraded +truthfully to `retained: false` instead of preserving an earlier success claim. + +This caller-owned attempt tree is scanner-transport evidence for the whole +live run. It is separate from the per-frame movable Digital ICE acquisition +archive described below; neither archive substitutes for the other. + +## Structure + +The camera capture route (Scanlight light source plus a tethered camera, +under `negpy/infrastructure/capture/` and `negpy/services/capture/`) lives +beside NegPy's single-frame `ScannerBackend` protocol, not inside it, +because it is a different acquisition workflow with its own hardware +lifecycle. Roll scanning follows the same shape rather than sitting inside +the plain-SANE scanner packages: + +- `negpy/infrastructure/roll/coolscanpy_roll.py` is the hardware adapter. + It is the only module that imports coolscanpy, and it translates + coolscanpy's typed exceptions to plain `RuntimeError`s the way + `SaneBackend` already does for the plain scanner (see Error handling + below). `negpy/infrastructure/roll/settings.py` holds the sidebar's + persisted settings, `RollScanSettings`, mirroring `ScannerSettings` and + `ScanlightSettings`. +- `negpy/services/roll/service.py` defines `RollScanningService`, which + orchestrates the coolscanpy device and roll lifecycle and writes results + to disk across the three output tiers described below. It never imports + coolscanpy itself, only the adapter above. +- `negpy/infrastructure/roll/repair.py` is the Tier-2 repair engine seam: + an `available()` check and a `register_engine()` entry point. The + `fauxice_bridge` registers the portable Digital ICE engine when its pinned + core runtime is installed. The macOS parity build treats a missing + registration as a frozen-smoke failure rather than silently disabling + repair. See "Output tiers" below. +- `negpy/services/roll/positive.py` renders Tier 3. It calls + `ImageProcessor.run_pipeline`, the same in-memory entry point NegPy's own + export pipeline uses after decoding a file, directly on a Tier-2 buffer. +- `negpy/services/roll/exact_color.py` is the separate fail-closed seam for + portable Nikon-exact color. It contains no builder tables or CML4 math. +- `negpy/services/roll/portable_builder.py` is an evidence/replay bridge. It + applies three pre-F builder LUTs captured and validated together by the + Stage-3 Windows oracle in the pinned order + `F[B_c(i)]`, using the 131,072-byte fixed LS5000.md3 post-F table. It + hashes both the repaired source and computed CML Stage-1 input. It is not + the final macOS-native per-scan builder. +- `negpy/services/roll/portable_cms.py` is the production adapter for the + verified DLL-free CML4 Stage-1/Stage-2 evaluator. It loads nine pinned + binary tables (2,506,760 bytes total), verifies every SHA-256 before use, + and evaluates full frames in bounded chunks. Its integer evaluator is a + byte-identical copy of the independently validated oracle source. +- `negpy/desktop/workers/roll_worker.py` defines `RollWorker`, a `QObject` + moved to its own thread, mirroring `CaptureWorker`. It opens a device's + roll extension lazily the first time a preview or batch scan names it, + and holds that reservation open across calls instead of reopening it + each time. +- `negpy/desktop/view/sidebar/coolscan_roll.py` defines + `CoolscanRollSidebar`, described below. + +## What this integration writes + +A batch scan can produce up to three tiers of output per slot, plus one +receipt. Each tier is a separate on/off setting in the sidebar, and any +combination is valid. A new profile enables all three, selects Hybrid repair, +and selects Nikon-exact color; existing saved choices are preserved. See +"Output tiers" below for what each tier is and what happens when a tier cannot +be produced. + +The filename pattern is the same Jinja2 template the plain Scan panel +uses, with `date` and `seq` variables. For roll scanning, `seq` is the +frame's physical slot number rather than an incrementing counter. Slot +numbers are already a stable identity for a given roll, so re-scanning one +bad slot overwrites that slot's old files instead of accumulating a second +copy beside them. This holds for every tier. + +Only a written Tier 1 (unrepaired) RGB TIFF is handed to NegPy's asset +discovery once a batch scan finishes. Tier 2 and Tier 3 are written to +disk when selected, but neither is opened as a NegPy asset automatically. +The receipt is not opened as an asset either. The scanner-native Digital ICE +prepass and infrared-validity array are not part of that Tier-1 TIFF pair. +They are consumed from the frame-bound acquisition while the scan is being +written and retained in the movable acquisition archive described below. +Hybrid output retains its disclosure mask and verified receipt, but that is +provenance for the result, not a replacement input from which a later repair +can be reconstructed. + +### Movable Digital ICE acquisition archive + +When Tier 1 includes infrared, NegPy also tries to retain the complete input +set needed to reconstruct that frame's `RepairAcquisition`. The frame receipt +records the result under `outputs.repair_acquisition_evidence`. A successful +retention has `retained: true`, `replayable: true`, and schema +`negpy.dice-acquisition-replay-v1`. + +The retained set contains: + +- `acquisition-binding.json`, represented by a `binding` object with `path`, + `bytes`, and `sha256`; +- `prepass.rgbi16.npy` and `ir-validity.npy`, listed under `artifacts` as the + scanner-native prepass and validity map; and +- the Tier-1 RGB and infrared TIFFs, listed under `sources` in + `upright-storage` orientation. + +Artifact and source rows include relative paths as well as their content and +layout bindings. The binding's `replay.requires` list names all five required +inputs: `storage_rgb_tiff`, `storage_ir_tiff`, `prepass_rgbi`, `ir_validity`, +and `acquisition_provenance`. Replay stacks the upright-storage RGB and +infrared planes, then applies `rot90(k=-1)` to reconstruct the scanner-native +main RGBI array. The loader re-derives the producer acquisition identity and +evidence hash before returning it. + +`load_repair_acquisition_evidence` accepts the retained binding file. Relative +source paths make the archive movable when the whole output archive—including +the Tier-1 TIFFs and its hidden evidence directories—is moved together. Moving +only the binding directory, or only the TIFF pair, is incomplete. After a +move, pass `acquisition-binding.json` at its new location; the capture-time +absolute `binding.path` in the outer frame receipt does not relocate itself. +The contract describes its authenticity as `integrity-bound-not-signed`: +hashes and canonical producer bindings detect changed or mismatched inputs, +but do not claim a cryptographic publisher signature. + +Retention is fail-soft for the irreplaceable Tier-1 output. If any unique +input cannot be validated or retained, the RGB/IR TIFF pair can still be +published while `retained: false`, `replayable: false`, and an `unavailable` +status explain why repair replay was withheld. That TIFF pair remains useful +as the scanner master, but is still insufficient by itself for parity repair. + +## Output tiers + +Tier 1 is the unrepaired capture: the scanner-linear RGB and its aligned +infrared plane, exactly as coolscanpy returned them. This is the archival +master, and the only tier the scanner itself can reproduce. It is written +as `.tif` plus `_IR.tif`, the same infrared suffix the +plain Scan panel already uses. It defaults on. + +Tier 2 is the repaired capture: Tier 1 with infrared-guided dust and +scratch repair applied, still scanner-linear and still a negative. It is +written as `_repaired.tif` plus `_repaired_IR.tif`. +That infrared sidecar is Tier 1's own infrared plane, unchanged, not a +repaired version of it. Repair consumes infrared to find defects; it does +not produce a new infrared image. Keeping the original infrared plane is +still useful archival evidence, but it is not sufficient for a later parity +repair without the bound prepass, validity data, and acquisition provenance. + +Repair runs in one of two modes. Exact mode uses the pinned portable Digital +ICE runtime over the bound scanner-native main pass, 285-dpi prepass, and +infrared-validity evidence. Hybrid mode additionally routes severe +zero-signal regions, where the infrared channel gives no usable reading, to +the separately pinned inpainting runtime. The receipt records the selected +backend, resolved mode, hashes, and (for hybrid) disclosure evidence. Runtime +depends heavily on backend, frame geometry, and the hybrid routing/model, so +the integration does not promise a fixed time or make a blanket +byte-identical claim for every hybrid environment. + +Tier 3 is the positive. The Roll Scanning service and its persisted +**Positive color** sidebar choice default to `nikon-exact` for the Nikon C-41 +parity workflow. `negpy-approximate` runs only when explicitly selected and +remains visibly labeled as a preview path: Tier 2 is inverted through NegPy's +own negative-to-positive rendering pipeline, the same pipeline a freshly +imported negative reaches the first time it is opened in NegPy. It is never +claimed or labeled as Nikon Scan parity. + +The service also exposes a fail-closed `nikon-exact` mode. NegPy ships two +separate DLL-free adapters: `PortableStage1Builder`, which applies either a +validated Stage-3 replay receipt or a freshly derived native receipt, and +`PortableCMSOnEvaluator` for the two captured CML4 stages. Neither uses a Wine +process, CML4 DLL, scanner, VM, or external repository at runtime. + +The native receipt boundary accepts the 97-dpi density result only as an +explicit pair on the returned frame: `Frame.nikon_density_evidence` plus +`Frame.nikon_density_ownership`, mirrored by +`Frame.receipt.nikon_density_ownership`. The ownership receipt binds the same +reservation and batch, preview bytes and preview identity, transport table, +reviewed and fresh registration fingerprints, frame attempt, one-based batch +index, and selected slot. A new preview or reservation, re-registration, film +movement, eject/refeed, changed transport identity, missing field, or mismatch +between the frame and its public receipt makes exact color unavailable. There +is deliberately no generic Roll/session evidence cache that can be promoted +later. + +The pinned Coolscan release gives each C-41 frame +`nikon_exact_builder_evidence`, which binds its settled 285-dpi analyzer +raster and final exposure triplet to the same ownership pair. `Frame` +construction revalidates the ownership, density, builder, and Digital ICE +bindings before NegPy can consume them. A clean build verifies that API and +the sealed Coolscan capture-bundle hash before packaging, so a stale pin cannot +silently publish an app that lacks the native exact path. The Stage-3 replay +route remains independently usable with an explicit validated replay receipt. + +The Stage-3 replay bridge is deliberately not presented as the production +builder architecture. The macOS-native path derives fresh pre-F LUTs for each +acquisition from coolscanpy's settled, frame-bound meter and analyzer evidence. +Reusing a previously captured Windows pre-F set is useful for evidence-bound +replay and regression testing only. + +The builder receipt can only be created by the file-backed trusted loader. It +uses stable, non-symlink reads and requires the complete PASS summary, an empty +error list, the exact 15-file artifact inventory, the pinned LS5000.md3 module +and resource, and complete observer source/executable provenance. The immutable +builder envelope binds the raw Stage-3 PASS report and its +SHA-256, the three 131,072-byte pre-F LUT blobs and their hashes, and the +pinned fixed post-F LUT identity. The builder computes the Stage-1 input only +after Fauxice repair; that future full-frame hash is output evidence in the +builder-application receipt, not something the earlier prescan receipt is +asked to predict. The CMS receipt separately binds that computed Stage-1 +input and final output. NegPy independently checks each receipt and content +binding before writing. Missing, malformed, unattested, or tampered evidence +produces an `unavailable` receipt entry; it never falls back to the +approximate renderer under an exact label. + +The frozen app and wheel also carry the original 4,014-byte portable-oracle +validation receipt, not only a copied summary of its result. Startup checks it +against SHA-256 +`edf6f3f89158810f1de4ce3b4ff8938326bc50e1b3035af59af472258e7d95e8` +and a closed schema before the CMS adapter can exist. The production CMS +receipt derives and binds the verified totals: 12 events, 265,440 active +16-bit values with zero mismatches, and 698,880 full-payload bytes with zero +mismatched bytes. + +Native same-acquisition builder evidence is an acquisition artifact, not a +side effect of rendering Tier 3. Whenever a frame carries it, NegPy validates +and stages the canonical density receipt, frame-ownership receipt, analyzer +raster, combined builder-evidence JSON, and three derived pre-F blobs before +attempting repair or the positive. The frame sidecar records that result under +`outputs.native_color_evidence`, including when Tier 3 was not selected or +failed. A successful native exact positive points at that already retained +artifact. A successful replay exact positive instead retains its raw Stage-3 +JSON and three pre-F blobs as part of the exact result. Every retained path, +byte size, and SHA-256 is recorded, and the frame receipt is published last. +The transaction rolls back errors observed by the process; it is not described +as a power-loss durability guarantee. These color-builder artifacts do not +replace Digital ICE's bound 285-dpi prepass, infrared-validity data, or repair +provenance, so Tier 1 TIFF and IR files alone still cannot reconstruct a later +parity repair. + +A successful exact positive also embeds Nikon Scan's exact 492-byte +`Nikon Adobe RGB 4.0.0.3000` profile in TIFF tag 34675. The source-embedded +profile is checked against SHA-256 +`a8d0d753bd6129357cc2647435ce675e8637a679eb526fa180fba460874ce1d3` +before use, and the sidecar binds its name, byte size, and hash. Unrepaired, +repaired, and `negpy-approximate` TIFFs keep their existing untagged behavior. +The exact sidecar also records the finished TIFF's file hash, decoded-pixel +hash, geometry, bit depth, planar layout, and embedded-profile hash after +reopening it. Here “exact Nikon color” means the color-stage RGB values and +ICC identity are exact; it does not claim that NegPy's TIFF compression, +secondary pages, or unrelated application metadata are byte-identical to a +Nikon Scan-written TIFF container. + +The positive is written as `_positive.tif`. Tier 3 always +derives from Tier 2's result in memory, never from a Tier 1 or Tier 2 file +already on disk, and never from Tier 1 directly. Selecting Tier 3 without +Tier 2 still runs repair; the repaired result is just not written to disk +on its own. Tier 3 is enabled in the first-run parity defaults; a saved user +choice to disable it is preserved. + +Repair, when it can run, always runs before inversion: capture, then +repair, then invert. That way Tier 3 benefits from whatever Tier 2 was +able to fix, rather than inverting an uncorrected frame. + +The first-run parity defaults enable all three tiers, choose Hybrid repair, +and choose Nikon-exact color. Tier 1 remains enabled because it is the archival +scanner output. Repaired and Positive run while the frame-bound 285-dpi +prepass, infrared-validity data, acquisition provenance, and native +color-builder evidence are still available. Existing saved settings continue +to win over these defaults, and users may opt out of derived tiers to reduce +compute or storage. A later Tier-1-only reprocessing job must not invent the +missing evidence or label its result exact. + +The macOS parity build includes the pinned portable Digital ICE core and the +bridge auto-registers it. Hybrid additionally needs its separately pinned +companion interpreter/model manifest. `write_frame` still checks registration +and evidence before attempting Tier 2, and records a plain unavailable status +instead of losing lower tiers. Since Tier 3 depends on Tier 2's in-memory +result, it degrades the same way whenever repair is unavailable, even if Tier +2 itself was not selected for writing. Tier 1 still writes when selected, +regardless of Tier-2 or Tier-3 failure. + +The receipt records, for every tier, whether it was written, and a status +explaining why not when it was not. A successful Tier 2 write also records +the repair engine's name and version, along with which mode ran. A +successful approximate Tier 3 write records its color-mode label, rendering +path, process mode, render intent, and whether auto exposure was on. A +successful exact write instead records the input/output content hashes, the +bound ICC profile identity, and both embedded receipt payloads plus their +SHA-256 bindings. Every Tier 3 +receipt entry also carries the Tier 2 provenance that fed it, so it stays +self-contained even when Tier 2 was not itself written to disk. + +Writing every tier is not free of storage cost. At 4000 dpi, one frame's +three tiers together take up roughly half a gigabyte on disk. A long roll +scanned at every tier adds up quickly, so plan storage accordingly. + +## Error handling + +coolscanpy raises a typed exception hierarchy rooted at +`PyCoolscanError`, covering things like a roll that no longer matches its +last preview, a slot that needs manual approval before it can be scanned, +and a requested safe stop. NegPy's existing scanner code has no typed +exception vocabulary of its own for coolscanpy's own failures; +`SaneBackend` reports every failure as a plain `RuntimeError` with a +human-readable message, and that message is what ends up in a status +label. This integration follows the same convention: every coolscanpy +exception is flattened to a `RuntimeError` at the boundary, with the +original exception preserved as `__cause__` for any caller that wants to +branch on it. + +`RollScanningService` itself raises one exception type of its own, +`RollScanningError`, for a lifecycle misuse that has nothing to do with +coolscanpy: calling `open_roll()` twice without closing the first +reservation, or calling `preview()`/`scan_many()` before `open_roll()` at +all. It subclasses `RuntimeError` and mirrors `CaptureError` on the camera +route, which exists for the same reason on that side. + +`Roll.safe_stop()` lets the frame already in flight finish and only +refuses the next one, by raising `SafeStopRequested`. `RollWorker` +distinguishes that deliberate stop from a real failure with +`coolscanpy_roll.is_safe_stop()`, which checks whether a caught +`RuntimeError`'s `__cause__` is coolscanpy's `SafeStopRequested`. A stop +reported this way ends the batch scan the same way a cancelled plain scan +already does: quietly, not as an error. + +## The integration point + +Everything that touches coolscanpy itself lives in one file, +`negpy/infrastructure/roll/coolscanpy_roll.py`. `open_roll()` in that +file is the single place a device handle gets resolved for the roll +workflow. It currently calls `coolscanpy.open()` directly. Everything +else, including `negpy/services/roll/service.py`, the worker, and the +sidebar, only ever talks to the `RollHandle` that function returns. + +That module is marked in a comment as the intended target for a future +change. NegPy's maintainer has a generic SANE-based coolscan route planned +upstream, which is expected to add a real backend-selection seam to +`ScannerService` (today `ScannerService._get_backend()` just hardcodes +`SaneBackend()`, with no selection mechanism at all). When that seam +lands, re-pointing this roll integration at it should only require +changing how `open_roll()` resolves a device. The roll workflow built on +top of it, the exception translation in the same file, the whole of +`service.py`, and the worker and sidebar built on top of that, should not +need to change. + +## The desktop panel + +`CoolscanRollSidebar` lives in the Scan tab as a third collapsible +section, "Roll Scanning", next to "Scanner (SANE)" and "Camera Scanning". +It is built unconditionally and hidden behind a setup hint when +coolscanpy is not importable, the same way `ScanlightSidebar` handles a +missing gphoto2. + +The panel covers device selection, a whole-roll preview rendered as a +thumbnail contact sheet in the app's central viewing area, a spacing-offset +spinner and an approve button for whichever slot is selected, and a batch +scan of every selected slot with a progress bar and a Safe Stop button. +Select All Frames makes a full-roll batch explicit instead of requiring +every thumbnail to be clicked individually. The preview can be toggled +between the scanner negative and an auto-toned positive display; this is a +display-only transform and never changes the Nikon-exact saved TIFF. + +Eject Roll uses the same proven SANE `scanimage --eject` transport action +as NegPy's ordinary scanner panel. It first closes the coolscanpy roll +reservation so the direct SANE command never competes for the scanner. +Successful ejection discards the contact sheet because the film's +registration is no longer valid. If the command returns an error, the +panel deliberately disables preview and further eject attempts for the +rest of that app session: the physical result is uncertain and a blind +retry could move film twice. + +The 30-second software wait is only a bound on how long NegPy waits for +`scanimage` to confirm the action; it is not proof that the transport failed. +If that wait expires, the button action may already have been dispatched. +The panel keeps the operation latched, reports the outcome as uncertain, and +requires a physical check rather than a software retry. + +The Output section has three checkboxes, one per tier, plus a repair-mode +dropdown that governs Tier 2. Any combination of the three checkboxes is +valid, and at least one must be checked before Scan Selected enables. +Unchecking Unrepaired shows a warning below the checkboxes: Tier 1 is the +only tier the scanner itself can reproduce, so turning it off is a real +tradeoff, not just another setting. + +Safe Stop calls +`Roll.safe_stop()` through the worker, which is coolscanpy's own name for +this action and its own contract: the frame already in flight always +finishes, and only the next one is refused. A Scan/Stop toggle button, the +shape the plain Scan and Camera Scanning panels use for their own cancel +action, would imply the click stops the frame in progress. Safe Stop +cannot promise that, so it gets a name and a button that say what it +actually does. + +The panel keeps a roll reservation open for the life of the app session, +including across a tab switch. It closes the current reservation only +when the user picks a different device, opening the new one in its place. +Eject Roll also closes it immediately before handing transport control to +SANE. There is no separate close-without-eject button. This differs from +the camera route, where the tethered session is deliberately released +once neither the live-view window nor the calibration window is open, +because some camera bodies get stuck in a tethered-capture state if a +session is left open past the window that used it. coolscanpy's roll +reservation has no equivalent failure mode, so there is nothing to +protect against by closing it early. + +The panel does not yet expose a material picker and opens a roll as color +negative. Coolscanpy's B&W batch route is implemented and covered offline, +but its first live macOS validation with conventional silver B&W film remains +outstanding. Exposing that choice in NegPy waits on that hardware acceptance, +not on a missing fine-scan implementation. + +A contact sheet thumbnail is a raw scanner preview frame, not a NegPy +pipeline buffer, so it is converted to a displayable image with a small +local helper rather than going through NegPy's color-managed +`ImageConverter`. Raw display exposure uses a robust high percentile, while +positive display uses optical-density inversion, robust per-channel +endpoints, and a display gamma. Hot pixels therefore cannot wash out the +entire contact sheet. `ImageConverter` is not used because it assumes a +working-space buffer a scanner's preview array is not. diff --git a/docs/FAUXICE_IR_REPAIR.md b/docs/FAUXICE_IR_REPAIR.md new file mode 100644 index 00000000..bb61a46f --- /dev/null +++ b/docs/FAUXICE_IR_REPAIR.md @@ -0,0 +1,181 @@ +# Optional IR dust repair (digital-fauxice) + +`negpy.services.repair` provides the repair runtime used by the roll workflow +and by the optional post-import file helper. It calls +[digital-fauxice](https://github.com/rohanpandula/digital-fauxice), an +independently reverse-engineered implementation whose pinned exact path was +validated sample-for-sample against Nikon output. Hybrid is a separate, +explicitly disclosed model-assisted path and is not labeled byte-exact Nikon +output. NegPy does not implement the repair math itself; it validates and +binds the acquisition, runtime, output, and receipts around the engine. + +The standalone post-import helper is off unless a caller explicitly enables +it. The LS-5000 roll sidebar is different: its new-user Color + DICE workflow +enables Repaired and Positive and selects Hybrid by default while preserving +any previously saved choices. Nothing in the normal image render path runs +repair implicitly. + +## What it needs + +Digital ICE does not repair a single scan. It needs two RGBI captures of the +same physical frame: a 285 dpi prepass and a 4000 dpi main scan, with focus, +exposure, frame position, and crop held fixed between them. The prepass +carries per-frame calibration the main pass depends on. It is not a +downsampled convenience copy and cannot be reconstructed from the main scan +after the fact. + +NegPy's generic single-frame/import pipeline captures no paired prepass. On a +frame imported that way, this module reports a skipped status rather than +inventing one. The current LS-5000 roll workflow can supply a real, frame-bound +prepass, main pass, infrared-validity array, and acquisition provenance from +the reviewed coolscanpy overlay. The two Tier-1 TIFFs alone do not retain all +of those inputs and cannot later be promoted to an exact repair acquisition. + +## Retained roll-acquisition replay archive + +For an infrared-bearing Tier-1 roll frame, NegPy attempts to retain those +otherwise transient inputs at write time. The frame receipt exposes the +result as `outputs.repair_acquisition_evidence`. Success is explicit: +`retained` and `replayable` are both `true`, and `schema` is +`negpy.dice-acquisition-replay-v1`. + +The archive adds three files to the RGB/IR TIFF pair: + +- `acquisition-binding.json`, represented by a `binding` object with `path`, + `bytes`, and `sha256`; +- `prepass.rgbi16.npy`, the scanner-native RGBI prepass; and +- `ir-validity.npy`, the scanner-native per-pixel validity map. + +The receipt lists the two NPY inputs under `artifacts` and the Tier-1 RGB and +infrared TIFFs under `sources`. Each row binds its expected layout and content +and includes a relative path. The binding declares the source TIFFs as +`upright-storage` and its `replay.requires` contract names five requirements: +`storage_rgb_tiff`, `storage_ir_tiff`, `prepass_rgbi`, `ir_validity`, and +`acquisition_provenance`. + +To reconstruct scanner-native main RGBI, replay stacks the upright-storage +RGB and infrared planes and applies `rot90(k=-1)`. The +`load_repair_acquisition_evidence` loader stable-reads the binding and every +referenced input, verifies their hashes and geometry, and re-derives the +Coolscan producer acquisition identity and evidence hash. Relative paths let +an operator move the whole output archive and replay it in the new location; +the TIFFs and hidden evidence directories must stay together. Copying only +the TIFF pair or only the binding directory does not satisfy the contract. +After a move, pass `acquisition-binding.json` at its new location; the +capture-time absolute `binding.path` in the outer frame receipt is not a +relocation pointer. + +The replay advertises `integrity-bound-not-signed`. Its hashes and canonical +producer binding detect substitution or corruption, but are not a publisher +signature. Retention is fail-soft: if validation or writing fails, Tier 1 can +still survive while the receipt records `retained: false`, +`replayable: false`, and an `unavailable` reason. That state must never be +treated as replayable or upgraded from the TIFF pair later. + +## Install + +Two optional dependency groups exist in `pyproject.toml`, alongside the +existing `scanner` group that carries `python-sane`: + +``` +uv sync --group fauxice # core engine only, exact mode +uv sync --group fauxice-hybrid # adds the hybrid companion +``` + +Neither package is on PyPI. Both install from a pinned digital-fauxice GitHub +release. The local macOS parity app bundles the core runtime and its frozen +smoke fails if the roll repair bridge does not register. A source installation +without the optional group can still run NegPy, but repair reports unavailable. + +Hybrid mode additionally needs a separately installed IOPaint 1.6.0 runtime +and the `big-lama.pt` weights, in their own virtual environment. Neither is +installed by the `fauxice-hybrid` group. See digital-fauxice's own +`hybrid/docs/hybrid-repair.md` for that setup; NegPy only points at the +paths once they exist. + +## Two modes + +The setting offers `exact` and `hybrid`. The standalone post-import helper +defaults to `exact`; the LS-5000 roll sidebar's first-run Color + DICE workflow +defaults to `hybrid` and visibly labels its model-generated fill. Existing +saved roll choices are preserved. + +`exact` reproduces Nikon's own Digital ICE output value for value. The +engine's validation compared two complete frames against Nikon's real +output, 68,447,316 16-bit samples per frame, with zero mismatches. + +`hybrid` additionally routes the frame's worst damage, the regions where the +engine's own defect signal maxes out, to a LaMa inpainting model, and +composites the result back into the exact output. It needs the separate +`fauxice-hybrid` package and its IOPaint runtime. If hybrid is requested but +either is missing, or its validated run fails, repair falls back to `exact` +and the receipt records why. Explicit cancellation remains cancellation; it +does not silently start a potentially long exact repair afterward. + +Do not use the old whole-frame timing numbers as a promise: runtime varies with +selected backend, frame geometry, routed damage, model runtime, and host +configuration. Hybrid can be materially slower because it invokes the +separately pinned inpainting environment for routed regions; the desktop +reports progress and supports cancellation instead of estimating a fixed +completion time. + +## Progress and cancellation + +`repair_ir_dust` and `repair_frame_files` accept an optional `progress` +callback and `cancel` event, in the same shape `ScannerService.run_scan` +already uses. Exact forwards cooperative progress and cancellation to the +engine. Hybrid reports coarse verified-run checkpoints and polls the external +process; cancellation terminates its process group and returns a cancelled +result instead of leaving the CLI running in the background. + +## Output and provenance + +A repair never rewrites the source master. NegPy keys stored edits by the +source file's content hash, so silently changing that file would orphan +every edit already saved against it. The post-import file helper writes a new +companion, `_FAUXICE.tif`, next to the untouched original; its +`_IR.tif` companion is only ever opened for reading. The roll workflow instead +uses its Tier-2 names and frame receipt described in +`COOLSCANPY_ROLL_SCANNING.md`. + +Every post-import file-helper call also writes +`_FAUXICE.json`, whether or not the repair was applied, so a caller +can show why. It records: + +- the status: applied, skipped, or unavailable, and a plain-text reason +- the mode requested and the mode actually used +- the engine's package name and installed version +- the compute backend requested and the one actually used +- for a hybrid run, the routing counts from the tool's own receipt (how + many regions were routed, how many pixels were synthesized, out of how + many pixels in the frame) and the SHA-256 of the disclosure mask PNG + +## Hybrid fill is disclosed, not hidden + +Hybrid mode does not claim its filled pixels are recovered film information. +They are generated by a model. Every synthesized pixel is recorded in a mask +PNG written alongside the output (`_FAUXICE_SYNTH.png`), where a +white pixel marks a synthesized one. NegPy stable-reads and validates the +canonical hybrid receipt, output and mask hashes, geometry, routing counts, +synthesis accounting, runtime manifests, model identity, and the receipt's +within-budget verdict. It does not turn those checks into a broader claim that +the entire hybrid image is byte-identical Nikon output. + +## Limits + +This only works on color negative film with a real infrared channel +captured on the same physical frame as the visible-light scan. Traditional +silver-based black and white film and some Kodachrome stocks block infrared +light outright, so neither this module nor Nikon's own Digital ICE can +clean them; there is no IR signal to read. + +The validated profile is narrow: a Nikon Super Coolscan 5000 ED running +Digital ICE Normal at the resolution metrics used in digital-fauxice's own +validation. An acquisition outside that profile is rejected before any +processing runs. + +Ordinary imports and generic one-pass scans still have no paired prepass and +therefore report a skipped status. The LS-5000 roll acquisition path supplies +the required pair and binds it to one frame. If that binding, validity data, +runtime, or receipt is missing or inconsistent, repair fails closed; it does +not reconstruct calibration from a main scan or from Tier-1 TIFFs. diff --git a/macos/NegPy.entitlements b/macos/NegPy.entitlements new file mode 100644 index 00000000..7b677ae2 --- /dev/null +++ b/macos/NegPy.entitlements @@ -0,0 +1,8 @@ + + + + + com.apple.security.cs.allow-unsigned-executable-memory + + + diff --git a/negpy/assets/portable_builder/ls5000-fixed-output-lut-u16le.bin b/negpy/assets/portable_builder/ls5000-fixed-output-lut-u16le.bin new file mode 100644 index 00000000..8914b4f2 Binary files /dev/null and b/negpy/assets/portable_builder/ls5000-fixed-output-lut-u16le.bin differ diff --git a/negpy/assets/portable_cms/cml4-stage1-clut0.bin b/negpy/assets/portable_cms/cml4-stage1-clut0.bin new file mode 100644 index 00000000..04190dc3 Binary files /dev/null and b/negpy/assets/portable_cms/cml4-stage1-clut0.bin differ diff --git a/negpy/assets/portable_cms/cml4-stage1-input-lut0.bin b/negpy/assets/portable_cms/cml4-stage1-input-lut0.bin new file mode 100644 index 00000000..35ec089f Binary files /dev/null and b/negpy/assets/portable_cms/cml4-stage1-input-lut0.bin differ diff --git a/negpy/assets/portable_cms/cml4-stage1-output-lut0.bin b/negpy/assets/portable_cms/cml4-stage1-output-lut0.bin new file mode 100644 index 00000000..a53409bd Binary files /dev/null and b/negpy/assets/portable_cms/cml4-stage1-output-lut0.bin differ diff --git a/negpy/assets/portable_cms/cml4-stage2-clut0.bin b/negpy/assets/portable_cms/cml4-stage2-clut0.bin new file mode 100644 index 00000000..9ee2ace9 Binary files /dev/null and b/negpy/assets/portable_cms/cml4-stage2-clut0.bin differ diff --git a/negpy/assets/portable_cms/cml4-stage2-input-lut0.bin b/negpy/assets/portable_cms/cml4-stage2-input-lut0.bin new file mode 100644 index 00000000..e38b10d3 Binary files /dev/null and b/negpy/assets/portable_cms/cml4-stage2-input-lut0.bin differ diff --git a/negpy/assets/portable_cms/cml4-stage2-output-lut0.bin b/negpy/assets/portable_cms/cml4-stage2-output-lut0.bin new file mode 100644 index 00000000..49836f48 Binary files /dev/null and b/negpy/assets/portable_cms/cml4-stage2-output-lut0.bin differ diff --git a/negpy/assets/portable_cms/lch-atan-u16le.bin b/negpy/assets/portable_cms/lch-atan-u16le.bin new file mode 100644 index 00000000..f9b2e215 Binary files /dev/null and b/negpy/assets/portable_cms/lch-atan-u16le.bin differ diff --git a/negpy/assets/portable_cms/lch-reciprocal-u16le.bin b/negpy/assets/portable_cms/lch-reciprocal-u16le.bin new file mode 100644 index 00000000..ce5bffbf Binary files /dev/null and b/negpy/assets/portable_cms/lch-reciprocal-u16le.bin differ diff --git a/negpy/assets/portable_cms/lch-sincos-i16le.bin b/negpy/assets/portable_cms/lch-sincos-i16le.bin new file mode 100644 index 00000000..78f4edb5 Binary files /dev/null and b/negpy/assets/portable_cms/lch-sincos-i16le.bin differ diff --git a/negpy/assets/portable_cms/portable-oracle-receipt.json b/negpy/assets/portable_cms/portable-oracle-receipt.json new file mode 100644 index 00000000..89b9812c --- /dev/null +++ b/negpy/assets/portable_cms/portable-oracle-receipt.json @@ -0,0 +1,192 @@ +{ + "stage1": { + "per_event": { + "30": { + "mismatched_u16": 0, + "total_u16": 32448, + "max_abs": 0, + "mae": 0.0, + "full_payload": { + "mismatched_u16": 0, + "total_u16": 32448, + "max_abs": 0, + "mae": 0.0 + } + }, + "34": { + "mismatched_u16": 0, + "total_u16": 16848, + "max_abs": 0, + "mae": 0.0, + "full_payload": { + "mismatched_u16": 0, + "total_u16": 32448, + "max_abs": 0, + "mae": 0.0 + } + }, + "38": { + "mismatched_u16": 0, + "total_u16": 22464, + "max_abs": 0, + "mae": 0.0, + "full_payload": { + "mismatched_u16": 0, + "total_u16": 22464, + "max_abs": 0, + "mae": 0.0 + } + }, + "42": { + "mismatched_u16": 0, + "total_u16": 11664, + "max_abs": 0, + "mae": 0.0, + "full_payload": { + "mismatched_u16": 0, + "total_u16": 22464, + "max_abs": 0, + "mae": 0.0 + } + }, + "47": { + "mismatched_u16": 0, + "total_u16": 32448, + "max_abs": 0, + "mae": 0.0, + "full_payload": { + "mismatched_u16": 0, + "total_u16": 32448, + "max_abs": 0, + "mae": 0.0 + } + }, + "52": { + "mismatched_u16": 0, + "total_u16": 16848, + "max_abs": 0, + "mae": 0.0, + "full_payload": { + "mismatched_u16": 0, + "total_u16": 32448, + "max_abs": 0, + "mae": 0.0 + } + } + }, + "total": { + "mismatched_u16": 0, + "total_u16": 132720, + "max_abs": 0, + "mae": 0.0 + }, + "full_payload_total": { + "mismatched_u16": 0, + "total_u16": 174720, + "max_abs": 0, + "mae": 0.0 + } + }, + "stage2": { + "per_event": { + "31": { + "mismatched_u16": 0, + "total_u16": 32448, + "max_abs": 0, + "mae": 0.0, + "full_payload": { + "mismatched_u16": 0, + "total_u16": 32448, + "max_abs": 0, + "mae": 0.0 + } + }, + "35": { + "mismatched_u16": 0, + "total_u16": 16848, + "max_abs": 0, + "mae": 0.0, + "full_payload": { + "mismatched_u16": 0, + "total_u16": 32448, + "max_abs": 0, + "mae": 0.0 + } + }, + "39": { + "mismatched_u16": 0, + "total_u16": 22464, + "max_abs": 0, + "mae": 0.0, + "full_payload": { + "mismatched_u16": 0, + "total_u16": 22464, + "max_abs": 0, + "mae": 0.0 + } + }, + "43": { + "mismatched_u16": 0, + "total_u16": 11664, + "max_abs": 0, + "mae": 0.0, + "full_payload": { + "mismatched_u16": 0, + "total_u16": 22464, + "max_abs": 0, + "mae": 0.0 + } + }, + "48": { + "mismatched_u16": 0, + "total_u16": 32448, + "max_abs": 0, + "mae": 0.0, + "full_payload": { + "mismatched_u16": 0, + "total_u16": 32448, + "max_abs": 0, + "mae": 0.0 + } + }, + "53": { + "mismatched_u16": 0, + "total_u16": 16848, + "max_abs": 0, + "mae": 0.0, + "full_payload": { + "mismatched_u16": 0, + "total_u16": 32448, + "max_abs": 0, + "mae": 0.0 + } + } + }, + "total": { + "mismatched_u16": 0, + "total_u16": 132720, + "max_abs": 0, + "mae": 0.0 + }, + "full_payload_total": { + "mismatched_u16": 0, + "total_u16": 174720, + "max_abs": 0, + "mae": 0.0 + } + }, + "all_12_events": { + "mismatched_u16": 0, + "total_u16": 265440, + "max_abs": 0, + "mae": 0.0 + }, + "all_12_full_payloads": { + "mismatched_u16": 0, + "total_u16": 349440, + "max_abs": 0, + "mae": 0.0, + "total_bytes": 698880, + "mismatched_bytes": 0 + } +} diff --git a/negpy/desktop/controller.py b/negpy/desktop/controller.py index f2cb75a8..6bbe9867 100644 --- a/negpy/desktop/controller.py +++ b/negpy/desktop/controller.py @@ -28,7 +28,12 @@ ThumbnailUpdateTask, ThumbnailWorker, ) -from negpy.desktop.workers.scan_worker import BatchRequest, RollPreviewRequest, ScanRequest, ScanWorker +from negpy.desktop.workers.scan_worker import ( + BatchRequest, + RollPreviewRequest as ScanRollPreviewRequest, + ScanRequest, + ScanWorker, +) from negpy.desktop.workers.stitch import StitchTask, StitchWorker from negpy.features.stitch.models import stitch_hash, stitch_name from negpy.desktop.workers.capture_worker import ( @@ -37,6 +42,11 @@ CaptureWorker, LiveViewRequest, ) +from negpy.desktop.workers.roll_worker import ( + RollBatchScanRequest, + RollPreviewRequest as CoolscanRollPreviewRequest, + RollWorker, +) from negpy.domain.models import ( ColorSpace, ExportFormat, @@ -53,6 +63,7 @@ ) from negpy.services.assets.half_frame import base_hash, slice_half from negpy.services.assets.sidecar import load_or_promote, write_sidecar +from negpy.services.roll.service import RollScanningError from negpy.features.exposure.logic import ( calculate_wb_shifts, calculate_wb_shifts_from_log, @@ -76,6 +87,7 @@ from negpy.kernel.system.config import APP_CONFIG from negpy.kernel.system.logging import get_logger from negpy.services.rendering.preview_manager import PreviewManager +from negpy.services.repair.fauxice_hybrid_runner import HybridRuntimeConfig from negpy.services.view.coordinate_mapping import CoordinateMapping logger = get_logger(__name__) @@ -213,7 +225,7 @@ class AppController(QObject): scan_batch_finished = pyqtSignal(list) # batch: all completed rgb paths scan_batch_requested = pyqtSignal(BatchRequest) scan_eject_requested = pyqtSignal(str) - scan_roll_preview_requested = pyqtSignal(RollPreviewRequest) + scan_roll_preview_requested = pyqtSignal(ScanRollPreviewRequest) scan_roll_preview_ready = pyqtSignal(object) # one RollPreview per strip slot scan_roll_preview_finished = pyqtSignal() capture_light_requested = pyqtSignal(int, int, int, int, str) @@ -241,8 +253,32 @@ class AppController(QObject): connection_polled = pyqtSignal(dict) # {usb_ok, usb_model, light_ok, light_detail} poll_light_temp_requested = pyqtSignal(str) # light port (temp-only poll, runs even mid-live-view) light_temp_polled = pyqtSignal(object) # Scanlight LED temperature °C, or None - - def __init__(self, session_manager: DesktopSessionManager): + roll_devices_requested = pyqtSignal() + roll_devices_ready = pyqtSignal(list) # list[coolscanpy.DeviceInfo] + roll_opened = pyqtSignal(str) # device_id now open + roll_preview_requested = pyqtSignal(CoolscanRollPreviewRequest) + roll_preview_ready = pyqtSignal(list) # list[coolscanpy.Thumbnail] + roll_spacing_offset_requested = pyqtSignal(int, int) + roll_spacing_offset_set = pyqtSignal(int, int) + roll_approve_requested = pyqtSignal(int) + roll_approved = pyqtSignal(int) + roll_scan_requested = pyqtSignal(RollBatchScanRequest) + roll_progress = pyqtSignal(float, str) + roll_frame_written = pyqtSignal(object) # RollFrameOutput + roll_finished = pyqtSignal(list) # list[RollFrameOutput] + roll_cancelled = pyqtSignal() + roll_eject_requested = pyqtSignal(str) + roll_ejected = pyqtSignal(bool) + roll_eject_error = pyqtSignal(str) + roll_error = pyqtSignal(str) + roll_status = pyqtSignal(str) + + def __init__( + self, + session_manager: DesktopSessionManager, + *, + hybrid_runtime: HybridRuntimeConfig | None = None, + ): super().__init__() self.session = session_manager self.state: AppState = session_manager.state @@ -260,6 +296,8 @@ def __init__(self, session_manager: DesktopSessionManager): self._pending_scanned_file: Optional[str] = None self._gpu_fallback_notified = False self._cleaned_up = False + self._roll_shutdown_complete = False + self._shutdown_block_reason: str | None = None self._active_batch: Optional[str] = None self._active_batch_title = "" self._active_batch_abortable = False @@ -327,6 +365,12 @@ def __init__(self, session_manager: DesktopSessionManager): # Scanning tab polls or the user acts. self._capture_thread_started = False + self.roll_thread = QThread() + self.roll_worker = RollWorker(hybrid_runtime=hybrid_runtime) + self.roll_worker.moveToThread(self.roll_thread) + # Same lazy-start reasoning as _capture_thread_started (_ensure_roll_thread). + self._roll_thread_started = False + self.canvas: Any = None self._is_rendering = False self._pending_render_task: Any = None @@ -531,6 +575,25 @@ def _connect_signals(self) -> None: self.capture_worker.poll_status.connect(self.connection_polled.emit) self.poll_light_temp_requested.connect(self.capture_worker.poll_light_temp) self.capture_worker.light_temp_polled.connect(self.light_temp_polled.emit) + self.roll_devices_requested.connect(self.roll_worker.list_devices) + self.roll_worker.devices_ready.connect(self.roll_devices_ready.emit) + self.roll_worker.opened.connect(self.roll_opened.emit) + self.roll_preview_requested.connect(self.roll_worker.run_preview) + self.roll_worker.preview_ready.connect(self.roll_preview_ready.emit) + self.roll_spacing_offset_requested.connect(self.roll_worker.set_spacing_offset) + self.roll_worker.spacing_offset_set.connect(self.roll_spacing_offset_set.emit) + self.roll_approve_requested.connect(self.roll_worker.approve) + self.roll_worker.approved.connect(self.roll_approved.emit) + self.roll_scan_requested.connect(self.roll_worker.run_batch_scan) + self.roll_worker.progress.connect(self.roll_progress.emit) + self.roll_worker.frame_written.connect(self.roll_frame_written.emit) + self.roll_worker.finished.connect(self._on_roll_scan_finished) + self.roll_worker.cancelled.connect(self.roll_cancelled.emit) + self.roll_eject_requested.connect(self.roll_worker.eject) + self.roll_worker.ejected.connect(self.roll_ejected.emit) + self.roll_worker.eject_error.connect(self.roll_eject_error.emit) + self.roll_worker.error.connect(self.roll_error.emit) + self.roll_worker.status.connect(self.roll_status.emit) self.session.active_file_changing.connect(lambda: self._update_thumbnail_from_state(force_readback=True)) self.session.session_emptied.connect(self._render_memo.clear) @@ -2068,7 +2131,7 @@ def start_batch(self, req: BatchRequest) -> None: self.scan_started.emit() self.scan_batch_requested.emit(req) - def start_roll_preview(self, req: RollPreviewRequest) -> None: + def start_roll_preview(self, req: ScanRollPreviewRequest) -> None: """Preview strip slots (results via scan_roll_preview_ready, then scan_roll_preview_finished). No scan_started — preview is dialog-local.""" self.scan_worker.prepare_scan() @@ -2264,6 +2327,80 @@ def _on_capture_finished(self, paths: list) -> None: self._pending_scanned_file = paths[0] self.request_asset_discovery(list(paths)) + # ── Roll scanning integration ───────────────────────────────────── + + def _ensure_roll_thread(self) -> None: + """Start the roll worker's thread on first use (lazy), mirroring + `_ensure_capture_thread`.""" + if not self._roll_thread_started: + self.roll_thread.start() + self._roll_thread_started = True + + def request_roll_devices(self) -> None: + self._ensure_roll_thread() + self.roll_devices_requested.emit() + + def start_coolscan_roll_preview(self, req: CoolscanRollPreviewRequest) -> None: + self._ensure_roll_thread() + self.roll_preview_requested.emit(req) + + def set_roll_spacing_offset(self, slot: int, offset_rows: int) -> None: + self._ensure_roll_thread() + self.roll_spacing_offset_requested.emit(slot, offset_rows) + + def approve_roll_slot(self, slot: int) -> None: + self._ensure_roll_thread() + self.roll_approve_requested.emit(slot) + + def start_roll_scan(self, req: RollBatchScanRequest) -> None: + self.roll_worker.prepare_batch() + self._ensure_roll_thread() + self.roll_scan_requested.emit(req) + + def roll_safe_stop(self) -> None: + self.roll_worker.safe_stop() + + def eject_roll(self, device_id: str) -> None: + """Release and eject the selected Coolscan roll on its worker thread.""" + + self._ensure_roll_thread() + self.roll_eject_requested.emit(device_id) + + @property + def shutdown_block_reason(self) -> str | None: + return self._shutdown_block_reason + + def request_shutdown(self) -> bool: + """Prove the scanner reservation closed before the UI may exit.""" + + if self._roll_shutdown_complete: + return True + try: + self.roll_worker.shutdown() + except Exception as error: + logger.exception("application shutdown blocked by scanner teardown") + self._shutdown_block_reason = str(error) + self.roll_status.emit("Cannot exit: the Coolscan reservation is not safely closed.") + return False + self._roll_shutdown_complete = True + self._shutdown_block_reason = None + return True + + def _on_roll_scan_finished(self, outputs: list) -> None: + """Feed newly-written roll frames into NegPy's file list, like the plain Scan and + Camera Scanning routes already do. Each output is already one complete per-slot RGB + TIFF (coolscanpy's roll engine writes one frame per slot, not an R/G/B triplet to + merge), so unlike `_on_capture_finished` this needs no rgbscan_mode/process_mode + bookkeeping -- just discovery. + + Only Tier 1 (`rgb_path`) is discovered here, and only when it was actually written -- + the roll sidebar's output-tier setting can leave it `None`. Tier 2 and Tier 3 are not + opened as NegPy assets by this seam; see docs/COOLSCANPY_ROLL_SCANNING.md.""" + self.roll_finished.emit(outputs) + rgb_paths = [o.rgb_path for o in outputs if o.rgb_path] + if rgb_paths: + self.request_asset_discovery(rgb_paths) + def effective_output_icc(self) -> Optional[str]: """Output profile the preview proofs through: a custom override, else the profile for the selected export color space. None means no proof (Same as Source).""" @@ -3192,6 +3329,8 @@ def cleanup(self) -> None: """ if self._cleaned_up: return + if not self.request_shutdown(): + raise RollScanningError(self._shutdown_block_reason or "Coolscan reservation is not safely closed") self._cleaned_up = True self._render_debounce.stop() self._cursor_readout_timer.stop() @@ -3223,6 +3362,9 @@ def cleanup(self) -> None: if self.capture_thread.isRunning(): self.capture_thread.quit() self.capture_thread.wait() + if self.roll_thread.isRunning(): + self.roll_thread.quit() + self.roll_thread.wait() self.render_worker.destroy_all() # All GPU-touching threads are now joined; release the wgpu device. diff --git a/negpy/desktop/frozen_entry.py b/negpy/desktop/frozen_entry.py new file mode 100644 index 00000000..aac6eccb --- /dev/null +++ b/negpy/desktop/frozen_entry.py @@ -0,0 +1,272 @@ +"""Minimal frozen auxiliary entry points for scanner and packaging checks. + +This module intentionally imports only the standard library at module load. +The real executable dispatches here before importing the Qt desktop, so a +desktop-only dependency or startup regression cannot block the isolated +LS-5000 worker or the offline post-build smoke. +""" + +from __future__ import annotations + +import sys + + +CAPTURE_HELPER_FLAG = "--ls5000-capture-helper" +LIVE_ACCEPTANCE_FLAG = "--ls5000-live-acceptance" +PACKAGING_SMOKE_FLAG = "--negpy-packaging-smoke" +PACKAGING_SMOKE_OK = "NegPy frozen scanner packaging smoke passed" + + +def _require_registered_repair_engine() -> None: + """Import the production bridge and reject a silently unavailable engine.""" + + from negpy.infrastructure.roll import fauxice_bridge + from negpy.infrastructure.roll import repair as roll_repair + + if not fauxice_bridge.engine_available() or not roll_repair.available(): + raise RuntimeError("the frozen Digital ICE repair bridge did not register an engine") + + +def _require_pinned_hybrid_runtime(*, loader=None) -> str: + """Load and independently rehash the configured hybrid model weights.""" + + import hashlib + import os + import re + import stat + from pathlib import Path + + if loader is None: + from negpy.services.repair.hybrid_runtime_manifest import ( + load_default_hybrid_runtime_manifest, + ) + + loader = load_default_hybrid_runtime_manifest + + runtime = loader() + if runtime is None: + raise RuntimeError("the pinned hybrid runtime is not installed") + try: + runtime.validate_files() + except ValueError as error: + raise RuntimeError(f"the pinned hybrid runtime is invalid: {error}") from error + + expected = runtime.model_weights_sha256 + if not isinstance(expected, str) or re.fullmatch(r"[0-9a-f]{64}", expected) is None: + raise RuntimeError("the hybrid model weights pin is not a lowercase SHA-256") + try: + weights = Path(runtime.model_weights) + linked = os.lstat(weights) + if stat.S_ISLNK(linked.st_mode) or not stat.S_ISREG(linked.st_mode): + raise RuntimeError("the hybrid model weights must be a regular non-symlink file") + flags = os.O_RDONLY | getattr(os, "O_CLOEXEC", 0) | getattr(os, "O_NOFOLLOW", 0) + descriptor = os.open(weights, flags) + try: + opened = os.fstat(descriptor) + identity = ( + opened.st_dev, + opened.st_ino, + opened.st_size, + opened.st_mtime_ns, + opened.st_ctime_ns, + ) + if not stat.S_ISREG(opened.st_mode) or identity != ( + linked.st_dev, + linked.st_ino, + linked.st_size, + linked.st_mtime_ns, + linked.st_ctime_ns, + ): + raise RuntimeError("the hybrid model weights changed while opening") + digest = hashlib.sha256() + while block := os.read(descriptor, 1024 * 1024): + digest.update(block) + after_read = os.fstat(descriptor) + finally: + os.close(descriptor) + after_path = os.lstat(weights) + except RuntimeError: + raise + except (OSError, TypeError, ValueError) as error: + raise RuntimeError(f"the hybrid model weights could not be verified: {error}") from error + + for metadata in (after_read, after_path): + if ( + metadata.st_dev, + metadata.st_ino, + metadata.st_size, + metadata.st_mtime_ns, + metadata.st_ctime_ns, + ) != identity: + raise RuntimeError("the hybrid model weights changed while hashing") + actual = digest.hexdigest() + if actual != expected: + raise RuntimeError(f"the hybrid model weights SHA-256 does not match the pin: expected {expected}, got {actual}") + return actual + + +def run_packaging_smoke_checks() -> None: + """Validate frozen scanner/color/DICE resources without opening a device.""" + + import ctypes + import hashlib + import importlib + from pathlib import Path + + from llvmlite.binding import check_jit_execution + + if not getattr(sys, "frozen", False): + raise RuntimeError("the packaging smoke is only valid in a frozen application") + meipass = getattr(sys, "_MEIPASS", None) + if not isinstance(meipass, str) or not meipass: + raise RuntimeError("the frozen application has no PyInstaller resource root") + resource_root = Path(meipass) + + # This is the exact hardened-runtime operation used by Numba's LLVM JIT: + # allocate an anonymous RW page and transition it to RX. Keep it explicit + # so the post-sign smoke catches missing executable-memory entitlements. + check_jit_execution() + + from coolscanpy import ( + DigitalIceAcquisition, + DigitalIceAcquisitionEvidence, + build_digital_ice_acquisition_evidence, + ) + from coolscanpy.protocol.ls5000_single_pass.bundle import verify_capture_bundle + from coolscanpy.protocol.ls5000_single_pass.density import ( + NikonDensityEvidence, + NikonExactBuilderEvidence, + build_nikon_density_evidence, + build_nikon_exact_builder_evidence, + ) + from coolscanpy.protocol.ls5000_single_pass.usb_backend import bundled_libusb_path + from negpy.services.repair.fauxice_hybrid_runner import ( + HybridRuntimeConfig, + run_hybrid_repair, + ) + from negpy.services.roll.nikon_icc import ( + NIKON_ADOBE_RGB_PROFILE_BYTES, + NIKON_ADOBE_RGB_PROFILE_SHA256, + nikon_adobe_rgb_profile, + ) + from negpy.services.roll.portable_builder import PortableStage1Builder + from negpy.services.roll.portable_cms import PortableCMSOnEvaluator + + required_callables = ( + DigitalIceAcquisition, + DigitalIceAcquisitionEvidence, + build_digital_ice_acquisition_evidence, + NikonDensityEvidence, + NikonExactBuilderEvidence, + build_nikon_density_evidence, + build_nikon_exact_builder_evidence, + HybridRuntimeConfig, + run_hybrid_repair, + ) + if not all(callable(value) for value in required_callables): + raise RuntimeError("a required Coolscan or DICE runtime API is unavailable") + for module_name in ( + "portable_digital_ice.backend", + "portable_digital_ice.fast_cpu.engine", + "portable_digital_ice.metal_backend.engine", + ): + importlib.import_module(module_name) + _require_registered_repair_engine() + _require_pinned_hybrid_runtime() + + bundle_sha256 = verify_capture_bundle(require_python_sources=False) + if len(bundle_sha256) != 64: + raise RuntimeError("the packaged Coolscan capture bundle identity is invalid") + + expected_libusb = resource_root / "coolscanpy" / "_native" / "libusb-1.0.dylib" + libusb_path = bundled_libusb_path() + if libusb_path != expected_libusb.resolve(strict=True): + raise RuntimeError(f"Coolscan resolved an unexpected frozen libusb: {libusb_path}") + libusb = ctypes.CDLL(str(libusb_path)) + if not all(hasattr(libusb, symbol) for symbol in ("libusb_init", "libusb_exit")): + raise RuntimeError("the packaged libusb is missing required entry points") + + # Constructors validate every exact LUT's size and pinned SHA-256, the + # portable evaluator source identity, and the strict 12-event receipt. + PortableStage1Builder() + PortableCMSOnEvaluator() + + profile = nikon_adobe_rgb_profile() + if len(profile) != NIKON_ADOBE_RGB_PROFILE_BYTES or hashlib.sha256(profile).hexdigest() != NIKON_ADOBE_RGB_PROFILE_SHA256: + raise RuntimeError("the exact Nikon output ICC profile failed validation") + + expected_icc = { + "AdobeCompat-v4.icc", + "DisplayP3-v4.icc", + "GrayGamma2.2.icc", + "ProPhoto-v4.icc", + "RGBScan.icc", + "Rec2020-v4.icc", + "sRGB-v4.icc", + } + icc_dir = resource_root / "icc" + packaged_icc = {path.name for path in icc_dir.glob("*.icc") if path.is_file()} + if packaged_icc != expected_icc: + raise RuntimeError("the frozen app's ICC asset set is incomplete or unexpected") + + +def dispatch_packaging_smoke(argv: list[str]) -> bool: + """Run the private post-build smoke before desktop initialization.""" + + if len(argv) < 2 or argv[1] != PACKAGING_SMOKE_FLAG: + return False + if len(argv) != 2: + raise RuntimeError("the packaging smoke does not accept arguments") + run_packaging_smoke_checks() + print(PACKAGING_SMOKE_OK, flush=True) + return True + + +def dispatch_capture_helper(argv: list[str]) -> bool: + """Run the frozen LS-5000 worker before desktop initialization.""" + + if len(argv) < 2 or argv[1] != CAPTURE_HELPER_FLAG: + return False + + from coolscanpy.protocol.ls5000_single_pass.capture_process import ( + CAPTURE_HELPER_FLAG as COOLSCAN_CAPTURE_HELPER_FLAG, + ) + from coolscanpy.protocol.ls5000_single_pass.worker import main as worker_main + + if COOLSCAN_CAPTURE_HELPER_FLAG != CAPTURE_HELPER_FLAG: + raise RuntimeError("NegPy and Coolscan helper flags do not match") + worker_main(argv[2:]) + return True + + +def dispatch_live_acceptance(argv: list[str]) -> bool: + """Run the one-shot LS-5000 acceptance before desktop initialization.""" + + if len(argv) < 2 or argv[1] != LIVE_ACCEPTANCE_FLAG: + return False + + from negpy.services.roll.live_acceptance import main as live_acceptance_main + + exit_code = live_acceptance_main(argv[2:]) + if exit_code: + raise SystemExit(exit_code) + return True + + +def dispatch_frozen_auxiliary(argv: list[str]) -> bool: + """Dispatch a helper/smoke command, or leave normal desktop startup alone.""" + + return dispatch_packaging_smoke(argv) or dispatch_live_acceptance(argv) or dispatch_capture_helper(argv) + + +__all__ = [ + "CAPTURE_HELPER_FLAG", + "LIVE_ACCEPTANCE_FLAG", + "PACKAGING_SMOKE_FLAG", + "PACKAGING_SMOKE_OK", + "dispatch_capture_helper", + "dispatch_frozen_auxiliary", + "dispatch_live_acceptance", + "dispatch_packaging_smoke", + "run_packaging_smoke_checks", +] diff --git a/negpy/desktop/main.py b/negpy/desktop/main.py index 578e15d4..e9db79c1 100644 --- a/negpy/desktop/main.py +++ b/negpy/desktop/main.py @@ -1,25 +1,39 @@ import os import sys +import tempfile +from typing import Callable -from PyQt6.QtCore import Qt, qInstallMessageHandler +from PyQt6.QtCore import QEvent, QObject, Qt, qInstallMessageHandler from PyQt6.QtGui import QIcon -from PyQt6.QtWidgets import QApplication, QProxyStyle, QStyle +from PyQt6.QtWidgets import QApplication, QMessageBox, QProgressDialog, QProxyStyle, QStyle -from negpy.desktop.controller import AppController -from negpy.desktop.session import DesktopSessionManager -from negpy.desktop.view.main_window import MainWindow -from negpy.infrastructure.storage.repository import StorageRepository -from negpy.services.assets.crosstalk import CrosstalkProfiles -from negpy.services.assets.gear import GearProfiles +from negpy.desktop.frozen_entry import ( + dispatch_capture_helper as _dispatch_capture_helper, + dispatch_packaging_smoke as _dispatch_packaging_smoke, +) from negpy.kernel.system.config import APP_CONFIG, BASE_USER_DIR from negpy.kernel.system.logging import get_logger, setup_logging from negpy.kernel.system.override import apply as apply_override from negpy.kernel.system.override import load_or_create as load_override from negpy.kernel.system.parallel import configure_cpu_parallel from negpy.kernel.system.paths import get_resource_path +from negpy.services.assets.crosstalk import CrosstalkProfiles +from negpy.services.assets.gear import GearProfiles +from negpy.services.repair.hybrid_runtime_manifest import ( + HybridRuntimeManifestError, + load_default_hybrid_runtime_manifest, +) logger = get_logger(__name__) +# A Finder-launched frozen macOS app can be stopped by the first access to +# ~/Documents before Qt has created a foreground window to host the TCC prompt. +# The handoff process below has no user-data side effects beyond a temporary, +# empty access probe; after consent it re-execs so the real process retains the +# existing pre-QApplication override/RHI/UI-scale ordering. +_MACOS_DOCUMENTS_READY_ENV = "NEGPY_MACOS_DOCUMENTS_READY" +_MACOS_ACCESS_PROBE_PREFIX = ".negpy-access-" + # qtawesome paints toolbar icons into a null pixmap when a button is asked to # render before its first layout has given it valid geometry (e.g. while the # startup "Restore Session" dialog spins a modal loop). The paint is harmless @@ -56,6 +70,100 @@ def styleHint(self, hint, option=None, widget=None, returnData=None): return super().styleHint(hint, option, widget, returnData) +class _ApplicationShutdownGate(QObject): + """Consume every Qt quit request until scanner teardown is proven.""" + + def __init__(self, allow_shutdown: Callable[[], bool]) -> None: + super().__init__() + self._allow_shutdown = allow_shutdown + + def eventFilter(self, watched, event) -> bool: + if event.type() == QEvent.Type.Quit and not self._allow_shutdown(): + return True + return super().eventFilter(watched, event) + + +class _MacOSDocumentsHandoff(QProgressDialog): + """A startup-only progress window that cannot dismiss the active TCC request.""" + + def closeEvent(self, event) -> None: + event.ignore() + + def keyPressEvent(self, event) -> None: + if event.key() == Qt.Key.Key_Escape: + event.accept() + return + super().keyPressEvent(event) + + +def _probe_user_data_access(user_dir: str) -> None: + """Prove that the configured user-data directory is writable without leaving a file behind.""" + + os.makedirs(user_dir, exist_ok=True) + fd: int | None = None + probe_path: str | None = None + try: + fd, probe_path = tempfile.mkstemp(prefix=_MACOS_ACCESS_PROBE_PREFIX, dir=user_dir) + finally: + if fd is not None: + try: + os.close(fd) + finally: + if probe_path is not None: + os.unlink(probe_path) + + +def _show_macos_documents_handoff() -> QProgressDialog: + """Show the foreground, non-cancellable UI that makes a TCC prompt visible.""" + + handoff = _MacOSDocumentsHandoff("Opening your existing NegPy workspace…", None, 0, 0) + handoff.setWindowTitle("NegPy") + handoff.setCancelButton(None) + handoff.setAutoClose(False) + handoff.setAutoReset(False) + handoff.setWindowFlag(Qt.WindowType.WindowCloseButtonHint, False) + handoff.setWindowFlag(Qt.WindowType.WindowContextHelpButtonHint, False) + handoff.setWindowModality(Qt.WindowModality.ApplicationModal) + handoff.show() + return handoff + + +def _macos_frozen_documents_handoff() -> bool: + """Handle the visible first-process permission check for a frozen macOS bundle. + + Returns True only when access was denied/unavailable and ``main`` must stop. + On success this function replaces the process and does not return. The child + consumes the one-process environment sentinel and follows normal startup. + """ + + if not (sys.platform == "darwin" and getattr(sys, "frozen", False)): + return False + if os.environ.pop(_MACOS_DOCUMENTS_READY_ENV, None) == "1": + return False + + app = QApplication.instance() or QApplication(sys.argv) + _handoff = _show_macos_documents_handoff() + app.processEvents() + + try: + _probe_user_data_access(BASE_USER_DIR) + except OSError: + QMessageBox.critical( + None, + "NegPy needs Documents access", + "NegPy could not open your existing Documents/NegPy workspace. " + "Your existing files were not changed. Grant NegPy access to the Documents folder " + "in macOS Privacy & Security, then reopen the app.", + ) + return True + + env = os.environ.copy() + env[_MACOS_DOCUMENTS_READY_ENV] = "1" + executable = os.path.abspath(sys.executable) + os.execve(executable, [executable, *sys.argv[1:]], env) + raise AssertionError("os.execve unexpectedly returned") + + def _install_exception_hook() -> None: """Log every unhandled exception — especially ones raised inside a Qt slot — to the file log and show a non-fatal notice, instead of letting PyQt call qFatal() and abort with a native crash @@ -101,10 +209,47 @@ def _bootstrap_environment() -> None: GearProfiles.ensure_user_dir() +def _load_desktop_hybrid_runtime(): + """Load the optional external companion without risking desktop startup.""" + + try: + runtime = load_default_hybrid_runtime_manifest() + except HybridRuntimeManifestError as error: + logger.error("Hybrid runtime disabled: %s", error) + return None + if runtime is None: + logger.info("Hybrid runtime is not installed; exact Digital ICE remains available") + else: + try: + runtime.validate_files() + except ValueError as error: + logger.error("Hybrid runtime disabled: %s", error) + return None + logger.info("Loaded pinned external hybrid runtime") + return runtime + + def main() -> None: """ Desktop entry point. """ + if _dispatch_packaging_smoke(sys.argv) or _dispatch_capture_helper(sys.argv): + return + + # A successful macOS handoff replaces this process. A denied handoff has + # already shown a visible explanation and must not initialize a silent, + # empty replacement workspace. + if _macos_frozen_documents_handoff(): + return + + # Keep desktop-only imports behind the helper dispatch. A UI regression + # must never prevent the frozen capture worker or packaging smoke from + # starting in their isolated process. + from negpy.desktop.controller import AppController + from negpy.desktop.session import DesktopSessionManager + from negpy.desktop.view.main_window import MainWindow + from negpy.infrastructure.storage.repository import StorageRepository + override_cfg = load_override(APP_CONFIG.override_toml_path) setup_logging(level=override_cfg.log_level_int) _install_exception_hook() # log unhandled slot exceptions to negpy.log instead of aborting @@ -153,9 +298,16 @@ def main() -> None: app.setStyleSheet(load_stylesheet()) session_manager = DesktopSessionManager(repo) - controller = AppController(session_manager) + controller = AppController( + session_manager, + hybrid_runtime=_load_desktop_hybrid_runtime(), + ) window = MainWindow(controller) + shutdown_gate = _ApplicationShutdownGate( + window.request_shutdown_for_exit, + ) + app.installEventFilter(shutdown_gate) window.show() exit_code = app.exec() diff --git a/negpy/desktop/view/main_window.py b/negpy/desktop/view/main_window.py index e921ad47..7d5daa34 100644 --- a/negpy/desktop/view/main_window.py +++ b/negpy/desktop/view/main_window.py @@ -8,6 +8,7 @@ QApplication, QMainWindow, QMessageBox, + QStackedWidget, QStatusBar, QVBoxLayout, QWidget, @@ -162,7 +163,28 @@ def _restore_window_geometry(self) -> None: self.resize(w, h) self.move(x, y) + def request_shutdown_for_exit(self) -> bool: + """Gate both window-close and application-level quit requests.""" + + if not self.controller.request_shutdown(): + reason = self.controller.shutdown_block_reason or ("The scanner reservation has not closed safely yet.") + self.statusBar().showMessage("Exit blocked until the Coolscan closes safely.") + QMessageBox.critical( + self, + "Scanner still reserved", + "NegPy cannot exit while the Coolscan reservation is unresolved.\n\n" + f"{reason}\n\nWait for the current operation to stop, then close again.", + ) + self.show() + self.raise_() + self.activateWindow() + return False + return True + def closeEvent(self, event) -> None: + if not self.request_shutdown_for_exit(): + event.ignore() + return try: self.controller.session.repo.save_global_setting("window_geometry", [self.x(), self.y(), self.width(), self.height()]) except Exception: @@ -232,7 +254,9 @@ def _init_ui(self) -> None: self.loading_overlay = LoadingOverlay(self.canvas) self.loading_overlay.raise_() - self.central_layout.addWidget(self.canvas, stretch=1) + self.central_stack = QStackedWidget() + self.central_stack.addWidget(self.canvas) + self.central_layout.addWidget(self.central_stack, stretch=1) self.setCentralWidget(self.central_widget) @@ -245,6 +269,8 @@ def _init_ui(self) -> None: self.drawer.setAllowedAreas(Qt.DockWidgetArea.LeftDockWidgetArea | Qt.DockWidgetArea.RightDockWidgetArea) self.right_panel = RightPanel(self.controller) + self.roll_preview_workspace = self.right_panel.coolscan_roll_sidebar.preview_workspace + self.central_stack.addWidget(self.roll_preview_workspace) # Back-compat alias: tutorial, keyboard shortcuts, and _sync_tool_buttons reach feature sidebars here. self.controls_panel = self.right_panel.controls_panel @@ -358,6 +384,8 @@ def reset_panel_layout(self) -> None: def _connect_signals(self) -> None: """Wire controller and view.""" self.controller.session.state_changed.connect(self._update_title) + self.right_panel.coolscan_roll_sidebar.workspace_requested.connect(self._show_roll_preview_workspace) + self.roll_preview_workspace.back_requested.connect(self._show_image_workspace) # visibilityChanged only mirrors the button — it also fires on close/minimize, # so we persist in the toggle methods to avoid clobbering the saved state on exit. @@ -377,6 +405,7 @@ def _connect_signals(self) -> None: # Metadata updates only on persistent history changes or file selection self.controller.session.history_changed.connect(self._refresh_image_info) self.controller.session.file_selected.connect(lambda _: self._refresh_image_info()) + self.controller.session.file_selected.connect(lambda _: self._show_image_workspace()) self.controller.session.session_emptied.connect(self._on_session_emptied) self.canvas.clicked.connect(self.controller.handle_canvas_clicked) @@ -422,6 +451,12 @@ def _connect_signals(self) -> None: def _refresh_dashboard(self) -> None: self.toolbar.refresh_gpu_status() + def _show_roll_preview_workspace(self) -> None: + self.central_stack.setCurrentWidget(self.roll_preview_workspace) + + def _show_image_workspace(self) -> None: + self.central_stack.setCurrentWidget(self.canvas) + def _display_buffer_for_canvas(self, buffer): if isinstance(buffer, GPUTexture): buffer = buffer.readback() diff --git a/negpy/desktop/view/sidebar/coolscan_roll.py b/negpy/desktop/view/sidebar/coolscan_roll.py new file mode 100644 index 00000000..29fb1c55 --- /dev/null +++ b/negpy/desktop/view/sidebar/coolscan_roll.py @@ -0,0 +1,932 @@ +"""Coolscan roll-scanning sidebar. + +Device selection, a whole-roll preview rendered as a thumbnail contact +sheet, per-slot spacing-offset nudge and approval, and one-button batch +fine-scan with progress and a safe-stop control. Mirrors the structure of +`ScanlightSidebar` (gating via `_apply_gating()`, settings persisted the +same way, one status line + progress bar) as closely as the workflow +allows -- preview/approve/batch-scan has no camera-route analogue, so the +contact sheet and per-slot controls below are new, built the simplest way +that fits the surrounding code. +""" + +from __future__ import annotations + +import dataclasses +from typing import TYPE_CHECKING + +import numpy as np +import qtawesome as qta +from PyQt6.QtCore import QSize, Qt, pyqtSignal, pyqtSlot +from PyQt6.QtGui import QColor, QIcon, QImage, QPixmap +from PyQt6.QtWidgets import ( + QAbstractItemView, + QCheckBox, + QComboBox, + QFileDialog, + QFormLayout, + QHBoxLayout, + QLabel, + QLineEdit, + QListWidget, + QListWidgetItem, + QMessageBox, + QProgressBar, + QPushButton, + QDoubleSpinBox, + QSpinBox, + QVBoxLayout, + QWidget, +) + +from negpy.desktop.view.styles.templates import section_subheader +from negpy.desktop.view.styles.theme import THEME +from negpy.infrastructure.roll import coolscanpy_roll +from negpy.infrastructure.roll.repair import RepairMode +from negpy.infrastructure.roll.settings import RollScanSettings +from negpy.services.roll.exact_color import PositiveColorMode +from negpy.services.roll.service import RollFrameOutput + +if TYPE_CHECKING: + import coolscanpy + + from negpy.desktop.workers.roll_worker import RollBatchScanRequest + +_SLOT_ROLE = Qt.ItemDataRole.UserRole +_WARN_COLOR = "#C8922E" # matches ScanlightSidebar's advisory tone +_THUMBNAIL_SIZE = QSize(220, 150) + + +def _thumbnail_rgb8( + image: "np.ndarray", + *, + positive: bool = False, +) -> np.ndarray: + """Tone a scanner-linear roll thumbnail for display without mutating it. + + The 97-dpi index is scanner-linear negative transmission, not a display + image. A peak-normalize followed by ``255 - value`` makes ordinary C-41 + frames look pale cyan and badly overexposed, especially when one hot + pixel or rail sets the peak. Raw display therefore ignores only the + extreme highlight tail. Positive display converts transmission to + optical density, applies robust per-channel endpoints, then a display + gamma. This is intentionally an auto-toned review preview; the saved + Nikon-exact positive still comes from the acquisition-bound builder/CMS. + """ + arr = np.asarray(image) + if arr.ndim == 2: + arr = np.stack([arr] * 3, axis=-1) + if arr.ndim != 3 or arr.shape[2] < 3: + raise ValueError("roll thumbnail must be a grayscale or RGB image") + rgb = np.asarray(arr[..., :3], dtype=np.float64) + if not rgb.size: + return np.empty((*rgb.shape[:2], 3), dtype=np.uint8) + finite = np.where(np.isfinite(rgb), rgb, 0.0) + finite = np.maximum(finite, 0.0) + + if not positive: + white = max(float(np.percentile(finite, 99.5)), 1.0) + display = np.clip(finite / white, 0.0, 1.0) + else: + channel_white = np.maximum( + np.percentile(finite, 99.5, axis=(0, 1)), + 1.0, + ) + transmission = np.clip( + finite / channel_white, + 1.0 / 65535.0, + 1.0, + ) + density = -np.log10(transmission) + black = np.percentile(density, 1.0, axis=(0, 1)) + white = np.percentile(density, 99.0, axis=(0, 1)) + span = np.maximum(white - black, np.finfo(np.float64).eps) + display = np.clip((density - black) / span, 0.0, 1.0) + display = np.sqrt(display) + + return np.ascontiguousarray(np.rint(display * 255.0).astype(np.uint8)) + + +def _thumbnail_pixmap(image: "np.ndarray", *, positive: bool = False) -> QPixmap: + """A `coolscanpy.Thumbnail.image` array as a displayable QPixmap.""" + arr = _thumbnail_rgb8(image, positive=positive) + h, w = arr.shape[:2] + qimg = QImage(arr.data, w, h, w * 3, QImage.Format.Format_RGB888).copy() # copy: detach from arr's buffer + return QPixmap.fromImage(qimg) + + +class RollPreviewWorkspace(QWidget): + """Center-stage contact sheet for the whole-roll workflow.""" + + back_requested = pyqtSignal() + + def __init__(self) -> None: + super().__init__() + layout = QVBoxLayout(self) + layout.setContentsMargins(24, 18, 24, 18) + layout.setSpacing(12) + + header = QHBoxLayout() + title_col = QVBoxLayout() + title_col.setSpacing(2) + title = QLabel("Roll preview") + title.setStyleSheet(f"color: {THEME.text_primary}; font-size: 20px; font-weight: 600;") + subtitle = QLabel("Review, approve, and select frames here. Display mode does not change captured or saved files.") + subtitle.setStyleSheet(f"color: {THEME.text_muted}; font-size: {THEME.font_size_small}px;") + subtitle.setWordWrap(True) + title_col.addWidget(title) + title_col.addWidget(subtitle) + header.addLayout(title_col, 1) + + mode_label = QLabel("Show as") + mode_label.setStyleSheet(f"color: {THEME.text_secondary};") + self.display_mode_combo = QComboBox() + self.display_mode_combo.setObjectName("roll_preview_display_mode") + self.display_mode_combo.addItem("Positive preview (auto tone)", True) + self.display_mode_combo.addItem("Negative (raw)", False) + self.display_mode_combo.setMinimumHeight(40) + self.display_mode_combo.setMinimumWidth(180) + self.display_mode_combo.setToolTip( + "Display only: Positive preview applies optical-density auto tone for exposure and framing review. " + "Negative shows the scanner-linear thumbnail. Neither changes capture bytes or the saved Nikon-exact color." + ) + self.back_btn = QPushButton("Back to image editor") + self.back_btn.setMinimumHeight(40) + self.back_btn.clicked.connect(lambda _checked=False: self.back_requested.emit()) + header.addWidget(mode_label) + header.addWidget(self.display_mode_combo) + header.addWidget(self.back_btn) + layout.addLayout(header) + + self.contact_sheet = QListWidget() + self.contact_sheet.setObjectName("roll_preview_contact_sheet") + self.contact_sheet.setViewMode(QListWidget.ViewMode.IconMode) + self.contact_sheet.setResizeMode(QListWidget.ResizeMode.Adjust) + self.contact_sheet.setMovement(QListWidget.Movement.Static) + self.contact_sheet.setWrapping(True) + self.contact_sheet.setSpacing(12) + self.contact_sheet.setSelectionMode(QAbstractItemView.SelectionMode.ExtendedSelection) + self.contact_sheet.setIconSize(_THUMBNAIL_SIZE) + self.contact_sheet.setMinimumHeight(360) + self.contact_sheet.setToolTip( + "Click a slot to nudge its spacing or approve it; select several, then use Scan Selected in the controls panel." + ) + layout.addWidget(self.contact_sheet, 1) + + +class CoolscanRollSidebar(QWidget): + """Whole-roll preview, spacing/approval and batch fine-scan panel.""" + + workspace_requested = pyqtSignal() + + def __init__(self, controller) -> None: + super().__init__() + self.controller = controller + self._settings: RollScanSettings = self._load_settings() + self._devices: list = [] + self._devices_loaded = False + self._thumbnails: dict[int, "coolscanpy.Thumbnail"] = {} + self._scanning = False + self._stopping = False # safe-stop acknowledged, waiting for the in-flight frame + self._preview_pending = False + self._eject_pending = False + self._eject_latched = False + self._eject_failed = False + self._active_scan_request: RollBatchScanRequest | None = None + + self.preview_workspace = RollPreviewWorkspace() + self.contact_sheet = self.preview_workspace.contact_sheet + self.preview_display_combo = self.preview_workspace.display_mode_combo + self._init_ui() + self._connect_signals() + + # ── settings persistence ────────────────────────────────────────── + + def _load_settings(self) -> RollScanSettings: + data = self.controller.session.repo.get_global_setting("roll_scan_settings", default={}) + if isinstance(data, dict) and data: + try: + known = {f.name for f in dataclasses.fields(RollScanSettings)} + return RollScanSettings(**{k: v for k, v in data.items() if k in known}) + except Exception: + pass + return RollScanSettings.defaults() + + def _save_settings(self) -> None: + self.controller.session.repo.save_global_setting("roll_scan_settings", dataclasses.asdict(self._settings)) + + def _update_settings_from_ui(self) -> None: + updated = dataclasses.replace( + self._settings, + last_device_id=self._current_device_id() or self._settings.last_device_id, + output_folder=self.folder_edit.text().strip(), + filename_pattern=self.pattern_edit.text().strip() or RollScanSettings.defaults().filename_pattern, + write_unrepaired=self.write_unrepaired_check.isChecked(), + write_repaired=self.write_repaired_check.isChecked(), + write_positive=self.write_positive_check.isChecked(), + repair_mode=self.repair_mode_combo.currentData() or RollScanSettings.defaults().repair_mode, + positive_mode=self.positive_mode_combo.currentData() or PositiveColorMode.NIKON_EXACT.value, + hybrid_synthesis_limit_percent=self.hybrid_synthesis_limit_spin.value(), + ) + if updated == self._settings: + return + self._settings = updated + self._save_settings() + self._apply_gating() + + # ── UI construction ─────────────────────────────────────────────── + + def _init_ui(self) -> None: + layout = QVBoxLayout(self) + layout.setContentsMargins(5, 0, 5, 5) + layout.setSpacing(10) + + self.preview_btn = QPushButton(qta.icon("fa5s.images", color=THEME.text_primary), " Preview Roll") + self.preview_btn.setObjectName("scan_btn") + self.preview_btn.setFixedHeight(40) + layout.addWidget(self.preview_btn) + + self.gate_hint = QLabel("") + self.gate_hint.setStyleSheet(f"color: {_WARN_COLOR}; font-size: {THEME.font_size_small}px;") + self.gate_hint.setWordWrap(True) + layout.addWidget(self.gate_hint) + + # ── DEVICE ────────────────────────────────────────── + layout.addWidget(section_subheader("DEVICE")) + self._setup_hint = QLabel( + "Roll scanning needs coolscanpy, an optional dependency: `pip install coolscanpy`. See docs/COOLSCANPY_ROLL_SCANNING.md." + ) + self._setup_hint.setWordWrap(True) + self._setup_hint.setStyleSheet(f"color: {_WARN_COLOR}; font-size: {THEME.font_size_small}px;") + self._setup_hint.setVisible(not coolscanpy_roll.available()) + layout.addWidget(self._setup_hint) + + device_row = QHBoxLayout() + self.device_combo = QComboBox() + self.device_combo.setToolTip("Select a Coolscan device") + self.device_combo.addItem("Detecting devices…", None) + self.refresh_btn = QPushButton() + self.refresh_btn.setIcon(qta.icon("fa5s.redo", color=THEME.text_secondary)) + self.refresh_btn.setToolTip("Refresh device list") + self.refresh_btn.setFixedWidth(32) + self.eject_btn = QPushButton(qta.icon("fa5s.eject", color=THEME.text_secondary), " Eject Roll…") + self.eject_btn.setToolTip( + "Release the scanner reservation and eject the loaded roll. This invalidates the current preview and frame registration." + ) + device_row.addWidget(self.device_combo, 1) + device_row.addWidget(self.refresh_btn) + device_row.addWidget(self.eject_btn) + layout.addLayout(device_row) + + # ── OUTPUT ────────────────────────────────────────── + layout.addWidget(section_subheader("OUTPUT")) + out_form = QFormLayout() + out_form.setSpacing(6) + folder_row = QHBoxLayout() + self.folder_edit = QLineEdit(self._settings.output_folder) + self.folder_edit.setPlaceholderText("Output folder…") + self.folder_browse = QPushButton("…") + self.folder_browse.setFixedWidth(32) + folder_row.addWidget(self.folder_edit) + folder_row.addWidget(self.folder_browse) + out_form.addRow("Folder", folder_row) + self.pattern_edit = QLineEdit(self._settings.filename_pattern) + self.pattern_edit.setToolTip( + 'Jinja2 template. Variables: {{ date }}, {{ seq }} (the slot number).\nExample: {{ date }}_{{ "%03d" % seq }}' + ) + out_form.addRow("Filename", self.pattern_edit) + + # Three independent tiers, not a single three-way choice -- any combination is + # valid. Unrepaired defaults on (see RollScanSettings): it is the archival + # master and the only tier the scanner itself can reproduce. + tiers_row = QHBoxLayout() + self.write_unrepaired_check = QCheckBox("Unrepaired") + self.write_unrepaired_check.setChecked(self._settings.write_unrepaired) + self.write_unrepaired_check.setToolTip( + "Tier 1, the archival master: the frame exactly as captured, with no repair applied. " + "This is the only tier the scanner can reproduce -- turning it off loses data that " + "cannot be recreated from the other two tiers." + ) + self.write_repaired_check = QCheckBox("Repaired") + self.write_repaired_check.setChecked(self._settings.write_repaired) + self.write_repaired_check.setToolTip( + "Tier 2: the unrepaired capture with infrared-guided dust/scratch repair applied, " + "still scanner-linear. It needs the frame-bound scanner prepass and validity evidence; " + "the Tier 1 TIFFs alone cannot recreate parity repair later." + ) + self.write_positive_check = QCheckBox("Positive") + self.write_positive_check.setChecked(self._settings.write_positive) + self.write_positive_check.setToolTip( + "Tier 3: the repaired capture converted through the Positive color path selected " + "below. Nikon exact is the parity default; NegPy approximate is an explicitly " + "labeled preview choice. Tier 2 must be producible first." + ) + tiers_row.addWidget(self.write_unrepaired_check) + tiers_row.addWidget(self.write_repaired_check) + tiers_row.addWidget(self.write_positive_check) + out_form.addRow("Write", tiers_row) + + self.positive_mode_combo = QComboBox() + self.positive_mode_combo.addItem("Nikon C-41 exact (parity)", PositiveColorMode.NIKON_EXACT.value) + self.positive_mode_combo.addItem("NegPy approximate (preview)", PositiveColorMode.NEGPY_APPROXIMATE.value) + self.positive_mode_combo.setToolTip( + "Color path for Tier 3. Nikon exact is the fail-closed parity path and requires " + "frame-bound builder evidence. NegPy approximate is a selectable preview path " + "and is never labeled Nikon-exact." + ) + idx = self.positive_mode_combo.findData(self._settings.positive_mode) + if idx >= 0: + self.positive_mode_combo.setCurrentIndex(idx) + out_form.addRow("Positive color", self.positive_mode_combo) + + self.repair_mode_combo = QComboBox() + self.repair_mode_combo.addItem("Exact (Nikon parity)", RepairMode.EXACT.value) + self.repair_mode_combo.addItem("Hybrid (generative severe-defect fill)", RepairMode.HYBRID.value) + self.repair_mode_combo.setToolTip( + "Governs Tier 2, and Tier 3 through it. Hybrid uses a generative inpainting model for " + "severe zero-signal regions and records every synthesized pixel in a disclosure mask. " + "Those fills are not recovered film data and are not bit-deterministic Nikon output." + ) + idx = self.repair_mode_combo.findData(self._settings.repair_mode) + if idx >= 0: + self.repair_mode_combo.setCurrentIndex(idx) + out_form.addRow("Repair mode", self.repair_mode_combo) + + self.hybrid_synthesis_limit_spin = QDoubleSpinBox() + self.hybrid_synthesis_limit_spin.setRange(0.0, 100.0) + self.hybrid_synthesis_limit_spin.setDecimals(1) + self.hybrid_synthesis_limit_spin.setSuffix("%") + self.hybrid_synthesis_limit_spin.setValue(self._settings.hybrid_synthesis_limit_percent) + self.hybrid_synthesis_limit_spin.setToolTip( + "Maximum portion of a frame Hybrid may synthesize. This is a ceiling, not a target; " + "the pinned runtime's own ceiling remains in force if it is lower. Set 0% to forbid synthesis." + ) + out_form.addRow("Hybrid ceiling", self.hybrid_synthesis_limit_spin) + + layout.addLayout(out_form) + + self.tier_hint = QLabel("") + self.tier_hint.setStyleSheet(f"color: {_WARN_COLOR}; font-size: {THEME.font_size_small}px;") + self.tier_hint.setWordWrap(True) + layout.addWidget(self.tier_hint) + + self.hybrid_guidance = QLabel("") + self.hybrid_guidance.setStyleSheet(f"color: {THEME.text_muted}; font-size: {THEME.font_size_small}px;") + self.hybrid_guidance.setWordWrap(True) + layout.addWidget(self.hybrid_guidance) + + # ── PROGRESS / STATUS ──────────────────────────────── + self.progress_bar = QProgressBar() + self.progress_bar.setVisible(False) + self.progress_bar.setRange(0, 100) + layout.addWidget(self.progress_bar) + self.status_label = QLabel("") + self.status_label.setStyleSheet(f"color: {THEME.text_muted}; font-size: {THEME.font_size_small}px;") + self.status_label.setWordWrap(True) + self.status_label.setVisible(False) + layout.addWidget(self.status_label) + + # The images live in the main center workspace; the sidebar contains + # only the actions and details needed to operate on that selection. + layout.addWidget(section_subheader("FRAME REVIEW")) + self.open_preview_workspace_btn = QPushButton(qta.icon("fa5s.th", color=THEME.text_secondary), " Open Roll Preview") + self.open_preview_workspace_btn.setMinimumHeight(40) + self.open_preview_workspace_btn.setToolTip("Show the roll thumbnails in the main workspace") + layout.addWidget(self.open_preview_workspace_btn) + selection_row = QHBoxLayout() + self.select_all_btn = QPushButton("Select All Frames") + self.select_all_btn.setToolTip("Select every frame found by the current roll preview") + self.clear_selection_btn = QPushButton("Clear Selection") + selection_row.addWidget(self.select_all_btn) + selection_row.addWidget(self.clear_selection_btn) + layout.addLayout(selection_row) + + slot_row = QFormLayout() + slot_row.setSpacing(6) + self.slot_label = QLabel("—") + slot_row.addRow("Slot", self.slot_label) + offset_row = QHBoxLayout() + self.offset_spin = QSpinBox() + self.offset_spin.setRange(-500, 500) + self.offset_spin.setToolTip("Nudge this slot's transport boundary, in native rows at preview resolution") + self.offset_apply_btn = QPushButton("Apply") + offset_row.addWidget(self.offset_spin, 1) + offset_row.addWidget(self.offset_apply_btn) + slot_row.addRow("Spacing offset", offset_row) + layout.addLayout(slot_row) + + self.approve_btn = QPushButton(qta.icon("fa5s.check", color=THEME.text_primary), " Approve Slot") + self.approve_btn.setToolTip("This slot's transport origin was not confidently automatic; approve it before it can be scanned.") + layout.addWidget(self.approve_btn) + + # ── SCAN ──────────────────────────────────────────── + self.scan_btn = QPushButton(qta.icon("fa5s.camera-retro", color=THEME.text_primary), " Scan Selected") + self.scan_btn.setObjectName("scan_btn") + self.scan_btn.setFixedHeight(40) + layout.addWidget(self.scan_btn) + + self.safe_stop_btn = QPushButton(qta.icon("fa5s.stop", color=THEME.text_secondary), " Safe Stop") + self.safe_stop_btn.setToolTip( + "Finish the frame in flight, then stop before the next one -- the transport can't be aborted mid-pull." + ) + self.safe_stop_btn.setEnabled(False) + layout.addWidget(self.safe_stop_btn) + + self._show_slot_detail(None) + self._apply_gating() + layout.addStretch() + + def _connect_signals(self) -> None: + self.refresh_btn.clicked.connect(self._on_refresh) + self.eject_btn.clicked.connect(self._on_eject_clicked) + self.device_combo.currentIndexChanged.connect(self._on_device_changed) + self.folder_browse.clicked.connect(self._on_browse_folder) + for w in (self.folder_edit, self.pattern_edit): + w.editingFinished.connect(self._update_settings_from_ui) + for cb in (self.write_unrepaired_check, self.write_repaired_check, self.write_positive_check): + cb.toggled.connect(self._update_settings_from_ui) + self.positive_mode_combo.currentIndexChanged.connect(self._update_settings_from_ui) + self.repair_mode_combo.currentIndexChanged.connect(self._update_settings_from_ui) + self.hybrid_synthesis_limit_spin.valueChanged.connect(self._update_settings_from_ui) + self.preview_btn.clicked.connect(self._on_preview_clicked) + self.open_preview_workspace_btn.clicked.connect(lambda _checked=False: self.workspace_requested.emit()) + self.preview_display_combo.currentIndexChanged.connect(self._on_preview_display_changed) + self.contact_sheet.itemSelectionChanged.connect(self._on_selection_changed) + self.select_all_btn.clicked.connect(self.contact_sheet.selectAll) + self.clear_selection_btn.clicked.connect(self.contact_sheet.clearSelection) + self.offset_apply_btn.clicked.connect(self._on_apply_offset) + self.approve_btn.clicked.connect(self._on_approve) + self.scan_btn.clicked.connect(self._on_scan_clicked) + self.safe_stop_btn.clicked.connect(self._on_safe_stop_clicked) + + self.controller.roll_devices_ready.connect(self._on_devices_ready) + self.controller.roll_opened.connect(self._on_opened) + self.controller.roll_preview_ready.connect(self._on_preview_ready) + self.controller.roll_spacing_offset_set.connect(self._on_spacing_offset_set) + self.controller.roll_approved.connect(self._on_approved) + self.controller.roll_progress.connect(self._on_progress) + self.controller.roll_frame_written.connect(self._on_frame_written) + self.controller.roll_finished.connect(self._on_finished) + self.controller.roll_cancelled.connect(self._on_cancelled) + self.controller.roll_ejected.connect(self._on_ejected) + self.controller.roll_eject_error.connect(self._on_eject_error) + self.controller.roll_error.connect(self._on_error) + self.controller.roll_status.connect(self._on_status) + + # ── activation hook ─────────────────────────────────────────────── + + def on_activated(self) -> None: + """Called when the Scan tab is switched to.""" + self._setup_hint.setVisible(not coolscanpy_roll.available()) + self._apply_gating() + if not self._devices_loaded: + self._request_devices() + + # ── devices ─────────────────────────────────────────────────────── + + def _request_devices(self) -> None: + if not coolscanpy_roll.available(): + return + self.device_combo.clear() + self.device_combo.addItem("Detecting devices…", None) + self.device_combo.setEnabled(False) + self.controller.request_roll_devices() + + def _on_refresh(self) -> None: + self._devices_loaded = False + self._request_devices() + + def _on_eject_clicked(self) -> None: + device_id = self._current_device_id() + if not device_id or self._scanning or self._preview_pending or self._eject_pending or self._eject_latched or self._eject_failed: + return + answer = QMessageBox.question( + self, + "Eject roll?", + "Eject the loaded roll?\n\n" + "The current preview and frame registration will be discarded. " + "If eject reports an uncertain outcome, do not press it again.", + QMessageBox.StandardButton.Yes | QMessageBox.StandardButton.No, + QMessageBox.StandardButton.No, + ) + if answer != QMessageBox.StandardButton.Yes: + return + self._eject_pending = True + self._eject_latched = True + self._clear_contact_sheet() + self._set_status("Ejecting roll… Do not press Eject again; wait for the film or a result.") + self._apply_gating() + self.controller.eject_roll(device_id) + + @pyqtSlot(list) + def _on_devices_ready(self, devices: "list[coolscanpy.DeviceInfo]") -> None: + self._devices = devices + self._devices_loaded = True + self.device_combo.clear() + self.device_combo.setEnabled(True) + + if not devices: + self.device_combo.addItem("No Coolscan devices detected", None) + self.device_combo.setEnabled(False) + self._apply_gating() + return + + for d in devices: + label = f"{d.vendor} {d.model}" if getattr(d, "vendor", "") else d.model + self.device_combo.addItem(label, d.id) + + if self._settings.last_device_id: + idx = self.device_combo.findData(self._settings.last_device_id) + if idx >= 0: + self.device_combo.setCurrentIndex(idx) + + self._apply_gating() + + def _on_device_changed(self, _index: int) -> None: + self._clear_contact_sheet() + self._update_settings_from_ui() + self._apply_gating() + + def _current_device_id(self) -> str | None: + return self.device_combo.currentData() + + @pyqtSlot(str) + def _on_opened(self, device_id: str) -> None: + self._set_status(f"Roll open on {device_id}.") + + # ── preview / contact sheet ───────────────────────────────────── + + def _clear_contact_sheet(self) -> None: + self.contact_sheet.clear() + self._thumbnails = {} + self._show_slot_detail(None) + + def _on_preview_clicked(self) -> None: + device_id = self._current_device_id() + if not device_id or self._scanning or self._preview_pending or self._eject_pending or self._eject_failed: + return + from negpy.desktop.workers.roll_worker import RollPreviewRequest + + self._preview_pending = True + self.workspace_requested.emit() + self._clear_contact_sheet() + self._set_status("Reading roll transport…") + self.progress_bar.setVisible(True) + self.progress_bar.setValue(0) + self._apply_gating() + self.controller.start_coolscan_roll_preview(RollPreviewRequest(device_id=device_id)) + + @pyqtSlot(list) + def _on_preview_ready(self, thumbnails: "list[coolscanpy.Thumbnail]") -> None: + self.progress_bar.setVisible(False) + if self._eject_pending or self._eject_failed or (self._eject_latched and not self._preview_pending): + self._preview_pending = False + self._clear_contact_sheet() + self._apply_gating() + return + self._preview_pending = False + self._eject_latched = False + self._thumbnails = {t.slot: t for t in thumbnails} + self.contact_sheet.clear() + for t in sorted(self._thumbnails): + self._add_slot_item(self._thumbnails[t]) + self._show_slot_detail(None) + self.workspace_requested.emit() + self._apply_gating() + + def _add_slot_item(self, thumb: "coolscanpy.Thumbnail") -> None: + pixmap = _thumbnail_pixmap(thumb.image, positive=self._preview_is_positive()).scaled( + _THUMBNAIL_SIZE, Qt.AspectRatioMode.KeepAspectRatio, Qt.TransformationMode.SmoothTransformation + ) + item = QListWidgetItem(QIcon(pixmap), f"Slot {thumb.slot}" + (" ⚠" if thumb.needs_approval else "")) + item.setData(_SLOT_ROLE, thumb.slot) + if thumb.needs_approval: + item.setForeground(QColor(_WARN_COLOR)) + self.contact_sheet.addItem(item) + + def _preview_is_positive(self) -> bool: + return bool(self.preview_display_combo.currentData()) + + def _on_preview_display_changed(self) -> None: + for i in range(self.contact_sheet.count()): + item = self.contact_sheet.item(i) + if item is None: + continue + thumb = self._thumbnails.get(item.data(_SLOT_ROLE)) + if thumb is None: + continue + pixmap = _thumbnail_pixmap(thumb.image, positive=self._preview_is_positive()).scaled( + _THUMBNAIL_SIZE, + Qt.AspectRatioMode.KeepAspectRatio, + Qt.TransformationMode.SmoothTransformation, + ) + item.setIcon(QIcon(pixmap)) + + def _on_selection_changed(self) -> None: + current = self.contact_sheet.currentItem() + slot = current.data(_SLOT_ROLE) if current is not None else None + self._show_slot_detail(self._thumbnails.get(slot) if slot is not None else None) + self._apply_gating() + + def _show_slot_detail(self, thumb: "coolscanpy.Thumbnail | None") -> None: + if thumb is None: + self.slot_label.setText("—") + self.offset_spin.setEnabled(False) + self.offset_apply_btn.setEnabled(False) + self.approve_btn.setVisible(False) + return + self.slot_label.setText(str(thumb.slot)) + self.offset_spin.setEnabled(True) + self.offset_apply_btn.setEnabled(True) + self.offset_spin.blockSignals(True) + self.offset_spin.setValue(int(thumb.spacing_offset)) + self.offset_spin.blockSignals(False) + self.approve_btn.setVisible(bool(thumb.needs_approval)) + + def _selected_slots(self) -> list[int]: + return sorted({item.data(_SLOT_ROLE) for item in self.contact_sheet.selectedItems()}) + + # ── spacing offset / approval ──────────────────────────────────── + + def _on_apply_offset(self) -> None: + current = self.contact_sheet.currentItem() + if current is None: + return + slot = current.data(_SLOT_ROLE) + self.controller.set_roll_spacing_offset(slot, self.offset_spin.value()) + + @pyqtSlot(int, int) + def _on_spacing_offset_set(self, slot: int, offset_rows: int) -> None: + thumb = self._thumbnails.get(slot) + if thumb is not None: + self._thumbnails[slot] = dataclasses.replace(thumb, spacing_offset=offset_rows) + self._set_status(f"Slot {slot}: spacing offset set to {offset_rows}.") + + def _on_approve(self) -> None: + current = self.contact_sheet.currentItem() + if current is None: + return + self.controller.approve_roll_slot(current.data(_SLOT_ROLE)) + + @pyqtSlot(int) + def _on_approved(self, slot: int) -> None: + thumb = self._thumbnails.get(slot) + if thumb is not None: + self._thumbnails[slot] = dataclasses.replace(thumb, needs_approval=False) + for i in range(self.contact_sheet.count()): + item = self.contact_sheet.item(i) + if item is not None and item.data(_SLOT_ROLE) == slot: + item.setText(f"Slot {slot}") + item.setData(Qt.ItemDataRole.ForegroundRole, None) # back to the theme's default text color + break + current = self.contact_sheet.currentItem() + if current is not None and current.data(_SLOT_ROLE) == slot: + self.approve_btn.setVisible(False) + self._set_status(f"Slot {slot} approved.") + self._apply_gating() + + # ── scan ────────────────────────────────────────────────────────── + + def _on_scan_clicked(self) -> None: + device_id = self._current_device_id() + slots = self._selected_slots() + output_folder = self.folder_edit.text().strip() + if ( + not device_id + or not slots + or not output_folder + or not self._any_tier_selected() + or self._scanning + or self._preview_pending + or self._eject_pending + or self._eject_latched + or self._eject_failed + ): + return + from negpy.desktop.workers.roll_worker import RollBatchScanRequest + + self._update_settings_from_ui() + self._save_settings() + req = RollBatchScanRequest( + device_id=device_id, + slots=tuple(slots), + output_folder=output_folder, + filename_pattern=self.pattern_edit.text().strip() or RollScanSettings.defaults().filename_pattern, + write_unrepaired=self.write_unrepaired_check.isChecked(), + write_repaired=self.write_repaired_check.isChecked(), + write_positive=self.write_positive_check.isChecked(), + repair_mode=self.repair_mode_combo.currentData() or RollScanSettings.defaults().repair_mode, + positive_mode=self.positive_mode_combo.currentData() or PositiveColorMode.NIKON_EXACT.value, + hybrid_synthesis_limit_percent=self.hybrid_synthesis_limit_spin.value(), + ) + self._active_scan_request = req + self.set_scanning(True) + self.controller.start_roll_scan(req) + + def _on_safe_stop_clicked(self) -> None: + self._stopping = True + self.safe_stop_btn.setEnabled(False) + self._set_status("Stopping the current frame safely…") + self.controller.roll_safe_stop() + + @pyqtSlot(float, str) + def _on_progress(self, fraction: float, message: str) -> None: + self.progress_bar.setVisible(True) + self.progress_bar.setValue(int(fraction * 100)) + if not self._stopping: + self._set_status(message) + + @pyqtSlot(object) + def _on_frame_written(self, output: RollFrameOutput) -> None: + if not self._stopping: + self._set_status(f"Wrote slot {output.slot}.") + + @pyqtSlot(list) + def _on_finished(self, outputs: list[RollFrameOutput]) -> None: + issues = self._completion_issues(outputs) + self._active_scan_request = None + self.set_scanning(False) + if issues: + self._set_status("Completed with issues — " + "; ".join(issues)) + else: + self._set_status(f"Scanned {len(outputs)} frame(s).") + + @pyqtSlot() + def _on_cancelled(self) -> None: + self._active_scan_request = None + self.set_scanning(False) + self._set_status("Stopped.") + + @pyqtSlot(bool) + def _on_ejected(self, triggered: bool) -> None: + self._eject_pending = False + self._eject_latched = True + self._clear_contact_sheet() + if triggered: + self._set_status("Eject started. Wait until the roll is fully out; preview registration cleared.") + else: + self._eject_failed = True + self._set_status("This device reported no eject action. Check the film physically; do not retry in this session.") + self._apply_gating() + + @pyqtSlot(str) + def _on_eject_error(self, message: str) -> None: + self._eject_pending = False + self._eject_latched = True + self._eject_failed = True + self._clear_contact_sheet() + self._set_status(f"Eject outcome uncertain: {message}. Check the film physically; do not retry in this session.") + self._apply_gating() + + @pyqtSlot(str) + def _on_error(self, msg: str) -> None: + self._active_scan_request = None + self._preview_pending = False + self.set_scanning(False) + self.progress_bar.setVisible(False) + self._set_status(f"Error: {msg}") + + @pyqtSlot(str) + def _on_status(self, msg: str) -> None: + if not self._stopping: + self._set_status(msg) + + def _set_status(self, text: str) -> None: + self.status_label.setText(text) + self.status_label.setVisible(bool(text)) + + def _completion_issues( + self, + outputs: list[RollFrameOutput], + ) -> list[str]: + """Describe requested outputs that the fail-closed service withheld.""" + + request = self._active_scan_request + if request is None: + return [] + issues: list[str] = [] + completed_slots = {output.slot for output in outputs} + missing_slots = [slot for slot in request.slots if slot not in completed_slots] + if missing_slots: + issues.append("slot(s) " + ", ".join(str(slot) for slot in missing_slots) + " did not complete") + for output in outputs: + prefix = f"slot {output.slot}: " + if request.write_unrepaired and not output.rgb_path: + issues.append(prefix + "requested unrepaired output unavailable") + if request.write_repaired and not output.repaired_rgb_path: + issues.append(prefix + "requested repaired output unavailable") + repair_was_needed = request.write_repaired or request.write_positive + if ( + repair_was_needed + and request.repair_mode == RepairMode.HYBRID.value + and not (output.native_synthesis_mask_path and output.hybrid_receipt_path) + ): + issues.append(prefix + "Hybrid repair degraded or unavailable") + if request.write_positive and not output.positive_path: + if request.positive_mode == PositiveColorMode.NIKON_EXACT.value: + issues.append(prefix + "Nikon exact positive unavailable") + else: + issues.append(prefix + "requested positive output unavailable") + return issues + + # ── browse ──────────────────────────────────────────────────────── + + def _on_browse_folder(self) -> None: + folder = QFileDialog.getExistingDirectory(self, "Select Output Folder") + if folder: + self.folder_edit.setText(folder) + self._update_settings_from_ui() + + # ── gating ──────────────────────────────────────────────────────── + + def _missing_for_preview(self) -> list[str]: + m = [] + if not coolscanpy_roll.available(): + m.append("install coolscanpy") + if not self._current_device_id(): + m.append("select a device") + if self._eject_failed: + m.append("restart after physically checking the uncertain eject") + return m + + def _any_tier_selected(self) -> bool: + return self.write_unrepaired_check.isChecked() or self.write_repaired_check.isChecked() or self.write_positive_check.isChecked() + + def _missing_for_scan(self) -> list[str]: + m = list(self._missing_for_preview()) + if not self._selected_slots(): + m.append("select at least one slot") + if not self.folder_edit.text().strip(): + m.append("choose an output folder") + if not self._any_tier_selected(): + m.append("select at least one output tier") + return m + + def _update_tier_hint(self) -> None: + """Makes the archival tradeoff legible instead of a silent default: Tier 1 is the + only tier the scanner itself can reproduce, so turning it off is a real choice + with a real cost, not just another checkbox.""" + if not self.write_unrepaired_check.isChecked(): + self.tier_hint.setText( + "Unrepaired (Tier 1) is off. If this frame ever needs to be re-scanned, only " + "the scanner can reproduce it -- Repaired and Positive are both derived from it." + ) + self.tier_hint.setVisible(True) + else: + self.tier_hint.setText("") + self.tier_hint.setVisible(False) + + def _update_hybrid_guidance(self) -> None: + """Explain the choice without silently opting a frame into AI fill.""" + + if self.repair_mode_combo.currentData() == RepairMode.HYBRID.value: + ceiling = self.hybrid_synthesis_limit_spin.value() + self.hybrid_guidance.setText( + f"Hybrid is recommended only when the scanner finds severe zero-signal infrared areas. " + f"It keeps Exact repair elsewhere, discloses every generated pixel, and is capped at {ceiling:.1f}%." + ) + else: + self.hybrid_guidance.setText( + "Exact is recommended for normal dust and scratches with usable infrared, and for Nikon-parity-only output. " + "Use Hybrid only when infrared has a severe zero-signal area that Exact cannot repair." + ) + + def _apply_gating(self) -> None: + missing_preview = self._missing_for_preview() + missing_scan = self._missing_for_scan() + registration_locked = self._preview_pending or self._eject_pending or self._eject_latched or self._eject_failed + self.preview_btn.setEnabled(not missing_preview and not self._scanning and not self._preview_pending and not self._eject_pending) + self.scan_btn.setEnabled(not missing_scan and not self._scanning and not registration_locked) + self.safe_stop_btn.setEnabled(self._scanning) + self.device_combo.setEnabled(not self._scanning and not self._preview_pending and not self._eject_pending) + self.refresh_btn.setEnabled(not self._scanning and not self._preview_pending and not self._eject_pending) + self.contact_sheet.setEnabled(not self._scanning and not registration_locked) + self.open_preview_workspace_btn.setEnabled(not self._eject_failed) + self.select_all_btn.setEnabled(bool(self._thumbnails) and not self._scanning and not registration_locked) + self.clear_selection_btn.setEnabled(bool(self._selected_slots()) and not self._scanning and not registration_locked) + self.eject_btn.setEnabled( + bool(self._current_device_id()) + and not self._scanning + and not self._preview_pending + and not self._eject_pending + and not self._eject_latched + and not self._eject_failed + ) + self._update_tier_hint() + self._update_hybrid_guidance() + if missing_scan: + self.gate_hint.setText("To scan: " + ", ".join(missing_scan) + ".") + self.gate_hint.setVisible(True) + else: + self.gate_hint.setText("") + self.gate_hint.setVisible(False) + + # ── state helpers ───────────────────────────────────────────────── + + def set_scanning(self, active: bool) -> None: + self._scanning = active + if active: + self._stopping = False + self.progress_bar.setVisible(True) + self.progress_bar.setValue(0) + self._apply_gating() diff --git a/negpy/desktop/view/sidebar/right_panel.py b/negpy/desktop/view/sidebar/right_panel.py index cfe085c8..46e9c9a4 100644 --- a/negpy/desktop/view/sidebar/right_panel.py +++ b/negpy/desktop/view/sidebar/right_panel.py @@ -101,8 +101,12 @@ def wrap_scroll(widget: QWidget) -> QScrollArea: self.scanlight_sidebar = ScanlightSidebar(self.controller) - # One "Scan" tab hosting both the SANE scanner and the RGB-Scan capture as - # collapsible sections (mirrors the "Colour — Lab, Toning" tab). + from negpy.desktop.view.sidebar.coolscan_roll import CoolscanRollSidebar + + self.coolscan_roll_sidebar = CoolscanRollSidebar(self.controller) + + # One "Scan" tab hosting the SANE scanner, the RGB-Scan capture and Coolscan roll + # scanning as collapsible sections (mirrors the "Colour — Lab, Toning" tab). self.scan_page = self._build_scan_page() # Tab descriptors: workflow control-group pages first, then Export / Metadata / Scan. @@ -213,8 +217,10 @@ def _resize_splitter_for_analysis(self, expanded: bool) -> None: self.splitter.setSizes([top, max(0, total - top)]) def _build_scan_page(self) -> QWidget: - """The 'Scan' tab hosts two collapsible sections (like Color's Lab / Toning): the - SANE flatbed/film scanner on top, the RGB-Scan trichromatic capture below.""" + """The 'Scan' tab hosts three collapsible sections (like Color's Lab / Toning): the + SANE flatbed/film scanner, the RGB-Scan trichromatic capture, and Coolscan roll + scanning -- each its own acquisition route, each gated on its own optional + dependency (SANE, gphoto2, coolscanpy respectively).""" repo = self.controller.session.repo def make(title: str, key: str, icon_name: str, content: QWidget, default_expanded: bool) -> CollapsibleSection: @@ -227,6 +233,7 @@ def make(title: str, key: str, icon_name: str, content: QWidget, default_expande self.scan_sane_section = make("Scanner (SANE)", "scan_sane", "fa5s.camera-retro", self.scan_sidebar, False) self.scan_rgb_section = make("Camera Scanning", "scan_rgb", "fa5s.camera", self.scanlight_sidebar, True) + self.scan_roll_section = make("Roll Scanning", "scan_roll", "fa5s.images", self.coolscan_roll_sidebar, False) page = QWidget() page_layout = QVBoxLayout(page) @@ -235,6 +242,7 @@ def make(title: str, key: str, icon_name: str, content: QWidget, default_expande page_layout.setSpacing(8) page_layout.addWidget(self.scan_sane_section) page_layout.addWidget(self.scan_rgb_section) + page_layout.addWidget(self.scan_roll_section) return page def apply_shortcut_tooltips(self) -> None: @@ -291,12 +299,15 @@ def _switch_tab(self, index: int) -> None: self._suspended_retouch_tool = None # Trigger device detection + gating refresh when the Scan tab is selected — it now - # hosts both the SANE scanner and the RGB-Scan capture as collapsible sections. + # hosts the SANE scanner, the RGB-Scan capture and Coolscan roll scanning as + # collapsible sections. if index == self._scan_index: if hasattr(self.scan_sidebar, "on_activated"): self.scan_sidebar.on_activated() if hasattr(self.scanlight_sidebar, "on_activated"): self.scanlight_sidebar.on_activated() + if hasattr(self.coolscan_roll_sidebar, "on_activated"): + self.coolscan_roll_sidebar.on_activated() def reveal_section(self, section_attr: str) -> None: """Switch to the tab containing the given ControlsPanel section.""" diff --git a/negpy/desktop/workers/roll_worker.py b/negpy/desktop/workers/roll_worker.py new file mode 100644 index 00000000..627fb1a5 --- /dev/null +++ b/negpy/desktop/workers/roll_worker.py @@ -0,0 +1,323 @@ +"""Background worker for coolscanpy roll scanning. Mirrors CaptureWorker. + +Owns one open RollScanningService reservation (a coolscanpy `Device` plus +its `Roll` extension), held across preview/approve/scan calls and opened +lazily for whichever device the sidebar asks for -- mirrors how +CaptureWorker's `_acquire_camera()` opens and holds one libgphoto2 session +rather than reconnecting for every call. +""" + +from __future__ import annotations + +import dataclasses +import math +import threading +from dataclasses import dataclass + +from PyQt6.QtCore import QObject, pyqtSignal, pyqtSlot + +from negpy.infrastructure.roll import coolscanpy_roll +from negpy.infrastructure.roll.repair import RepairCancelled +from negpy.infrastructure.roll.settings import RollScanSettings +from negpy.kernel.system.logging import get_logger +from negpy.services.repair.fauxice_hybrid_runner import HybridRuntimeConfig +from negpy.services.roll.service import RollFrameOutput, RollScanningService + +logger = get_logger(__name__) + + +class RollShutdownBlocked(RuntimeError): + """The scanner worker has not reached a state that is safe to close.""" + + +@dataclass(frozen=True) +class RollPreviewRequest: + device_id: str + slots: tuple[int, ...] = () # empty = the whole roll + + +@dataclass(frozen=True) +class RollBatchScanRequest: + device_id: str + slots: tuple[int, ...] + output_folder: str + filename_pattern: str + # Which of the three output tiers to write -- see + # `RollScanningService.write_frame`. Defaults mirror `RollScanSettings`: + # first-run requests retain Tier 1 and produce Hybrid repair plus the + # Nikon-exact positive while their frame-bound evidence is available. + write_unrepaired: bool = RollScanSettings.defaults().write_unrepaired + write_repaired: bool = RollScanSettings.defaults().write_repaired + write_positive: bool = RollScanSettings.defaults().write_positive + repair_mode: str = RollScanSettings.defaults().repair_mode + positive_mode: str = RollScanSettings.defaults().positive_mode + hybrid_synthesis_limit_percent: float = RollScanSettings.defaults().hybrid_synthesis_limit_percent + + def __post_init__(self) -> None: + value = self.hybrid_synthesis_limit_percent + if ( + isinstance(value, bool) + or not isinstance(value, (int, float)) + or not math.isfinite(float(value)) + or not 0.0 <= float(value) <= 100.0 + ): + raise ValueError("hybrid_synthesis_limit_percent must be finite and in [0, 100]") + object.__setattr__(self, "hybrid_synthesis_limit_percent", float(value)) + + +class RollWorker(QObject): + """Drives one coolscanpy Device + Roll reservation off the UI thread.""" + + devices_ready = pyqtSignal(list) # list[coolscanpy.DeviceInfo] + opened = pyqtSignal(str) # device_id now open + closed = pyqtSignal() + preview_ready = pyqtSignal(list) # list[coolscanpy.Thumbnail] + spacing_offset_set = pyqtSignal(int, int) # slot, offset_rows actually applied + approved = pyqtSignal(int) # slot + progress = pyqtSignal(float, str) # 0.0..1.0, message + frame_written = pyqtSignal(object) # RollFrameOutput + finished = pyqtSignal(list) # list[RollFrameOutput] written this batch + cancelled = pyqtSignal() + ejected = pyqtSignal(bool) + eject_error = pyqtSignal(str) + error = pyqtSignal(str) + status = pyqtSignal(str) + + def __init__( + self, + *, + hybrid_runtime: HybridRuntimeConfig | None = None, + ) -> None: + super().__init__() + self._hybrid_runtime = hybrid_runtime + self._service = RollScanningService(hybrid_runtime=hybrid_runtime) + self._open_device_id: str | None = None + self._operation_lock = threading.RLock() + self._shutdown_requested = threading.Event() + + def _reject_if_shutting_down(self) -> None: + if self._shutdown_requested.is_set(): + raise RollShutdownBlocked("scanner teardown has started; no new operation may begin") + + # ----- device / roll lifecycle ----- + + @pyqtSlot() + def list_devices(self) -> None: + try: + with self._operation_lock: + self._reject_if_shutting_down() + devices = self._service.list_devices() + self.devices_ready.emit(devices) + except Exception as e: + logger.exception("roll device listing failed") + self.error.emit(str(e)) + self.devices_ready.emit([]) + + def _ensure_open(self, device_id: str) -> None: + """Open `device_id`'s roll extension if it isn't already the one held. + + Switching to a different device closes the previous reservation + first -- coolscanpy allows only one `Roll` open per device, and a + stale reservation on the old device would otherwise sit open with + nothing left able to use it. + """ + self._reject_if_shutting_down() + if self._open_device_id == device_id: + return + if self._open_device_id is not None: + self._service.close() + self._open_device_id = None + self.status.emit("Opening roll…") + self._service.open_roll(device_id) + self._open_device_id = device_id + self.opened.emit(device_id) + + @pyqtSlot() + def close_roll(self) -> bool: + try: + with self._operation_lock: + self._service.close() + except Exception as error: + logger.exception("error closing roll") + self.status.emit("Scanner close is unresolved; the reservation remains open.") + self.error.emit(f"Close roll: {error}") + return False + else: + self._open_device_id = None + self.closed.emit() + return True + + @pyqtSlot(str) + def eject(self, device_id: str) -> None: + """Release the roll reservation, then issue one intentional eject.""" + + try: + with self._operation_lock: + self._reject_if_shutting_down() + if self._open_device_id is not None and self._open_device_id != device_id: + raise RuntimeError(f"open roll belongs to {self._open_device_id!r}, not requested device {device_id!r}") + if self._open_device_id is not None: + self._service.close() + self._open_device_id = None + self.closed.emit() + self.status.emit("Ejecting roll…") + triggered = self._service.eject(device_id) + self.ejected.emit(triggered) + except Exception as error: + logger.exception("roll eject failed") + self.eject_error.emit(str(error)) + + # ----- preview / approval ----- + + @pyqtSlot(RollPreviewRequest) + def run_preview(self, req: RollPreviewRequest) -> None: + try: + with self._operation_lock: + self._reject_if_shutting_down() + self._ensure_open(req.device_id) + self.status.emit("Reading roll transport…") + thumbnails = self._service.preview( + req.slots or None, + on_progress=lambda p: self.progress.emit( + p.fraction, + p.message, + ), + ) + self.preview_ready.emit(thumbnails) + self.status.emit(f"Previewed {len(thumbnails)} slot(s).") + except Exception as e: + logger.exception("roll preview failed") + self.error.emit(f"Preview: {e}") + + @pyqtSlot(int, int) + def set_spacing_offset(self, slot: int, offset_rows: int) -> None: + try: + with self._operation_lock: + self._reject_if_shutting_down() + self._service.set_spacing_offset(slot, offset_rows) + self.spacing_offset_set.emit(slot, offset_rows) + except Exception as e: + logger.exception("set_spacing_offset failed") + self.error.emit(f"Spacing offset: {e}") + + @pyqtSlot(int) + def approve(self, slot: int) -> None: + try: + with self._operation_lock: + self._reject_if_shutting_down() + self._service.approve(slot) + self.approved.emit(slot) + except Exception as e: + logger.exception("approve failed") + self.error.emit(f"Approve: {e}") + + # ----- scanning ----- + + def prepare_batch(self) -> None: + """Reset cancellation before the controller queues this batch.""" + + self._reject_if_shutting_down() + self._service.prepare_batch() + + @pyqtSlot(RollBatchScanRequest) + def run_batch_scan(self, req: RollBatchScanRequest) -> None: + written: list[RollFrameOutput] = [] + try: + with self._operation_lock: + self._reject_if_shutting_down() + self._ensure_open(req.device_id) + self.status.emit("Scanning…") + hybrid_runtime = self._hybrid_runtime + if hybrid_runtime is not None: + user_fraction = req.hybrid_synthesis_limit_percent / 100.0 + hybrid_runtime = dataclasses.replace( + hybrid_runtime, + max_synthesis_fraction=min( + user_fraction, + hybrid_runtime.max_synthesis_fraction, + ), + ) + for frame in self._service.scan_many( + req.slots, + on_progress=lambda p: self.progress.emit( + p.fraction, + p.message, + ), + ): + output = self._service.write_frame( + frame, + req.output_folder, + req.filename_pattern, + write_unrepaired=req.write_unrepaired, + write_repaired=req.write_repaired, + write_positive=req.write_positive, + repair_mode=req.repair_mode, + positive_mode=req.positive_mode, + hybrid_runtime=hybrid_runtime, + on_repair_progress=lambda fraction: self.progress.emit( + fraction, + "Applying Digital ICE repair…", + ), + ) + written.append(output) + self.frame_written.emit(output) + self.finished.emit(written) + except Exception as e: + if coolscanpy_roll.is_safe_stop(e) or isinstance( + e, + RepairCancelled, + ): + # safe_stop() was requested deliberately (see `safe_stop` below): the frame + # already in flight always finishes -- and is already written, already on + # disk, already in `written` -- so this is a clean stop, not a failure. + self.cancelled.emit() + return + logger.exception("roll batch scan failed") + self.error.emit(str(e)) + + def safe_stop(self) -> None: + """Request a graceful stop of an in-progress batch scan. + + Not a `@pyqtSlot`, called directly cross-thread like `CaptureWorker + .cancel()`: `RollScanningService.safe_stop()` only sets a + `threading.Event`, which is thread-safe on its own and needs no Qt + queueing. Unlike `CaptureWorker`, there is no separate local cancel + flag here -- gphoto2 has no stop primitive of its own, so + `CaptureWorker` keeps its own `threading.Event` and checks it + between channels; coolscanpy already exposes `Roll.safe_stop()` for + exactly this, so adding a second flag here would only duplicate it. + Either way the frame already in flight finishes; only the next one + is refused. + """ + self._service.safe_stop() + + def shutdown(self, *, timeout_seconds: float = 5.0) -> None: + """Stop any scan and release the roll, or block application exit. + + The operation lock prevents teardown from racing the scanner-owning + worker path. A timed-out or failed close leaves the service and + device identity intact so the user can retry instead of abandoning + an ownership-uncertain helper/reservation. + """ + + self._shutdown_requested.set() + self.safe_stop() + acquired = self._operation_lock.acquire(timeout=timeout_seconds) + if not acquired: + message = "the current scanner operation is still stopping; application exit is blocked" + self.status.emit(message) + raise RollShutdownBlocked(message) + try: + self._service.close() + except Exception as error: + logger.exception("error closing roll on shutdown") + self.status.emit("Scanner close is unresolved; application exit is blocked.") + self.error.emit(f"Shutdown: {error}") + raise + else: + was_open = self._open_device_id is not None + self._open_device_id = None + if was_open: + self.closed.emit() + finally: + self._operation_lock.release() diff --git a/negpy/infrastructure/roll/__init__.py b/negpy/infrastructure/roll/__init__.py new file mode 100644 index 00000000..c390fe95 --- /dev/null +++ b/negpy/infrastructure/roll/__init__.py @@ -0,0 +1,8 @@ +"""Coolscan roll-feeder capture layer -- no Qt, no NegPy file model. + +Drives a Nikon Coolscan LS-5000 plus SA-21/SA-30 roll feeder through the +optional `coolscanpy` package: whole-roll preview, per-slot spacing +correction and approval, and batch fine-scanning with receipts. Entirely +absent when `coolscanpy` is not installed -- see +`coolscanpy_roll.available()`. +""" diff --git a/negpy/infrastructure/roll/coolscanpy_roll.py b/negpy/infrastructure/roll/coolscanpy_roll.py new file mode 100644 index 00000000..528510b4 --- /dev/null +++ b/negpy/infrastructure/roll/coolscanpy_roll.py @@ -0,0 +1,227 @@ +"""Optional coolscanpy-backed roll-scanning adapter for the Nikon Coolscan LS-5000. + +coolscanpy (https://github.com/rohanpandula/coolscanpy) is a standalone, +SANE-free library that talks to an LS-5000 plus SA-21/SA-30 roll feeder +directly over USB: whole-roll preview, per-slot spacing correction, and +batch fine-scanning with receipts. NegPy consumes it exactly like +python-sane or gphoto2 elsewhere in this package -- an optional dependency, +imported lazily, entirely absent by default. See `available()`. + +This module owns every place coolscanpy itself gets imported or driven for +the roll workflow: opening a device, opening its roll extension, and +translating coolscanpy's exceptions. `negpy.services.roll.service` builds +the file-writing workflow on top of it but never imports coolscanpy +directly, matching how `negpy.services.capture.service` never imports +`gphoto2` directly either. + +-------------------------------------------------------------------------- +INTEGRATION POINT for a future re-point +-------------------------------------------------------------------------- +`open_roll()` below is the one place a device handle gets resolved for the +roll workflow. It currently goes straight to `coolscanpy.open()`. NegPy's +maintainer has a generic SANE-based coolscan route planned upstream, which +is expected to add a real backend-selection seam to +`negpy.services.scanning.service.ScannerService` (today `_get_backend()` +just hardcodes `SaneBackend()`). When that seam lands, re-pointing this +adapter at it should only mean changing how `open_roll()` resolves a +device -- everything built on top of the returned `RollHandle` (this +module's exception translation, and all of +`negpy/services/roll/service.py`) is unaffected, since it never talks to +coolscanpy except through this one function and the `RollHandle` it +returns. +-------------------------------------------------------------------------- +""" + +from __future__ import annotations + +import importlib.util +from pathlib import Path +from typing import TYPE_CHECKING, Iterable, Iterator + +if TYPE_CHECKING: + import coolscanpy + from coolscanpy.protocol.ls5000_single_pass.capture_process import ( + ManualFrameApproval as CoolscanManualFrameApproval, + ) + from coolscanpy.types import ProgressCallback as CoolscanProgressCallback + + +def available() -> bool: + """True if the optional `coolscanpy` dependency is importable. + + A cheap, side-effect-free presence check -- mirrors the + `ScanSidebar._sane_available()` check the plain Scan panel already uses + for python-sane, so callers (a future roll panel, a service, a test) can + degrade gracefully without ever importing coolscanpy just to ask whether + it exists. + """ + return importlib.util.find_spec("coolscanpy") is not None + + +class RollHandle: + """One open coolscanpy `Device` + `Roll`, released together on `close()`. + + Construct via `open_roll()`, not directly. Every method here is a thin + forward to the underlying `coolscanpy.Roll`, translating coolscanpy's + typed exceptions (rooted at `PyCoolscanError`) to plain `RuntimeError`s + -- matching how `SaneBackend.scan()` already reports failures elsewhere + in this package (a message string the scan worker forwards verbatim to + the GUI's status label). NegPy's scanner layer has no typed exception + vocabulary of its own for a caller to catch selectively; a caller that + wants coolscanpy's own richer hierarchy can still recover it from + `__cause__`, since every translation chains `from error`. + """ + + def __init__(self, device: "coolscanpy.Device", roll: "coolscanpy.Roll") -> None: + self._device = device + self._roll = roll + + def preview( + self, slots: Iterable[int] | None = None, *, on_progress: "CoolscanProgressCallback | None" = None + ) -> "list[coolscanpy.Thumbnail]": + """One whole-roll transport read. See `coolscanpy.Roll.preview`.""" + try: + return self._roll.preview(slots, on_progress=on_progress) + except Exception as error: + raise _translate(error) from error + + def restore_preview_session( + self, + payload: str, + slots: Iterable[int] | None = None, + ) -> "list[coolscanpy.Thumbnail]": + """Restore one content-verified preview without scanner I/O.""" + + try: + return self._roll.restore_preview_session(payload, slots) + except Exception as error: + raise _translate(error) from error + + def set_spacing_offset(self, slot: int, offset_rows: int) -> None: + try: + self._roll.set_spacing_offset(slot, offset_rows) + except Exception as error: + raise _translate(error) from error + + def approve(self, slot: int) -> "CoolscanManualFrameApproval": + """Approve one reviewed slot and return Coolscanpy's bound receipt.""" + + try: + return self._roll.approve(slot) + except Exception as error: + raise _translate(error) from error + + def needs_approval(self, slot: int) -> bool: + try: + return self._roll.needs_approval(slot) + except Exception as error: + raise _translate(error) from error + + def scan_many(self, slots: Iterable[int], *, on_progress: "CoolscanProgressCallback | None" = None) -> Iterator["coolscanpy.Frame"]: + """Batch fine-scan `slots` in one transport reservation. + + A `RuntimeError` raised here whose `__cause__` is a + `coolscanpy.SafeStopRequested` means `safe_stop()` was already + called deliberately and should be treated as a clean stop, not + surfaced as a failure -- see `is_safe_stop()`. + """ + try: + yield from self._roll.scan_many(slots, on_progress=on_progress) + except Exception as error: + raise _translate(error) from error + + def safe_stop(self) -> None: + """Request a graceful stop; the frame already in flight still finishes.""" + self._roll.safe_stop() + + def close(self) -> None: + """Release the roll, then its device after ownership is confirmed.""" + + self._roll.close() + self._device.close() + + def __enter__(self) -> "RollHandle": + return self + + def __exit__(self, *exc: object) -> None: + self.close() + + +def list_devices() -> "list[coolscanpy.DeviceInfo]": + """Enumerate attached LS-5000 units. Empty list, never raises, if none found.""" + import coolscanpy + + return coolscanpy.get_devices() + + +def eject(device_id: str) -> bool: + """Eject one direct-USB roll after its coolscanpy reservation is closed.""" + + from negpy.infrastructure.scanners.sane_backend import scanimage_eject_direct_usb + + return scanimage_eject_direct_usb(device_id) + + +def open_roll( + device_id: str | None = None, + *, + material: "coolscanpy.Material | None" = None, + attempts_root: str | Path | None = None, +) -> RollHandle: + """Open one LS-5000 and its roll-feeder extension. + + `device_id` follows `coolscanpy.open()`: omitted (`None`) picks "the one + attached unit", an explicit id (from `list_devices()`) disambiguates + when more than one is attached. `material` defaults to + `coolscanpy.Material.COLOR_NEGATIVE`, NegPy's live-accepted workflow. + Coolscanpy's host-native B&W fine-scan route is implemented and covered + without hardware, but still awaits its first live macOS acceptance with + conventional silver B&W film. + """ + import coolscanpy + + resolved_material = material if material is not None else coolscanpy.Material.COLOR_NEGATIVE + try: + device = coolscanpy.open(device_id or "ls5000") + except Exception as error: + raise _translate(error) from error + + try: + roll_kwargs = {"material": resolved_material} + if attempts_root is not None: + roll_kwargs["attempts_root"] = Path(attempts_root) + roll = device.roll(**roll_kwargs) + except Exception as error: + device.close() + raise _translate(error) from error + + return RollHandle(device, roll) + + +def _translate(error: BaseException) -> RuntimeError: + """Flatten any coolscanpy failure to a plain `RuntimeError`. + + Covers the typed `PyCoolscanError` hierarchy (`RollMismatch`, + `FingerprintRefused`, `ManualReviewRequired`, `SafeStopRequested`, ...) + and the handful of plain `ValueError`/`RuntimeError` coolscanpy itself + raises (e.g. "no roll adapter is attached"), so nothing coolscanpy-shaped + ever crosses out of this module. `raise _translate(error) from error` + always preserves the original as `__cause__` -- see `is_safe_stop()`. + """ + return RuntimeError(str(error)) + + +def is_safe_stop(error: BaseException) -> bool: + """True if `error` is this module's translation of a deliberate + `RollHandle.safe_stop()` outcome, not a genuine failure. + + `error` is normally the `RuntimeError` a caller of `scan_many()` just + caught; every translation in this module chains `from error`, so the + original `coolscanpy.SafeStopRequested` (when that is what happened) + is always sitting in `__cause__`. Imports coolscanpy lazily like the + rest of this module -- safe here because an error this module raised + already means coolscanpy was reachable. + """ + import coolscanpy + + return isinstance(getattr(error, "__cause__", None), coolscanpy.SafeStopRequested) diff --git a/negpy/infrastructure/roll/fauxice_bridge.py b/negpy/infrastructure/roll/fauxice_bridge.py new file mode 100644 index 00000000..782abb3d --- /dev/null +++ b/negpy/infrastructure/roll/fauxice_bridge.py @@ -0,0 +1,161 @@ +"""Wire scanner-native portable Digital ICE into the roll repair seam.""" + +from __future__ import annotations + +import hashlib +import io +import threading +from typing import Callable + +import numpy as np +from PIL import Image + +from negpy.infrastructure.roll import repair as roll_repair +from negpy.infrastructure.roll.repair import ( + RepairAcquisition, + RepairMode, + RepairResult, +) + +from negpy.services.repair.fauxice_ir_repair import ( + FauxiceRepairCancelled, + FauxiceRepairConfig, + engine_available, + repair_ir_dust, +) + + +def _raw_rgb_sha256(rgb: np.ndarray) -> str: + canonical = np.array(rgb, dtype=" bytes: + stream = io.BytesIO() + Image.fromarray(mask.astype(np.uint8) * 255, mode="L").save( + stream, + format="PNG", + optimize=False, + compress_level=9, + ) + return stream.getvalue() + + +class _FauxiceEngine: + def repair( + self, + acquisition: RepairAcquisition, + mode: RepairMode, + *, + hybrid_runtime=None, + progress: Callable[[float], None] | None = None, + cancel: threading.Event | None = None, + ) -> RepairResult: + from negpy.services.repair.fauxice_ir_repair import ( + RepairMode as FauxiceMode, + RepairStatus, + _engine_version, + ) + + fauxice_mode = FauxiceMode.HYBRID if mode is RepairMode.HYBRID else FauxiceMode.EXACT + config = FauxiceRepairConfig(enabled=True, mode=fauxice_mode) + main = acquisition.main_rgbi + try: + result = repair_ir_dust( + main[..., :3], + main[..., 3], + same_frame_id=acquisition.acquisition_id, + config=config, + prepass_rgbi=acquisition.prepass_rgbi, + validity_mask=acquisition.ir_validity, + hybrid_runtime=hybrid_runtime, + progress=progress, + cancel=cancel, + ) + except FauxiceRepairCancelled as error: + raise roll_repair.RepairCancelled(str(error)) from error + + if result.status is not RepairStatus.APPLIED or result.repaired_rgb16 is None: + raise RuntimeError(f"fauxice repair did not produce output: {result.reason}") + if result.mode_resolved is None: + raise RuntimeError("fauxice repair omitted its resolved mode") + resolved_mode = RepairMode(result.mode_resolved.value) + native_repaired = np.asarray(result.repaired_rgb16) + if native_repaired.dtype != np.uint16 or native_repaired.shape != (*main.shape[:2], 3) or not native_repaired.flags.c_contiguous: + raise RuntimeError("fauxice repair returned invalid scanner-native RGB geometry") + native_output_sha256 = _raw_rgb_sha256(native_repaired) + if result.native_output_rgb_sha256 is not None and result.native_output_rgb_sha256 != native_output_sha256: + raise RuntimeError("fauxice native output SHA-256 changed") + storage_rgb = acquisition.storage_rgb(native_repaired) + + mask_fields: dict[str, object] = {} + if resolved_mode is RepairMode.HYBRID: + routed_native_mask = result.hybrid_mask + routed_native_mask_png = result.hybrid_mask_png + if ( + routed_native_mask is None + or routed_native_mask_png is None + or result.hybrid_mask_sha256 is None + or result.hybrid_receipt is None + or result.hybrid_receipt_sha256 is None + ): + raise RuntimeError("hybrid repair omitted its verified disclosure evidence") + routed_native_mask = np.asarray(routed_native_mask) + if ( + routed_native_mask.dtype != np.bool_ + or routed_native_mask.shape != main.shape[:2] + or not routed_native_mask.flags.c_contiguous + ): + raise RuntimeError("hybrid disclosure mask geometry is invalid") + if hashlib.sha256(routed_native_mask_png).hexdigest() != result.hybrid_mask_sha256: + raise RuntimeError("hybrid native mask SHA-256 changed") + if hashlib.sha256(result.hybrid_receipt).hexdigest() != result.hybrid_receipt_sha256: + raise RuntimeError("hybrid receipt SHA-256 changed") + native_mask = np.ascontiguousarray(routed_native_mask & acquisition.ir_validity) + native_mask_png = _encode_mask_png(native_mask) + storage_mask = acquisition.storage_mask(native_mask) + storage_mask_png = _encode_mask_png(storage_mask) + mask_fields = { + "native_synthesis_mask_png": native_mask_png, + "native_synthesis_mask_sha256": hashlib.sha256(native_mask_png).hexdigest(), + "native_synthesis_mask_shape": tuple(native_mask.shape), + "routed_native_synthesis_mask_png": routed_native_mask_png, + "routed_native_synthesis_mask_sha256": result.hybrid_mask_sha256, + "routed_native_synthesis_mask_shape": tuple(routed_native_mask.shape), + "storage_synthesis_mask_png": storage_mask_png, + "storage_synthesis_mask_sha256": hashlib.sha256(storage_mask_png).hexdigest(), + "storage_synthesis_mask_shape": tuple(storage_mask.shape), + "synthesis_mask_transform": acquisition.storage_transform, + "synthesis_fraction": float(np.count_nonzero(native_mask)) / float(native_mask.size), + "routing_counts": result.hybrid_routing_counts, + "hybrid_receipt": result.hybrid_receipt, + "hybrid_receipt_sha256": result.hybrid_receipt_sha256, + "hybrid_provenance_class": result.hybrid_provenance_class, + "hybrid_receipt_output_rgb_sha256": (result.hybrid_receipt_output_rgb_sha256), + } + + return RepairResult( + rgb=storage_rgb, + engine="digital-fauxice", + engine_version=result.engine_version or _engine_version() or "unknown", + mode_requested=mode, + mode_resolved=resolved_mode, + reason=result.reason, + acquisition_id=acquisition.acquisition_id, + slot=acquisition.slot, + reservation_id=acquisition.reservation_id, + evidence_sha256=acquisition.evidence_sha256, + backend_requested=result.backend_requested, + backend_used=result.backend_used, + backend_selection_reason=result.backend_selection_reason, + native_output_rgb_sha256=native_output_sha256, + storage_output_rgb_sha256=_raw_rgb_sha256(storage_rgb), + **mask_fields, + ) + + +if engine_available(): + roll_repair.register_engine(_FauxiceEngine()) + + +__all__ = ["_FauxiceEngine"] diff --git a/negpy/infrastructure/roll/repair.py b/negpy/infrastructure/roll/repair.py new file mode 100644 index 00000000..af66a691 --- /dev/null +++ b/negpy/infrastructure/roll/repair.py @@ -0,0 +1,281 @@ +"""Tier-2 dust/scratch repair engine seam for roll-scanned RGBI frames. + +The roll adapter writes a captured frame in up to three tiers (see +`negpy.services.roll.service.write_frame`): the unrepaired RGBI capture, the +same frame with infrared-guided dust/scratch repair applied, and a positive +rendered from the repaired frame. This module is the plug point for the +middle tier -- it does not repair anything itself. + +This seam has no intrinsic implementation: `available()` stays false and +`repair()` raises until a bridge calls `register_engine()`. NegPy's +`fauxice_bridge` registers the installed portable Digital ICE runtime during +roll-service import. The explicit registration boundary keeps Tier 2 honest: +a build that cannot load the engine reports repair as unavailable rather than +writing a copy of Tier 1 under a repaired label. See +`docs/COOLSCANPY_ROLL_SCANNING.md` for the intended capture -> repair -> +invert ordering and which project is expected to fill this slot. +""" + +from __future__ import annotations + +import hashlib +import re +import threading +from dataclasses import dataclass +from enum import StrEnum +from typing import TYPE_CHECKING, Callable, Protocol + +import numpy as np + +if TYPE_CHECKING: + from negpy.services.repair.fauxice_hybrid_runner import HybridRuntimeConfig + + +DIGITAL_ICE_STORAGE_TRANSFORM = "rot90-k1-scanner-native-to-upright-v1" + + +class RepairCancelled(RuntimeError): + """A caller-requested repair stop; never a degradable repair failure.""" + + +def _raw_sha256(array: np.ndarray, *, dtype: np.dtype) -> str: + canonical = np.array(array, dtype=dtype, order="C", copy=True) + return hashlib.sha256(memoryview(canonical).cast("B")).hexdigest() + + +class RepairMode(StrEnum): + """A Tier-2 repair variant. + + EXACT heals only the pixels the infrared channel confidently flags as a + defect. HYBRID additionally routes severe zero-signal regions (where the + infrared channel carries no usable signal at all, e.g. a dense scratch) + to a separately pinned inpainting runtime. Receipts bind the backend, + resolved mode, output hashes, and hybrid disclosure evidence rather than + making a blanket reproducibility claim across different runtimes. + """ + + EXACT = "exact" + HYBRID = "hybrid" + + +@dataclass(frozen=True) +class RepairAcquisition: + """One hash-bound scanner-native prepass/main pair ready for repair.""" + + acquisition_id: str + slot: int + reservation_id: str + capture_attempt_id: str + storage_transform: str + evidence_sha256: str + main_rgbi_sha256: str + prepass_rgbi_sha256: str + ir_validity_sha256: str + main_rgbi: np.ndarray + prepass_rgbi: np.ndarray + ir_validity: np.ndarray + + def __post_init__(self) -> None: + if re.fullmatch(r"dice-[0-9a-f]{64}", self.acquisition_id) is None: + raise ValueError("repair acquisition identity is malformed") + if type(self.slot) is not int or not 1 <= self.slot <= 40: + raise ValueError("repair acquisition slot must be in 1..40") + for label, value in ( + ("reservation identity", self.reservation_id), + ("capture-attempt identity", self.capture_attempt_id), + ): + if type(value) is not str or not value.strip(): + raise ValueError(f"repair {label} must be non-empty") + if self.storage_transform != DIGITAL_ICE_STORAGE_TRANSFORM: + raise ValueError("repair acquisition storage transform is unsupported") + for label, digest in ( + ("evidence", self.evidence_sha256), + ("main RGBI", self.main_rgbi_sha256), + ("prepass RGBI", self.prepass_rgbi_sha256), + ("IR validity", self.ir_validity_sha256), + ): + if re.fullmatch(r"[0-9a-f]{64}", digest) is None: + raise ValueError(f"repair acquisition {label} SHA-256 is malformed") + + main = np.array(self.main_rgbi, dtype=" "RepairAcquisition": + """Test/helper constructor; production passes producer-bound hashes.""" + + return cls( + acquisition_id=acquisition_id, + slot=slot, + reservation_id=reservation_id, + capture_attempt_id=capture_attempt_id, + storage_transform=storage_transform, + evidence_sha256=evidence_sha256, + main_rgbi_sha256=_raw_sha256(main_rgbi, dtype=np.dtype(" np.ndarray: + native = np.asarray(scanner_native_rgb) + if native.dtype != np.uint16 or native.shape != (*self.main_rgbi.shape[:2], 3): + raise ValueError("repair output must match scanner-native main RGB geometry") + return np.ascontiguousarray(np.rot90(native, k=1, axes=(0, 1))) + + def storage_mask(self, scanner_native_mask: np.ndarray) -> np.ndarray: + native = np.asarray(scanner_native_mask) + if native.dtype != np.bool_ or native.shape != self.main_rgbi.shape[:2]: + raise ValueError("repair mask must match scanner-native main geometry") + return np.ascontiguousarray(np.rot90(native, k=1, axes=(0, 1))) + + +@dataclass(frozen=True) +class RepairResult: + """One frame's Tier-2 RGB, plus what the receipt needs to audit or + reproduce it. The infrared plane is not part of this result -- Tier 2 + retains Tier 1's own infrared plane unchanged (see `service.write_frame`), + since repair consumes infrared to find defects rather than producing a + repaired version of it.""" + + rgb: np.ndarray + engine: str + engine_version: str + mode_requested: RepairMode + mode_resolved: RepairMode + reason: str + acquisition_id: str | None = None + slot: int | None = None + reservation_id: str | None = None + evidence_sha256: str | None = None + backend_requested: str | None = None + backend_used: str | None = None + backend_selection_reason: str | None = None + native_output_rgb_sha256: str | None = None + storage_output_rgb_sha256: str | None = None + native_synthesis_mask_png: bytes | None = None + native_synthesis_mask_sha256: str | None = None + native_synthesis_mask_shape: tuple[int, int] | None = None + routed_native_synthesis_mask_png: bytes | None = None + routed_native_synthesis_mask_sha256: str | None = None + routed_native_synthesis_mask_shape: tuple[int, int] | None = None + storage_synthesis_mask_png: bytes | None = None + storage_synthesis_mask_sha256: str | None = None + storage_synthesis_mask_shape: tuple[int, int] | None = None + synthesis_mask_transform: str | None = None + synthesis_fraction: float | None = None + routing_counts: dict[str, int] | None = None + hybrid_receipt: bytes | None = None + hybrid_receipt_sha256: str | None = None + hybrid_provenance_class: str | None = None + hybrid_receipt_output_rgb_sha256: str | None = None + + @property + def degraded(self) -> bool: + return self.mode_requested is not self.mode_resolved + + +class RepairEngine(Protocol): + """What a repair implementation must provide to `register_engine()`.""" + + def repair( + self, + acquisition: RepairAcquisition, + mode: RepairMode, + *, + hybrid_runtime: "HybridRuntimeConfig | None" = None, + progress: Callable[[float], None] | None = None, + cancel: threading.Event | None = None, + ) -> RepairResult: ... + + +_engine: RepairEngine | None = None + + +def available() -> bool: + """True once a repair engine has been registered. + + Cheap and side-effect-free, mirroring `coolscanpy_roll.available()`, so + `RollScanningService.write_frame` can decide whether Tier 2 (and, since + Tier 3 is defined as Tier 2 inverted, Tier 3 too) can be produced before + doing any work. + """ + return _engine is not None + + +def register_engine(engine: RepairEngine) -> None: + """The integration point a future repair implementation is expected to + call -- e.g. at import time, guarded the same way `coolscanpy_roll` + guards its own optional dependency -- to make Tier 2 and Tier 3 + available. Nothing in this codebase calls this today; see the module + docstring.""" + global _engine + _engine = engine + + +def unregister_engine() -> None: + """Reverts to the unavailable state. Mainly a test teardown hook.""" + global _engine + _engine = None + + +def repair( + acquisition: RepairAcquisition, + mode: RepairMode, + *, + hybrid_runtime: "HybridRuntimeConfig | None" = None, + progress: Callable[[float], None] | None = None, + cancel: threading.Event | None = None, +) -> RepairResult: + """Repair one frame's RGB using its infrared plane. + + Raises `RuntimeError` if no engine is registered -- callers on the + roll-scanning write path check `available()` first so a missing engine + degrades Tier 2/3 with a clear status instead of raising mid-batch. + """ + if _engine is None: + raise RuntimeError("no dust-repair engine is registered; Tier 2 and Tier 3 are unavailable") + return _engine.repair( + acquisition, + mode, + hybrid_runtime=hybrid_runtime, + progress=progress, + cancel=cancel, + ) diff --git a/negpy/infrastructure/roll/settings.py b/negpy/infrastructure/roll/settings.py new file mode 100644 index 00000000..bf17736a --- /dev/null +++ b/negpy/infrastructure/roll/settings.py @@ -0,0 +1,64 @@ +"""Persisted Roll Scanning panel settings (stored as a global setting dict).""" + +from __future__ import annotations + +from dataclasses import dataclass +import math + +from negpy.infrastructure.roll.repair import RepairMode +from negpy.services.roll.exact_color import PositiveColorMode + + +@dataclass(frozen=True) +class RollScanSettings: + """Sticky settings for the Roll Scanning sidebar. + + Persisted via the session repo under the `roll_scan_settings` key, + mirroring `ScannerSettings` and `ScanlightSettings`. `last_device_id` + empty means no device remembered yet; per-slot spacing offsets and + approvals are not persisted here at all -- they live on the open + coolscanpy `Roll` for as long as it stays open, and `preview()` resets + them on its own the moment it re-reads the transport. + + `write_unrepaired`, `write_repaired` and `write_positive` select which of + the three output tiers a batch scan writes (see + `negpy.services.roll.service.write_frame`); any combination is valid, and + they are independent, not a single three-way choice. `write_unrepaired` + defaults on because it is the archival master and the only tier the + scanner itself can reproduce. For a new user, repaired and positive also + default on in Hybrid + Nikon-exact mode so the complete parity workflow + runs while its frame-bound prepass, infrared-validity data, acquisition + provenance, and native color-builder evidence are still available. Saved + user choices override these first-run defaults. `repair_mode` governs Tier + 2 (and, through it, Tier 3) whenever a repair engine is registered; see + `negpy.infrastructure.roll.repair`. `positive_mode` chooses the Tier-3 + color path. The roll-scanning parity workflow defaults to fail-closed Nikon + exact color; NegPy's approximate renderer remains an explicit choice. + """ + + last_device_id: str = "" + output_folder: str = "" + filename_pattern: str = '{{ date }}_{{ "%03d" % seq }}' + write_unrepaired: bool = True + write_repaired: bool = True + write_positive: bool = True + repair_mode: str = RepairMode.HYBRID.value + positive_mode: str = PositiveColorMode.NIKON_EXACT.value + # This is a user-selected ceiling, not a target percentage. The pinned + # runtime remains the final hard limit; the worker takes the lower value. + hybrid_synthesis_limit_percent: float = 10.0 + + def __post_init__(self) -> None: + value = self.hybrid_synthesis_limit_percent + if ( + isinstance(value, bool) + or not isinstance(value, (int, float)) + or not math.isfinite(float(value)) + or not 0.0 <= float(value) <= 100.0 + ): + raise ValueError("hybrid_synthesis_limit_percent must be finite and in [0, 100]") + object.__setattr__(self, "hybrid_synthesis_limit_percent", float(value)) + + @classmethod + def defaults(cls) -> "RollScanSettings": + return cls() diff --git a/negpy/infrastructure/scanners/sane_backend.py b/negpy/infrastructure/scanners/sane_backend.py index c631f6f0..79fe8a74 100644 --- a/negpy/infrastructure/scanners/sane_backend.py +++ b/negpy/infrastructure/scanners/sane_backend.py @@ -1,7 +1,10 @@ import math +import re +import shutil import subprocess import sys import threading +from pathlib import Path from typing import Callable import numpy as np @@ -56,6 +59,12 @@ # "out of documents"; the eject has already fired, so that exit is expected. _EJECT_TIMEOUT_S = 30.0 _EJECT_BENIGN_STDERR_MARKERS = ("out of documents", "no documents", "no more documents") +_SCANIMAGE_FALLBACK_PATHS = ( + "/opt/homebrew/bin/scanimage", + "/usr/local/bin/scanimage", + "/usr/bin/scanimage", +) +_DIRECT_USB_DEVICE_ID = re.compile(r"usb:(?P[1-9][0-9]*):(?P
[1-9][0-9]*)\Z") # Standard SANE option-unit enum values. Kept local so capability detection # does not require importing the optional python-sane extension. @@ -314,6 +323,18 @@ def _find_eject_option(opt) -> str | None: return None +def _scanimage_executable() -> str: + """Resolve scanimage even when a frozen Finder app has a minimal PATH.""" + + discovered = shutil.which("scanimage") + if discovered: + return discovered + for candidate in _SCANIMAGE_FALLBACK_PATHS: + if Path(candidate).is_file(): + return candidate + raise RuntimeError("Cannot eject: `scanimage` (sane-utils) is not installed") + + def _scanimage_eject(device_id: str) -> None: """Press the vendor eject button via `scanimage --eject`. @@ -325,7 +346,7 @@ def _scanimage_eject(device_id: str) -> None: """ try: proc = subprocess.run( - ["scanimage", "-d", device_id, "--eject"], + [_scanimage_executable(), "-d", device_id, "--eject"], capture_output=True, text=True, timeout=_EJECT_TIMEOUT_S, @@ -333,7 +354,10 @@ def _scanimage_eject(device_id: str) -> None: except FileNotFoundError as exc: raise RuntimeError("Cannot eject: `scanimage` (sane-utils) is not installed") from exc except subprocess.TimeoutExpired as exc: - raise RuntimeError(f"Eject timed out after {_EJECT_TIMEOUT_S:g}s for {device_id!r}") from exc + raise RuntimeError( + f"Eject confirmation timed out after {_EJECT_TIMEOUT_S:g}s for {device_id!r}; " + "the button action may already have been dispatched. Check the film physically and do not retry" + ) from exc if proc.returncode == 0: return @@ -344,6 +368,28 @@ def _scanimage_eject(device_id: str) -> None: raise RuntimeError(f"scanimage --eject failed for {device_id!r}: {detail}") +def scanimage_eject_direct_usb(device_id: str) -> bool: + """Eject a direct-USB Coolscan device through its exact SANE topology. + + The accepted roll path identifies the scanner as ``usb:BUS:ADDRESS`` + because it captures directly over USB. The proven eject path is + coolscan3's ``scanimage --eject`` action, whose device name carries the + same bus and address as zero-padded decimal fields. Reject every other + spelling so an Eject button can never drift onto a different scanner. + """ + + match = _DIRECT_USB_DEVICE_ID.fullmatch(device_id) + if match is None: + raise ValueError(f"direct Coolscan eject requires usb:BUS:ADDRESS, got {device_id!r}") + bus = int(match.group("bus")) + address = int(match.group("address")) + if bus > 999 or address > 999: + raise ValueError(f"direct Coolscan eject topology is out of range: {device_id!r}") + sane_id = f"coolscan3:usb:libusb:{bus:03d}:{address:03d}" + _scanimage_eject(sane_id) + return True + + def _sane_container_depth(requested_depth: int) -> int: """The container SANE ships `requested_depth` samples in — 8 or 16 bits. diff --git a/negpy/kernel/system/paths.py b/negpy/kernel/system/paths.py index 3e68923d..31fdcebf 100644 --- a/negpy/kernel/system/paths.py +++ b/negpy/kernel/system/paths.py @@ -46,6 +46,14 @@ def get_default_user_dir() -> str: if env_path: return os.path.abspath(env_path) + # A frozen macOS app must create a foreground Qt window before its first + # Documents access so macOS can present the Files & Folders prompt. Keep + # resolution pure here; the desktop startup handoff performs the first + # deliberate write/access probe once that window is visible. + if sys.platform == "darwin": + home = Path(os.path.expanduser("~")) + return str((home / "Documents" / "NegPy").absolute()) + docs_dir: Optional[Path] = None if sys.platform == "win32": @@ -94,9 +102,6 @@ def get_default_user_dir() -> str: except Exception: pass - elif sys.platform == "darwin": - docs_dir = Path.home() / "Documents" - home = Path(os.path.expanduser("~")) if not docs_dir: docs_dir = home / "Documents" diff --git a/negpy/services/assets/crosstalk.py b/negpy/services/assets/crosstalk.py index 0fc091b5..a2f88647 100644 --- a/negpy/services/assets/crosstalk.py +++ b/negpy/services/assets/crosstalk.py @@ -1,5 +1,6 @@ import os import re +import stat import tomllib from typing import List, Optional @@ -7,6 +8,7 @@ from negpy.kernel.system.paths import get_resource_path DEFAULT_NAME = "Default" +_MAX_PROFILE_BYTES = 64 * 1024 # ponytail: 2-line helpers duplicated from contact_sheet_templates; extract to a @@ -69,9 +71,31 @@ def _scan() -> dict: @staticmethod def _parse_file(path: str) -> Optional[tuple]: """Parses a .toml file to (name, flat 9-float list), or None if invalid.""" + descriptor: Optional[int] = None try: - with open(path, "rb") as f: - data = tomllib.load(f) + linked = os.lstat(path) + if stat.S_ISLNK(linked.st_mode) or not stat.S_ISREG(linked.st_mode): + return None + if linked.st_size < 0 or linked.st_size > _MAX_PROFILE_BYTES: + return None + + flags = os.O_RDONLY | getattr(os, "O_CLOEXEC", 0) | getattr(os, "O_NOFOLLOW", 0) | getattr(os, "O_NONBLOCK", 0) + descriptor = os.open(path, flags) + with os.fdopen(descriptor, "rb") as f: + descriptor = None + before = os.fstat(f.fileno()) + if not stat.S_ISREG(before.st_mode) or not os.path.samestat(linked, before) or before.st_size != linked.st_size: + return None + payload = f.read(_MAX_PROFILE_BYTES + 1) + after = os.fstat(f.fileno()) + if ( + len(payload) != before.st_size + or len(payload) > _MAX_PROFILE_BYTES + or (before.st_dev, before.st_ino, before.st_size, before.st_mtime_ns) + != (after.st_dev, after.st_ino, after.st_size, after.st_mtime_ns) + ): + return None + data = tomllib.loads(payload.decode("utf-8")) rows = data.get("matrix") if not isinstance(rows, list) or len(rows) != 3: return None @@ -88,6 +112,9 @@ def _parse_file(path: str) -> Optional[tuple]: return name, flat except Exception: return None + finally: + if descriptor is not None: + os.close(descriptor) @staticmethod def list_profiles() -> List[str]: diff --git a/negpy/services/repair/__init__.py b/negpy/services/repair/__init__.py new file mode 100644 index 00000000..9807d1e8 --- /dev/null +++ b/negpy/services/repair/__init__.py @@ -0,0 +1,25 @@ +"""Optional post-import repair steps. No Qt dependencies.""" + +from negpy.services.repair.fauxice_hybrid_runner import HybridRuntimeConfig +from negpy.services.repair.fauxice_ir_repair import ( + FauxiceRepairConfig, + FauxiceRepairResult, + RepairMode, + RepairStatus, + engine_available, + hybrid_available, + repair_frame_files, + repair_ir_dust, +) + +__all__ = [ + "FauxiceRepairConfig", + "FauxiceRepairResult", + "HybridRuntimeConfig", + "RepairMode", + "RepairStatus", + "engine_available", + "hybrid_available", + "repair_frame_files", + "repair_ir_dust", +] diff --git a/negpy/services/repair/fauxice_hybrid_runner.py b/negpy/services/repair/fauxice_hybrid_runner.py new file mode 100644 index 00000000..4cfc0018 --- /dev/null +++ b/negpy/services/repair/fauxice_hybrid_runner.py @@ -0,0 +1,1071 @@ +"""Subprocess bridge to the optional ``fauxce-hybrid`` companion tool. + +``fauxce-hybrid`` is a separate, independently optional package from the core +``portable_digital_ice`` engine (see ``fauxice_ir_repair.py``). It has no +importable "run a repair" function of its own; ``src/fauxce_hybrid/cli.py`` +in the upstream project keeps that logic private and exposes only the +``fauxce-hybrid`` console script. Shelling out to that script, exactly as its +own docs show, is therefore the calling contract, not a shortcut around one. + +The hybrid CLI needs the same paired 285 dpi prepass + 4000 dpi main RGBI +acquisition as the core engine (it runs the core engine internally to get the +``at_floor_mask`` it routes on), so it never relaxes the prepass requirement +described in ``fauxice_ir_repair.py``. + +IOPaint and the LaMa model weights are never invoked by NegPy directly. They +run inside the fauxce-hybrid subprocess, pointed at a pinned interpreter and +hash-verified weights that ``HybridRuntimeConfig`` names but never bundles. +""" + +from __future__ import annotations + +import hashlib +import io +import json +import math +import os +import re +import signal +import stat +import subprocess +import tempfile +import threading +import time + +import numpy as np +import numpy.typing as npt +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Callable, Sequence + +from PIL import Image + + +class HybridRunError(RuntimeError): + """The fauxce-hybrid subprocess did not produce a usable result.""" + + +class HybridRunCancelled(HybridRunError): + """The caller cancelled the external hybrid process and its children.""" + + +def _stable_regular_sha256(path: Path, *, label: str) -> str: + before = os.lstat(path) + if stat.S_ISLNK(before.st_mode) or not stat.S_ISREG(before.st_mode): + raise ValueError(f"{label} must be a regular non-symlink file") + flags = os.O_RDONLY + if hasattr(os, "O_CLOEXEC"): + flags |= os.O_CLOEXEC + if hasattr(os, "O_NOFOLLOW"): + flags |= os.O_NOFOLLOW + if hasattr(os, "O_NONBLOCK"): + flags |= os.O_NONBLOCK + descriptor = os.open(path, flags) + try: + opened = os.fstat(descriptor) + identity = ( + opened.st_dev, + opened.st_ino, + opened.st_size, + opened.st_mtime_ns, + opened.st_ctime_ns, + ) + if not stat.S_ISREG(opened.st_mode) or identity != ( + before.st_dev, + before.st_ino, + before.st_size, + before.st_mtime_ns, + before.st_ctime_ns, + ): + raise ValueError(f"{label} changed while it was opened") + digest = hashlib.sha256() + remaining = opened.st_size + while remaining: + block = os.read(descriptor, min(1024 * 1024, remaining)) + if not block: + raise ValueError(f"{label} changed while it was hashed") + digest.update(block) + remaining -= len(block) + # Bound the read by the size captured at open. One extra byte detects + # growth without following an attacker-controlled, never-ending file. + if os.read(descriptor, 1): + raise ValueError(f"{label} changed while it was hashed") + after_read = os.fstat(descriptor) + finally: + os.close(descriptor) + after_path = os.lstat(path) + for metadata in (after_read, after_path): + if ( + metadata.st_dev, + metadata.st_ino, + metadata.st_size, + metadata.st_mtime_ns, + metadata.st_ctime_ns, + ) != identity: + raise ValueError(f"{label} changed while it was hashed") + return digest.hexdigest() + + +@dataclass(frozen=True) +class HybridRuntimeConfig: + """Where the pinned IOPaint interpreter and hash-verified weights live. + + None of these paths are discovered automatically. fauxce-hybrid's own + docs (``hybrid/docs/hybrid-repair.md`` in the digital-fauxice repository) + require the caller to install IOPaint 1.6.0 into its own virtualenv and + to supply the measured SHA-256 of ``big-lama.pt`` rather than trusting a + filename; this config just carries those caller-verified values through. + """ + + hybrid_python: Path + executable: Path + core_source_manifest_sha256: str + hybrid_source_manifest_sha256: str + iopaint_python: Path + iopaint_executable: Path + iopaint_source_manifest_sha256: str + model_dir: Path + model_weights: Path + model_weights_sha256: str + inpaint_device: str = "cpu" + inpaint_threads: int = 1 + inpaint_seed: int = 0 + max_synthesis_fraction: float = 0.02 + + def __post_init__(self) -> None: + for field_name in ( + "hybrid_python", + "executable", + "iopaint_python", + "iopaint_executable", + "model_dir", + "model_weights", + ): + value = Path(getattr(self, field_name)) + if not value.is_absolute(): + raise ValueError(f"{field_name} must be an absolute explicit path") + object.__setattr__(self, field_name, value) + for field_name in ( + "core_source_manifest_sha256", + "hybrid_source_manifest_sha256", + "iopaint_source_manifest_sha256", + "model_weights_sha256", + ): + if re.fullmatch(r"[0-9a-f]{64}", getattr(self, field_name)) is None: + raise ValueError(f"{field_name} must be a lowercase SHA-256 digest") + if self.inpaint_device not in {"cpu", "cuda", "mps"}: + raise ValueError("inpaint_device must be cpu, cuda, or mps") + if type(self.inpaint_threads) is not int or self.inpaint_threads < 1: + raise ValueError("inpaint_threads must be a positive integer") + if type(self.inpaint_seed) is not int or self.inpaint_seed < 0: + raise ValueError("inpaint_seed must be a non-negative integer") + if ( + isinstance(self.max_synthesis_fraction, bool) + or not isinstance(self.max_synthesis_fraction, (int, float)) + or not math.isfinite(float(self.max_synthesis_fraction)) + or not 0.0 <= float(self.max_synthesis_fraction) <= 1.0 + ): + raise ValueError("max_synthesis_fraction must be finite and in [0, 1]") + object.__setattr__(self, "max_synthesis_fraction", float(self.max_synthesis_fraction)) + + def validate_files(self) -> None: + """Fail closed unless every configured runtime artifact is present.""" + + executables = ( + ("hybrid_python", self.hybrid_python), + ("executable", self.executable), + ("iopaint_python", self.iopaint_python), + ("iopaint_executable", self.iopaint_executable), + ) + for label, path in executables: + try: + resolved = path.resolve(strict=True) + mode = resolved.stat().st_mode + except OSError as error: + raise ValueError(f"{label} is unavailable: {error}") from error + if not stat.S_ISREG(mode) or not os.access(path, os.X_OK): + raise ValueError(f"{label} must resolve to an executable regular file") + try: + model_dir = self.model_dir.resolve(strict=True) + except OSError as error: + raise ValueError(f"model_dir is unavailable: {error}") from error + if not model_dir.is_dir(): + raise ValueError("model_dir must resolve to a directory") + try: + weights = self.model_weights.resolve(strict=True) + weights_mode = weights.stat().st_mode + except OSError as error: + raise ValueError(f"model_weights is unavailable: {error}") from error + if not stat.S_ISREG(weights_mode): + raise ValueError("model_weights must resolve to a regular file") + try: + measured_weights_sha256 = _stable_regular_sha256( + self.model_weights, + label="model_weights", + ) + except OSError as error: + raise ValueError(f"model_weights is unavailable: {error}") from error + if measured_weights_sha256 != self.model_weights_sha256: + raise ValueError("model_weights SHA-256 does not match the pinned runtime") + + +@dataclass(frozen=True) +class HybridRunResult: + """What one fauxce-hybrid subprocess call produced, read back into memory. + + The mask is carried as bytes rather than a path. The CLI files remain in + the caller-owned per-run scratch child until the caller removes it; only + values copied into this result are independent of that directory. + """ + + hybrid_rgb16: npt.NDArray[np.uint16] + synth_mask_png: bytes + synth_mask_sha256: str + synth_mask: npt.NDArray[np.bool_] + receipt: bytes + receipt_sha256: str + acquisition_manifest_sha256: str + main_rgbi_sha256: str + prepass_rgbi_sha256: str + output_rgb16_sha256: str + provenance_class: str + synthesis_fraction: float | None + engine_version: str | None + backend_requested: str | None + backend_used: str | None + backend_selection_reason: str | None + routing_counts: dict[str, int] | None + + +ReceiptVerifier = Callable[[Path, HybridRuntimeConfig], Any] +VerificationRunner = Callable[..., "subprocess.CompletedProcess[str]"] +_MAX_SUBPROCESS_LOG_BYTES = 16 * 1024 * 1024 + + +@dataclass(frozen=True) +class _ReceiptVerification: + receipt_sha256: str + model_weights_rehashed: bool + + +_EXTERNAL_RECEIPT_VERIFIER = r""" +import json +import sys +from pathlib import Path + +from fauxce_hybrid.receipts import verify_receipt + +receipt_path = Path(sys.argv[1]) +weights_path = Path(sys.argv[2]) +weights_sha256 = sys.argv[3] + +def resolve_weights(attestation): + if getattr(attestation, "sha256", None) != weights_sha256: + raise ValueError("receipt model weights do not match the pinned runtime") + return weights_path + +verified = verify_receipt( + receipt_path, + model_weights_resolver=resolve_weights, + require_model_weights=True, +) +document = { + "model_weights_rehashed": bool(verified.model_weights_rehashed), + "receipt_sha256": verified.receipt_sha256, + "schema": "negpy.external-fauxce-receipt-verification-v1", +} +sys.stdout.write(json.dumps(document, sort_keys=True, separators=(",", ":"))) +""".strip() + + +def _process_group_exists(process_group: int) -> bool: + try: + os.killpg(process_group, 0) + except ProcessLookupError: + return False + except (AttributeError, PermissionError): + return True + return True + + +def _terminate_process_group(process: subprocess.Popen) -> None: + process_group = process.pid + try: + os.killpg(process_group, signal.SIGTERM) + except (AttributeError, OSError): + try: + process.terminate() + except OSError: + pass + deadline = time.monotonic() + 3.0 + while _process_group_exists(process_group) and time.monotonic() < deadline: + time.sleep(0.05) + # The group leader may have exited on TERM while an IOPaint/model child + # ignored it. Kill the group after grace regardless of leader state. + try: + os.killpg(process_group, signal.SIGKILL) + except (AttributeError, OSError): + try: + process.kill() + except OSError: + pass + try: + process.wait(timeout=3.0) + except (OSError, subprocess.TimeoutExpired): + pass + + +def _sanitized_subprocess_environment() -> dict[str, str]: + environment = dict(os.environ) + for name in ( + "PYTHONHOME", + "PYTHONINSPECT", + "PYTHONPATH", + "PYTHONSTARTUP", + "PYTHONUSERBASE", + ): + environment.pop(name, None) + for name in tuple(environment): + if name.startswith(("DYLD_", "LD_")): + environment.pop(name, None) + environment.update( + { + "LANG": "C", + "LC_ALL": "C", + "MKL_NUM_THREADS": "1", + "NUMEXPR_NUM_THREADS": "1", + "OMP_NUM_THREADS": "1", + "OPENBLAS_NUM_THREADS": "1", + "PYTHONHASHSEED": "0", + "PYTHONNOUSERSITE": "1", + } + ) + return environment + + +def _raise_if_cancelled( + cancel: threading.Event | None, + *, + phase: str, +) -> None: + if cancel is not None and cancel.is_set(): + raise HybridRunCancelled(f"fauxce-hybrid cancelled {phase}") + + +def _run_command( + argv: Sequence[str], + *, + timeout_seconds: float, + cancel: threading.Event | None, + runner: VerificationRunner | None, + label: str, +) -> subprocess.CompletedProcess[str]: + if cancel is not None and cancel.is_set(): + raise HybridRunCancelled(f"{label} cancelled before launch") + if runner is not None: + try: + completed = runner( + list(argv), + capture_output=True, + env=_sanitized_subprocess_environment(), + text=True, + timeout=timeout_seconds, + ) + except (OSError, subprocess.SubprocessError) as error: + raise HybridRunError(f"could not launch {label}: {error}") from error + if cancel is not None and cancel.is_set(): + raise HybridRunCancelled(f"{label} cancelled") + return completed + + process: subprocess.Popen | None = None + completed_normally = False + try: + with ( + tempfile.TemporaryFile(mode="w+t", encoding="utf-8") as stdout_file, + tempfile.TemporaryFile( + mode="w+t", + encoding="utf-8", + ) as stderr_file, + ): + process = subprocess.Popen( + list(argv), + stdin=subprocess.DEVNULL, + stdout=stdout_file, + stderr=stderr_file, + env=_sanitized_subprocess_environment(), + text=True, + start_new_session=True, + ) + deadline = time.monotonic() + timeout_seconds + while process.poll() is None: + if cancel is not None and cancel.is_set(): + _terminate_process_group(process) + raise HybridRunCancelled(f"{label} cancelled") + if time.monotonic() >= deadline: + _terminate_process_group(process) + raise HybridRunError(f"{label} exceeded its {timeout_seconds:g}s timeout") + if ( + os.fstat(stdout_file.fileno()).st_size > _MAX_SUBPROCESS_LOG_BYTES + or os.fstat(stderr_file.fileno()).st_size > _MAX_SUBPROCESS_LOG_BYTES + ): + _terminate_process_group(process) + raise HybridRunError(f"{label} output exceeded its safe size limit") + time.sleep(0.05) + if ( + os.fstat(stdout_file.fileno()).st_size > _MAX_SUBPROCESS_LOG_BYTES + or os.fstat(stderr_file.fileno()).st_size > _MAX_SUBPROCESS_LOG_BYTES + ): + raise HybridRunError(f"{label} output exceeded its safe size limit") + stdout_file.seek(0) + stderr_file.seek(0) + completed = subprocess.CompletedProcess( + list(argv), + process.returncode, + stdout_file.read(), + stderr_file.read(), + ) + if _process_group_exists(process.pid): + _terminate_process_group(process) + raise HybridRunError(f"{label} left child processes running after exit") + completed_normally = True + return completed + except (HybridRunError, HybridRunCancelled): + raise + except (OSError, subprocess.SubprocessError) as error: + raise HybridRunError(f"could not launch {label}: {error}") from error + finally: + if process is not None and not completed_normally: + _terminate_process_group(process) + + +def _raw_sha256(array: np.ndarray, *, dtype: np.dtype) -> str: + canonical = np.array(array, dtype=dtype, order="C", copy=True) + return hashlib.sha256(memoryview(canonical).cast("B")).hexdigest() + + +def _canonical_rgbi(array: np.ndarray, *, label: str) -> np.ndarray: + value = np.asarray(array) + if value.ndim != 3 or value.shape[2] != 4 or value.dtype != np.uint16: + raise HybridRunError(f"{label} must be an HxWx4 uint16 array") + canonical = np.array(value, dtype=" bytes: + try: + return json.dumps( + document, + sort_keys=True, + separators=(",", ":"), + ensure_ascii=False, + allow_nan=False, + ).encode("utf-8") + except (TypeError, ValueError, RecursionError) as error: + raise HybridRunError(f"acquisition manifest is not canonical JSON: {error}") from error + + +def _canonical_receipt_bytes(document: object) -> bytes: + try: + return ( + json.dumps( + document, + sort_keys=True, + separators=(",", ":"), + ensure_ascii=True, + allow_nan=False, + ).encode("utf-8") + + b"\n" + ) + except (TypeError, ValueError, RecursionError) as error: + raise HybridRunError(f"hybrid receipt is not canonical JSON: {error}") from error + + +def _stable_regular_bytes(path: Path, *, maximum_bytes: int, label: str) -> bytes: + descriptor: int | None = None + try: + linked = os.lstat(path) + if stat.S_ISLNK(linked.st_mode) or not stat.S_ISREG(linked.st_mode): + raise HybridRunError(f"{label} must be a regular non-symlink file") + if linked.st_size < 0 or linked.st_size > maximum_bytes: + raise HybridRunError(f"{label} exceeds its safe size limit") + flags = os.O_RDONLY | getattr(os, "O_CLOEXEC", 0) | getattr(os, "O_NOFOLLOW", 0) | getattr(os, "O_NONBLOCK", 0) + descriptor = os.open(path, flags) + with os.fdopen(descriptor, "rb", closefd=True) as handle: + descriptor = None + before = os.fstat(handle.fileno()) + if not stat.S_ISREG(before.st_mode) or ( + before.st_dev, + before.st_ino, + before.st_size, + before.st_mtime_ns, + before.st_ctime_ns, + ) != ( + linked.st_dev, + linked.st_ino, + linked.st_size, + linked.st_mtime_ns, + linked.st_ctime_ns, + ): + raise HybridRunError(f"{label} changed while being opened") + payload = handle.read(maximum_bytes + 1) + after = os.fstat(handle.fileno()) + if ( + len(payload) != before.st_size + or len(payload) > maximum_bytes + or ( + before.st_dev, + before.st_ino, + before.st_size, + before.st_mtime_ns, + before.st_ctime_ns, + ) + != ( + after.st_dev, + after.st_ino, + after.st_size, + after.st_mtime_ns, + after.st_ctime_ns, + ) + ): + raise HybridRunError(f"{label} changed while being read") + return payload + except HybridRunError: + raise + except OSError as error: + raise HybridRunError(f"could not securely read {label}: {error}") from error + finally: + if descriptor is not None: + os.close(descriptor) + + +def _decode_mask_png(payload: bytes, *, expected_shape: tuple[int, int]) -> np.ndarray: + try: + with Image.open(io.BytesIO(payload)) as image: + if image.format != "PNG": + raise HybridRunError("synthesis disclosure mask is not a PNG") + if image.size != (expected_shape[1], expected_shape[0]): + raise HybridRunError("synthesis disclosure mask geometry is invalid") + decoded = np.asarray(image.convert("L")) + except HybridRunError: + raise + except Exception as error: + raise HybridRunError(f"synthesis disclosure mask PNG is invalid: {error}") from error + if decoded.shape != expected_shape or decoded.dtype != np.uint8: + raise HybridRunError("synthesis disclosure mask geometry is invalid") + unique = np.unique(decoded) + if not np.all(np.isin(unique, np.array([0, 255], dtype=np.uint8))): + raise HybridRunError("synthesis disclosure mask must be binary") + mask = np.ascontiguousarray(decoded != 0) + mask.setflags(write=False) + return mask + + +def _default_receipt_verifier( + path: Path, + runtime: HybridRuntimeConfig, + *, + runner: VerificationRunner | None = None, + cancel: threading.Event | None = None, +) -> _ReceiptVerification: + """Verify with the explicitly configured Python-3.12 companion. + + NegPy itself supports Python 3.13 while fauxce-hybrid intentionally uses + a separate Python 3.12 environment. Importing the verifier in-process + would therefore make a correctly installed companion appear unavailable. + The external verifier returns only a small attestation; NegPy then reads + and checks every result artifact itself before publishing it. + """ + + completed = _run_command( + [ + str(runtime.hybrid_python), + "-I", + "-c", + _EXTERNAL_RECEIPT_VERIFIER, + str(path), + str(runtime.model_weights), + runtime.model_weights_sha256, + ], + timeout_seconds=300.0, + cancel=cancel, + runner=runner, + label="external hybrid receipt verifier", + ) + if completed.returncode != 0: + detail = (completed.stderr or completed.stdout or "").strip() + raise HybridRunError("external hybrid receipt verification failed: " + detail[-2000:]) + try: + document = json.loads(completed.stdout) + except (TypeError, json.JSONDecodeError) as error: + raise HybridRunError("external hybrid receipt verifier returned invalid JSON") from error + if ( + not isinstance(document, dict) + or document.get("schema") != "negpy.external-fauxce-receipt-verification-v1" + or re.fullmatch(r"[0-9a-f]{64}", document.get("receipt_sha256", "")) is None + or type(document.get("model_weights_rehashed")) is not bool + ): + raise HybridRunError("external hybrid receipt verifier returned an invalid attestation") + return _ReceiptVerification( + receipt_sha256=document["receipt_sha256"], + model_weights_rehashed=document["model_weights_rehashed"], + ) + + +def _decode_npy_rgb16( + payload: bytes, + *, + expected_shape: tuple[int, int, int], +) -> np.ndarray: + stream = io.BytesIO(payload) + try: + version = np.lib.format.read_magic(stream) + if version == (1, 0): + shape, fortran_order, dtype = np.lib.format.read_array_header_1_0(stream) + elif version == (2, 0): + shape, fortran_order, dtype = np.lib.format.read_array_header_2_0(stream) + else: + raise HybridRunError(f"hybrid output NPY version {version!r} is unsupported") + if tuple(shape) != expected_shape or dtype != np.dtype(" HybridRunResult: + """Run one frame through the fauxce-hybrid CLI and read its outputs back. + + ``scratch_dir`` must resolve to a directory, but it may already contain + other entries. This function creates one exclusive + ``negpy-hybrid-run-*`` child and then creates the CLI's ``--out`` path + inside it (the CLI refuses a pre-existing output path). The caller owns + both the parent and per-run child lifetime; nothing here deletes them, so + an ephemeral caller should wrap the parent in + ``tempfile.TemporaryDirectory()``. + """ + + main = _canonical_rgbi(main_rgbi, label="main_rgbi") + prepass = _canonical_rgbi(prepass_rgbi, label="prepass_rgbi") + try: + runtime.validate_files() + except ValueError as error: + raise HybridRunError(f"hybrid runtime is unavailable: {error}") from error + try: + scratch_root = Path(scratch_dir).resolve(strict=True) + except OSError as error: + raise HybridRunError(f"hybrid scratch directory is unavailable: {error}") from error + if not scratch_root.is_dir(): + raise HybridRunError("hybrid scratch path must resolve to a directory") + if type(same_frame_id) is not str or not same_frame_id.strip(): + raise HybridRunError("same_frame_id must be non-empty") + if cancel is not None and cancel.is_set(): + raise HybridRunCancelled("fauxce-hybrid cancelled before launch") + if progress is not None: + progress(0.0) + main_sha256 = _raw_sha256(main, dtype=np.dtype(" dict[str, Any]: + artifacts = receipt.get("artifacts") + if not isinstance(artifacts, list): + raise HybridRunError("hybrid receipt artifacts are malformed") + matches = [artifact for artifact in artifacts if isinstance(artifact, dict) and artifact.get("role") == role] + if len(matches) != 1: + raise HybridRunError(f"hybrid receipt must contain exactly one {role!r} artifact") + return matches[0] + + +def _validate_receipt_bindings( + receipt: dict[str, Any], + *, + runtime: HybridRuntimeConfig, + backend: str, + acquisition_manifest_sha256: str, + main: np.ndarray, + prepass: np.ndarray, + hybrid_rgb16: np.ndarray, + synthesis_mask: np.ndarray, + model_weights_rehashed: bool, +) -> None: + if receipt.get("schema") != "fauxce-hybrid-receipt-v2": + raise HybridRunError("hybrid receipt schema is unsupported") + inputs = receipt.get("inputs") + if not isinstance(inputs, dict): + raise HybridRunError("hybrid receipt inputs are malformed") + for role, expected in (("main", main), ("prepass", prepass)): + row = inputs.get(role) + if not isinstance(row, dict): + raise HybridRunError(f"hybrid receipt {role} input is missing") + if row.get("canonical_encoding") != "uint16_little_endian_c_order": + raise HybridRunError(f"hybrid receipt {role} encoding is unsupported") + if row.get("shape") != list(expected.shape): + raise HybridRunError(f"hybrid receipt {role} geometry changed") + if row.get("raw_sha256") != _raw_sha256(expected, dtype=np.dtype(" frame_pixels + or counts["final_regions"] > counts["synthesis_pixels"] + or (counts["final_regions"] == 0) != (pixel_count == 0) + ): + raise HybridRunError("hybrid receipt routing counts changed") + + output_hash = _raw_sha256(hybrid_rgb16, dtype=np.dtype(" dict[str, Any]: + """Pull the few fields NegPy's sidecar cares about out of hybrid-receipt.json. + + Reads defensively (``.get`` all the way down): this module is a + provenance consumer, not the receipt's verifier. ``fauxce-hybrid`` itself + is the authority on whether a given receipt is internally consistent. + + ``routing_counts`` comes from the receipt's ``routing.counts`` object + (``fauxce-hybrid-receipt-v2.schema.json``): the disclosed region/pixel + counts behind the single ``synthesis_fraction`` float, e.g. "13 regions, + 16137 pixels of 22815772". ``None`` when the receipt is missing any of + the expected keys, rather than publishing a partial count. + """ + + core = receipt.get("core", {}) if isinstance(receipt, dict) else {} + backend = core.get("backend", {}) if isinstance(core, dict) else {} + synthesis = receipt.get("synthesis", {}) if isinstance(receipt, dict) else {} + routing = receipt.get("routing", {}) if isinstance(receipt, dict) else {} + counts = routing.get("counts", {}) if isinstance(routing, dict) else {} + routing_counts = ( + {key: counts[key] for key in _ROUTING_COUNT_KEYS} + if isinstance(counts, dict) and all(key in counts for key in _ROUTING_COUNT_KEYS) + else None + ) + return { + "synthesis_fraction": synthesis.get("fraction"), + "engine_version": core.get("version"), + "backend_requested": backend.get("requested"), + "backend_used": backend.get("used"), + "backend_selection_reason": backend.get("reason"), + "routing_counts": routing_counts, + } + + +__all__ = [ + "HybridRunError", + "HybridRunCancelled", + "HybridRunResult", + "HybridRuntimeConfig", + "run_hybrid_repair", +] diff --git a/negpy/services/repair/fauxice_ir_repair.py b/negpy/services/repair/fauxice_ir_repair.py new file mode 100644 index 00000000..11ff5cab --- /dev/null +++ b/negpy/services/repair/fauxice_ir_repair.py @@ -0,0 +1,636 @@ +"""Optional post-import infrared dust repair via the digital-fauxice engine. + +This wraps ``portable-digital-ice`` (import name ``portable_digital_ice``), a +byte-exact, independently reverse-engineered reimplementation of the Nikon +Coolscan's Digital ICE infrared dust repair. It is an optional dependency: +every function here degrades to a reported status instead of raising when the +engine, or its optional ``fauxce-hybrid`` companion, is not installed. + +Read this before wiring a caller to this module. The input contract is +narrower than "an RGB master plus its IR sidecar": + +The engine repairs one main scan using a *second*, earlier capture of the +same physical frame: a 285 dpi RGBI prepass that supplies per-frame +calibration the main pass depends on. Its own docs are explicit that this +is not a resampling convenience and cannot be reconstructed after the fact: +"Reconstructing or guessing it from the main scan would move outside the +byte-exact claim." NegPy's generic import/scanning writer still lacks that +paired capture, but the Coolscan roll path now supplies its scanner-bound +285 dpi prepass, validity mask, and main RGBI acquisition directly. Calls +without that real prepass report ``SKIPPED`` rather than fabricate one. See +``docs/FAUXICE_IR_REPAIR.md`` for the exact/hybrid split +(``fauxice_hybrid_runner.py`` covers the external hybrid path). + +This is a one-shot repair, not a per-render pipeline stage. It accepts an +optional progress callback and cancellation event, mirroring the shape +``ScannerService.run_scan`` already uses for its own long operation +(``negpy/services/scanning/service.py``), so a future background worker can +wrap a repair call the same way ``ScanWorker`` wraps a scan. + +A repaired frame is published as a new companion file +(``_FAUXICE.tif``) next to the untouched original, never as an +in-place rewrite: NegPy keys stored edits by +the source file's content hash (see ``StorageRepository`` and +``negpy/services/assets/sidecar.py``), so silently rewriting the master +would orphan any edits already saved against it. The original ``_IR.tif`` +companion is only ever opened for reading here. +""" + +from __future__ import annotations + +import importlib.util +import hashlib +import json +import os +import subprocess +import tempfile +import threading +from dataclasses import dataclass +from enum import Enum +from pathlib import Path +from typing import Callable + +import numpy as np +import numpy.typing as npt +import tifffile + +from negpy.kernel.system.logging import get_logger +from negpy.services.repair.fauxice_hybrid_runner import ( + HybridRunCancelled, + HybridRunError, + HybridRuntimeConfig, + run_hybrid_repair, +) + +logger = get_logger(__name__) + +ENGINE_DISTRIBUTION = "portable-digital-ice" +ENGINE_IMPORT_NAME = "portable_digital_ice" +HYBRID_IMPORT_NAME = "fauxce_hybrid" +ENGINE_HOME = "https://github.com/rohanpandula/digital-fauxice" + +IR_SIDECAR_SUFFIX = "_IR" +REPAIR_OUTPUT_SUFFIX = "_FAUXICE" +REPAIR_SIDECAR_SUFFIX = "_FAUXICE" +SYNTH_MASK_SUFFIX = "_FAUXICE_SYNTH" + +# The engine currently validates exactly one acquisition profile (see its +# own profile.py: LS5000Selector8NormalProfile). These are not arbitrary +# defaults; they are the only dpi pair it accepts today. +PREPASS_DPI = 285 +MAIN_DPI = 4000 + + +class RepairStatus(str, Enum): + APPLIED = "applied" + SKIPPED = "skipped" + UNAVAILABLE = "unavailable" + + +class RepairMode(str, Enum): + EXACT = "exact" + HYBRID = "hybrid" + + +class FauxiceRepairCancelled(RuntimeError): + """A cooperative stop requested by the caller.""" + + +def engine_available() -> bool: + """True if the core digital-fauxice engine is importable. + + Uses ``find_spec`` rather than a real import so an availability check + never pays the engine's import cost (or runs any stub's side effects) + just to answer "is this installed." + """ + return importlib.util.find_spec(ENGINE_IMPORT_NAME) is not None + + +def hybrid_available(runtime: HybridRuntimeConfig | None = None) -> bool: + """True only when an explicit external hybrid runtime was supplied. + + The companion deliberately runs in a separate Python environment, so an + in-process import probe is both irrelevant and wrong on NegPy's supported + Python 3.13 runtime. ``HybridRuntimeConfig`` is validated when built and + carries every executable/path/hash needed at the process boundary. + """ + + if runtime is None: + return False + try: + runtime.validate_files() + except ValueError: + return False + return True + + +def _engine_version() -> str | None: + try: + from importlib.metadata import PackageNotFoundError, version + + return version(ENGINE_DISTRIBUTION) + except PackageNotFoundError: + return None + except Exception: + return None + + +def _format_fraction(value: float | None) -> str: + return "unknown" if value is None else f"{value:.4%}" + + +@dataclass(frozen=True) +class FauxiceRepairConfig: + """Off-by-default settings for the post-import Digital ICE repair step. + + This mirrors NegPy's existing "bool = False" feature-flag idiom (see + ``RetouchConfig.ir_dust_remove`` in ``negpy/features/retouch/models.py``) + without living inside ``WorkspaceConfig``. ``WorkspaceConfig`` holds + cheap per-render recipe parameters ``DarkroomEngine`` replays on every + slider drag; this step is an explicit, one-shot external process a + caller (a worker, a CLI, a future sidebar action) triggers deliberately, + so it gets its own config object instead. + """ + + enabled: bool = False + mode: RepairMode = RepairMode.EXACT + backend: str = "auto" + + +@dataclass(frozen=True) +class FauxiceRepairResult: + """The outcome of one ``repair_ir_dust`` call.""" + + status: RepairStatus + reason: str + mode_requested: RepairMode + mode_resolved: RepairMode | None = None + repaired_rgb16: npt.NDArray[np.uint16] | None = None + engine_version: str | None = None + backend_requested: str | None = None + backend_used: str | None = None + backend_selection_reason: str | None = None + hybrid_mask_png: bytes | None = None + hybrid_mask_sha256: str | None = None + hybrid_mask: npt.NDArray[np.bool_] | None = None + hybrid_synthesis_fraction: float | None = None + hybrid_routing_counts: dict[str, int] | None = None + hybrid_receipt: bytes | None = None + hybrid_receipt_sha256: str | None = None + hybrid_provenance_class: str | None = None + hybrid_receipt_output_rgb_sha256: str | None = None + native_output_rgb_sha256: str | None = None + + def provenance(self) -> dict: + """JSON-serializable provenance record; the sidecar's payload.""" + + payload: dict = { + "kind": "negpy.fauxice-ir-repair", + "version": 1, + "status": self.status.value, + "reason": self.reason, + "mode_requested": self.mode_requested.value, + "mode_resolved": self.mode_resolved.value if self.mode_resolved is not None else None, + "engine": { + "package": ENGINE_IMPORT_NAME, + "version": self.engine_version, + "backend_requested": self.backend_requested, + "backend_used": self.backend_used, + "backend_selection_reason": self.backend_selection_reason, + }, + } + if self.mode_resolved is RepairMode.HYBRID: + payload["hybrid"] = { + "package": HYBRID_IMPORT_NAME, + "disclosure_mask_sha256": self.hybrid_mask_sha256, + "synthesis_fraction": self.hybrid_synthesis_fraction, + # Region/pixel counts straight from the tool's own receipt + # (routing.counts); None when the receipt did not carry the + # expected keys rather than guessing at partial data. + "routing_counts": self.hybrid_routing_counts, + } + return payload + + +def _disabled_result(mode: RepairMode) -> FauxiceRepairResult: + return FauxiceRepairResult( + status=RepairStatus.SKIPPED, + reason="disabled by configuration (FauxiceRepairConfig.enabled is False)", + mode_requested=mode, + ) + + +def _unavailable_result(mode: RepairMode) -> FauxiceRepairResult: + return FauxiceRepairResult( + status=RepairStatus.UNAVAILABLE, + reason=( + f"{ENGINE_IMPORT_NAME} is not installed; install the optional " + f"'fauxice' dependency group from {ENGINE_HOME} " + "(see docs/FAUXICE_IR_REPAIR.md)" + ), + mode_requested=mode, + ) + + +def repair_ir_dust( + rgb: npt.NDArray[np.uint16], + ir: npt.NDArray[np.uint16], + *, + same_frame_id: str, + config: FauxiceRepairConfig, + prepass_rgbi: npt.NDArray[np.uint16] | None = None, + validity_mask: npt.NDArray[np.bool_] | None = None, + hybrid_runtime: HybridRuntimeConfig | None = None, + hybrid_subprocess_runner: Callable[..., "subprocess.CompletedProcess[str]"] | None = None, + progress: Callable[[float], None] | None = None, + cancel: threading.Event | None = None, +) -> FauxiceRepairResult: + """Repair infrared-flagged dust in ``rgb`` using the digital-fauxice engine. + + ``rgb`` (HxWx3 uint16) and ``ir`` (HxW uint16) are the two halves of + NegPy's own single-pass capture (``negpy/services/scanning/writer.py`` + writes them from one ``ScanResult``), so reassembling them into one + HxWx4 RGBI array is a lossless relabeling, not a reconstruction. + + ``prepass_rgbi`` is different in kind: seeing the module docstring's + prepass note before wiring a caller to this function. Pass ``None`` + (the default) unless a real 285 dpi acquisition of the same frame is + available; passing anything reconstructed or guessed would silently + break the engine's exactness claim. + + Malformed array shapes/dtypes raise ``ValueError`` immediately (a caller + bug, not an expected runtime state). Missing preconditions (feature + disabled, engine not installed, no prepass, hybrid unavailable) never + raise; they come back as a status on the result, so a caller can show + it to a user without a try/except. + + ``progress`` and ``cancel`` follow ``ScannerService.run_scan``'s own + shape (a ``0.0``-``1.0`` callback plus a ``threading.Event``) so a + background worker can wire them the same way ``ScanWorker`` wires a + scan. The exact engine polls its native cooperative callback; the hybrid + runner polls the event while its isolated process group runs, terminates + that group on cancellation, and reports coarse phase progress. + """ + + mode_requested = config.mode + + if not config.enabled: + return _disabled_result(mode_requested) + if not engine_available(): + return _unavailable_result(mode_requested) + + if rgb.ndim != 3 or rgb.shape[2] != 3 or rgb.dtype != np.uint16: + raise ValueError("rgb must be an HxWx3 uint16 array") + if ir.ndim != 2 or ir.dtype != np.uint16: + raise ValueError("ir must be an HxW uint16 array") + if ir.shape != rgb.shape[:2]: + raise ValueError(f"ir shape {ir.shape} does not match rgb shape {rgb.shape[:2]}") + if validity_mask is not None and validity_mask.shape != rgb.shape[:2]: + raise ValueError("validity_mask must match rgb's height and width") + if prepass_rgbi is not None and (prepass_rgbi.ndim != 3 or prepass_rgbi.shape[2] != 4 or prepass_rgbi.dtype != np.uint16): + raise ValueError("prepass_rgbi must be an HxWx4 uint16 array") + if not same_frame_id.strip(): + raise ValueError("same_frame_id must be non-empty") + + if prepass_rgbi is None: + return FauxiceRepairResult( + status=RepairStatus.SKIPPED, + reason=( + "no paired 285 dpi prepass acquisition available for this frame; " + "Digital ICE requires one captured alongside the main scan and " + "cannot be reconstructed from it (see docs/FAUXICE_IR_REPAIR.md)" + ), + mode_requested=mode_requested, + ) + + if cancel is not None and cancel.is_set(): + raise FauxiceRepairCancelled("cancelled by caller before repair started") + + main_rgbi = np.dstack([rgb, ir]) + + mode_resolved = RepairMode.EXACT + hybrid_note = "" + hybrid_outcome = None + + if mode_requested is RepairMode.HYBRID: + if hybrid_runtime is None: + hybrid_note = "hybrid mode requested but no hybrid runtime is configured; degraded to exact repair. " + elif not hybrid_available(hybrid_runtime): + try: + hybrid_runtime.validate_files() + except ValueError as error: + detail = str(error) + else: # pragma: no cover - defensive against a racing runtime + detail = "runtime artifacts changed during validation" + hybrid_note = f"hybrid mode requested but the configured hybrid runtime is unavailable ({detail}); degraded to exact repair. " + elif cancel is not None and cancel.is_set(): + raise FauxiceRepairCancelled("cancelled before the hybrid run started") + else: + try: + with tempfile.TemporaryDirectory(prefix="negpy-fauxice-hybrid-") as scratch: + hybrid_outcome = run_hybrid_repair( + main_rgbi, + prepass_rgbi, + same_frame_id=same_frame_id, + backend=config.backend, + runtime=hybrid_runtime, + scratch_dir=Path(scratch), + runner=hybrid_subprocess_runner, + progress=progress, + cancel=cancel, + ) + mode_resolved = RepairMode.HYBRID + except HybridRunCancelled as error: + raise FauxiceRepairCancelled(f"hybrid repair cancelled: {error}") from error + except HybridRunError as error: + hybrid_note = f"hybrid mode requested but the fauxce-hybrid run failed ({error}); degraded to exact repair. " + + result_kwargs: dict + if mode_resolved is RepairMode.HYBRID and hybrid_outcome is not None: + repaired = hybrid_outcome.hybrid_rgb16 + result_kwargs = { + "engine_version": hybrid_outcome.engine_version, + "backend_requested": hybrid_outcome.backend_requested, + "backend_used": hybrid_outcome.backend_used, + "backend_selection_reason": hybrid_outcome.backend_selection_reason, + "hybrid_mask_png": hybrid_outcome.synth_mask_png, + "hybrid_mask_sha256": hybrid_outcome.synth_mask_sha256, + "hybrid_mask": hybrid_outcome.synth_mask, + "hybrid_synthesis_fraction": hybrid_outcome.synthesis_fraction, + "hybrid_routing_counts": hybrid_outcome.routing_counts, + "hybrid_receipt": hybrid_outcome.receipt, + "hybrid_receipt_sha256": hybrid_outcome.receipt_sha256, + "hybrid_provenance_class": hybrid_outcome.provenance_class, + "hybrid_receipt_output_rgb_sha256": (hybrid_outcome.output_rgb16_sha256), + "native_output_rgb_sha256": hybrid_outcome.output_rgb16_sha256, + } + region_note = "" + if result_kwargs["hybrid_routing_counts"] is not None: + region_note = f"; {result_kwargs['hybrid_routing_counts']['final_regions']} region(s) routed" + reason = ( + f"applied via {HYBRID_IMPORT_NAME} (engine {result_kwargs['engine_version']}, " + f"backend {result_kwargs['backend_used']}); synthesis fraction " + f"{_format_fraction(result_kwargs['hybrid_synthesis_fraction'])}{region_note}" + ) + else: + from portable_digital_ice import ProcessingCancelled + + try: + repaired, backend_info = _run_exact( + main_rgbi, + prepass_rgbi, + same_frame_id=same_frame_id, + backend=config.backend, + progress=progress, + cancel=cancel, + ) + except ProcessingCancelled as error: + raise FauxiceRepairCancelled(f"{hybrid_note}cancelled before completion: {error}") from error + except (ValueError, TypeError, RuntimeError) as error: + return FauxiceRepairResult( + status=RepairStatus.SKIPPED, + reason=f"{hybrid_note}engine rejected the acquisition: {error}", + mode_requested=mode_requested, + mode_resolved=None, + ) + result_kwargs = { + "engine_version": _engine_version(), + "native_output_rgb_sha256": hashlib.sha256( + np.array(repaired, dtype=" tuple[npt.NDArray[np.uint16], dict]: + """Call the installed engine directly. + + Only reached after ``engine_available()`` is true. The import is local + to this function, never at module scope, so importing this module never + requires the optional engine to be installed. + + ``progress``/``cancel`` are NegPy's own ``Callable[[float], None]`` / + ``threading.Event`` shapes; the engine's native callbacks take a + ``ProcessingProgress`` object and a zero-argument ``() -> bool`` poll + function respectively, so this is where the two conventions meet. + """ + + from portable_digital_ice import ( + AcquisitionEpoch, + ComputeBackend, + DualRGBIAcquisition, + ProcessingJob, + ProcessingMode, + RGBI16Frame, + ScannerModel, + process, + ) + + prepass_frame = RGBI16Frame(prepass_rgbi, AcquisitionEpoch.PREPASS, PREPASS_DPI, f"{same_frame_id}-prepass") + main_frame = RGBI16Frame(main_rgbi, AcquisitionEpoch.MAIN, MAIN_DPI, f"{same_frame_id}-main") + acquisition = DualRGBIAcquisition(prepass_frame, main_frame, same_frame_id) + job = ProcessingJob( + acquisition=acquisition, + scanner_model=ScannerModel.NIKON_SUPER_COOLSCAN_5000_ED, + mode=ProcessingMode.NORMAL, + selector=8, + resolution_metric=MAIN_DPI, + bit_depth=16, + focus_exposure_locked=True, + ) + + engine_progress = None + if progress is not None: + + def engine_progress(step: object) -> None: + total = getattr(step, "total", 0) or 0 + completed = getattr(step, "completed", 0) + progress(float(completed) / float(total) if total else 0.0) + + engine_cancelled = cancel.is_set if cancel is not None else None + + routed = process( + job, + backend=ComputeBackend(backend), + progress=engine_progress, + cancelled=engine_cancelled, + ) + selection = routed.selection + return routed.result.output_rgb16, { + "backend_requested": selection.requested.value, + "backend_used": selection.used.value, + "backend_selection_reason": selection.reason, + } + + +def default_ir_path(rgb_path: Path) -> Path: + """The sidecar path NegPy's own scanning writer produces: ``_IR.tif``.""" + return rgb_path.with_name(rgb_path.stem + IR_SIDECAR_SUFFIX + rgb_path.suffix) + + +def _load_array(path: Path) -> np.ndarray: + if path.suffix.lower() == ".npy": + return np.load(path, allow_pickle=False) + return np.asarray(tifffile.imread(path)) + + +def _load_validity_mask(path: Path) -> np.ndarray: + mask = _load_array(path) + if mask.ndim == 3: + mask = mask[:, :, 0] + return mask if mask.dtype == np.bool_ else mask.astype(bool) + + +def _atomic_write_bytes(path: Path, data: bytes) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + fd, tmp_name = tempfile.mkstemp(dir=str(path.parent), suffix=".part") + try: + with os.fdopen(fd, "wb") as handle: + handle.write(data) + os.replace(tmp_name, path) + except BaseException: + Path(tmp_name).unlink(missing_ok=True) + raise + + +def _atomic_write_json(path: Path, payload: dict) -> None: + _atomic_write_bytes(path, (json.dumps(payload, indent=2, default=str) + "\n").encode("utf-8")) + + +def _atomic_write_tiff(path: Path, rgb16: np.ndarray) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + fd, tmp_name = tempfile.mkstemp(dir=str(path.parent), suffix=".part.tif") + os.close(fd) + try: + tifffile.imwrite(tmp_name, rgb16, photometric="rgb", compression="lzw") + os.replace(tmp_name, path) + except BaseException: + Path(tmp_name).unlink(missing_ok=True) + raise + + +def _write_sidecar(path: Path, result: FauxiceRepairResult, *, rgb_path: Path, ir_path: Path) -> None: + payload = result.provenance() + payload["source"] = {"rgb": rgb_path.name, "ir": ir_path.name} + if result.status is RepairStatus.APPLIED: + output = {"repaired_rgb": rgb_path.stem + REPAIR_OUTPUT_SUFFIX + rgb_path.suffix} + if result.mode_resolved is RepairMode.HYBRID and result.hybrid_mask_png is not None: + output["disclosure_mask"] = rgb_path.stem + SYNTH_MASK_SUFFIX + ".png" + payload["output"] = output + _atomic_write_json(path, payload) + + +def repair_frame_files( + rgb_path: Path, + *, + config: FauxiceRepairConfig, + ir_path: Path | None = None, + prepass_path: Path | None = None, + validity_mask_path: Path | None = None, + same_frame_id: str | None = None, + hybrid_runtime: HybridRuntimeConfig | None = None, + hybrid_subprocess_runner: Callable[..., "subprocess.CompletedProcess[str]"] | None = None, + progress: Callable[[float], None] | None = None, + cancel: threading.Event | None = None, +) -> FauxiceRepairResult: + """Load a NegPy-imported RGB master plus its ``_IR`` companion and repair it. + + Always writes a provenance sidecar (``_FAUXICE.json``), even when + the result is not ``APPLIED``, so a caller can show why. Only writes the + repaired master (``_FAUXICE.tif``, and the disclosure mask PNG for + a hybrid run) when the result is ``APPLIED``; a skipped or unavailable + attempt otherwise leaves every existing file byte-for-byte untouched. + ``ir_path`` is never opened for writing. + + ``prepass_path``, if given, may be a ``.npy`` array or a TIFF holding + the paired 285 dpi acquisition (HxWx4 uint16). NegPy's own import + pipeline does not produce one today (see the module docstring), so this + parameter exists for whatever future acquisition path can supply it, and + for tests. + """ + + rgb_path = Path(rgb_path) + ir_path = Path(ir_path) if ir_path is not None else default_ir_path(rgb_path) + frame_id = same_frame_id or rgb_path.stem + sidecar_path = rgb_path.with_name(rgb_path.stem + REPAIR_SIDECAR_SUFFIX + ".json") + + if not config.enabled: + result = _disabled_result(config.mode) + elif not engine_available(): + result = _unavailable_result(config.mode) + elif not ir_path.is_file(): + result = FauxiceRepairResult( + status=RepairStatus.SKIPPED, + reason=f"no IR companion found at {ir_path.name}", + mode_requested=config.mode, + ) + else: + rgb = np.asarray(tifffile.imread(rgb_path)) + ir = np.asarray(tifffile.imread(ir_path)) + prepass = _load_array(Path(prepass_path)) if prepass_path is not None else None + validity_mask = _load_validity_mask(Path(validity_mask_path)) if validity_mask_path is not None else None + result = repair_ir_dust( + rgb, + ir, + same_frame_id=frame_id, + config=config, + prepass_rgbi=prepass, + validity_mask=validity_mask, + hybrid_runtime=hybrid_runtime, + hybrid_subprocess_runner=hybrid_subprocess_runner, + progress=progress, + cancel=cancel, + ) + + _write_sidecar(sidecar_path, result, rgb_path=rgb_path, ir_path=ir_path) + + if result.status is RepairStatus.APPLIED and result.repaired_rgb16 is not None: + output_path = rgb_path.with_name(rgb_path.stem + REPAIR_OUTPUT_SUFFIX + rgb_path.suffix) + _atomic_write_tiff(output_path, result.repaired_rgb16) + if result.hybrid_mask_png is not None: + mask_path = rgb_path.with_name(rgb_path.stem + SYNTH_MASK_SUFFIX + ".png") + _atomic_write_bytes(mask_path, result.hybrid_mask_png) + + return result + + +__all__ = [ + "ENGINE_HOME", + "ENGINE_IMPORT_NAME", + "HYBRID_IMPORT_NAME", + "FauxiceRepairConfig", + "FauxiceRepairResult", + "HybridRuntimeConfig", + "RepairMode", + "RepairStatus", + "default_ir_path", + "engine_available", + "hybrid_available", + "repair_frame_files", + "repair_ir_dust", +] diff --git a/negpy/services/repair/hybrid_runtime_manifest.py b/negpy/services/repair/hybrid_runtime_manifest.py new file mode 100644 index 00000000..e462a284 --- /dev/null +++ b/negpy/services/repair/hybrid_runtime_manifest.py @@ -0,0 +1,222 @@ +"""Strict operator-provisioned configuration for the external hybrid runtime.""" + +from __future__ import annotations + +import hashlib +import json +import math +import os +import re +import stat +from pathlib import Path + +from negpy.kernel.system.config import BASE_USER_DIR +from negpy.services.repair.fauxice_hybrid_runner import HybridRuntimeConfig + + +RUNTIME_MANIFEST_SCHEMA = "negpy.fauxce-hybrid-runtime.v2" +RUNTIME_MANIFEST_FILENAME = "fauxce-hybrid-runtime.json" +RUNTIME_MANIFEST_MAX_BYTES = 64 * 1024 +_SHA256_RE = re.compile(r"[0-9a-f]{64}") +_PATH_FIELDS = ( + "hybrid_python", + "executable", + "iopaint_python", + "iopaint_executable", + "model_dir", + "model_weights", +) +_HASH_FIELDS = ( + "core_source_manifest_sha256", + "hybrid_source_manifest_sha256", + "iopaint_source_manifest_sha256", + "model_weights_sha256", +) +_REQUIRED_FIELDS = frozenset( + { + "schema", + *_PATH_FIELDS, + *_HASH_FIELDS, + "inpaint_device", + "inpaint_threads", + "inpaint_seed", + "max_synthesis_fraction", + } +) + + +class HybridRuntimeManifestError(RuntimeError): + """The external runtime manifest or its independently retained pin failed.""" + + +def default_hybrid_runtime_manifest_path() -> Path: + return Path(BASE_USER_DIR) / RUNTIME_MANIFEST_FILENAME + + +def runtime_manifest_pin_path(manifest_path: Path) -> Path: + return manifest_path.with_name(manifest_path.name + ".sha256") + + +def _stable_regular_bytes(path: Path, *, maximum_bytes: int, label: str) -> bytes: + descriptor: int | None = None + try: + linked = os.lstat(path) + if stat.S_ISLNK(linked.st_mode) or not stat.S_ISREG(linked.st_mode): + raise HybridRuntimeManifestError(f"{label} must be a regular non-symlink file") + if linked.st_size < 0 or linked.st_size > maximum_bytes: + raise HybridRuntimeManifestError(f"{label} exceeds its safe size limit") + flags = os.O_RDONLY | getattr(os, "O_CLOEXEC", 0) | getattr(os, "O_NOFOLLOW", 0) | getattr(os, "O_NONBLOCK", 0) + descriptor = os.open(path, flags) + with os.fdopen(descriptor, "rb", closefd=True) as handle: + descriptor = None + before = os.fstat(handle.fileno()) + if ( + not stat.S_ISREG(before.st_mode) + or (before.st_dev, before.st_ino) != (linked.st_dev, linked.st_ino) + or before.st_size != linked.st_size + ): + raise HybridRuntimeManifestError(f"{label} changed while opening") + payload = handle.read(maximum_bytes + 1) + after = os.fstat(handle.fileno()) + if ( + len(payload) != before.st_size + or len(payload) > maximum_bytes + or (before.st_dev, before.st_ino, before.st_size, before.st_mtime_ns) + != (after.st_dev, after.st_ino, after.st_size, after.st_mtime_ns) + ): + raise HybridRuntimeManifestError(f"{label} changed while reading") + return payload + except HybridRuntimeManifestError: + raise + except OSError as error: + raise HybridRuntimeManifestError(f"could not read {label}: {error}") from error + finally: + if descriptor is not None: + os.close(descriptor) + + +def canonical_runtime_manifest_bytes(document: object) -> bytes: + try: + return ( + json.dumps( + document, + sort_keys=True, + separators=(",", ":"), + ensure_ascii=True, + allow_nan=False, + ).encode("utf-8") + + b"\n" + ) + except (TypeError, ValueError, RecursionError) as error: + raise HybridRuntimeManifestError(f"hybrid runtime manifest cannot be canonicalized: {error}") from error + + +def load_hybrid_runtime_manifest( + manifest_path: str | Path, + *, + expected_sha256: str, +) -> HybridRuntimeConfig: + """Load one canonical manifest whose digest came from a separate boundary.""" + + path = Path(manifest_path) + if _SHA256_RE.fullmatch(expected_sha256) is None: + raise HybridRuntimeManifestError("hybrid runtime manifest pin must be a lowercase SHA-256 digest") + payload = _stable_regular_bytes( + path, + maximum_bytes=RUNTIME_MANIFEST_MAX_BYTES, + label="hybrid runtime manifest", + ) + actual_sha256 = hashlib.sha256(payload).hexdigest() + if actual_sha256 != expected_sha256: + raise HybridRuntimeManifestError(f"hybrid runtime manifest SHA-256 mismatch: expected {expected_sha256}, got {actual_sha256}") + try: + document = json.loads(payload) + except (UnicodeDecodeError, json.JSONDecodeError) as error: + raise HybridRuntimeManifestError(f"hybrid runtime manifest is not valid JSON: {error}") from error + if not isinstance(document, dict): + raise HybridRuntimeManifestError("hybrid runtime manifest must be an object") + if canonical_runtime_manifest_bytes(document) != payload: + raise HybridRuntimeManifestError("hybrid runtime manifest is not canonical JSON") + fields = frozenset(document) + if fields != _REQUIRED_FIELDS: + missing = sorted(_REQUIRED_FIELDS - fields) + extra = sorted(fields - _REQUIRED_FIELDS) + raise HybridRuntimeManifestError(f"hybrid runtime manifest fields changed (missing={missing}, extra={extra})") + if document["schema"] != RUNTIME_MANIFEST_SCHEMA: + raise HybridRuntimeManifestError("hybrid runtime manifest schema is unsupported") + for field in _PATH_FIELDS: + if not isinstance(document[field], str) or not document[field]: + raise HybridRuntimeManifestError(f"{field} must be a non-empty path string") + for field in _HASH_FIELDS: + if not isinstance(document[field], str) or _SHA256_RE.fullmatch(document[field]) is None: + raise HybridRuntimeManifestError(f"{field} must be a lowercase SHA-256 digest") + if document["inpaint_device"] not in {"cpu", "cuda", "mps"}: + raise HybridRuntimeManifestError("inpaint_device must be cpu, cuda, or mps") + for field, minimum in (("inpaint_threads", 1), ("inpaint_seed", 0)): + value = document[field] + if type(value) is not int or value < minimum: + raise HybridRuntimeManifestError(f"{field} must be an integer greater than or equal to {minimum}") + maximum_fraction = document["max_synthesis_fraction"] + if ( + isinstance(maximum_fraction, bool) + or not isinstance(maximum_fraction, (int, float)) + or not math.isfinite(float(maximum_fraction)) + or not 0.0 <= float(maximum_fraction) <= 1.0 + ): + raise HybridRuntimeManifestError("max_synthesis_fraction must be finite and in [0, 1]") + try: + return HybridRuntimeConfig( + hybrid_python=Path(document["hybrid_python"]), + executable=Path(document["executable"]), + core_source_manifest_sha256=document["core_source_manifest_sha256"], + hybrid_source_manifest_sha256=document["hybrid_source_manifest_sha256"], + iopaint_python=Path(document["iopaint_python"]), + iopaint_executable=Path(document["iopaint_executable"]), + iopaint_source_manifest_sha256=document["iopaint_source_manifest_sha256"], + model_dir=Path(document["model_dir"]), + model_weights=Path(document["model_weights"]), + model_weights_sha256=document["model_weights_sha256"], + inpaint_device=document["inpaint_device"], + inpaint_threads=document["inpaint_threads"], + inpaint_seed=document["inpaint_seed"], + max_synthesis_fraction=float(maximum_fraction), + ) + except (TypeError, ValueError) as error: + raise HybridRuntimeManifestError(f"hybrid runtime manifest values are invalid: {error}") from error + + +def load_default_hybrid_runtime_manifest( + manifest_path: str | Path | None = None, +) -> HybridRuntimeConfig | None: + """Load the desktop runtime, or return ``None`` when it was never installed.""" + + path = default_hybrid_runtime_manifest_path() if manifest_path is None else Path(manifest_path) + pin_path = runtime_manifest_pin_path(path) + if not path.exists() and not pin_path.exists(): + return None + if not path.exists() or not pin_path.exists(): + raise HybridRuntimeManifestError("hybrid runtime manifest and its .sha256 pin must both exist") + pin_bytes = _stable_regular_bytes( + pin_path, + maximum_bytes=65, + label="hybrid runtime manifest pin", + ) + try: + pin = pin_bytes.decode("ascii").rstrip("\n") + except UnicodeDecodeError as error: + raise HybridRuntimeManifestError("hybrid runtime manifest pin is not ASCII") from error + if pin_bytes != (pin + "\n").encode("ascii") or _SHA256_RE.fullmatch(pin) is None: + raise HybridRuntimeManifestError("hybrid runtime manifest pin must be one lowercase SHA-256 plus newline") + return load_hybrid_runtime_manifest(path, expected_sha256=pin) + + +__all__ = [ + "HybridRuntimeManifestError", + "RUNTIME_MANIFEST_FILENAME", + "RUNTIME_MANIFEST_SCHEMA", + "canonical_runtime_manifest_bytes", + "default_hybrid_runtime_manifest_path", + "load_default_hybrid_runtime_manifest", + "load_hybrid_runtime_manifest", + "runtime_manifest_pin_path", +] diff --git a/negpy/services/roll/__init__.py b/negpy/services/roll/__init__.py new file mode 100644 index 00000000..6cdf3b68 --- /dev/null +++ b/negpy/services/roll/__init__.py @@ -0,0 +1,56 @@ +"""Roll-scanning orchestration -- drives a coolscanpy Device + Roll extension +through preview, spacing correction, approval and batch fine-scanning. + +No Qt dependencies (mirrors `negpy.services.capture` and +`negpy.services.scanning`). +""" + +from negpy.services.roll.exact_color import ( + BuilderReceipt, + ExactColorIntegrityError, + ExactColorResult, + ExactColorUnavailable, + FIXED_COMPOSITION_SHA256, + NATIVE_BUILDER_SCOPE, + NativeValidatedBuilderReceipt, + PositiveColorMode, + STAGE3_REPLAY_SCOPE, + Stage1BuilderResult, + ValidatedBuilderReceipt, + VerifiedBuilderApplicationReceipt, + VerifiedCMSReceipt, + VerifiedPortableCMSEvaluator, + VerifiedStage1Builder, + load_stage3_replay_builder_receipt, +) +from negpy.services.roll.native_builder import NativeBuilderEvidence, build_native_builder_receipt +from negpy.services.roll.portable_builder import PortableStage1Builder +from negpy.services.roll.portable_cms import PortableCMSOnEvaluator +from negpy.services.roll.service import RollFrameOutput, RollScanningError, RollScanningService, available + +__all__ = [ + "ExactColorIntegrityError", + "ExactColorResult", + "ExactColorUnavailable", + "FIXED_COMPOSITION_SHA256", + "BuilderReceipt", + "NATIVE_BUILDER_SCOPE", + "NativeBuilderEvidence", + "NativeValidatedBuilderReceipt", + "PositiveColorMode", + "PortableCMSOnEvaluator", + "PortableStage1Builder", + "RollFrameOutput", + "RollScanningError", + "RollScanningService", + "STAGE3_REPLAY_SCOPE", + "Stage1BuilderResult", + "ValidatedBuilderReceipt", + "VerifiedBuilderApplicationReceipt", + "VerifiedCMSReceipt", + "VerifiedPortableCMSEvaluator", + "VerifiedStage1Builder", + "available", + "build_native_builder_receipt", + "load_stage3_replay_builder_receipt", +] diff --git a/negpy/services/roll/deep_acceptance.py b/negpy/services/roll/deep_acceptance.py new file mode 100644 index 00000000..4accc2ec --- /dev/null +++ b/negpy/services/roll/deep_acceptance.py @@ -0,0 +1,2002 @@ +"""Offline, fail-closed audit of completed LS-5000 acceptance outputs. + +The live acceptance runner deliberately performs only a cheap publication +check while the scanner is reserved. This module composes the stronger +loaders used by the production write path after the scanner has been closed: +Digital ICE replay, retained native-builder re-derivation, portable CML4 +re-evaluation, Hybrid disclosure binding, and exact TIFF/ICC verification. + +Nothing in this module opens a scanner or imports a hardware backend. +""" + +from __future__ import annotations + +import hashlib +import hmac +import importlib +import json +import os +import re +import stat +from contextlib import contextmanager +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Iterator, NoReturn, Protocol, Sequence, cast + +import numpy as np + +from negpy.infrastructure.roll.repair import RepairAcquisition, RepairMode, RepairResult +from negpy.services.repair.hybrid_runtime_manifest import ( + HybridRuntimeManifestError, + load_default_hybrid_runtime_manifest, +) +from negpy.services.roll import exact_color, nikon_icc +from negpy.services.roll import service as roll_service +from negpy.services.roll.live_reservation import is_ignorable_finder_metadata_file +from negpy.services.roll.portable_builder import PortableStage1Builder +from negpy.services.roll.portable_cms import PortableCMSOnEvaluator +from negpy.services.roll.service import RollFrameOutput + + +class _FcntlAPI(Protocol): + LOCK_EX: int + LOCK_NB: int + LOCK_UN: int + + def flock(self, descriptor: int, operation: int) -> None: ... + + +try: + _fcntl_module = importlib.import_module("fcntl") +except ImportError: # pragma: no cover - this audit targets the macOS workflow + _fcntl: _FcntlAPI | None = None +else: + _fcntl = cast(_FcntlAPI, _fcntl_module) + + +SCHEMA = "negpy.ls5000-deep-acceptance.v1" +SLOTS = (1, 2, 3, 4, 5, 6) +# Historical fixture/export compatibility only. The production audit derives +# the actual approved slots from the sealed frame evidence. +APPROVED_SLOTS = frozenset((1, 6)) +_OUTPUT_FIELDS = ( + "rgb_path", + "ir_path", + "repaired_rgb_path", + "repaired_ir_path", + "positive_path", + "receipt_path", + "synthesis_mask_path", + "native_synthesis_mask_path", + "hybrid_receipt_path", +) +_SHA256_RE = re.compile(r"[0-9a-f]{64}") +_MAX_EVIDENCE_BYTES = 64 * 1024 * 1024 +_MAX_RECEIPT_BYTES = 16 * 1024 * 1024 + + +class _HybridRuntime(Protocol): + core_source_manifest_sha256: str + hybrid_source_manifest_sha256: str + iopaint_source_manifest_sha256: str + model_weights_sha256: str + inpaint_device: str + inpaint_threads: int + inpaint_seed: int + + def validate_files(self) -> None: ... + + +class DeepAcceptanceError(ValueError): + """A completed output failed an offline acceptance invariant.""" + + +@dataclass(frozen=True) +class _FrameAudit: + summary: dict[str, Any] + referenced_files: frozenset[Path] + receipt_path: Path + receipt: dict[str, Any] + ownership: object + builder_receipt: exact_color.NativeValidatedBuilderReceipt + output_artifacts: dict[str, str] + + +def _fail(label: str, message: str) -> NoReturn: + raise DeepAcceptanceError(f"{label}: {message}") + + +def _public_output_slot( + output: object, + *, + expected_slot: int | None, +) -> tuple[RollFrameOutput, int]: + if type(output) is not RollFrameOutput: + raise DeepAcceptanceError("output must be one RollFrameOutput") + slot = output.slot if expected_slot is None else expected_slot + if type(slot) is not int: + raise DeepAcceptanceError("expected slot must be an integer") + return output, slot + + +def _validated_runtime(runtime: _HybridRuntime | None) -> _HybridRuntime: + try: + active = runtime if runtime is not None else load_default_hybrid_runtime_manifest() + except (HybridRuntimeManifestError, OSError, TypeError, ValueError) as error: + raise DeepAcceptanceError(f"pinned Hybrid runtime manifest is invalid: {error}") from error + if active is None: + raise DeepAcceptanceError("the pinned Hybrid runtime is not installed") + try: + active.validate_files() + except (AttributeError, OSError, TypeError, ValueError) as error: + raise DeepAcceptanceError(f"pinned Hybrid runtime is invalid: {error}") from error + return active + + +def _color_dependencies( + builder: PortableStage1Builder | None, + evaluator: PortableCMSOnEvaluator | None, +) -> tuple[PortableStage1Builder, PortableCMSOnEvaluator]: + try: + active_builder = builder if builder is not None else PortableStage1Builder() + active_evaluator = evaluator if evaluator is not None else PortableCMSOnEvaluator() + except (exact_color.ExactColorUnavailable, OSError, TypeError, ValueError) as error: + raise DeepAcceptanceError(f"portable exact-color dependencies are unavailable: {error}") from error + return active_builder, active_evaluator + + +def _canonical_json( + value: object, + *, + newline: bool = False, + ensure_ascii: bool = False, +) -> bytes: + payload = json.dumps( + value, + sort_keys=True, + separators=(",", ":"), + ensure_ascii=ensure_ascii, + allow_nan=False, + ).encode("utf-8") + return payload + (b"\n" if newline else b"") + + +def _sha256(payload: bytes) -> str: + return hashlib.sha256(payload).hexdigest() + + +def _require_dict(value: object, *, label: str) -> dict[str, Any]: + if type(value) is not dict: + _fail(label, "must be an object") + return cast(dict[str, Any], value) + + +def _require_str(value: object, *, label: str) -> str: + if type(value) is not str or not value: + _fail(label, "must be a non-empty string") + return value + + +def _require_sha256(value: object, *, label: str) -> str: + if type(value) is not str or _SHA256_RE.fullmatch(value) is None: + _fail(label, "must be one lowercase SHA-256 digest") + return value + + +def _output_root(path: str | os.PathLike[str]) -> Path: + try: + candidate = Path(path) + linked = candidate.lstat() + resolved = candidate.resolve(strict=True) + except (OSError, TypeError, ValueError) as error: + raise DeepAcceptanceError(f"output directory is unavailable: {error}") from error + if stat.S_ISLNK(linked.st_mode) or not stat.S_ISDIR(linked.st_mode): + raise DeepAcceptanceError("output directory must be a real directory") + return resolved + + +def _regular_file( + path_value: str | os.PathLike[str], + *, + root: Path, + label: str, +) -> Path: + path = Path(path_value) + if not path.is_absolute(): + _fail(label, "path must be absolute") + lexical = Path(os.path.abspath(os.fspath(path))) + if not lexical.is_relative_to(root): + _fail(label, "escaped the output directory") + try: + current = root + for component in lexical.relative_to(root).parts[:-1]: + current /= component + parent_metadata = current.lstat() + if stat.S_ISLNK(parent_metadata.st_mode) or not stat.S_ISDIR(parent_metadata.st_mode): + _fail(label, "has a symlink or non-directory parent") + linked = lexical.lstat() + resolved = lexical.resolve(strict=True) + except OSError as error: + raise DeepAcceptanceError(f"{label}: unavailable: {error}") from error + if stat.S_ISLNK(linked.st_mode) or not stat.S_ISREG(linked.st_mode): + _fail(label, "must be a regular non-symlink file") + if resolved != lexical: + _fail(label, "resolved through a symbolic link") + return resolved + + +def _stable_bytes(path: Path, *, maximum_bytes: int, label: str) -> bytes: + try: + return roll_service._stable_regular_bytes( # noqa: SLF001 - production integrity boundary + str(path), + maximum_bytes=maximum_bytes, + label=label, + ) + except (OSError, ValueError) as error: + raise DeepAcceptanceError(f"{label}: {error}") from error + + +def _load_frame_receipt( + receipt_path: Path, + *, + label: str, +) -> tuple[dict[str, Any], bytes]: + payload = _stable_bytes( + receipt_path, + maximum_bytes=_MAX_RECEIPT_BYTES, + label=label, + ) + try: + document = roll_service._strict_json_loads(payload) # noqa: SLF001 + except (UnicodeDecodeError, ValueError, TypeError) as error: + raise DeepAcceptanceError(f"{label}: invalid JSON: {error}") from error + document = _require_dict(document, label=label) + try: + canonical = json.dumps( + document, + indent=2, + ensure_ascii=True, + allow_nan=False, + ).encode("utf-8") + except (TypeError, ValueError, RecursionError) as error: + raise DeepAcceptanceError(f"{label}: cannot canonicalize JSON: {error}") from error + if payload != canonical: + _fail(label, "is not in the production canonical frame-receipt encoding") + return document, payload + + +def _nested_receipt_value( + document: object, + *keys: str, + label: str, +) -> object: + current = document + for key in keys: + current = _require_dict(current, label=label).get(key) + return current + + +def _receipt_artifact_path( + document: object, + *keys: str, + root: Path, + label: str, +) -> Path: + value = _nested_receipt_value(document, *keys, label=label) + if type(value) is not str or not Path(value).is_absolute(): + _fail(label, "must be an absolute artifact path") + return _regular_file(value, root=root, label=label) + + +def _absolute_receipt_paths(value: object, *, key: str | None = None) -> set[str]: + found: set[str] = set() + if type(value) is dict: + for child_key, child in cast(dict[str, object], value).items(): + found.update(_absolute_receipt_paths(child, key=child_key)) + elif type(value) is list: + for child in value: + found.update(_absolute_receipt_paths(child, key=key)) + elif ( + type(value) is str + and key != "relative_path" + and (key == "path" or (key is not None and key.endswith("_path"))) + and Path(value).is_absolute() + ): + found.add(value) + return found + + +def _collect_completed_frame_files_locked( + output: RollFrameOutput, + *, + root: Path, + expected_slot: int, +) -> tuple[dict[str, Any], bytes, frozenset[Path]]: + """Resolve one exact completed receipt while its frame lock is held.""" + + label = f"slot {expected_slot}" + if type(output) is not RollFrameOutput or output.slot != expected_slot: + _fail(label, "RollFrameOutput identity is wrong") + receipt_path = _regular_file( + output.receipt_path, + root=root, + label=f"{label} frame receipt", + ) + receipt, receipt_bytes = _load_frame_receipt( + receipt_path, + label=f"{label} frame receipt", + ) + if ( + receipt.get("version") != 1 + or receipt.get("slot") != expected_slot + or receipt.get("dpi") != 4_000 + or receipt.get("depth") != 16 + or type(receipt.get("device_id")) is not str + or not receipt.get("device_id") + or receipt.get("device_model") != "Nikon LS-5000 ED 1.03" + ): + _fail(label, "frame or LS-5000 capture identity is wrong") + outputs = _require_dict(receipt.get("outputs"), label=f"{label} outputs") + unrepaired = _require_dict(outputs.get("unrepaired"), label=f"{label} unrepaired tier") + repaired = _require_dict(outputs.get("repaired"), label=f"{label} repaired tier") + positive = _require_dict(outputs.get("positive"), label=f"{label} positive tier") + dice = _require_dict( + outputs.get("repair_acquisition_evidence"), + label=f"{label} DICE evidence", + ) + native = _require_dict( + outputs.get("native_color_evidence"), + label=f"{label} native color evidence", + ) + if any(row.get("written") is not True for row in (unrepaired, repaired, positive)): + _fail(label, "all three output tiers must be complete") + if ( + repaired.get("mode_requested") != "hybrid" + or repaired.get("mode_resolved") != "hybrid" + or repaired.get("degraded") is not False + or positive.get("color_mode") != "nikon-exact" + or positive.get("exact_nikon_color") is not True + or positive.get("native_per_acquisition_builder") is not True + or positive.get("builder_validated") is not True + or positive.get("cms_verified") is not True + or dice.get("retained") is not True + or dice.get("replayable") is not True + or native.get("retained") is not True + or native.get("native_per_acquisition_builder") is not True + ): + _fail(label, "completed Hybrid/Nikon-exact evidence flags changed") + artifacts = _require_dict(receipt.get("artifacts"), label=f"{label} artifacts") + if set(artifacts) != {"rgb", "ir"}: + _fail(label, "frame artifact inventory is not exactly RGB and IR") + + required: list[Path] = [ + _receipt_artifact_path( + outputs, + "unrepaired", + "rgb_path", + root=root, + label=f"{label} RGB", + ), + _receipt_artifact_path( + outputs, + "unrepaired", + "ir_path", + root=root, + label=f"{label} IR", + ), + _receipt_artifact_path( + outputs, + "repaired", + "rgb_path", + root=root, + label=f"{label} repaired RGB", + ), + _receipt_artifact_path( + outputs, + "repaired", + "ir_path", + root=root, + label=f"{label} repaired IR", + ), + _receipt_artifact_path( + outputs, + "positive", + "rgb_path", + root=root, + label=f"{label} positive", + ), + _receipt_artifact_path( + outputs, + "repair_acquisition_evidence", + "binding", + "path", + root=root, + label=f"{label} DICE binding", + ), + ] + for key in ("prepass_rgbi", "ir_validity"): + required.append( + _receipt_artifact_path( + outputs, + "repair_acquisition_evidence", + "artifacts", + key, + "path", + root=root, + label=f"{label} DICE {key}", + ) + ) + for key in ("storage_rgb_tiff", "storage_ir_tiff"): + required.append( + _receipt_artifact_path( + outputs, + "repair_acquisition_evidence", + "sources", + key, + "path", + root=root, + label=f"{label} DICE {key}", + ) + ) + for branch, name in ( + (("applied_final", "storage"), "storage mask"), + (("applied_final", "native"), "native mask"), + (("routed_raw", "native"), "routed native mask"), + ): + required.append( + _receipt_artifact_path( + outputs, + "repaired", + "disclosure_mask", + *branch, + "path", + root=root, + label=f"{label} {name}", + ) + ) + for key, name in ( + ("hybrid_receipt", "Hybrid receipt"), + ("hybrid_evidence_binding", "Hybrid binding"), + ): + required.append( + _receipt_artifact_path( + outputs, + "repaired", + key, + "path", + root=root, + label=f"{label} {name}", + ) + ) + + retained = _require_dict( + native.get("retained_builder_evidence"), + label=f"{label} retained native builder", + ) + if positive.get("retained_builder_evidence") != retained: + _fail(label, "positive and native tiers reference different builder evidence") + for key in ( + "builder_receipt", + "analyzer_rgb", + "evidence_receipt", + "frame_ownership_receipt", + "density_evidence_receipt", + ): + required.append( + _receipt_artifact_path( + retained, + key, + "path", + root=root, + label=f"{label} native {key}", + ) + ) + luts = retained.get("pre_f_luts") + if type(luts) is not list or len(luts) != 3: + _fail(label, "retained native builder must contain exactly three LUTs") + for index, row in enumerate(luts): + required.append( + _receipt_artifact_path( + row, + "path", + root=root, + label=f"{label} native LUT {index}", + ) + ) + + required_set = set(required) + if len(required_set) != 21: + _fail(label, "receipt-bound artifact paths alias or are incomplete") + recorded_absolute = _absolute_receipt_paths(outputs) + if recorded_absolute != {str(path) for path in required_set}: + missing = sorted(str(path) for path in required_set - {Path(item) for item in recorded_absolute}) + extra = sorted(recorded_absolute - {str(path) for path in required_set}) + raise DeepAcceptanceError(f"{label}: receipt path inventory changed (missing={missing}, extra={extra})") + + expected_output_paths = { + "rgb_path": required[0], + "ir_path": required[1], + "repaired_rgb_path": required[2], + "repaired_ir_path": required[3], + "positive_path": required[4], + "receipt_path": receipt_path, + "synthesis_mask_path": required[10], + "native_synthesis_mask_path": required[11], + "hybrid_receipt_path": required[13], + } + if set(expected_output_paths) != set(_OUTPUT_FIELDS): + raise AssertionError("internal RollFrameOutput field inventory changed") + for field, path in expected_output_paths.items(): + if getattr(output, field) != str(path): + _fail(label, f"RollFrameOutput {field} differs from its receipt") + + receipt_lock = _regular_file( + _receipt_lock_path(receipt_path), + root=root, + label=f"{label} frame receipt lock", + ) + return receipt, receipt_bytes, frozenset((*required_set, receipt_path, receipt_lock)) + + +def collect_completed_frame_files( + output: RollFrameOutput, + *, + output_dir: str | os.PathLike[str], + expected_slot: int | None = None, +) -> list[str]: + """Cheaply collect one completed frame's exact owned-file inventory. + + This hardware-inert checkpoint validates only the bounded frame receipt, + path bindings, regular files, and persistent frame lock. It deliberately + does not decode TIFF/PNG/NPY data, hash model or raster bytes, load the + Hybrid runtime, or recompute color. + """ + + checked_output, slot = _public_output_slot( + output, + expected_slot=expected_slot, + ) + root = _output_root(output_dir) + with _hold_frame_locks((checked_output,), root=root): + _, _, referenced = _collect_completed_frame_files_locked( + checked_output, + root=root, + expected_slot=slot, + ) + return sorted(str(path) for path in referenced) + + +def _artifact_row( + row_value: object, + *, + root: Path, + label: str, + expected_path: Path | None = None, + maximum_bytes: int = _MAX_EVIDENCE_BYTES, + digest_key: str = "sha256", +) -> tuple[Path, bytes]: + row = _require_dict(row_value, label=label) + path_value = row.get("path") + if type(path_value) is not str: + _fail(label, "has no path") + path = _regular_file(path_value, root=root, label=label) + if expected_path is not None and path != expected_path: + _fail(label, "path disagrees with the published RollFrameOutput") + payload = _stable_bytes(path, maximum_bytes=maximum_bytes, label=label) + if row.get("bytes") != len(payload): + _fail(label, "byte count changed") + digest = _require_sha256(row.get(digest_key), label=f"{label} SHA-256") + if not hmac.compare_digest(digest, _sha256(payload)): + _fail(label, "SHA-256 changed") + return path, payload + + +def _published_path( + output: RollFrameOutput, + field: str, + recorded: object, + *, + root: Path, + label: str, +) -> Path: + value = getattr(output, field) + if type(value) is not str or type(recorded) is not str or value != recorded: + _fail(label, "receipt and RollFrameOutput paths disagree") + return _regular_file(value, root=root, label=label) + + +def _array_artifact_matches(row_value: object, array: np.ndarray, *, label: str) -> None: + row = _require_dict(row_value, label=label) + contiguous = np.ascontiguousarray(array) + expected = { + "byte_length": contiguous.nbytes, + "dtype": str(contiguous.dtype), + "sha256": hashlib.sha256(memoryview(contiguous).cast("B")).hexdigest(), + "shape": list(contiguous.shape), + } + if row != expected: + _fail(label, "does not bind the decoded array") + + +def _decode_tiff(path: Path, shape: tuple[int, ...], *, label: str) -> np.ndarray: + try: + return roll_service._stable_tiff_array( # noqa: SLF001 - production integrity boundary + str(path), + expected_shape=shape, + label=label, + ) + except (OSError, ValueError) as error: + raise DeepAcceptanceError(f"{label}: {error}") from error + + +def _bound_archive_path( + row_value: object, + *, + binding_path: Path, + root: Path, + label: str, +) -> Path: + row = _require_dict(row_value, label=label) + relative = row.get("relative_path") + if type(relative) is not str or not relative or Path(relative).is_absolute(): + _fail(label, "has no safe relative path") + return _regular_file( + binding_path.parent / relative, + root=root, + label=label, + ) + + +def _validate_dice_archive_paths( + dice: dict[str, Any], + *, + binding_path: Path, + binding_bytes: bytes, + root: Path, + rgb_path: Path, + ir_path: Path, + referenced: set[Path], + label: str, +) -> None: + """Bind sidecar paths to the exact relative members consumed by replay.""" + + try: + binding = roll_service._strict_json_loads(binding_bytes) # noqa: SLF001 + except (UnicodeDecodeError, ValueError, TypeError) as error: + raise DeepAcceptanceError(f"{label}: DICE binding JSON is invalid: {error}") from error + binding = _require_dict(binding, label=f"{label} DICE binding") + if binding_path.name != "acquisition-binding.json" or binding_path.parent.parent != root / ".negpy-dice-acquisition": + _fail(label, "DICE evidence escaped its production archive directory") + bound_acquisition = _require_dict(binding.get("acquisition"), label=f"{label} bound DICE acquisition") + if ( + dice.get("acquisition_id") != bound_acquisition.get("acquisition_id") + or dice.get("replay") != binding.get("replay") + or dice.get("schema") != binding.get("schema") + ): + _fail(label, "DICE sidecar identity or replay contract changed") + outer_artifacts = _require_dict(dice.get("artifacts"), label=f"{label} DICE artifacts") + bound_artifacts = _require_dict(binding.get("artifacts"), label=f"{label} bound DICE artifacts") + outer_sources = _require_dict(dice.get("sources"), label=f"{label} DICE sources") + bound_sources = _require_dict(binding.get("sources"), label=f"{label} bound DICE sources") + if set(outer_artifacts) != {"prepass_rgbi", "ir_validity"} or set(bound_artifacts) != {"prepass_rgbi", "ir_validity"}: + _fail(label, "DICE artifact inventory changed") + if set(outer_sources) != {"storage_rgb_tiff", "storage_ir_tiff"} or set(bound_sources) != {"storage_rgb_tiff", "storage_ir_tiff"}: + _fail(label, "DICE source inventory changed") + + expected_artifact_names = { + "prepass_rgbi": "prepass.rgbi16.npy", + "ir_validity": "ir-validity.npy", + } + for key, expected_name in expected_artifact_names.items(): + outer = _require_dict(outer_artifacts.get(key), label=f"{label} DICE {key}") + bound = _require_dict(bound_artifacts.get(key), label=f"{label} bound DICE {key}") + if {name: value for name, value in outer.items() if name != "path"} != bound: + _fail(label, f"DICE {key} sidecar differs from the replay binding") + outer_path, _ = _artifact_row( + outer, + root=root, + label=f"{label} DICE {key}", + digest_key="file_sha256", + ) + bound_path = _bound_archive_path( + bound, + binding_path=binding_path, + root=root, + label=f"{label} bound DICE {key}", + ) + if outer_path != bound_path: + _fail(label, f"DICE {key} path differs from the replay binding") + if bound_path.parent != binding_path.parent or bound_path.name != expected_name: + _fail(label, f"DICE {key} escaped its production archive directory") + referenced.add(bound_path) + + for key, expected_path in ( + ("storage_rgb_tiff", rgb_path), + ("storage_ir_tiff", ir_path), + ): + outer = _require_dict(outer_sources.get(key), label=f"{label} DICE {key}") + bound = _require_dict(bound_sources.get(key), label=f"{label} bound DICE {key}") + if {name: value for name, value in outer.items() if name != "path"} != bound: + _fail(label, f"DICE {key} sidecar differs from the replay binding") + recorded = outer.get("path") + if type(recorded) is not str: + _fail(label, f"DICE {key} has no path") + outer_path = _regular_file( + recorded, + root=root, + label=f"{label} DICE {key}", + ) + bound_path = _bound_archive_path( + bound, + binding_path=binding_path, + root=root, + label=f"{label} bound DICE {key}", + ) + if outer_path != expected_path or bound_path != expected_path: + _fail(label, f"DICE {key} does not bind the published Tier-1 TIFF") + + +def _stored_receipt_hash(document: object) -> str: + return _sha256(_canonical_json(document)) + + +def _receipt_lock_path(receipt_path: Path) -> Path: + """Return the persistent transaction-lock path for one frame receipt.""" + + lock_name = hashlib.sha256(str(receipt_path).encode("utf-8")).hexdigest() + return receipt_path.parent / ".negpy-locks" / f"{lock_name}.lock" + + +@contextmanager +def _hold_frame_locks( + outputs: Sequence[RollFrameOutput], + *, + root: Path, +) -> Iterator[tuple[Path, ...]]: + """Hold every production frame lock for the entire audit snapshot.""" + + if _fcntl is None: + raise DeepAcceptanceError("safe frame locking is unavailable") + lock_paths: set[Path] = set() + for index, output in enumerate(outputs, start=1): + if type(output) is not RollFrameOutput or type(output.receipt_path) is not str: + _fail(f"frame {index}", "RollFrameOutput identity is wrong") + receipt_path = _regular_file( + output.receipt_path, + root=root, + label=f"frame {index} receipt", + ) + lock_paths.add( + _regular_file( + _receipt_lock_path(receipt_path), + root=root, + label=f"frame {index} receipt lock", + ) + ) + + held: list[tuple[Path, int, tuple[int, int]]] = [] + try: + for path in sorted(lock_paths): + before = path.lstat() + flags = os.O_RDWR | getattr(os, "O_CLOEXEC", 0) | getattr(os, "O_NOFOLLOW", 0) + descriptor = os.open(path, flags) + try: + opened = os.fstat(descriptor) + identity = (opened.st_dev, opened.st_ino) + if not stat.S_ISREG(opened.st_mode) or identity != (before.st_dev, before.st_ino): + raise DeepAcceptanceError(f"frame lock changed while opening: {path}") + try: + _fcntl.flock(descriptor, _fcntl.LOCK_EX | _fcntl.LOCK_NB) + except BlockingIOError as error: + raise DeepAcceptanceError(f"frame output is busy in another process: {path}") from error + visible = path.lstat() + if identity != (visible.st_dev, visible.st_ino): + raise DeepAcceptanceError(f"frame lock changed while acquiring ownership: {path}") + except BaseException: + os.close(descriptor) + raise + held.append((path, descriptor, identity)) + yield tuple(path for path, _, _ in held) + for path, _, identity in held: + visible = path.lstat() + if not stat.S_ISREG(visible.st_mode) or identity != (visible.st_dev, visible.st_ino): + raise DeepAcceptanceError(f"frame lock changed during deep acceptance: {path}") + except OSError as error: + raise DeepAcceptanceError(f"cannot hold frame output locks: {error}") from error + finally: + for _, descriptor, _ in reversed(held): + try: + _fcntl.flock(descriptor, _fcntl.LOCK_UN) + finally: + os.close(descriptor) + + +def _require_receipt_copy( + document: object, + digest: object, + expected_document: dict[str, Any], + expected_digest: str, + *, + label: str, +) -> None: + if document != expected_document: + _fail(label, "payload differs from a fresh production evaluation") + if digest != expected_digest or _stored_receipt_hash(document) != expected_digest: + _fail(label, "SHA-256 differs from a fresh production evaluation") + + +def _runtime_receipt_binding( + receipt: dict[str, Any], + *, + acquisition: RepairAcquisition, + repaired: dict[str, Any], + runtime: _HybridRuntime, + label: str, +) -> None: + inputs = _require_dict(receipt.get("inputs"), label=f"{label} inputs") + main = _require_dict(inputs.get("main"), label=f"{label} main input") + prepass = _require_dict(inputs.get("prepass"), label=f"{label} prepass input") + geometry = _require_dict(inputs.get("geometry"), label=f"{label} geometry") + provenance = _require_dict(inputs.get("provenance"), label=f"{label} provenance") + main_rgbi = np.asarray(getattr(acquisition, "main_rgbi")) + prepass_rgbi = np.asarray(getattr(acquisition, "prepass_rgbi")) + if main != { + "canonical_encoding": "uint16_little_endian_c_order", + "raw_sha256": getattr(acquisition, "main_rgbi_sha256"), + "shape": list(main_rgbi.shape), + }: + _fail(label, "main input does not bind the retained acquisition") + if prepass != { + "canonical_encoding": "uint16_little_endian_c_order", + "raw_sha256": getattr(acquisition, "prepass_rgbi_sha256"), + "shape": list(prepass_rgbi.shape), + }: + _fail(label, "prepass input does not bind the retained acquisition") + if geometry.get("mask_shape") != list(main_rgbi.shape[:2]) or geometry.get("output_shape") != [*main_rgbi.shape[:2], 3]: + _fail(label, "geometry does not bind the retained acquisition") + assertion = { + "assertions": { + "focus_exposure_locked": True, + "same_frame_id": getattr(acquisition, "acquisition_id"), + }, + "inputs": { + "main": {"raw_sha256": getattr(acquisition, "main_rgbi_sha256")}, + "prepass": {"raw_sha256": getattr(acquisition, "prepass_rgbi_sha256")}, + }, + "provenance_class": "caller_asserted_bare_npy", + "schema": "negpy.fauxce-hybrid-acquisition-assertion-v1", + } + if provenance.get("basis") != "caller_asserted" or provenance.get("source_manifest_sha256") != _sha256(_canonical_json(assertion)): + _fail(label, "acquisition assertion binding changed") + + core = _require_dict(receipt.get("core"), label=f"{label} core") + backend = _require_dict(core.get("backend"), label=f"{label} backend") + generation = _require_dict(receipt.get("generation"), label=f"{label} generation") + if core.get("source_manifest_sha256") != getattr(runtime, "core_source_manifest_sha256") or generation.get( + "hybrid_source_manifest_sha256" + ) != getattr(runtime, "hybrid_source_manifest_sha256"): + _fail(label, "source manifests differ from the pinned runtime") + if core.get("version") != repaired.get("engine_version"): + _fail(label, "Hybrid core version differs from the repaired receipt") + if ( + backend.get("requested") != repaired.get("backend_requested") + or backend.get("used") != repaired.get("backend_used") + or backend.get("reason") != repaired.get("backend_selection_reason") + ): + _fail(label, "backend disclosure differs from the repaired receipt") + + inpainting = _require_dict(receipt.get("inpainting"), label=f"{label} inpainting") + if type(inpainting.get("invoked")) is not bool: + _fail(label, "inpainting invocation disclosure is malformed") + if inpainting["invoked"]: + model = _require_dict(inpainting.get("model"), label=f"{label} model") + tool = _require_dict(inpainting.get("tool"), label=f"{label} tool") + run = _require_dict(inpainting.get("runtime"), label=f"{label} runtime") + if ( + model.get("weights_sha256") != getattr(runtime, "model_weights_sha256") + or tool.get("iopaint_source_manifest_sha256") != getattr(runtime, "iopaint_source_manifest_sha256") + or run.get("device") != getattr(runtime, "inpaint_device") + or run.get("threads") != getattr(runtime, "inpaint_threads") + or run.get("seed") != getattr(runtime, "inpaint_seed") + ): + _fail(label, "inpainting runtime differs from the pinned runtime") + + +def _hybrid_result( + repaired: dict[str, Any], + *, + acquisition: RepairAcquisition, + repaired_rgb: np.ndarray, + output: RollFrameOutput, + root: Path, + runtime: _HybridRuntime, + referenced: set[Path], + label: str, +) -> RepairResult: + disclosure = _require_dict(repaired.get("disclosure_mask"), label=f"{label} disclosure mask") + applied = _require_dict(disclosure.get("applied_final"), label=f"{label} applied mask") + routed = _require_dict(disclosure.get("routed_raw"), label=f"{label} routed mask") + storage_row = _require_dict(applied.get("storage"), label=f"{label} storage mask") + native_row = _require_dict(applied.get("native"), label=f"{label} native mask") + routed_row = _require_dict(routed.get("native"), label=f"{label} routed native mask") + hybrid_row = _require_dict(repaired.get("hybrid_receipt"), label=f"{label} Hybrid receipt") + binding_row = _require_dict( + repaired.get("hybrid_evidence_binding"), + label=f"{label} Hybrid binding", + ) + + storage_path = _published_path( + output, + "synthesis_mask_path", + storage_row.get("path"), + root=root, + label=f"{label} storage mask", + ) + native_path = _published_path( + output, + "native_synthesis_mask_path", + native_row.get("path"), + root=root, + label=f"{label} native mask", + ) + receipt_path = _published_path( + output, + "hybrid_receipt_path", + hybrid_row.get("path"), + root=root, + label=f"{label} Hybrid receipt", + ) + storage_path, storage_png = _artifact_row( + storage_row, + root=root, + label=f"{label} storage mask", + expected_path=storage_path, + ) + native_path, native_png = _artifact_row( + native_row, + root=root, + label=f"{label} native mask", + expected_path=native_path, + ) + routed_path, routed_png = _artifact_row( + routed_row, + root=root, + label=f"{label} routed native mask", + ) + receipt_path, receipt_bytes = _artifact_row( + hybrid_row, + root=root, + label=f"{label} Hybrid receipt", + expected_path=receipt_path, + maximum_bytes=_MAX_RECEIPT_BYTES, + ) + binding_path, binding_bytes = _artifact_row( + binding_row, + root=root, + label=f"{label} Hybrid binding", + maximum_bytes=_MAX_RECEIPT_BYTES, + ) + evidence_directory = receipt_path.parent + hybrid_digest = _require_sha256(hybrid_row.get("sha256"), label=f"{label} Hybrid receipt SHA-256") + expected_hidden_paths = { + receipt_path: "hybrid-receipt.json", + native_path: "synth-mask-applied-scanner-native.png", + routed_path: "synth-mask-routed-scanner-native.png", + binding_path: "negpy-binding.json", + } + if evidence_directory.name != hybrid_digest or any( + path.parent != evidence_directory or path.name != expected_name for path, expected_name in expected_hidden_paths.items() + ): + _fail(label, "Hybrid evidence escaped its content-addressed directory") + referenced.update((storage_path, native_path, routed_path, receipt_path, binding_path)) + + try: + receipt_document = roll_service._strict_json_loads( # noqa: SLF001 + receipt_bytes + ) + binding_document = roll_service._strict_json_loads( # noqa: SLF001 + binding_bytes + ) + except (UnicodeDecodeError, ValueError, TypeError) as error: + raise DeepAcceptanceError(f"{label}: retained Hybrid JSON is invalid: {error}") from error + receipt_document = _require_dict(receipt_document, label=f"{label} Hybrid receipt") + binding_document = _require_dict(binding_document, label=f"{label} Hybrid binding") + if _canonical_json(receipt_document, newline=True, ensure_ascii=True) != receipt_bytes: + _fail(label, "Hybrid receipt is not canonical") + if _canonical_json(binding_document) != binding_bytes: + _fail(label, "Hybrid retained binding is not canonical") + + engine = _require_str(repaired.get("engine"), label=f"{label} repair engine") + engine_version = _require_str(repaired.get("engine_version"), label=f"{label} repair engine version") + reason = _require_str(repaired.get("reason"), label=f"{label} repair reason") + try: + result = RepairResult( + rgb=repaired_rgb, + engine=engine, + engine_version=engine_version, + mode_requested=RepairMode(repaired.get("mode_requested")), + mode_resolved=RepairMode(repaired.get("mode_resolved")), + reason=reason, + acquisition_id=repaired.get("acquisition_id"), + slot=repaired.get("slot"), + reservation_id=repaired.get("reservation_id"), + evidence_sha256=repaired.get("evidence_sha256"), + backend_requested=repaired.get("backend_requested"), + backend_used=repaired.get("backend_used"), + backend_selection_reason=repaired.get("backend_selection_reason"), + native_output_rgb_sha256=repaired.get("native_output_rgb_sha256"), + storage_output_rgb_sha256=repaired.get("storage_output_rgb_sha256"), + native_synthesis_mask_png=native_png, + native_synthesis_mask_sha256=native_row.get("sha256"), + native_synthesis_mask_shape=tuple(native_row.get("shape", ())), + routed_native_synthesis_mask_png=routed_png, + routed_native_synthesis_mask_sha256=routed_row.get("sha256"), + routed_native_synthesis_mask_shape=tuple(routed_row.get("shape", ())), + storage_synthesis_mask_png=storage_png, + storage_synthesis_mask_sha256=storage_row.get("sha256"), + storage_synthesis_mask_shape=tuple(storage_row.get("shape", ())), + synthesis_mask_transform=applied.get("transform"), + synthesis_fraction=applied.get("fraction"), + routing_counts=routed.get("routing_counts"), + hybrid_receipt=receipt_bytes, + hybrid_receipt_sha256=hybrid_row.get("sha256"), + hybrid_provenance_class=hybrid_row.get("provenance_class"), + hybrid_receipt_output_rgb_sha256=hybrid_row.get("verified_output_rgb_sha256"), + ) + except (TypeError, ValueError) as error: + raise DeepAcceptanceError(f"{label}: Hybrid result receipt is malformed: {error}") from error + try: + roll_service._validate_repair_result_binding( # noqa: SLF001 + acquisition, + result, + requested_mode=RepairMode.HYBRID, + ) + except (TypeError, ValueError) as error: + raise DeepAcceptanceError(f"{label}: Hybrid result binding failed: {error}") from error + if result.engine != "digital-fauxice" or result.degraded: + _fail(label, "repair was not non-degraded digital-fauxice Hybrid") + try: + applied_mask = roll_service._decode_binary_mask( # noqa: SLF001 + native_png, + expected_shape=acquisition.main_rgbi.shape[:2], + label=f"{label} final scanner-native disclosure mask", + ) + except (OSError, TypeError, ValueError) as error: + raise DeepAcceptanceError(f"{label}: final disclosure mask could not be decoded: {error}") from error + applied_pixel_count = int(np.count_nonzero(applied_mask)) + del applied_mask + if applied.get("pixel_count") != applied_pixel_count: + _fail(label, "outer synthesis pixel count differs from the decoded mask") + + expected_binding = { + "acquisition": { + "acquisition_id": getattr(acquisition, "acquisition_id"), + "capture_attempt_id": getattr(acquisition, "capture_attempt_id"), + "evidence_sha256": getattr(acquisition, "evidence_sha256"), + "ir_validity_sha256": getattr(acquisition, "ir_validity_sha256"), + "main_rgbi_sha256": getattr(acquisition, "main_rgbi_sha256"), + "prepass_rgbi_sha256": getattr(acquisition, "prepass_rgbi_sha256"), + "reservation_id": getattr(acquisition, "reservation_id"), + "slot": getattr(acquisition, "slot"), + "storage_transform": getattr(acquisition, "storage_transform"), + }, + "hybrid_receipt_sha256": result.hybrid_receipt_sha256, + "hybrid_receipt_output_rgb_sha256": result.hybrid_receipt_output_rgb_sha256, + "native_output_rgb_sha256": result.native_output_rgb_sha256, + "native_synthesis_mask_sha256": result.native_synthesis_mask_sha256, + "routed_native_synthesis_mask_sha256": result.routed_native_synthesis_mask_sha256, + "provenance_class": result.hybrid_provenance_class, + "schema": "negpy.dice-hybrid-retained-evidence-v2", + "storage_output_rgb_sha256": result.storage_output_rgb_sha256, + "storage_synthesis_mask_sha256": result.storage_synthesis_mask_sha256, + "synthesis_mask_transform": result.synthesis_mask_transform, + } + if binding_document != expected_binding: + _fail(label, "Hybrid retained binding changed") + _runtime_receipt_binding( + receipt_document, + acquisition=acquisition, + repaired=repaired, + runtime=runtime, + label=label, + ) + return result + + +def _retained_native_paths( + retained_value: object, + *, + root: Path, + label: str, + referenced: set[Path], +) -> Path: + retained = _require_dict(retained_value, label=label) + if retained.get("scope") != exact_color.NATIVE_BUILDER_SCOPE or retained.get("native_per_acquisition_builder") is not True: + _fail(label, "is not native per-acquisition builder evidence") + receipt_path, _ = _artifact_row(retained.get("builder_receipt"), root=root, label=f"{label} receipt") + if receipt_path.name != "native-builder-receipt.json": + _fail(label, "native builder receipt filename changed") + if receipt_path.parent.parent != root / ".negpy-native-builder": + _fail(label, "native builder evidence escaped its production directory") + referenced.add(receipt_path) + directory = receipt_path.parent + expected_rows = { + "analyzer_rgb": directory / "analyzer-rgb-u16le.bin", + "evidence_receipt": directory / "native-builder-evidence.json", + "frame_ownership_receipt": (directory / "nikon-density-frame-ownership.json"), + "density_evidence_receipt": directory / "nikon-density-evidence.json", + } + for key, expected_path in expected_rows.items(): + path, _ = _artifact_row( + retained.get(key), + root=root, + label=f"{label} {key}", + expected_path=expected_path, + ) + referenced.add(path) + luts = retained.get("pre_f_luts") + if type(luts) is not list or len(luts) != 3: + _fail(label, "must retain exactly three pre-F LUTs") + for index, (channel, row) in enumerate(zip(("r", "g", "b"), luts, strict=True)): + row_document = _require_dict(row, label=f"{label} pre-F LUT {index}") + if row_document.get("channel") != channel: + _fail(label, "pre-F LUT channel order changed") + path, _ = _artifact_row( + row_document, + root=root, + label=f"{label} pre-F LUT {index}", + expected_path=directory / f"builder-preF-{channel}.bin", + ) + referenced.add(path) + return receipt_path + + +def _validate_frame( + output: RollFrameOutput, + *, + root: Path, + expected_slot: int, + builder: PortableStage1Builder, + evaluator: PortableCMSOnEvaluator, + runtime: _HybridRuntime, +) -> _FrameAudit: + label = f"slot {expected_slot}" + receipt, receipt_bytes, completed_files = _collect_completed_frame_files_locked( + output, + root=root, + expected_slot=expected_slot, + ) + receipt_path = _regular_file(output.receipt_path, root=root, label=f"{label} frame receipt") + receipt_lock_path = _regular_file( + _receipt_lock_path(receipt_path), + root=root, + label=f"{label} frame receipt lock", + ) + smear = receipt.get("transport_smear") + if not isinstance(smear, dict): + # Compatibility with an early compact receipt spelling. + if receipt.get("transport_smear_verdict") != "clean": + _fail(label, "transport-smear verdict is not clean") + elif smear.get("verdict") != "clean": + _fail(label, "transport-smear verdict is not clean") + + outputs = _require_dict(receipt.get("outputs"), label=f"{label} outputs") + unrepaired = _require_dict(outputs.get("unrepaired"), label=f"{label} unrepaired tier") + repaired = _require_dict(outputs.get("repaired"), label=f"{label} repaired tier") + positive = _require_dict(outputs.get("positive"), label=f"{label} positive tier") + if any(entry.get("written") is not True for entry in (unrepaired, repaired, positive)): + _fail(label, "all three tiers were not written") + if repaired.get("mode_requested") != "hybrid" or repaired.get("mode_resolved") != "hybrid" or repaired.get("degraded") is not False: + _fail(label, "repair was not non-degraded Hybrid") + if ( + positive.get("color_mode") != "nikon-exact" + or positive.get("exact_nikon_color") is not True + or positive.get("inversion_path") != "native-per-acquisition-builder-and-verified-portable-cms" + or positive.get("native_per_acquisition_builder") is not True + or positive.get("native_builder_scope") != exact_color.NATIVE_BUILDER_SCOPE + or positive.get("builder_validated") is not True + or positive.get("cms_verified") is not True + ): + _fail(label, "positive is not validated native Nikon exact color") + + rgb_path = _published_path( + output, + "rgb_path", + unrepaired.get("rgb_path"), + root=root, + label=f"{label} RGB", + ) + ir_path = _published_path( + output, + "ir_path", + unrepaired.get("ir_path"), + root=root, + label=f"{label} IR", + ) + repaired_rgb_path = _published_path( + output, + "repaired_rgb_path", + repaired.get("rgb_path"), + root=root, + label=f"{label} repaired RGB", + ) + repaired_ir_path = _published_path( + output, + "repaired_ir_path", + repaired.get("ir_path"), + root=root, + label=f"{label} repaired IR", + ) + positive_path = _published_path( + output, + "positive_path", + positive.get("rgb_path"), + root=root, + label=f"{label} positive", + ) + referenced = { + receipt_path, + receipt_lock_path, + rgb_path, + ir_path, + repaired_rgb_path, + repaired_ir_path, + positive_path, + } + + dice = _require_dict( + outputs.get("repair_acquisition_evidence"), + label=f"{label} DICE acquisition evidence", + ) + if dice.get("retained") is not True or dice.get("replayable") is not True or dice.get("schema") != "negpy.dice-acquisition-replay-v1": + _fail(label, "DICE acquisition evidence is not retained and replayable") + binding_path, binding_bytes = _artifact_row( + dice.get("binding"), + root=root, + label=f"{label} DICE acquisition binding", + maximum_bytes=_MAX_RECEIPT_BYTES, + ) + referenced.add(binding_path) + _validate_dice_archive_paths( + dice, + binding_path=binding_path, + binding_bytes=binding_bytes, + root=root, + rgb_path=rgb_path, + ir_path=ir_path, + referenced=referenced, + label=label, + ) + try: + acquisition = roll_service.load_repair_acquisition_evidence(binding_path) + except (OSError, TypeError, ValueError) as error: + raise DeepAcceptanceError(f"{label}: DICE replay failed: {error}") from error + # Production frame validation is also used for ordinary dated output + # names. The live-acceptance inventory separately enforces its fixed + # ``acceptance_slotNN`` pattern; the archive itself is content-addressed + # against whichever basename RollScanningService actually published. + expected_dice_token = _sha256( + f"{acquisition.acquisition_id}\0{rgb_path.stem}".encode("utf-8") + ) + if binding_path.parent.name != expected_dice_token: + _fail(label, "DICE evidence directory is not content-addressed") + acquisition_entry = _require_dict(repaired.get("acquisition"), label=f"{label} repaired acquisition") + expected_acquisition = { + "acquisition_id": acquisition.acquisition_id, + "slot": acquisition.slot, + "reservation_id": acquisition.reservation_id, + "capture_attempt_id": acquisition.capture_attempt_id, + "evidence_sha256": acquisition.evidence_sha256, + "storage_transform": acquisition.storage_transform, + "main_rgbi_sha256": acquisition.main_rgbi_sha256, + "prepass_rgbi_sha256": acquisition.prepass_rgbi_sha256, + "ir_validity_sha256": acquisition.ir_validity_sha256, + } + if acquisition_entry != expected_acquisition: + _fail(label, "repaired acquisition differs from retained DICE evidence") + + storage_shape = ( + acquisition.main_rgbi.shape[1], + acquisition.main_rgbi.shape[0], + ) + rgb = _decode_tiff(rgb_path, (*storage_shape, 3), label=f"{label} RGB") + infrared = _decode_tiff(ir_path, storage_shape, label=f"{label} IR") + expected_storage = np.rot90(acquisition.main_rgbi, k=1, axes=(0, 1)) + if not np.array_equal(rgb, expected_storage[..., :3]) or not np.array_equal(infrared, expected_storage[..., 3]): + _fail(label, "Tier-1 TIFFs differ from the replayed DICE acquisition") + artifacts = _require_dict(receipt.get("artifacts"), label=f"{label} artifacts") + if set(artifacts) != {"rgb", "ir"}: + _fail(label, "frame artifact inventory is not exactly RGB and IR") + _array_artifact_matches(artifacts.get("rgb"), rgb, label=f"{label} RGB artifact") + _array_artifact_matches(artifacts.get("ir"), infrared, label=f"{label} IR artifact") + del rgb, expected_storage + + repaired_rgb = _decode_tiff(repaired_rgb_path, (*storage_shape, 3), label=f"{label} repaired RGB") + repaired_ir = _decode_tiff(repaired_ir_path, storage_shape, label=f"{label} repaired IR") + if not np.array_equal(repaired_ir, infrared): + _fail(label, "repaired IR is not the unchanged Tier-1 IR") + del repaired_ir, infrared + + repair_result = _hybrid_result( + repaired, + acquisition=acquisition, + repaired_rgb=repaired_rgb, + output=output, + root=root, + runtime=runtime, + referenced=referenced, + label=label, + ) + if ( + positive.get("repair_engine") != repair_result.engine + or positive.get("repair_engine_version") != repair_result.engine_version + or positive.get("repair_mode") != str(repair_result.mode_resolved) + ): + _fail(label, "positive repair provenance differs from the repaired tier") + acquisition_id = acquisition.acquisition_id + acquisition_reservation_id = acquisition.reservation_id + acquisition_capture_attempt_id = acquisition.capture_attempt_id + repair_storage_output_rgb_sha256 = repair_result.storage_output_rgb_sha256 + hybrid_receipt_sha256 = repair_result.hybrid_receipt_sha256 + del repair_result, acquisition + + native_evidence = _require_dict( + outputs.get("native_color_evidence"), + label=f"{label} native color evidence", + ) + if ( + native_evidence.get("retained") is not True + or native_evidence.get("native_per_acquisition_builder") is not True + or native_evidence.get("scope") != exact_color.NATIVE_BUILDER_SCOPE + ): + _fail(label, "native color evidence was not retained") + retained = native_evidence.get("retained_builder_evidence") + receipt_file = _retained_native_paths( + retained, + root=root, + label=f"{label} native builder evidence", + referenced=referenced, + ) + if positive.get("retained_builder_evidence") != retained: + _fail(label, "positive and native evidence reference different builders") + try: + builder_receipt = exact_color.load_native_builder_receipt(receipt_file) + except exact_color.ExactColorUnavailable as error: + raise DeepAcceptanceError(f"{label}: native builder reload failed: {error}") from error + if ( + builder_receipt.slot != expected_slot + or builder_receipt.reservation_id != acquisition_reservation_id + or builder_receipt.capture_attempt_id != acquisition_capture_attempt_id + ): + _fail(label, "native builder belongs to another acquisition") + expected_builder_document = exact_color.builder_receipt_payload(builder_receipt) + if ( + native_evidence.get("builder_receipt") != expected_builder_document + or native_evidence.get("builder_receipt_sha256") != builder_receipt.sha256 + ): + _fail(label, "native color evidence does not bind the reloaded builder") + + try: + color = exact_color.evaluate_exact_color( + repaired_rgb, + builder_receipt=builder_receipt, + builder=builder, + evaluator=evaluator, + ) + except exact_color.ExactColorUnavailable as error: + raise DeepAcceptanceError(f"{label}: exact color replay failed: {error}") from error + application = color.builder_application_receipt + if application is None: + _fail(label, "exact color replay omitted its builder application receipt") + _require_receipt_copy( + positive.get("builder_receipt"), + positive.get("builder_receipt_sha256"), + exact_color.receipt_payload(color.builder_receipt), + color.builder_receipt.sha256, + label=f"{label} builder receipt", + ) + _require_receipt_copy( + positive.get("builder_application_receipt"), + positive.get("builder_application_receipt_sha256"), + exact_color.receipt_payload(application), + application.sha256, + label=f"{label} builder application receipt", + ) + _require_receipt_copy( + positive.get("cms_receipt"), + positive.get("cms_receipt_sha256"), + exact_color.receipt_payload(color.cms_receipt), + color.cms_receipt.sha256, + label=f"{label} CMS receipt", + ) + if ( + positive.get("repaired_input_rgb_sha256") != color.source_rgb_sha256 + or positive.get("input_rgb_sha256") != color.input_rgb_sha256 + or positive.get("stage1_input_rgb_sha256") != color.input_rgb_sha256 + or positive.get("output_rgb_sha256") != color.output_rgb_sha256 + or repaired.get("storage_output_rgb_sha256") != repair_storage_output_rgb_sha256 + ): + _fail(label, "color or repair content hashes changed") + try: + profile = nikon_icc.nikon_adobe_rgb_profile() + except nikon_icc.NikonICCProfileError as error: + raise DeepAcceptanceError(f"{label}: pinned Nikon ICC profile is unavailable: {error}") from error + try: + tiff_binding = roll_service._verify_exact_positive_tiff( # noqa: SLF001 + str(positive_path), + expected_rgb=color.rgb, + expected_icc=profile, + ) + except exact_color.ExactColorIntegrityError as error: + raise DeepAcceptanceError(f"{label}: exact positive TIFF failed: {error}") from error + if tiff_binding != positive.get("tiff_artifact"): + _fail(label, "positive TIFF binding differs from its sidecar") + if positive.get("icc_profile") != nikon_icc.profile_receipt_binding(): + _fail(label, "positive ICC receipt differs from the pinned Nikon profile") + + try: + from coolscanpy.protocol.ls5000_single_pass.density import ( + NikonDensityFrameOwnershipReceipt, + ) + + ownership_document = roll_service._strict_json_loads( # noqa: SLF001 + builder_receipt.frame_ownership_receipt + ) + ownership = NikonDensityFrameOwnershipReceipt.from_dict(ownership_document) + except (ImportError, TypeError, ValueError) as error: + raise DeepAcceptanceError(f"{label}: density ownership is invalid: {error}") from error + outer_ownership = receipt.get("nikon_density_ownership") + compact_ownership = { + "reservation_id": ownership.reservation_id, + "batch_session_id": ownership.batch_session_id, + "preview_sha256": ownership.preview_sha256, + "preview_identity_sha256": ownership.preview_identity_sha256, + "transport_table_sha256": ownership.transport_table_sha256, + "reviewed_fingerprint_sha256": ownership.reviewed_fingerprint_sha256, + "fresh_fingerprint_sha256": ownership.fresh_fingerprint_sha256, + "frame_capture_attempt_id": ownership.frame_capture_attempt_id, + "frame_index": ownership.frame_index, + "frame_total": ownership.frame_total, + "selected_slots": list(ownership.selected_slots), + "selected_slot": ownership.selected_slot, + } + if outer_ownership not in (compact_ownership, ownership.to_dict()): + _fail(label, "public frame receipt and retained density ownership disagree") + if ( + ownership.selected_slot != expected_slot + or ownership.reservation_id != acquisition_reservation_id + or ownership.frame_capture_attempt_id != acquisition_capture_attempt_id + or receipt.get("reviewed_fingerprint_sha256") != ownership.reviewed_fingerprint_sha256 + or receipt.get("fresh_fingerprint_sha256") != ownership.fresh_fingerprint_sha256 + ): + _fail(label, "frame, acquisition, and density ownership disagree") + if ( + builder_receipt.preview_sha256 != ownership.preview_sha256 + or builder_receipt.preview_identity_sha256 != ownership.preview_identity_sha256 + or builder_receipt.batch_session_id != ownership.batch_session_id + ): + _fail(label, "native builder and density ownership disagree") + + if referenced != set(completed_files): + missing = sorted(str(path) for path in set(completed_files) - referenced) + extra = sorted(str(path) for path in referenced - set(completed_files)) + raise DeepAcceptanceError(f"{label}: deep artifact inventory changed (missing={missing}, extra={extra})") + output_artifacts: dict[str, str] = {} + for field in _OUTPUT_FIELDS: + path = getattr(output, field) + if type(path) is not str: + _fail(label, "RollFrameOutput omitted a completed artifact") + output_artifacts[field] = path + + return _FrameAudit( + summary={ + "slot": expected_slot, + "frame_receipt": { + "path": str(receipt_path), + "bytes": len(receipt_bytes), + "sha256": _sha256(receipt_bytes), + }, + "acquisition_id": acquisition_id, + "reservation_id": acquisition_reservation_id, + "capture_attempt_id": acquisition_capture_attempt_id, + "transport_identity_sha256": ownership.transport_identity_sha256, + "preview_identity_sha256": ownership.preview_identity_sha256, + "builder_receipt_sha256": builder_receipt.sha256, + "cms_receipt_sha256": color.cms_receipt.sha256, + "positive_file_sha256": tiff_binding["file_sha256"], + "hybrid_receipt_sha256": hybrid_receipt_sha256, + "referenced_file_count": len(referenced), + "referenced_files": sorted(str(path) for path in referenced), + }, + referenced_files=frozenset(referenced), + receipt_path=receipt_path, + receipt=receipt, + ownership=ownership, + builder_receipt=builder_receipt, + output_artifacts=output_artifacts, + ) + + +def validate_completed_frame( + output: RollFrameOutput, + *, + output_dir: str | os.PathLike[str], + expected_slot: int | None = None, + builder: PortableStage1Builder | None = None, + evaluator: PortableCMSOnEvaluator | None = None, + hybrid_runtime: _HybridRuntime | None = None, +) -> dict[str, Any]: + """Deeply validate one already-published frame without touching hardware.""" + + checked_output, slot = _public_output_slot( + output, + expected_slot=expected_slot, + ) + root = _output_root(output_dir) + runtime = _validated_runtime(hybrid_runtime) + active_builder, active_evaluator = _color_dependencies(builder, evaluator) + with _hold_frame_locks((checked_output,), root=root): + audit = _validate_frame( + checked_output, + root=root, + expected_slot=slot, + builder=active_builder, + evaluator=active_evaluator, + runtime=runtime, + ) + return { + "schema": SCHEMA, + "status": "passed", + "scope": "frame", + "output_dir": str(root), + **audit.summary, + } + + +def _validate_manual_approval( + frame: _FrameAudit, + *, + expected: bool | None, +) -> dict[str, Any] | None: + slot = frame.summary["slot"] + approval_value = frame.receipt.get("manual_approval") + if expected is not None and (approval_value is not None) is not expected: + _fail(f"slot {slot}", "manual approval presence is wrong") + if approval_value is None: + return None + approval = _require_dict(approval_value, label=f"slot {slot} manual approval") + if set(approval) != { + "reviewed_fingerprint_sha256", + "slot", + "spacing_offset", + "thumbnail_sha256", + "reviewed_lookup_row", + "reviewed_native_origin", + "review_reasons", + }: + _fail(f"slot {slot}", "manual approval fields changed") + review_reasons = approval.get("review_reasons") + if type(review_reasons) is not list or not review_reasons or any(type(reason) is not str or not reason for reason in review_reasons): + _fail(f"slot {slot}", "manual approval reasons are malformed") + try: + from coolscanpy.types import ApprovalReceipt + + parsed = ApprovalReceipt( + reviewed_fingerprint_sha256=approval["reviewed_fingerprint_sha256"], + slot=approval["slot"], + spacing_offset=approval["spacing_offset"], + thumbnail_sha256=approval["thumbnail_sha256"], + reviewed_lookup_row=approval["reviewed_lookup_row"], + reviewed_native_origin=approval["reviewed_native_origin"], + review_reasons=tuple(review_reasons), + ) + except (ImportError, KeyError, TypeError, ValueError) as error: + raise DeepAcceptanceError(f"slot {slot}: manual approval is invalid: {error}") from error + ownership = frame.ownership + if ( + parsed.slot != slot + or parsed.spacing_offset != frame.receipt.get("spacing_offset") + or parsed.reviewed_fingerprint_sha256 != getattr(ownership, "reviewed_fingerprint_sha256") + ): + _fail(f"slot {slot}", "manual approval does not bind the reviewed frame") + return { + "binding_sha256": parsed.binding_sha256, + "boundary_offset_rows": parsed.spacing_offset, + "review_reasons": list(parsed.review_reasons), + "reviewed_fingerprint_sha256": parsed.reviewed_fingerprint_sha256, + "reviewed_lookup_row": parsed.reviewed_lookup_row, + "reviewed_native_origin": parsed.reviewed_native_origin, + "schema_version": 1, + "slot": parsed.slot, + "thumbnail_sha256": parsed.thumbnail_sha256, + } + + +def _inventory( + root: Path, + frames: Sequence[_FrameAudit], + *, + allowed_output_lock_name: str | None, +) -> dict[str, Any]: + expected_files = set().union(*(frame.referenced_files for frame in frames)) + if allowed_output_lock_name is not None: + if ( + type(allowed_output_lock_name) is not str + or not allowed_output_lock_name + or Path(allowed_output_lock_name).name != allowed_output_lock_name + or allowed_output_lock_name in {".", ".."} + ): + raise DeepAcceptanceError("allowed output lock name must be one safe basename") + expected_files.add(root / allowed_output_lock_name) + + found_files: set[Path] = set() + found_directories = {root} + try: + for directory, names, files in os.walk(root, topdown=True, followlinks=False): + parent = Path(directory) + for name in names: + child = parent / name + metadata = child.lstat() + if stat.S_ISLNK(metadata.st_mode) or not stat.S_ISDIR(metadata.st_mode): + _fail("inventory", f"directory entry is unsafe: {child}") + found_directories.add(child.resolve(strict=True)) + for name in files: + child = parent / name + metadata = child.lstat() + if is_ignorable_finder_metadata_file(name, metadata): + continue + if stat.S_ISLNK(metadata.st_mode) or not stat.S_ISREG(metadata.st_mode): + _fail("inventory", f"file entry is unsafe: {child}") + found_files.add(child.resolve(strict=True)) + except OSError as error: + raise DeepAcceptanceError(f"inventory walk failed: {error}") from error + + try: + expected_files = {path.resolve(strict=True) for path in expected_files} + except OSError as error: + raise DeepAcceptanceError(f"inventory expected file is unavailable: {error}") from error + expected_directories = {root} + for path in expected_files: + parent = path.parent + while parent != root: + expected_directories.add(parent) + parent = parent.parent + expected_directories.add(root) + if found_files != expected_files: + missing = sorted(str(path.relative_to(root)) for path in expected_files - found_files) + extra = sorted(str(path.relative_to(root)) for path in found_files - expected_files) + raise DeepAcceptanceError(f"inventory file set changed (missing={missing}, extra={extra})") + if found_directories != expected_directories: + missing = sorted(str(path.relative_to(root)) for path in expected_directories - found_directories) + extra = sorted(str(path.relative_to(root)) for path in found_directories - expected_directories) + raise DeepAcceptanceError(f"inventory directory set changed (missing={missing}, extra={extra})") + allowed_lock_path = (root / allowed_output_lock_name).resolve(strict=True) if allowed_output_lock_name is not None else None + visible = [ + path for path in found_files if path != allowed_lock_path and not any(part.startswith(".") for part in path.relative_to(root).parts) + ] + if len(visible) != 42: + raise DeepAcceptanceError(f"visible inventory contains {len(visible)} files; expected 42") + return { + "regular_file_count": len(found_files), + "directory_count": len(found_directories), + "visible_file_count": len(visible), + "allowed_output_lock_path": (str(allowed_lock_path) if allowed_lock_path is not None else None), + "exact": True, + } + + +def _run_receipt_binding( + run_receipt_path: str | os.PathLike[str] | None, + *, + root: Path, + frames: Sequence[_FrameAudit], + approvals: dict[int, dict[str, Any]], +) -> dict[str, Any] | None: + if run_receipt_path is None: + return None + path = Path(run_receipt_path).absolute() + if path.is_relative_to(root): + raise DeepAcceptanceError("run receipt must be outside the output directory") + payload = _stable_bytes( + path, + maximum_bytes=_MAX_RECEIPT_BYTES, + label="live run receipt", + ) + try: + document = roll_service._strict_json_loads(payload) # noqa: SLF001 + except (UnicodeDecodeError, ValueError, TypeError) as error: + raise DeepAcceptanceError(f"live run receipt is invalid JSON: {error}") from error + document = _require_dict(document, label="live run receipt") + if _canonical_json(document, newline=True, ensure_ascii=True) != payload: + raise DeepAcceptanceError("live run receipt is not canonical JSON") + rows = document.get("frames") + settings = document.get("settings") + close = document.get("close") + output_lease = document.get("output_lease") + operation_state = document.get("operation_state") + deep_result = document.get("deep_acceptance") + expected_device_id = frames[0].receipt.get("device_id") if frames else None + approved_slots = sorted(approvals) + if ( + document.get("schema") != "negpy.ls5000-live-acceptance.v2" + or document.get("status") != "succeeded" + or document.get("phase") != "succeeded" + or document.get("device_id") != expected_device_id + or document.get("output_dir") != str(root) + or document.get("slots") != list(SLOTS) + or document.get("approved_slots") != approved_slots + or settings + != { + "write_unrepaired": True, + "write_repaired": True, + "write_positive": True, + "repair_mode": "hybrid", + "positive_mode": "nikon-exact", + "filename_pattern": 'acceptance_slot{{ "%02d" % seq }}', + } + or close + != { + "iterator": {"attempted": True, "succeeded": True}, + "roll": {"attempted": True, "succeeded": True}, + } + or type(output_lease) is not dict + or output_lease.get("acquired") is not True + or output_lease.get("release_attempted") is not True + or output_lease.get("released") is not True + or type(operation_state) is not dict + or operation_state.get("batch_exhausted") is not True + or operation_state.get("verified_slots") != list(SLOTS) + or type(deep_result) is not dict + or deep_result.get("status") != "passed" + or deep_result.get("slots") != list(SLOTS) + or document.get("retry_count") != 0 + or document.get("eject_requested") is not False + or type(rows) is not list + or len(rows) != 6 + ): + raise DeepAcceptanceError("live run receipt did not record one successful six-frame run") + for frame, row_value in zip(frames, rows, strict=True): + row = _require_dict(row_value, label="live run frame") + artifacts = _require_dict(row.get("artifacts"), label="live run frame artifacts") + if ( + row.get("slot") != frame.summary["slot"] + or row.get("expected_slot") != frame.summary["slot"] + or row.get("frame_receipt_sha256") != frame.summary["frame_receipt"]["sha256"] + or artifacts != frame.output_artifacts + ): + _fail("live run receipt", "frame sidecar binding changed") + + reviewed_row = _require_dict( + document.get("reviewed_approval"), + label="live run reviewed approval", + ) + if set(reviewed_row) != { + "path", + "expected_sha256", + "verified_sha256", + "bytes", + "reviewed_fingerprint_sha256", + "contact_sheet", + }: + _fail("live run receipt", "reviewed-approval binding fields changed") + reviewed_path_value = reviewed_row.get("path") + if type(reviewed_path_value) is not str or not Path(reviewed_path_value).is_absolute(): + _fail("live run receipt", "reviewed-approval path is not absolute") + reviewed_path = Path(reviewed_path_value).absolute() + if reviewed_path.is_relative_to(root): + _fail("live run receipt", "reviewed approval must be outside output") + reviewed_bytes = _stable_bytes( + reviewed_path, + maximum_bytes=64 * 1024, + label="reviewed approval", + ) + reviewed_digest = _sha256(reviewed_bytes) + if ( + reviewed_row.get("bytes") != len(reviewed_bytes) + or reviewed_row.get("expected_sha256") != reviewed_digest + or reviewed_row.get("verified_sha256") != reviewed_digest + ): + _fail("live run receipt", "reviewed-approval content binding changed") + try: + reviewed_document = roll_service._strict_json_loads( # noqa: SLF001 + reviewed_bytes + ) + except (UnicodeDecodeError, ValueError, TypeError) as error: + raise DeepAcceptanceError(f"live run reviewed approval is invalid JSON: {error}") from error + reviewed_document = _require_dict( + reviewed_document, + label="live run reviewed approval", + ) + if _canonical_json(reviewed_document, newline=True, ensure_ascii=True) != reviewed_bytes: + _fail("live run receipt", "reviewed approval is not canonical JSON") + expected_approval_rows = [approvals[slot] for slot in approved_slots] + reviewed_contact = _require_dict( + reviewed_document.get("contact_sheet"), + label="reviewed contact sheet", + ) + if ( + reviewed_document.get("schema") != "negpy.ls5000-reviewed-approval.v1" + or reviewed_document.get("approvals") != expected_approval_rows + or reviewed_document.get("reviewed_fingerprint_sha256") != getattr(frames[0].ownership, "reviewed_fingerprint_sha256") + or reviewed_row.get("reviewed_fingerprint_sha256") != reviewed_document.get("reviewed_fingerprint_sha256") + or reviewed_row.get("contact_sheet") + != { + "path": reviewed_contact.get("path"), + "sha256": reviewed_contact.get("sha256"), + } + ): + _fail("live run receipt", "reviewed approvals differ from frame receipts") + return {"path": str(path), "bytes": len(payload), "sha256": _sha256(payload)} + + +def _validate_six_frame_batch_locked( + outputs: Sequence[RollFrameOutput], + *, + output_dir: str | os.PathLike[str], + run_receipt_path: str | os.PathLike[str] | None = None, + allowed_output_lock_name: str | None = None, + builder: PortableStage1Builder | None = None, + evaluator: PortableCMSOnEvaluator | None = None, + hybrid_runtime: _HybridRuntime | None = None, +) -> dict[str, Any]: + """Deeply validate one exact six-frame batch and its recursive inventory.""" + + if isinstance(outputs, (str, bytes, bytearray)): + raise DeepAcceptanceError("batch must contain exactly six RollFrameOutput values") + try: + frame_outputs = tuple(outputs) + except TypeError as error: + raise DeepAcceptanceError("batch must contain exactly six RollFrameOutput values") from error + if len(frame_outputs) != 6 or any(type(output) is not RollFrameOutput for output in frame_outputs): + raise DeepAcceptanceError("batch must contain exactly six RollFrameOutput values") + if tuple(output.slot for output in frame_outputs) != SLOTS: + raise DeepAcceptanceError("batch slots must be exactly 1,2,3,4,5,6 in order") + root = _output_root(output_dir) + runtime = _validated_runtime(hybrid_runtime) + active_builder, active_evaluator = _color_dependencies(builder, evaluator) + frames = tuple( + _validate_frame( + output, + root=root, + expected_slot=slot, + builder=active_builder, + evaluator=active_evaluator, + runtime=runtime, + ) + for slot, output in zip(SLOTS, frame_outputs, strict=True) + ) + + common = None + density_receipt_sha256 = None + device_identity = None + approval_rows: dict[int, dict[str, Any]] = {} + for index, frame in enumerate(frames, start=1): + ownership = frame.ownership + if ( + getattr(ownership, "frame_index") != index + or getattr(ownership, "frame_total") != 6 + or tuple(getattr(ownership, "selected_slots")) != SLOTS + or getattr(ownership, "selected_slot") != index + ): + _fail(f"slot {index}", "batch coordinates are wrong") + approval_row = _validate_manual_approval(frame, expected=None) + if approval_row is not None: + approval_rows[index] = approval_row + candidate = ( + getattr(ownership, "reservation_id"), + getattr(ownership, "batch_session_id"), + getattr(ownership, "preview_sha256"), + getattr(ownership, "preview_identity_sha256"), + getattr(ownership, "transport_table_sha256"), + getattr(ownership, "transport_identity_sha256"), + getattr(ownership, "reviewed_fingerprint_sha256"), + getattr(ownership, "fresh_fingerprint_sha256"), + ) + if common is None: + common = candidate + elif candidate != common: + raise DeepAcceptanceError("batch frames disagree on reservation, preview, transport, or fingerprint identity") + current_density = frame.builder_receipt.density_evidence_receipt_sha256 + if density_receipt_sha256 is None: + density_receipt_sha256 = current_density + elif current_density != density_receipt_sha256: + raise DeepAcceptanceError("batch frames disagree on Nikon density evidence") + current_device = ( + frame.receipt.get("device_id"), + frame.receipt.get("device_model"), + ) + if device_identity is None: + device_identity = current_device + elif current_device != device_identity: + raise DeepAcceptanceError("batch frames disagree on scanner identity") + + inventory = _inventory( + root, + frames, + allowed_output_lock_name=allowed_output_lock_name, + ) + run_receipt = _run_receipt_binding( + run_receipt_path, + root=root, + frames=frames, + approvals=approval_rows, + ) + assert common is not None + approved_slots = sorted(approval_rows) + result: dict[str, Any] = { + "schema": SCHEMA, + "status": "passed", + "scope": "six-frame-batch", + "output_dir": str(root), + "slots": list(SLOTS), + "approved_slots": approved_slots, + "reservation_id": common[0], + "preview_sha256": common[2], + "preview_identity_sha256": common[3], + "transport_identity_sha256": common[5], + "reviewed_fingerprint_sha256": common[6], + "fresh_fingerprint_sha256": common[7], + "density_evidence_receipt_sha256": density_receipt_sha256, + "frames": [frame.summary for frame in frames], + "referenced_files": sorted(str(path) for path in set().union(*(frame.referenced_files for frame in frames))), + "inventory": inventory, + } + if run_receipt is not None: + result["run_receipt"] = run_receipt + assert device_identity is not None + result.update( + device_id=device_identity[0], + device_model=device_identity[1], + manual_approval_bindings=[ + { + "slot": slot, + "binding_sha256": approval_rows[slot]["binding_sha256"], + } + for slot in approved_slots + ], + ) + return result + + +def validate_six_frame_batch( + outputs: Sequence[RollFrameOutput], + *, + output_dir: str | os.PathLike[str], + run_receipt_path: str | os.PathLike[str] | None = None, + allowed_output_lock_name: str | None = None, + builder: PortableStage1Builder | None = None, + evaluator: PortableCMSOnEvaluator | None = None, + hybrid_runtime: _HybridRuntime | None = None, +) -> dict[str, Any]: + """Deeply validate one locked, exact six-frame batch and its inventory.""" + + if isinstance(outputs, (str, bytes, bytearray)): + raise DeepAcceptanceError("batch must contain exactly six RollFrameOutput values") + try: + frame_outputs = tuple(outputs) + except TypeError as error: + raise DeepAcceptanceError("batch must contain exactly six RollFrameOutput values") from error + if len(frame_outputs) != 6 or any(type(output) is not RollFrameOutput for output in frame_outputs): + raise DeepAcceptanceError("batch must contain exactly six RollFrameOutput values") + if tuple(output.slot for output in frame_outputs) != SLOTS: + raise DeepAcceptanceError("batch slots must be exactly 1,2,3,4,5,6 in order") + root = _output_root(output_dir) + with _hold_frame_locks(frame_outputs, root=root): + return _validate_six_frame_batch_locked( + frame_outputs, + output_dir=root, + run_receipt_path=run_receipt_path, + allowed_output_lock_name=allowed_output_lock_name, + builder=builder, + evaluator=evaluator, + hybrid_runtime=hybrid_runtime, + ) + + +__all__ = [ + "DeepAcceptanceError", + "SCHEMA", + "collect_completed_frame_files", + "validate_completed_frame", + "validate_six_frame_batch", +] diff --git a/negpy/services/roll/exact_color.py b/negpy/services/roll/exact_color.py new file mode 100644 index 00000000..9e3c17e7 --- /dev/null +++ b/negpy/services/roll/exact_color.py @@ -0,0 +1,1463 @@ +"""Fail-closed boundary for portable Nikon exact-color evaluation. + +This module deliberately contains no builder or CML4 math. It is the narrow +seam between NegPy's repaired uint16 RGB buffer and two separately validated +operations: + +* a Stage-3 evidence/replay bridge or a fresh, identity-bound native builder + receipt using three validated pre-F LUTs and the pinned LS5000.md3 post-F + composition; and +* a verified portable CMS evaluator, injected by the caller. NegPy ships one + adapter for the captured CML4 Stage-1/Stage-2 transform. + +Receipts are immutable bytes rather than mutable dictionaries. NegPy owns the +aggregate builder envelope; the embedded Stage-3 validation and CMS payloads +retain their producing components' schemas. Every payload has an exact +content hash and explicit attestation. Results are bound to repaired input, +computed Stage-1 input, and final output before any TIFF is written. + +The native receipt exists only when distinct 97-dpi density, 285-dpi analyzer, +and final-exposure evidence are explicitly bound to the same frame. +""" + +from __future__ import annotations + +import hashlib +import hmac +import json +import math +import os +import re +import stat +import struct +from dataclasses import dataclass, field +from enum import StrEnum +from pathlib import Path +from types import MappingProxyType +from typing import Any, Final, Mapping, Protocol, cast + +import numpy as np + + +_SHA256 = re.compile(r"^[0-9a-f]{64}$") +_MAX_RECEIPT_BYTES = 1024 * 1024 +_BUILDER_CHANNELS = ("r", "g", "b") +_PRE_F_LUT_BYTES = 65_536 * 2 +# Keep this in lockstep with native_builder: both producer-side and serialized +# receipt validation accept only the two proven LS-5000 97-dpi geometries. +_SUPPORTED_DENSITY_SOURCE_WIRE_BYTES = frozenset((6_250_496, 5_804_032)) +BUILDER_RECEIPT_SCHEMA = "negpy.validated-stage1-builder" +STAGE3_REPORT_SCHEMA = "nikonre.ls5000_stage3_validation" +FIXED_COMPOSITION_SHA256 = "8729cae5a7aa551ae35926b80d097d73d92ecb6bde471130d6344c6c10ecbe7a" +STAGE3_REPLAY_SCOPE = "stage3-captured-pref-evidence-replay-bridge" +NATIVE_BUILDER_SCOPE = "ls5000-native-per-acquisition-stage1-builder" +NATIVE_BUILDER_ALGORITHM_ID = "ls5000-md3-prescan-to-pref-v1" +NATIVE_RESOURCE_SHA256 = "cd934185df496f071d307ba4f96a2a2b6ac31c3c85efc62a7fa1e3216fdba70c" + +CMS_RECEIPT_KIND: Final = "negpy.portable-cms-on-receipt" +CMS_RECEIPT_VERSION: Final = 1 +CMS_ALGORITHM_ID: Final = "cml4-captured-optimized-stage1-stage2-v1" +CMS_SCOPE: Final = "captured-cml4-stage1-stage2-only" +CMS_ORACLE_SOURCE_SHA256: Final = "2a9bad4b89cefb9fcb2bebbc59009ea0248e1ea93897cfa99c2c320c7f675490" +CMS_VALIDATION_RECEIPT_SHA256: Final = "edf6f3f89158810f1de4ce3b4ff8938326bc50e1b3035af59af472258e7d95e8" +CMS_ASSET_SHA256: Mapping[str, str] = MappingProxyType( + { + "lch-atan-u16le.bin": "56b8ac82456941a0a8aad6d7de2c79b21785529037b504582731ce6abcd143b1", + "lch-sincos-i16le.bin": "dc8e71681bc46e60e33448171865fc96770dc17d3cfd0db835f6173e6fed7a35", + "lch-reciprocal-u16le.bin": "6959ef7deeb57dc96eb4653fe13b4d37fcb4250c28be15654b8e8dd236849042", + "cml4-stage1-clut0.bin": "a2abbc1e76dc037e6b364a58b483ddf01b9dde7c0e072f85885cb1f8c9dcbf1c", + "cml4-stage1-input-lut0.bin": "1a487c024ceaf83018c8ab0e405e9c12b25a0500a2ae6e3465e20b29638729d7", + "cml4-stage1-output-lut0.bin": "fe87ce159ec126597f9fb605b57cebec9b0264da1ca16d1725efe34db2e4fd2b", + "cml4-stage2-clut0.bin": "d14b7c76091552bf03899327ca6a7c74c712a0423e429de8f8cc8203d5c98da3", + "cml4-stage2-input-lut0.bin": "60fa510492f2adad2be9b00107b3d7ed7188a798f29748c3401560b70be3a248", + "cml4-stage2-output-lut0.bin": "b88574377d1a0cf47fe8806335641db2ca35197817cd055a4cd55141708c8291", + } +) + +_STAGE3_MODULE_BYTES = 1_052_672 +_STAGE3_MODULE_SHA256 = "45afd6fb61a9517ff95d1896dc4257779c319310c2e8bbe75f3b4f3dada920af" +_STAGE3_RESOURCE_BYTES = 1_024 +_STAGE3_RESOURCE_SHA256 = "cd934185df496f071d307ba4f96a2a2b6ac31c3c85efc62a7fa1e3216fdba70c" +_STAGE3_RESOURCE_VA = "0x100ce578" +_STAGE3_FIXED_ARTIFACT_BYTES = { + "analyzer-desc.bin": 96, + "analyzer-pixels.bin": 281 * 425 * 3 * 2, + "builder-args.bin": 204, + **{f"builder-control-{axis}-{channel}.bin": 256 for channel in _BUILDER_CHANNELS for axis in ("x", "y")}, + **{f"builder-preF-{channel}.bin": _PRE_F_LUT_BYTES for channel in _BUILDER_CHANNELS}, +} +_STAGE3_DYNAMIC_ARTIFACTS = { + "callback-buffer.bin", + "debugger-session.log", + "stage3-capture-state.json", +} +_STAGE3_ARTIFACTS = frozenset(_STAGE3_FIXED_ARTIFACT_BYTES) | _STAGE3_DYNAMIC_ARTIFACTS +_STAGE3_CALLBACK_BYTES = frozenset({281 * 425 * 3 * 2, 281 * 425 * 4 * 2}) +_TRUSTED_BUILDER_RECEIPT_TOKEN = object() +_TRUSTED_NATIVE_BUILDER_RECEIPT_TOKEN = object() +_TRUSTED_CMS_RECEIPT_TOKEN = object() + + +class ExactColorUnavailable(RuntimeError): + """Exact Nikon color cannot be produced from the supplied evidence.""" + + +class ExactColorIntegrityError(ExactColorUnavailable): + """An evaluator result or receipt is malformed or not content-bound.""" + + +class PositiveColorMode(StrEnum): + """Receipt-visible Tier-3 color paths; exact never implies fallback.""" + + NEGPY_APPROXIMATE = "negpy-approximate" + NIKON_EXACT = "nikon-exact" + + +@dataclass(frozen=True) +class ValidatedBuilderReceipt: + """File-validated Stage-3 replay evidence and exact builder artifacts. + + This is an evidence/replay bridge. It is not the future macOS-native, + per-acquisition builder, and direct caller construction is never trusted. + """ + + payload: bytes + sha256: str + stage3_receipt: bytes + stage3_receipt_sha256: str + pre_f_luts: tuple[bytes, bytes, bytes] + pre_f_lut_sha256: tuple[str, str, str] + fixed_composition_sha256: str + evidence_filenames: tuple[str, str, str, str] + _factory_token: object = field(repr=False, compare=False) + + @property + def attested(self) -> bool: + return self._factory_token is _TRUSTED_BUILDER_RECEIPT_TOKEN + + +@dataclass(frozen=True) +class NativeValidatedBuilderReceipt: + """Fresh pre-F LUTs derived from identity-bound native scan evidence.""" + + payload: bytes + sha256: str + evidence_payload: bytes + evidence_sha256: str + frame_ownership_receipt: bytes + frame_ownership_receipt_sha256: str + density_evidence_receipt: bytes + density_evidence_receipt_sha256: str + analyzer_rgb: bytes + analyzer_rgb_sha256: str + analyzer_shape: tuple[int, int, int] + pre_f_luts: tuple[bytes, bytes, bytes] + pre_f_lut_sha256: tuple[str, str, str] + fixed_composition_sha256: str + session_id: str + reservation_id: str + batch_session_id: str + preview_sha256: str + preview_identity_sha256: str + capture_attempt_id: str + scan_identity: str + slot: int + _factory_token: object = field(repr=False, compare=False) + + @property + def attested(self) -> bool: + return self._factory_token is _TRUSTED_NATIVE_BUILDER_RECEIPT_TOKEN + + +BuilderReceipt = ValidatedBuilderReceipt | NativeValidatedBuilderReceipt + + +@dataclass(frozen=True) +class VerifiedBuilderApplicationReceipt: + """Immutable receipt for applying validated builder artifacts to one frame.""" + + payload: bytes + sha256: str + attested: bool + + +@dataclass(frozen=True) +class Stage1BuilderResult: + """Computed CML Stage-1 input plus bindings to source and builder evidence.""" + + rgb: np.ndarray + source_rgb_sha256: str + stage1_input_rgb_sha256: str + builder_receipt: BuilderReceipt + application_receipt: VerifiedBuilderApplicationReceipt + + +@dataclass(frozen=True) +class VerifiedCMSReceipt: + """Opaque receipt issued only by the verified portable CMS adapter.""" + + payload: bytes + sha256: str + _factory_token: object = field(repr=False, compare=False) + + @property + def attested(self) -> bool: + return self._factory_token is _TRUSTED_CMS_RECEIPT_TOKEN + + +@dataclass(frozen=True) +class ExactColorResult: + """Final RGB plus receipts and hashes binding both sides of evaluation.""" + + rgb: np.ndarray + input_rgb_sha256: str + output_rgb_sha256: str + builder_receipt: BuilderReceipt + cms_receipt: VerifiedCMSReceipt + source_rgb_sha256: str | None = None + builder_application_receipt: VerifiedBuilderApplicationReceipt | None = None + + +class VerifiedStage1Builder(Protocol): + """Apply independently validated builder artifacts at the exact boundary.""" + + def apply( + self, + rgb: np.ndarray, + *, + builder_receipt: BuilderReceipt, + ) -> Stage1BuilderResult: ... + + +class VerifiedPortableCMSEvaluator(Protocol): + """Injected adapter around a separately verified portable CMS engine.""" + + def evaluate( + self, + rgb: np.ndarray, + *, + builder_receipt: BuilderReceipt, + ) -> ExactColorResult: ... + + +def rgb16_content_sha256(rgb: np.ndarray) -> str: + """Hash canonical RGB16 content, including its shape and byte order. + + Evaluator adapters should call this helper rather than reimplementing the + content-identity grammar. Shape is included so equal byte strings with a + different raster geometry cannot share an identity. + """ + + _validate_rgb16(rgb, label="RGB input") + canonical = np.asarray(rgb, dtype=" ValidatedBuilderReceipt: + """Load one real Stage-3 PASS report and its three adjacent pre-F LUTs. + + The four files are read as one fail-closed snapshot using non-symlink + descriptor reads. The report is then checked against the complete + Stage-3 PASS/provenance/artifact contract before a trusted replay receipt + can exist. Callers cannot replace this with an ``attested=True`` claim. + """ + + report_path = Path(stage3_report_path).expanduser().absolute() + lut_paths = tuple(report_path.parent / f"builder-preF-{channel}.bin" for channel in _BUILDER_CHANNELS) + report_blob, report_identity = _stable_read_non_symlink( + report_path, + label="Stage-3 validation report", + max_bytes=_MAX_RECEIPT_BYTES, + ) + lut_snapshots = tuple( + _stable_read_non_symlink( + path, + label=f"Stage-3 pre-F {channel} LUT", + expected_bytes=_PRE_F_LUT_BYTES, + ) + for channel, path in zip(_BUILDER_CHANNELS, lut_paths, strict=True) + ) + all_paths = (report_path, *lut_paths) + all_identities = (report_identity, *(identity for _, identity in lut_snapshots)) + _assert_paths_unchanged(all_paths, all_identities) + + pre_f_luts = ( + lut_snapshots[0][0], + lut_snapshots[1][0], + lut_snapshots[2][0], + ) + pre_f_hashes = ( + hashlib.sha256(pre_f_luts[0]).hexdigest(), + hashlib.sha256(pre_f_luts[1]).hexdigest(), + hashlib.sha256(pre_f_luts[2]).hexdigest(), + ) + report = _parse_json_object(report_blob, label="Stage-3 validation") + _validate_stage3_report(report, pre_f_hashes) + report_hash = hashlib.sha256(report_blob).hexdigest() + envelope = _canonical_json( + { + "fixed_composition": { + "lut_sha256": FIXED_COMPOSITION_SHA256, + "order": "F[B_c(i)]", + }, + "native_per_acquisition_builder": False, + "pre_f_luts": [ + { + "bytes": _PRE_F_LUT_BYTES, + "channel": channel, + "sha256": digest, + } + for channel, digest in zip(_BUILDER_CHANNELS, pre_f_hashes, strict=True) + ], + "schema": BUILDER_RECEIPT_SCHEMA, + "scope": STAGE3_REPLAY_SCOPE, + "stage3_receipt_sha256": report_hash, + "version": 1, + } + ) + receipt = ValidatedBuilderReceipt( + payload=envelope, + sha256=hashlib.sha256(envelope).hexdigest(), + stage3_receipt=report_blob, + stage3_receipt_sha256=report_hash, + pre_f_luts=pre_f_luts, + pre_f_lut_sha256=pre_f_hashes, + fixed_composition_sha256=FIXED_COMPOSITION_SHA256, + evidence_filenames=(report_path.name, lut_paths[0].name, lut_paths[1].name, lut_paths[2].name), + _factory_token=_TRUSTED_BUILDER_RECEIPT_TOKEN, + ) + builder_receipt_payload(receipt) + return receipt + + +def load_native_builder_receipt( + native_receipt_path: str | os.PathLike[str], +) -> NativeValidatedBuilderReceipt: + """Reload one retained native builder snapshot for an exact-color retry. + + Every acquisition-bound input is snapshotted through non-symlink file + descriptors with strict byte bounds, then revalidated as one native + receipt. Merely retaining three LUTs is insufficient: ownership, density, + analyzer, builder envelope, and LUTs all have to remain mutually bound. + """ + + receipt_path = Path(native_receipt_path).expanduser().absolute() + directory = receipt_path.parent + evidence_path = directory / "native-builder-evidence.json" + ownership_path = directory / "nikon-density-frame-ownership.json" + density_path = directory / "nikon-density-evidence.json" + analyzer_path = directory / "analyzer-rgb-u16le.bin" + lut_paths = tuple(directory / f"builder-preF-{channel}.bin" for channel in _BUILDER_CHANNELS) + paths = ( + receipt_path, + evidence_path, + ownership_path, + density_path, + analyzer_path, + *lut_paths, + ) + reads = ( + _stable_read_non_symlink( + receipt_path, + label="native builder receipt", + max_bytes=_MAX_RECEIPT_BYTES, + ), + _stable_read_non_symlink( + evidence_path, + label="native builder evidence", + max_bytes=_MAX_RECEIPT_BYTES, + ), + _stable_read_non_symlink( + ownership_path, + label="native frame-ownership receipt", + max_bytes=_MAX_RECEIPT_BYTES, + ), + _stable_read_non_symlink( + density_path, + label="native density-evidence receipt", + max_bytes=_MAX_RECEIPT_BYTES, + ), + _stable_read_non_symlink( + analyzer_path, + label="native analyzer RGB", + expected_bytes=425 * 281 * 3 * 2, + ), + *( + _stable_read_non_symlink( + path, + label=f"native pre-F {channel} LUT", + expected_bytes=_PRE_F_LUT_BYTES, + ) + for channel, path in zip( + _BUILDER_CHANNELS, + lut_paths, + strict=True, + ) + ), + ) + _assert_paths_unchanged( + paths, + tuple(identity for _, identity in reads), + label="native builder evidence", + ) + ( + receipt_blob, + evidence_blob, + ownership_blob, + density_blob, + analyzer_blob, + pre_f_r, + pre_f_g, + pre_f_b, + ) = (payload for payload, _ in reads) + + envelope = _parse_json_object(receipt_blob, label="native builder receipt") + if _canonical_json(envelope) != receipt_blob: + raise ExactColorIntegrityError("native builder receipt is not canonical JSON") + # Import lazily: native_builder owns the pinned derivation and imports this + # boundary for its receipt factory. Re-running that derivation is what + # prevents a writable evidence directory from self-attesting new LUTs. + from negpy.services.roll import native_builder + + rebuilt = native_builder.rebuild_retained_native_builder_receipt( + envelope_payload=receipt_blob, + evidence_payload=evidence_blob, + frame_ownership_receipt=ownership_blob, + density_evidence_receipt=density_blob, + analyzer_rgb=analyzer_blob, + pre_f_luts=(pre_f_r, pre_f_g, pre_f_b), + ) + if directory.name != rebuilt.sha256: + raise ExactColorIntegrityError("native builder evidence is outside its content-addressed directory") + return rebuilt + + +def _issue_verified_cms_receipt(payload: bytes) -> VerifiedCMSReceipt: + """Issue a CMS receipt from the hash-verified production adapter only.""" + + if type(payload) is not bytes or not payload or len(payload) > _MAX_RECEIPT_BYTES: + raise ExactColorIntegrityError("CMS receipt payload is missing or unbounded") + return VerifiedCMSReceipt( + payload=payload, + sha256=hashlib.sha256(payload).hexdigest(), + _factory_token=_TRUSTED_CMS_RECEIPT_TOKEN, + ) + + +def _issue_native_builder_receipt( + *, + payload: bytes, + evidence_payload: bytes, + frame_ownership_receipt: bytes, + frame_ownership_receipt_sha256: str, + density_evidence_receipt: bytes, + density_evidence_receipt_sha256: str, + analyzer_rgb: bytes, + analyzer_rgb_sha256: str, + analyzer_shape: tuple[int, int, int], + pre_f_luts: tuple[bytes, bytes, bytes], + session_id: str, + reservation_id: str, + batch_session_id: str, + preview_sha256: str, + preview_identity_sha256: str, + capture_attempt_id: str, + scan_identity: str, + slot: int, +) -> NativeValidatedBuilderReceipt: + """Issue a native receipt after the package-local builder closes its gates.""" + + receipt = NativeValidatedBuilderReceipt( + payload=payload, + sha256=hashlib.sha256(payload).hexdigest(), + evidence_payload=evidence_payload, + evidence_sha256=hashlib.sha256(evidence_payload).hexdigest(), + frame_ownership_receipt=frame_ownership_receipt, + frame_ownership_receipt_sha256=frame_ownership_receipt_sha256, + density_evidence_receipt=density_evidence_receipt, + density_evidence_receipt_sha256=density_evidence_receipt_sha256, + analyzer_rgb=analyzer_rgb, + analyzer_rgb_sha256=analyzer_rgb_sha256, + analyzer_shape=analyzer_shape, + pre_f_luts=pre_f_luts, + pre_f_lut_sha256=( + hashlib.sha256(pre_f_luts[0]).hexdigest(), + hashlib.sha256(pre_f_luts[1]).hexdigest(), + hashlib.sha256(pre_f_luts[2]).hexdigest(), + ), + fixed_composition_sha256=FIXED_COMPOSITION_SHA256, + session_id=session_id, + reservation_id=reservation_id, + batch_session_id=batch_session_id, + preview_sha256=preview_sha256, + preview_identity_sha256=preview_identity_sha256, + capture_attempt_id=capture_attempt_id, + scan_identity=scan_identity, + slot=slot, + _factory_token=_TRUSTED_NATIVE_BUILDER_RECEIPT_TOKEN, + ) + builder_receipt_payload(receipt) + return receipt + + +def evaluate_exact_color( + rgb: np.ndarray, + *, + builder_receipt: BuilderReceipt | None, + builder: VerifiedStage1Builder | None, + evaluator: VerifiedPortableCMSEvaluator | None, +) -> ExactColorResult: + """Evaluate and independently validate one exact-color result. + + Nothing falls back to NegPy's approximate renderer here. Missing or + invalid evidence raises ExactColorUnavailable and callers must expose + that state rather than label any other output exact. + """ + + _validate_rgb16(rgb, label="exact-color input") + if builder_receipt is None: + raise ExactColorUnavailable("validated Stage-3 replay or native builder receipt is not supplied") + if builder is None: + raise ExactColorUnavailable("verified Stage-1 builder applicator is not supplied") + if evaluator is None: + raise ExactColorUnavailable("verified portable CMS evaluator is not supplied") + builder_receipt_payload(builder_receipt) + + source_snapshot = np.array(rgb, dtype=np.uint16, order="C", copy=True) + expected_source = rgb16_content_sha256(source_snapshot) + source_snapshot.setflags(write=False) + try: + builder_result = builder.apply(source_snapshot, builder_receipt=builder_receipt) + except ExactColorUnavailable: + raise + except Exception as error: + raise ExactColorUnavailable(f"Stage-1 builder applicator failed: {error}") from error + if not isinstance(builder_result, Stage1BuilderResult): + raise ExactColorIntegrityError("Stage-1 builder returned an invalid result type") + if builder_result.builder_receipt != builder_receipt: + raise ExactColorIntegrityError("Stage-1 builder substituted a different builder receipt") + builder_receipt_payload(builder_result.builder_receipt) + application_payload = _receipt_payload( + builder_result.application_receipt, + VerifiedBuilderApplicationReceipt, + label="builder application", + ) + if rgb16_content_sha256(source_snapshot) != expected_source: + raise ExactColorIntegrityError("Stage-1 builder mutated its repaired RGB input") + if builder_result.source_rgb_sha256 != expected_source: + raise ExactColorIntegrityError("builder source hash does not match the repaired RGB content") + _validate_rgb16(builder_result.rgb, label="Stage-1 builder output") + if builder_result.rgb.shape != source_snapshot.shape: + raise ExactColorIntegrityError("Stage-1 builder output geometry differs from its input") + stage1_snapshot = np.array(builder_result.rgb, dtype=np.uint16, order="C", copy=True) + expected_stage1 = rgb16_content_sha256(stage1_snapshot) + if builder_result.stage1_input_rgb_sha256 != expected_stage1: + raise ExactColorIntegrityError("builder output hash does not match the Stage-1 input content") + if application_payload.get("builder_receipt_sha256") != builder_receipt.sha256: + raise ExactColorIntegrityError("builder application receipt does not bind the builder receipt") + if application_payload.get("source_rgb_sha256") != expected_source: + raise ExactColorIntegrityError("builder application receipt does not bind its repaired RGB input") + if application_payload.get("stage1_input_rgb_sha256") != expected_stage1: + raise ExactColorIntegrityError("builder application receipt does not bind its Stage-1 output") + _validate_builder_application_receipt(application_payload, builder_receipt) + + stage1_snapshot.setflags(write=False) + try: + result = evaluator.evaluate(stage1_snapshot, builder_receipt=builder_receipt) + except ExactColorUnavailable: + raise + except Exception as error: + raise ExactColorUnavailable(f"portable CMS evaluator failed: {error}") from error + if not isinstance(result, ExactColorResult): + raise ExactColorIntegrityError("portable CMS evaluator returned an invalid result type") + if result.builder_receipt != builder_receipt: + raise ExactColorIntegrityError("evaluator result substituted a different builder receipt") + builder_receipt_payload(result.builder_receipt) + cms_payload = _receipt_payload(result.cms_receipt, VerifiedCMSReceipt, label="CMS") + + if rgb16_content_sha256(stage1_snapshot) != expected_stage1: + raise ExactColorIntegrityError("portable CMS evaluator mutated its Stage-1 input") + if result.input_rgb_sha256 != expected_stage1: + raise ExactColorIntegrityError("evaluator input hash does not match the Stage-1 input content") + _validate_rgb16(result.rgb, label="exact-color output") + if result.rgb.shape != stage1_snapshot.shape: + raise ExactColorIntegrityError("exact-color output geometry differs from its input") + # Snapshot before hashing: the external evaluator may retain its original + # array, but it cannot change the content we bind and hand to the writer. + snapshot = np.array(result.rgb, dtype=np.uint16, order="C", copy=True) + expected_output = rgb16_content_sha256(snapshot) + if result.output_rgb_sha256 != expected_output: + raise ExactColorIntegrityError("evaluator output hash does not match the returned RGB content") + _validate_cms_receipt( + cms_payload, + builder_receipt_sha256=builder_receipt.sha256, + input_rgb_sha256=expected_stage1, + output_rgb_sha256=expected_output, + ) + + snapshot.setflags(write=False) + return ExactColorResult( + rgb=snapshot, + input_rgb_sha256=expected_stage1, + output_rgb_sha256=expected_output, + builder_receipt=result.builder_receipt, + cms_receipt=result.cms_receipt, + source_rgb_sha256=expected_source, + builder_application_receipt=builder_result.application_receipt, + ) + + +def receipt_payload( + receipt: BuilderReceipt | VerifiedBuilderApplicationReceipt | VerifiedCMSReceipt, +) -> dict[str, Any]: + """Return a JSON-safe copy for the outer NegPy scan receipt.""" + + if isinstance(receipt, (ValidatedBuilderReceipt, NativeValidatedBuilderReceipt)): + return builder_receipt_payload(receipt) + if type(receipt) is VerifiedBuilderApplicationReceipt: + expected = VerifiedBuilderApplicationReceipt + elif type(receipt) is VerifiedCMSReceipt: + expected = VerifiedCMSReceipt + else: + raise ExactColorIntegrityError("receipt has an invalid type") + return _receipt_payload(receipt, expected, label="receipt") + + +def builder_receipt_payload(receipt: BuilderReceipt) -> dict[str, Any]: + """Validate every artifact binding in a replay or native builder receipt.""" + + if type(receipt) is NativeValidatedBuilderReceipt: + return _native_builder_receipt_payload(receipt) + + if type(receipt) is not ValidatedBuilderReceipt or receipt._factory_token is not _TRUSTED_BUILDER_RECEIPT_TOKEN: + raise ExactColorIntegrityError("builder receipt was not produced by the trusted Stage-3 replay file loader") + payload = _receipt_payload(receipt, ValidatedBuilderReceipt, label="builder") + _validate_sha256(receipt.stage3_receipt_sha256, label="Stage-3 receipt") + if type(receipt.stage3_receipt) is not bytes or not receipt.stage3_receipt: + raise ExactColorIntegrityError("Stage-3 validation receipt is missing") + if not hmac.compare_digest( + hashlib.sha256(receipt.stage3_receipt).hexdigest(), + receipt.stage3_receipt_sha256, + ): + raise ExactColorIntegrityError("Stage-3 validation receipt does not match its SHA-256") + _validate_pre_f_artifacts(receipt) + stage3 = _parse_json_object(receipt.stage3_receipt, label="Stage-3 validation") + _validate_stage3_report(stage3, receipt.pre_f_lut_sha256) + + expected_luts = [ + { + "bytes": _PRE_F_LUT_BYTES, + "channel": channel, + "sha256": digest, + } + for channel, digest in zip(_BUILDER_CHANNELS, receipt.pre_f_lut_sha256, strict=True) + ] + if payload.get("schema") != BUILDER_RECEIPT_SCHEMA or type(payload.get("version")) is not int or payload.get("version") != 1: + raise ExactColorIntegrityError("builder receipt envelope schema is unsupported") + if payload.get("scope") != STAGE3_REPLAY_SCOPE or payload.get("native_per_acquisition_builder") is not False: + raise ExactColorIntegrityError("builder receipt does not identify the Stage-3 evidence/replay bridge") + if payload.get("stage3_receipt_sha256") != receipt.stage3_receipt_sha256: + raise ExactColorIntegrityError("builder receipt envelope does not bind the Stage-3 receipt") + if payload.get("pre_f_luts") != expected_luts: + raise ExactColorIntegrityError("builder receipt envelope does not bind the three pre-F LUTs") + if payload.get("fixed_composition") != { + "lut_sha256": FIXED_COMPOSITION_SHA256, + "order": "F[B_c(i)]", + }: + raise ExactColorIntegrityError("builder receipt envelope does not bind the fixed post-F composition") + if ( + type(receipt.evidence_filenames) is not tuple + or len(receipt.evidence_filenames) != 4 + or type(receipt.evidence_filenames[0]) is not str + or not receipt.evidence_filenames[0] + or receipt.evidence_filenames[1:] + != ( + "builder-preF-r.bin", + "builder-preF-g.bin", + "builder-preF-b.bin", + ) + ): + raise ExactColorIntegrityError("builder receipt evidence filenames are malformed") + return payload + + +def _native_builder_receipt_payload(receipt: NativeValidatedBuilderReceipt) -> dict[str, Any]: + if receipt._factory_token is not _TRUSTED_NATIVE_BUILDER_RECEIPT_TOKEN: + raise ExactColorIntegrityError("native builder receipt was not produced by the trusted native factory") + payload = _receipt_payload(receipt, NativeValidatedBuilderReceipt, label="native builder") + if type(receipt.evidence_payload) is not bytes or not receipt.evidence_payload: + raise ExactColorIntegrityError("native builder evidence payload is missing") + _validate_sha256(receipt.evidence_sha256, label="native builder evidence") + if not hmac.compare_digest(hashlib.sha256(receipt.evidence_payload).hexdigest(), receipt.evidence_sha256): + raise ExactColorIntegrityError("native builder evidence does not match its SHA-256") + evidence = _parse_json_object(receipt.evidence_payload, label="native builder evidence") + for label, blob, expected_sha256 in ( + ("frame ownership", receipt.frame_ownership_receipt, receipt.frame_ownership_receipt_sha256), + ("density evidence", receipt.density_evidence_receipt, receipt.density_evidence_receipt_sha256), + ): + _validate_sha256(expected_sha256, label=f"native {label}") + if type(blob) is not bytes or not blob or not hmac.compare_digest(hashlib.sha256(blob).hexdigest(), expected_sha256): + raise ExactColorIntegrityError(f"native {label} receipt does not match its SHA-256") + _validate_native_evidence_document(evidence, receipt) + if ( + type(receipt.analyzer_rgb) is not bytes + or type(receipt.analyzer_shape) is not tuple + or receipt.analyzer_shape != (425, 281, 3) + or len(receipt.analyzer_rgb) != 425 * 281 * 3 * 2 + ): + raise ExactColorIntegrityError("native builder analyzer snapshot has unsupported geometry") + _validate_sha256(receipt.analyzer_rgb_sha256, label="native builder analyzer RGB") + if not hmac.compare_digest(hashlib.sha256(receipt.analyzer_rgb).hexdigest(), receipt.analyzer_rgb_sha256): + raise ExactColorIntegrityError("native builder analyzer snapshot does not match its SHA-256") + _validate_pre_f_artifacts(receipt) + expected_luts = [ + {"bytes": _PRE_F_LUT_BYTES, "channel": channel, "sha256": digest} + for channel, digest in zip(_BUILDER_CHANNELS, receipt.pre_f_lut_sha256, strict=True) + ] + identity = { + "batch_session_id": receipt.batch_session_id, + "capture_attempt_id": receipt.capture_attempt_id, + "preview_identity_sha256": receipt.preview_identity_sha256, + "preview_sha256": receipt.preview_sha256, + "reservation_id": receipt.reservation_id, + "scan_identity": receipt.scan_identity, + "session_id": receipt.session_id, + "slot": receipt.slot, + } + if ( + set(payload) + != { + "algorithm", + "analyzer_source", + "density_source", + "evidence_sha256", + "fixed_composition", + "identity", + "native_per_acquisition_builder", + "pre_f_luts", + "schema", + "scope", + "version", + } + or payload.get("schema") != BUILDER_RECEIPT_SCHEMA + or type(payload.get("version")) is not int + or payload.get("version") != 1 + or payload.get("scope") != NATIVE_BUILDER_SCOPE + or payload.get("native_per_acquisition_builder") is not True + or payload.get("algorithm") != NATIVE_BUILDER_ALGORITHM_ID + or payload.get("density_source") != evidence.get("density_source") + or payload.get("analyzer_source") != evidence.get("analyzer_source") + or payload.get("identity") != identity + or payload.get("evidence_sha256") != receipt.evidence_sha256 + or payload.get("pre_f_luts") != expected_luts + or payload.get("fixed_composition") != {"lut_sha256": FIXED_COMPOSITION_SHA256, "order": "F[B_c(i)]"} + ): + raise ExactColorIntegrityError("native builder receipt envelope is malformed or incompletely bound") + if evidence.get("analyzer_source", {}).get("rgb_sha256") != receipt.analyzer_rgb_sha256: + raise ExactColorIntegrityError("native builder evidence does not bind its analyzer snapshot") + return payload + + +def _validate_native_evidence_document( + evidence: dict[str, Any], + receipt: NativeValidatedBuilderReceipt, +) -> None: + expected_identity = { + "batch_session_id": receipt.batch_session_id, + "capture_attempt_id": receipt.capture_attempt_id, + "preview_identity_sha256": receipt.preview_identity_sha256, + "preview_sha256": receipt.preview_sha256, + "reservation_id": receipt.reservation_id, + "scan_identity": receipt.scan_identity, + "session_id": receipt.session_id, + "slot": receipt.slot, + } + if set(evidence) != { + "algorithm", + "analyzer_source", + "calibration", + "density_evidence", + "density_source", + "frame_ownership", + "identity", + "resource", + "schema", + "version", + }: + raise ExactColorIntegrityError("native builder evidence fields are incomplete") + if ( + evidence.get("schema") != "negpy.native-stage1-builder-evidence" + or type(evidence.get("version")) is not int + or evidence.get("version") != 1 + or evidence.get("identity") != expected_identity + or evidence.get("resource") != {"bytes": 1_024, "sha256": NATIVE_RESOURCE_SHA256} + or evidence.get("algorithm") + != { + "curve": "ls5000-md3-10010c30-pref-v1", + "parameter_derivation": "ls5000-md3-100100d0-to-1000f470-v1", + } + ): + raise ExactColorIntegrityError("native builder evidence provenance is unsupported") + ownership = evidence.get("frame_ownership") + if type(ownership) is not dict or set(ownership) != { + "schema_version", + "scope", + "binding_status", + "session_reservation_retained", + "reservation_id", + "batch_session_id", + "preview_sha256", + "preview_identity_sha256", + "transport_table_sha256", + "transport_identity_sha256", + "reviewed_fingerprint_sha256", + "fresh_fingerprint_sha256", + "frame_capture_attempt_id", + "frame_index", + "frame_total", + "selected_slots", + "selected_slot", + }: + raise ExactColorIntegrityError("native frame-ownership evidence is malformed") + ownership_blob = _canonical_json(cast(dict[str, Any], ownership)) + if ( + ownership_blob != receipt.frame_ownership_receipt + or hashlib.sha256(ownership_blob).hexdigest() != receipt.frame_ownership_receipt_sha256 + or ownership.get("schema_version") != 1 + or ownership.get("scope") != "reservation-preview-frame" + or ownership.get("binding_status") != "proven-exact-reservation-preview-registration-and-transport" + or ownership.get("session_reservation_retained") is not True + or ownership.get("reservation_id") != receipt.reservation_id + or ownership.get("batch_session_id") != receipt.batch_session_id + or receipt.session_id != receipt.reservation_id + or receipt.reservation_id != receipt.batch_session_id + or ownership.get("preview_sha256") != receipt.preview_sha256 + or ownership.get("preview_identity_sha256") != receipt.preview_identity_sha256 + or ownership.get("frame_capture_attempt_id") != receipt.capture_attempt_id + or ownership.get("selected_slot") != receipt.slot + ): + raise ExactColorIntegrityError("native frame ownership belongs to another preview, reservation, or frame") + _validate_sha256(ownership.get("transport_table_sha256"), label="native transport table") + _validate_sha256(ownership.get("transport_identity_sha256"), label="native transport identity") + _validate_sha256(ownership.get("reviewed_fingerprint_sha256"), label="native reviewed fingerprint") + _validate_sha256(ownership.get("fresh_fingerprint_sha256"), label="native fresh fingerprint") + selected_slots = ownership.get("selected_slots") + frame_index = ownership.get("frame_index") + frame_total = ownership.get("frame_total") + if ( + type(selected_slots) is not list + or type(frame_index) is not int + or type(frame_total) is not int + or len(selected_slots) != frame_total + or not 1 <= frame_index <= frame_total + or selected_slots[frame_index - 1] != receipt.slot + ): + raise ExactColorIntegrityError("native frame ownership batch coordinates are malformed") + transport_material = { + "reservation_id": receipt.reservation_id, + "batch_session_id": receipt.batch_session_id, + "preview_sha256": receipt.preview_sha256, + "preview_identity_sha256": receipt.preview_identity_sha256, + "transport_table_sha256": ownership.get("transport_table_sha256"), + "reviewed_fingerprint_sha256": ownership.get("reviewed_fingerprint_sha256"), + "fresh_fingerprint_sha256": ownership.get("fresh_fingerprint_sha256"), + "selected_slots": selected_slots, + } + if hashlib.sha256(_canonical_json(transport_material)).hexdigest() != ownership.get("transport_identity_sha256"): + raise ExactColorIntegrityError("native frame ownership transport identity digest is invalid") + density_evidence = evidence.get("density_evidence") + density_evidence_document = _parse_json_object(receipt.density_evidence_receipt, label="native density evidence") + if ( + density_evidence != {"receipt_sha256": receipt.density_evidence_receipt_sha256, "scope": "reservation-preview"} + or set(density_evidence_document) + != { + "schema_version", + "scope", + "per_frame_binding_status", + "preview_identity_sha256", + "source_payload_bytes", + "calibration_binding", + "source_binding", + "exposure_binding", + "result", + } + or density_evidence_document.get("schema_version") != 1 + or density_evidence_document.get("scope") != "reservation-preview" + or density_evidence_document.get("per_frame_binding_status") != "requires-explicit-frame-ownership-receipt" + or density_evidence_document.get("source_payload_bytes") + not in _SUPPORTED_DENSITY_SOURCE_WIRE_BYTES + or density_evidence_document.get("preview_identity_sha256") != receipt.preview_identity_sha256 + ): + raise ExactColorIntegrityError("native density evidence belongs to another preview or reservation") + calibration_binding = density_evidence_document.get("calibration_binding") + source_binding = density_evidence_document.get("source_binding") + exposure_binding = density_evidence_document.get("exposure_binding") + density_result = density_evidence_document.get("result") + if not all(type(value) is dict for value in (calibration_binding, source_binding, exposure_binding, density_result)): + raise ExactColorIntegrityError("native density evidence bindings are malformed") + calibration_binding = cast(dict[str, Any], calibration_binding) + source_binding = cast(dict[str, Any], source_binding) + exposure_binding = cast(dict[str, Any], exposure_binding) + density_result = cast(dict[str, Any], density_result) + density_calibration = calibration_binding.get("calibration") + if type(density_calibration) is not dict: + raise ExactColorIntegrityError("native density calibration evidence is malformed") + binding_identity = ( + source_binding.get("session_id"), + source_binding.get("capture_attempt_id"), + source_binding.get("scan_identity"), + ) + if ( + binding_identity + != ( + exposure_binding.get("session_id"), + exposure_binding.get("capture_attempt_id"), + exposure_binding.get("scan_identity"), + ) + or binding_identity + != ( + density_result.get("session_id"), + density_result.get("capture_attempt_id"), + density_result.get("scan_identity"), + ) + or binding_identity + != ( + density_calibration.get("session_id"), + calibration_binding.get("capture_attempt_id"), + calibration_binding.get("scan_identity"), + ) + or binding_identity[0] != receipt.reservation_id + or binding_identity[2] != receipt.scan_identity + or source_binding.get("wire_sha256") != receipt.preview_sha256 + or density_result.get("source_wire_sha256") != receipt.preview_sha256 + or density_result.get("promotable") is not True + ): + raise ExactColorIntegrityError("native density evidence identity is inconsistent") + calibration = evidence.get("calibration") + if type(calibration) is not dict or set(calibration) != {"read8c_numerators_rgb"}: + raise ExactColorIntegrityError("native calibration evidence is malformed") + numerators = calibration.get("read8c_numerators_rgb") + density = evidence.get("density_source") + analyzer = evidence.get("analyzer_source") + if type(density) is not dict or set(density) != { + "arithmetic", + "child_buffer_sha256", + "densities_binary64_hex_rgb", + "f03_denominators_raw_10ns_rgb", + "resolution_dpi", + "wire_sha256", + }: + raise ExactColorIntegrityError("native density-source evidence is malformed") + if type(analyzer) is not dict or set(analyzer) != { + "final_f02_denominators_raw_10ns_rgb", + "geometry", + "rectangle_inclusive", + "resolution_dpi", + "rgb_bytes", + "rgb_sha256", + }: + raise ExactColorIntegrityError("native analyzer-source evidence is malformed") + density_denominators = density.get("f03_denominators_raw_10ns_rgb") + final_denominators = analyzer.get("final_f02_denominators_raw_10ns_rgb") + for label, values in ( + ("calibration numerators", numerators), + ("density f03 denominators", density_denominators), + ("final f02 denominators", final_denominators), + ): + if type(values) is not list or len(values) != 3 or any(type(value) is not int or not 1 <= value <= 0xFFFFFFFF for value in values): + raise ExactColorIntegrityError(f"native {label} are malformed") + if density_denominators == final_denominators: + raise ExactColorIntegrityError("native density f03 and final f02 exposure triplets were conflated") + density_hex = density.get("densities_binary64_hex_rgb") + try: + decoded_density = [float.fromhex(value) for value in density_hex] if type(density_hex) is list else [] + except (TypeError, ValueError) as error: + raise ExactColorIntegrityError("native density doubles are malformed") from error + if len(decoded_density) != 3 or any(not math.isfinite(value) for value in decoded_density): + raise ExactColorIntegrityError("native density doubles are malformed") + density_binary64 = [struct.pack(">d", value).hex() for value in decoded_density] + if ( + source_binding.get("resolution_dpi") != 97 + or source_binding.get("wire_sha256") != density.get("wire_sha256") + or source_binding.get("child_buffer_sha256") != density.get("child_buffer_sha256") + or density_calibration.get("numerators_rgb") != numerators + or exposure_binding.get("density_f03_exposures_raw_10ns_rgb") != density_denominators + or density_result.get("algorithm_id") != density.get("arithmetic") + or density_result.get("source_wire_sha256") != density.get("wire_sha256") + or density_result.get("source_child_buffer_sha256") != density.get("child_buffer_sha256") + or density_result.get("numerators_rgb") != numerators + or density_result.get("density_f03_denominators_raw_10ns_rgb") != density_denominators + or density_result.get("densities_rgb") != decoded_density + or density_result.get("density_binary64_be_hex_rgb") != density_binary64 + ): + raise ExactColorIntegrityError("native density receipt does not bind the builder inputs") + _validate_sha256(density.get("wire_sha256"), label="native density source wire") + _validate_sha256(density.get("child_buffer_sha256"), label="native density source child") + rectangle = analyzer.get("rectangle_inclusive") + if ( + density.get("arithmetic") != "ls5000-md3-10088810-layout1-u16-proven-inputs-macos-binary64-exact-v6" + or density.get("resolution_dpi") != 97 + or analyzer.get("resolution_dpi") != 285 + or analyzer.get("geometry") != [425, 281, 3] + or analyzer.get("rgb_bytes") != 425 * 281 * 3 * 2 + or type(rectangle) is not list + or len(rectangle) != 4 + or any(type(value) is not int for value in rectangle) + or not (0 <= rectangle[0] <= rectangle[2] < 425 and 0 <= rectangle[1] <= rectangle[3] < 281) + ): + raise ExactColorIntegrityError("native density/analyzer geometry or arithmetic is unsupported") + _validate_sha256(analyzer.get("rgb_sha256"), label="native analyzer RGB") + + +def _validate_builder_application_receipt( + payload: dict[str, Any], + receipt: BuilderReceipt, +) -> None: + common_ok = ( + payload.get("kind") == "negpy.verified-stage1-builder-application" + and type(payload.get("version")) is int + and payload.get("version") == 1 + and payload.get("fixed_composition") == {"lut_sha256": FIXED_COMPOSITION_SHA256, "order": "F[B_c(i)]"} + and payload.get("pre_f_lut_sha256") == dict(zip(_BUILDER_CHANNELS, receipt.pre_f_lut_sha256, strict=True)) + ) + if isinstance(receipt, ValidatedBuilderReceipt): + specific_ok = ( + payload.get("scope") == STAGE3_REPLAY_SCOPE + and payload.get("native_per_acquisition_builder") is False + and payload.get("stage3_receipt_sha256") == receipt.stage3_receipt_sha256 + ) + label = "validated Stage-3 artifacts" + elif isinstance(receipt, NativeValidatedBuilderReceipt): + specific_ok = ( + payload.get("scope") == NATIVE_BUILDER_SCOPE + and payload.get("native_per_acquisition_builder") is True + and payload.get("native_evidence_sha256") == receipt.evidence_sha256 + ) + label = "validated native builder artifacts" + else: # pragma: no cover - BuilderReceipt is a closed union + raise ExactColorIntegrityError("builder receipt has an invalid type") + if not common_ok or not specific_ok: + raise ExactColorIntegrityError(f"builder application receipt does not bind the {label}") + + +def _receipt_payload( + receipt: object, + expected_type: ( + type[ValidatedBuilderReceipt] + | type[NativeValidatedBuilderReceipt] + | type[VerifiedBuilderApplicationReceipt] + | type[VerifiedCMSReceipt] + ), + *, + label: str, +) -> dict[str, Any]: + if ( + not isinstance( + receipt, + ( + ValidatedBuilderReceipt, + NativeValidatedBuilderReceipt, + VerifiedBuilderApplicationReceipt, + VerifiedCMSReceipt, + ), + ) + or type(receipt) is not expected_type + ): + raise ExactColorIntegrityError(f"{label} receipt has an invalid type") + payload = receipt.payload + if type(payload) is not bytes or not payload or len(payload) > _MAX_RECEIPT_BYTES: + raise ExactColorIntegrityError(f"{label} receipt payload is missing or unbounded") + if type(receipt) is VerifiedCMSReceipt and receipt._factory_token is not _TRUSTED_CMS_RECEIPT_TOKEN: + raise ExactColorIntegrityError("CMS receipt was not issued by the trusted portable CMS adapter") + if receipt.attested is not True: + raise ExactColorIntegrityError(f"{label} receipt is not attested") + if type(receipt.sha256) is not str or not _SHA256.fullmatch(receipt.sha256): + raise ExactColorIntegrityError(f"{label} receipt SHA-256 is malformed") + if not hmac.compare_digest(hashlib.sha256(payload).hexdigest(), receipt.sha256): + raise ExactColorIntegrityError(f"{label} receipt payload does not match its SHA-256") + return _parse_json_object(payload, label=f"{label} receipt") + + +def _parse_json_object(payload: bytes, *, label: str) -> dict[str, Any]: + if type(payload) is not bytes or not payload or len(payload) > _MAX_RECEIPT_BYTES: + raise ExactColorIntegrityError(f"{label} payload is missing or unbounded") + try: + parsed = json.loads( + payload, + object_pairs_hook=_reject_duplicate_keys, + parse_constant=_reject_json_constant, + ) + except (UnicodeDecodeError, ValueError, RecursionError) as error: + raise ExactColorIntegrityError(f"{label} is not valid JSON: {error}") from error + if type(parsed) is not dict: + raise ExactColorIntegrityError(f"{label} must contain a JSON object") + return parsed + + +def _validate_sha256(value: object, *, label: str) -> None: + if type(value) is not str or not _SHA256.fullmatch(value): + raise ExactColorIntegrityError(f"{label} SHA-256 is malformed") + + +def _validate_pre_f_artifacts(receipt: BuilderReceipt) -> None: + if type(receipt.pre_f_luts) is not tuple or len(receipt.pre_f_luts) != 3: + raise ExactColorIntegrityError("builder receipt must contain exactly three pre-F LUT byte blobs") + if type(receipt.pre_f_lut_sha256) is not tuple or len(receipt.pre_f_lut_sha256) != 3: + raise ExactColorIntegrityError("builder receipt must contain exactly three pre-F LUT hashes") + for channel, blob, expected in zip( + _BUILDER_CHANNELS, + receipt.pre_f_luts, + receipt.pre_f_lut_sha256, + strict=True, + ): + if type(blob) is not bytes or len(blob) != _PRE_F_LUT_BYTES: + raise ExactColorIntegrityError(f"builder pre-F {channel} LUT must be exactly {_PRE_F_LUT_BYTES} bytes") + _validate_sha256(expected, label=f"builder pre-F {channel} LUT") + if not hmac.compare_digest(hashlib.sha256(blob).hexdigest(), expected): + raise ExactColorIntegrityError(f"builder pre-F {channel} LUT does not match its SHA-256") + _validate_sha256(receipt.fixed_composition_sha256, label="fixed composition LUT") + if receipt.fixed_composition_sha256 != FIXED_COMPOSITION_SHA256: + raise ExactColorIntegrityError("builder receipt names an unsupported fixed composition LUT") + + +def _validate_stage3_report( + report: dict[str, Any], + pre_f_lut_sha256: tuple[str, str, str], +) -> None: + if ( + report.get("schema") != STAGE3_REPORT_SCHEMA + or type(report.get("schema_version")) is not int + or report.get("schema_version") != 1 + or report.get("status") != "pass" + ): + raise ExactColorIntegrityError("Stage-3 validation report is not a supported PASS receipt") + if report.get("errors") != []: + raise ExactColorIntegrityError("Stage-3 validation report errors must be exactly empty") + if type(report.get("capture_directory")) is not str or not report["capture_directory"]: + raise ExactColorIntegrityError("Stage-3 validation report capture directory is malformed") + required_summary = { + "callback_analyzer_exact": True, + "builder_scalars_exact": 21, + "builder_scalars_total": 21, + "builder_controls_exact": 6, + "builder_controls_total": 6, + "builder_pref_channels_exact": 3, + "builder_pref_channels_total": 3, + "builder_pref_mismatched_u16": 0, + "builder_pref_total_u16": 196_608, + "captured_args_replay_channels_exact": 3, + "lifecycle_exact": True, + } + summary = report.get("summary") + if ( + type(summary) is not dict + or summary != required_summary + or any(type(summary[key]) is not type(value) for key, value in required_summary.items()) + ): + raise ExactColorIntegrityError("Stage-3 validation report does not close every builder gate") + + provenance = report.get("provenance") + if type(provenance) is not dict or set(provenance) != { + "module", + "observer_executable", + "observer_source", + "resource", + }: + raise ExactColorIntegrityError("Stage-3 validation report provenance fields are incomplete") + _validate_observer_provenance(provenance.get("observer_source"), label="observer source") + _validate_observer_provenance(provenance.get("observer_executable"), label="observer executable") + module = provenance.get("module") + if ( + type(module) is not dict + or set(module) != {"path", "bytes", "sha256"} + or type(module.get("path")) is not str + or not module["path"] + or type(module.get("bytes")) is not int + or module.get("bytes") != _STAGE3_MODULE_BYTES + or module.get("sha256") != _STAGE3_MODULE_SHA256 + ): + raise ExactColorIntegrityError("Stage-3 module provenance is not the pinned LS5000.md3") + resource = provenance.get("resource") + if ( + type(resource) is not dict + or resource + != { + "bytes": _STAGE3_RESOURCE_BYTES, + "sha256": _STAGE3_RESOURCE_SHA256, + "virtual_address": _STAGE3_RESOURCE_VA, + } + or type(resource.get("bytes")) is not int + ): + raise ExactColorIntegrityError("Stage-3 resource provenance is not the pinned LS5000.md3 block") + + artifacts = report.get("artifacts") + if type(artifacts) is not list: + raise ExactColorIntegrityError("Stage-3 validation report has no artifact inventory") + inventory: dict[str, dict[str, Any]] = {} + for row in artifacts: + if type(row) is not dict or set(row) != {"bytes", "name", "sha256"} or type(row.get("name")) is not str: + raise ExactColorIntegrityError("Stage-3 artifact inventory is malformed") + name = row["name"] + if name in inventory: + raise ExactColorIntegrityError(f"Stage-3 artifact inventory repeats {name!r}") + if type(row.get("bytes")) is not int or row["bytes"] <= 0: + raise ExactColorIntegrityError(f"Stage-3 artifact inventory has an invalid byte size for {name!r}") + _validate_sha256(row.get("sha256"), label=f"Stage-3 artifact {name}") + inventory[name] = row + if set(inventory) != _STAGE3_ARTIFACTS: + raise ExactColorIntegrityError("Stage-3 artifact inventory does not contain the exact required file set") + for name, expected_bytes in _STAGE3_FIXED_ARTIFACT_BYTES.items(): + if inventory[name]["bytes"] != expected_bytes: + raise ExactColorIntegrityError(f"Stage-3 artifact inventory has the wrong size for {name}") + if inventory["callback-buffer.bin"]["bytes"] not in _STAGE3_CALLBACK_BYTES: + raise ExactColorIntegrityError("Stage-3 callback artifact has unsupported geometry") + for channel, expected in zip(_BUILDER_CHANNELS, pre_f_lut_sha256, strict=True): + if inventory.get(f"builder-preF-{channel}.bin") != { + "bytes": _PRE_F_LUT_BYTES, + "name": f"builder-preF-{channel}.bin", + "sha256": expected, + }: + raise ExactColorIntegrityError(f"Stage-3 validation report does not bind builder-preF-{channel}.bin") + + +def _validate_observer_provenance(value: object, *, label: str) -> None: + if type(value) is not dict: + raise ExactColorIntegrityError(f"Stage-3 {label} provenance is malformed") + row = cast(dict[str, Any], value) + if ( + set(row) != {"path", "bytes", "sha256"} + or type(row.get("path")) is not str + or not row["path"] + or type(row.get("bytes")) is not int + or row["bytes"] <= 0 + ): + raise ExactColorIntegrityError(f"Stage-3 {label} provenance is malformed") + _validate_sha256(row.get("sha256"), label=f"Stage-3 {label}") + + +def _validate_cms_receipt( + payload: dict[str, Any], + *, + builder_receipt_sha256: str, + input_rgb_sha256: str, + output_rgb_sha256: str, +) -> None: + required_keys = { + "algorithm", + "assets", + "builder_receipt_sha256", + "chunk_pixels", + "dll_free", + "input_rgb_sha256", + "kind", + "oracle_source", + "output_rgb_sha256", + "scope", + "stage_order", + "upstream_builder_included", + "validation", + "version", + } + if set(payload) != required_keys: + raise ExactColorIntegrityError("CMS receipt does not contain the exact production contract") + if ( + payload.get("kind") != CMS_RECEIPT_KIND + or type(payload.get("version")) is not int + or payload.get("version") != CMS_RECEIPT_VERSION + or payload.get("algorithm") != CMS_ALGORITHM_ID + ): + raise ExactColorIntegrityError("CMS receipt kind, version, or algorithm is unsupported") + if payload.get("assets") != dict(CMS_ASSET_SHA256): + raise ExactColorIntegrityError("CMS receipt does not bind the exact nine transform assets") + if payload.get("oracle_source") != { + "path": "portable_oracle_evaluator.py", + "sha256": CMS_ORACLE_SOURCE_SHA256, + }: + raise ExactColorIntegrityError("CMS receipt does not bind the verified oracle source") + validation = payload.get("validation") + expected_validation = { + "events": 12, + "full_payload_mismatched_bytes": 0, + "full_payload_total_bytes": 698_880, + "mismatched_u16": 0, + "receipt_sha256": CMS_VALIDATION_RECEIPT_SHA256, + "total_u16": 265_440, + } + if ( + type(validation) is not dict + or validation != expected_validation + or any(type(validation[key]) is not type(value) for key, value in expected_validation.items()) + ): + raise ExactColorIntegrityError("CMS receipt does not bind the zero-mismatch 12-event validation") + if ( + payload.get("scope") != CMS_SCOPE + or payload.get("stage_order") != ["stage1", "stage2"] + or payload.get("dll_free") is not True + or payload.get("upstream_builder_included") is not False + ): + raise ExactColorIntegrityError("CMS receipt scope or stage contract is unsupported") + chunk_pixels = payload.get("chunk_pixels") + if type(chunk_pixels) is not int or not 1 <= chunk_pixels <= 262_144: + raise ExactColorIntegrityError("CMS receipt chunk size is invalid") + if ( + payload.get("builder_receipt_sha256") != builder_receipt_sha256 + or payload.get("input_rgb_sha256") != input_rgb_sha256 + or payload.get("output_rgb_sha256") != output_rgb_sha256 + ): + raise ExactColorIntegrityError("CMS receipt does not bind its builder, input, and output") + + +def _stable_read_non_symlink( + path: Path, + *, + label: str, + expected_bytes: int | None = None, + max_bytes: int | None = None, +) -> tuple[bytes, tuple[int, int, int, int, int]]: + byte_limit = expected_bytes if expected_bytes is not None else max_bytes + if byte_limit is None: + raise ExactColorIntegrityError(f"{label} read has no byte bound") + descriptor: int | None = None + try: + before = path.lstat() + if stat.S_ISLNK(before.st_mode) or not stat.S_ISREG(before.st_mode): + raise OSError("not a regular non-symlink file") + if expected_bytes is not None and before.st_size != expected_bytes: + raise ExactColorIntegrityError(f"{label} must be exactly {expected_bytes} bytes") + if max_bytes is not None and before.st_size > max_bytes: + raise ExactColorIntegrityError(f"{label} exceeds the bounded receipt size") + flags = os.O_RDONLY | getattr(os, "O_CLOEXEC", 0) | getattr(os, "O_NOFOLLOW", 0) | getattr(os, "O_NONBLOCK", 0) + descriptor = os.open(path, flags) + opened = os.fstat(descriptor) + chunks: list[bytes] = [] + total = 0 + while total <= byte_limit: + chunk = os.read( + descriptor, + min(1024 * 1024, byte_limit + 1 - total), + ) + if not chunk: + break + chunks.append(chunk) + total += len(chunk) + if total > byte_limit: + raise ExactColorIntegrityError(f"{label} exceeds its bounded byte length") + after = os.fstat(descriptor) + final = path.lstat() + except ExactColorIntegrityError: + raise + except OSError as error: + raise ExactColorUnavailable(f"{label} is unavailable: {path}: {error}") from error + finally: + if descriptor is not None: + os.close(descriptor) + identities = tuple(_file_identity(item) for item in (before, opened, after, final)) + if len(set(identities)) != 1: + raise ExactColorIntegrityError(f"{label} changed while being read: {path}") + payload = b"".join(chunks) + if len(payload) != before.st_size: + raise ExactColorIntegrityError(f"{label} changed byte length while being read: {path}") + return payload, identities[0] + + +def _assert_paths_unchanged( + paths: tuple[Path, ...], + identities: tuple[tuple[int, int, int, int, int], ...], + *, + label: str = "Stage-3 replay evidence", +) -> None: + for path, expected in zip(paths, identities, strict=True): + try: + current = path.lstat() + except OSError as error: + raise ExactColorUnavailable(f"{label} disappeared: {path}: {error}") from error + if stat.S_ISLNK(current.st_mode) or not stat.S_ISREG(current.st_mode) or _file_identity(current) != expected: + raise ExactColorIntegrityError(f"{label} changed during snapshot: {path}") + + +def _file_identity(value: os.stat_result) -> tuple[int, int, int, int, int]: + return ( + value.st_dev, + value.st_ino, + value.st_size, + value.st_mtime_ns, + value.st_ctime_ns, + ) + + +def _canonical_json(value: dict[str, Any]) -> bytes: + return json.dumps( + value, + sort_keys=True, + separators=(",", ":"), + ensure_ascii=False, + allow_nan=False, + ).encode("utf-8") + + +def _reject_duplicate_keys(pairs: list[tuple[str, Any]]) -> dict[str, Any]: + parsed: dict[str, Any] = {} + for key, value in pairs: + if key in parsed: + raise ValueError(f"duplicate receipt key {key!r}") + parsed[key] = value + return parsed + + +def _reject_json_constant(value: str) -> None: + raise ValueError(f"non-standard JSON constant {value!r}") + + +def _validate_rgb16(rgb: object, *, label: str) -> None: + if ( + not isinstance(rgb, np.ndarray) + or rgb.dtype != np.uint16 + or rgb.ndim != 3 + or rgb.shape[2] != 3 + or rgb.shape[0] == 0 + or rgb.shape[1] == 0 + ): + raise ExactColorIntegrityError(f"{label} must be a non-empty HxWx3 uint16 array") + + +__all__ = [ + "BUILDER_RECEIPT_SCHEMA", + "BuilderReceipt", + "CMS_ALGORITHM_ID", + "CMS_ASSET_SHA256", + "CMS_ORACLE_SOURCE_SHA256", + "CMS_RECEIPT_KIND", + "CMS_RECEIPT_VERSION", + "CMS_SCOPE", + "CMS_VALIDATION_RECEIPT_SHA256", + "ExactColorIntegrityError", + "ExactColorResult", + "ExactColorUnavailable", + "FIXED_COMPOSITION_SHA256", + "NATIVE_BUILDER_ALGORITHM_ID", + "NATIVE_BUILDER_SCOPE", + "NATIVE_RESOURCE_SHA256", + "NativeValidatedBuilderReceipt", + "PositiveColorMode", + "STAGE3_REPORT_SCHEMA", + "STAGE3_REPLAY_SCOPE", + "Stage1BuilderResult", + "ValidatedBuilderReceipt", + "VerifiedBuilderApplicationReceipt", + "VerifiedCMSReceipt", + "VerifiedPortableCMSEvaluator", + "VerifiedStage1Builder", + "builder_receipt_payload", + "evaluate_exact_color", + "load_native_builder_receipt", + "load_stage3_replay_builder_receipt", + "receipt_payload", + "rgb16_content_sha256", +] diff --git a/negpy/services/roll/live_acceptance.py b/negpy/services/roll/live_acceptance.py new file mode 100644 index 00000000..39861f58 --- /dev/null +++ b/negpy/services/roll/live_acceptance.py @@ -0,0 +1,1246 @@ +"""One-shot, fail-closed LS-5000 six-frame live acceptance runner.""" + +from __future__ import annotations + +import argparse +import dataclasses +import hashlib +import hmac +import json +import math +import os +import re +import signal +import stat +import threading +import uuid +from collections.abc import Callable +from datetime import UTC, datetime +from pathlib import Path +from typing import Any + +from negpy.infrastructure.roll.repair import RepairMode +from negpy.services.repair.hybrid_runtime_manifest import ( + load_default_hybrid_runtime_manifest, +) +from negpy.services.repair.fauxice_hybrid_runner import HybridRuntimeConfig +from negpy.services.roll.exact_color import PositiveColorMode +from negpy.services.roll.deep_acceptance import ( + collect_completed_frame_files, + validate_six_frame_batch, +) +from negpy.services.roll.live_reservation import ( + OUTPUT_LOCK_NAME, + ExclusiveReceiptReservation, + FixedOutputLease, + ReservationConflict, +) +from negpy.services.roll.live_evidence import ( + CaptureEvidenceSnapshot, + bind_capture_evidence_inventory, + snapshot_capture_evidence, + validate_six_frame_batch_capture_evidence, +) +from negpy.services.roll.live_review import ( + ValidatedReviewedApproval, + approval_payload, + load_reviewed_approval, + validate_restored_thumbnails, +) +from negpy.services.roll.service import RollFrameOutput, RollScanningService + + +SCHEMA = "negpy.ls5000-live-acceptance.v2" +SLOTS = (1, 2, 3, 4, 5, 6) +# Kept for backwards-compatible imports. The active approval set is derived +# from the restored preview and the SHA-pinned review document below. +APPROVED_SLOTS = (1, 6) +EXPECTED_DEVICE_MODEL = "Nikon LS-5000 ED 1.03" +FILENAME_PATTERN = 'acceptance_slot{{ "%02d" % seq }}' +_SESSION_MAX_BYTES = 64 * 1024 +_FRAME_RECEIPT_MAX_BYTES = 16 * 1024 * 1024 +_SHA256_RE = re.compile(r"[0-9a-f]{64}") +# The frozen app enumerates attached units with direct usb:BUS:ADDRESS ids. +_DIRECT_DEVICE_ID_RE = re.compile(r"usb:[0-9]{1,4}:[0-9]{1,4}") +_MAX_TELEMETRY_ERRORS = 16 + + +class LiveAcceptanceError(RuntimeError): + """The one-shot live acceptance run stopped without passing.""" + + def __init__(self, message: str, *, receipt: dict[str, Any]) -> None: + self.receipt = receipt + super().__init__(message) + + +@dataclasses.dataclass(frozen=True) +class LiveAcceptanceRequest: + device_id: str + preview_session_path: Path + preview_session_sha256: str + reviewed_approval_path: Path + reviewed_approval_sha256: str + output_dir: Path + run_receipt_path: Path + attempts_root_path: Path + confirm_live: bool + hybrid_runtime_manifest_path: Path | None = None + + +@dataclasses.dataclass(frozen=True) +class _SessionPayload: + text: str + sha256: str + size: int + + +def _utc_now() -> str: + return datetime.now(UTC).isoformat() + + +def _json_event(event: dict[str, Any]) -> None: + print( + json.dumps( + event, + sort_keys=True, + separators=(",", ":"), + ensure_ascii=True, + allow_nan=False, + ), + flush=True, + ) + + +def _read_stable_regular_file( + path: Path, + *, + maximum_bytes: int, + label: str, +) -> bytes: + descriptor: int | None = None + try: + linked = path.lstat() + if stat.S_ISLNK(linked.st_mode) or not stat.S_ISREG(linked.st_mode): + raise ValueError(f"{label} must be a regular non-symlink file") + if linked.st_size > maximum_bytes: + raise ValueError(f"{label} exceeds its safe size limit") + flags = os.O_RDONLY | getattr(os, "O_CLOEXEC", 0) | getattr(os, "O_NOFOLLOW", 0) | getattr(os, "O_NONBLOCK", 0) + descriptor = os.open(path, flags) + with os.fdopen(descriptor, "rb", closefd=True) as handle: + descriptor = None + opened = os.fstat(handle.fileno()) + if not stat.S_ISREG(opened.st_mode) or (opened.st_dev, opened.st_ino, opened.st_size) != ( + linked.st_dev, + linked.st_ino, + linked.st_size, + ): + raise ValueError(f"{label} changed while opening") + payload = handle.read(maximum_bytes + 1) + after = os.fstat(handle.fileno()) + final = path.lstat() + + def identity(item: os.stat_result) -> tuple[int, int, int, int, int]: + return ( + item.st_dev, + item.st_ino, + item.st_size, + item.st_mtime_ns, + item.st_ctime_ns, + ) + + if ( + len(payload) != opened.st_size + or len(payload) > maximum_bytes + or identity(opened) != identity(after) + or identity(opened) != identity(final) + ): + raise ValueError(f"{label} changed while reading") + return payload + except FileNotFoundError as error: + raise ValueError(f"{label} is missing: {path}") from error + except OSError as error: + raise ValueError(f"could not read {label}: {error}") from error + finally: + if descriptor is not None: + os.close(descriptor) + + +def _load_session(path: Path, expected_sha256: str) -> _SessionPayload: + if _SHA256_RE.fullmatch(expected_sha256) is None: + raise ValueError("preview-session pin must be a lowercase SHA-256 digest") + payload = _read_stable_regular_file( + path, + maximum_bytes=_SESSION_MAX_BYTES, + label="preview session", + ) + actual_sha256 = hashlib.sha256(payload).hexdigest() + if not hmac.compare_digest(actual_sha256, expected_sha256): + raise ValueError(f"preview-session SHA-256 mismatch: expected {expected_sha256}, got {actual_sha256}") + try: + text = payload.decode("utf-8") + except UnicodeDecodeError as error: + raise ValueError("preview session is not UTF-8") from error + return _SessionPayload(text=text, sha256=actual_sha256, size=len(payload)) + + +def _default_review_loader( + request: LiveAcceptanceRequest, + session: _SessionPayload, +) -> ValidatedReviewedApproval: + return load_reviewed_approval( + request.reviewed_approval_path, + request.reviewed_approval_sha256, + preview_session_path=request.preview_session_path, + preview_session_payload=session.text, + preview_session_sha256=session.sha256, + ) + + +def _paths_overlap(left: Path, right: Path) -> bool: + return left == right or left.is_relative_to(right) or right.is_relative_to(left) + + +def _preflight_paths(request: LiveAcceptanceRequest) -> tuple[Path, Path, Path]: + output = request.output_dir.resolve(strict=True) + linked_output = request.output_dir.lstat() + if stat.S_ISLNK(linked_output.st_mode) or not stat.S_ISDIR(linked_output.st_mode): + raise ValueError("output directory must be an existing non-symlink directory") + receipt = request.run_receipt_path.absolute() + receipt_parent = receipt.parent.resolve(strict=True) + if not receipt_parent.is_dir(): + raise ValueError("run-receipt parent must be a directory") + receipt = receipt_parent / receipt.name + if receipt == output or receipt.is_relative_to(output): + raise ValueError("run receipt must be outside the output directory") + attempts = request.attempts_root_path.resolve(strict=True) + linked_attempts = request.attempts_root_path.lstat() + if stat.S_ISLNK(linked_attempts.st_mode) or not stat.S_ISDIR(linked_attempts.st_mode): + raise ValueError("attempts root must be an existing non-symlink directory") + if _paths_overlap(attempts, output): + raise ValueError("attempts root must be disjoint from the output directory") + if receipt == attempts or receipt.is_relative_to(attempts): + raise ValueError("run receipt must be outside the attempts root") + return output, receipt, attempts + + +def _require_artifact(path_value: str | None, *, output_dir: Path, label: str) -> str: + if not path_value: + raise ValueError(f"{label} was not produced") + path = Path(path_value) + linked = path.lstat() + if stat.S_ISLNK(linked.st_mode) or not stat.S_ISREG(linked.st_mode): + raise ValueError(f"{label} must be a regular non-symlink file") + resolved = path.resolve(strict=True) + if not resolved.is_relative_to(output_dir): + raise ValueError(f"{label} escaped the output directory") + return str(resolved) + + +def _require_true(document: dict[str, Any], key: str, *, label: str) -> None: + if document.get(key) is not True: + raise ValueError(f"{label} is not verified") + + +def _validate_frame_output( + output: RollFrameOutput, + *, + expected_slot: int, + output_dir: Path, +) -> dict[str, Any]: + if output.slot != expected_slot: + raise ValueError(f"writer returned slot {output.slot}; expected {expected_slot}") + artifacts = { + field: _require_artifact( + getattr(output, field), + output_dir=output_dir, + label=f"slot {expected_slot} {field}", + ) + for field in ( + "rgb_path", + "ir_path", + "repaired_rgb_path", + "repaired_ir_path", + "positive_path", + "receipt_path", + "synthesis_mask_path", + "native_synthesis_mask_path", + "hybrid_receipt_path", + ) + } + receipt_path = Path(artifacts["receipt_path"]) + receipt_bytes = _read_stable_regular_file( + receipt_path, + maximum_bytes=_FRAME_RECEIPT_MAX_BYTES, + label=f"slot {expected_slot} frame receipt", + ) + try: + frame_receipt = json.loads(receipt_bytes) + except (UnicodeDecodeError, json.JSONDecodeError) as error: + raise ValueError(f"slot {expected_slot} frame receipt is invalid JSON") from error + if not isinstance(frame_receipt, dict) or frame_receipt.get("slot") != expected_slot: + raise ValueError(f"slot {expected_slot} frame receipt identity is wrong") + outputs = frame_receipt.get("outputs") + if not isinstance(outputs, dict): + raise ValueError(f"slot {expected_slot} frame receipt has no outputs object") + for tier in ("unrepaired", "repaired", "positive"): + entry = outputs.get(tier) + if not isinstance(entry, dict) or entry.get("written") is not True: + raise ValueError(f"slot {expected_slot} {tier} tier was not written") + repaired = outputs["repaired"] + if ( + repaired.get("mode_requested") != RepairMode.HYBRID.value + or repaired.get("mode_resolved") != RepairMode.HYBRID.value + or repaired.get("degraded") is not False + ): + raise ValueError(f"slot {expected_slot} did not complete non-degraded Hybrid repair") + positive = outputs["positive"] + if positive.get("color_mode") != PositiveColorMode.NIKON_EXACT.value: + raise ValueError(f"slot {expected_slot} positive is not Nikon exact") + for key in ( + "exact_nikon_color", + "native_per_acquisition_builder", + "builder_validated", + "cms_verified", + ): + _require_true(positive, key, label=f"slot {expected_slot} positive {key}") + for evidence_key, truth_keys in ( + ("repair_acquisition_evidence", ("retained", "replayable")), + ("native_color_evidence", ("retained",)), + ): + evidence = outputs.get(evidence_key) + if not isinstance(evidence, dict): + raise ValueError(f"slot {expected_slot} has no {evidence_key}") + for key in truth_keys: + _require_true( + evidence, + key, + label=f"slot {expected_slot} {evidence_key} {key}", + ) + return { + "slot": expected_slot, + "artifacts": artifacts, + "frame_receipt_sha256": hashlib.sha256(receipt_bytes).hexdigest(), + } + + +def _event_value(value: object) -> str | int | float | bool | None: + if value is None: + return None + if isinstance(value, (str, int, bool)): + return value + if isinstance(value, float): + return value if math.isfinite(value) else repr(value) + try: + return repr(value)[:512] + except Exception: + try: + name = type(value).__name__ + except Exception: + name = "object" + return f"" + + +def _progress_event(progress: object) -> dict[str, Any]: + event: dict[str, Any] = {"event": "scan_progress"} + for field in ("stage", "slot", "index", "total", "fraction", "message"): + value = getattr(progress, field, None) + if value is not None: + event[field] = _event_value(value) + return event + + +def _error_payload(error: BaseException) -> dict[str, str]: + try: + message = str(error)[:4096] + except Exception: + message = "" + return { + "type": type(error).__name__, + "message": message, + } + + +def _combine_error( + current: BaseException | None, + later: BaseException, + *, + label: str, +) -> BaseException: + if current is None: + return later + current_payload = _error_payload(current) + later_payload = _error_payload(later) + return RuntimeError( + f"{current_payload['type']}: {current_payload['message']}; {label}: {later_payload['type']}: {later_payload['message']}" + ) + + +def _default_service_factory( + *, + hybrid_runtime: HybridRuntimeConfig, +) -> RollScanningService: + return RollScanningService(hybrid_runtime=hybrid_runtime) + + +def run_live_acceptance( + request: LiveAcceptanceRequest, + *, + service_factory: Callable[..., Any] = _default_service_factory, + hybrid_runtime_loader: Callable[[Path | None], Any] = (load_default_hybrid_runtime_manifest), + review_loader: Callable[[LiveAcceptanceRequest, _SessionPayload], ValidatedReviewedApproval] = _default_review_loader, + frame_inventory_collector: Callable[..., list[str]] = (collect_completed_frame_files), + batch_validator: Callable[..., dict[str, Any]] = validate_six_frame_batch, + receipt_reserver: Callable[..., Any] = ExclusiveReceiptReservation.reserve, + output_lease_factory: Callable[..., Any] = FixedOutputLease.acquire, + evidence_lease_factory: Callable[..., Any] = FixedOutputLease.acquire, + emit: Callable[[dict[str, Any]], None] = _json_event, +) -> dict[str, Any]: + """Run one exact six-slot batch; any discrepancy ends it without retry.""" + + started_at = _utc_now() + run_id = uuid.uuid4().hex + phase = "starting" + frames: list[dict[str, Any]] = [] + outputs: list[RollFrameOutput] = [] + restored_slots: list[object] = [] + required_approval_slots: list[int] = [] + approved_slots: list[int] = [] + yielded_slots: list[object] = [] + committed_slots: list[object] = [] + verified_slots: list[int] = [] + telemetry_errors: list[dict[str, str]] = [] + telemetry_error_count = 0 + last_progress: dict[str, Any] | None = None + processing_slot: int | None = None + batch_prepared = False + batch_exhausted = False + transport_may_have_advanced_beyond_yielded = False + + session: _SessionPayload | None = None + review: ValidatedReviewedApproval | None = None + reviewed_approval_slots: tuple[int, ...] = () + runtime: Any = None + deep_batch: dict[str, Any] | None = None + output_dir: Path | None = None + attempts_root: Path | None = None + receipt_path = request.run_receipt_path.absolute() + receipt_reservation: Any = None + receipt_inode: tuple[int, int] | None = None + receipt_reserved = False + output_lease: Any = None + lease_acquired = False + lease_release_attempted = False + lease_released = False + evidence_lease: Any = None + evidence_lease_acquired = False + evidence_lease_release_attempted = False + evidence_lease_released = False + evidence_snapshot: CaptureEvidenceSnapshot | None = None + evidence_inventory: Any = None + capture_evidence: dict[str, object] | None = None + capture_evidence_error: dict[str, str] | None = None + inventory_snapshot: Any = None + owned_files: set[Path] = set() + service: Any = None + service_constructed = False + roll_opened = False + iterator: Any = None + iterator_created = False + iterator_close_attempted = False + iterator_close_succeeded = False + close_attempted = False + close_succeeded = False + operation_error: BaseException | None = None + + def record_telemetry_error(event_name: str, error: BaseException) -> None: + nonlocal telemetry_error_count + telemetry_error_count += 1 + if len(telemetry_errors) < _MAX_TELEMETRY_ERRORS: + telemetry_errors.append( + { + "event": event_name[:128], + **_error_payload(error), + } + ) + + def safe_emit(event: dict[str, Any]) -> None: + try: + emit(event) + except Exception as error: + record_telemetry_error( + str(event.get("event", "unknown")), + error, + ) + + def operation_state() -> dict[str, Any]: + return { + "phase": phase, + "service_constructed": service_constructed, + "roll_opened": roll_opened, + "requested_slots": list(SLOTS), + "restored_slots": list(restored_slots), + "required_approval_slots": list(required_approval_slots), + "approved_slots": list(approved_slots), + "batch_prepared": batch_prepared, + "yielded_slots": list(yielded_slots), + "committed_slots": list(committed_slots), + "verified_slots": list(verified_slots), + "processing_slot": processing_slot, + "batch_exhausted": batch_exhausted, + "last_progress": last_progress, + "transport_may_have_advanced_beyond_yielded": (transport_may_have_advanced_beyond_yielded), + } + + def receipt_document( + status: str, + *, + finished_at: str | None = None, + error: BaseException | None = None, + ) -> dict[str, Any]: + document: dict[str, Any] = { + "schema": SCHEMA, + "status": status, + "phase": phase, + "run_id": run_id, + "started_at": started_at, + "finished_at": finished_at, + "device_id": request.device_id, + "preview_session": { + "path": str(request.preview_session_path.absolute()), + "expected_sha256": request.preview_session_sha256, + "verified_sha256": session.sha256 if session is not None else None, + "bytes": session.size if session is not None else None, + }, + "reviewed_approval": { + "path": str(request.reviewed_approval_path.absolute()), + "expected_sha256": request.reviewed_approval_sha256, + "verified_sha256": review.sha256 if review is not None else None, + "bytes": review.byte_length if review is not None else None, + "reviewed_fingerprint_sha256": (review.reviewed_fingerprint_sha256 if review is not None else None), + "contact_sheet": ( + { + "path": str(review.contact_sheet_path), + "sha256": review.contact_sheet_sha256, + } + if review is not None + else None + ), + }, + "output_dir": str(output_dir or request.output_dir.absolute()), + "capture_evidence": ( + capture_evidence + if capture_evidence is not None + else { + "path": str(attempts_root or request.attempts_root_path.absolute()), + "retained": False, + "manifest_sha256": None, + "file_count": None, + "directory_count": None, + "total_bytes": None, + "files": None, + "snapshot_error": capture_evidence_error, + } + ), + "run_receipt": str(receipt_path), + "receipt_reservation": { + "reserved": receipt_reserved, + "inode": list(receipt_inode) if receipt_inode is not None else None, + }, + "output_lease": { + "acquired": lease_acquired, + "lock_name": OUTPUT_LOCK_NAME, + "release_attempted": lease_release_attempted, + "released": lease_released, + }, + "capture_evidence_lease": { + "acquired": evidence_lease_acquired, + "lock_name": OUTPUT_LOCK_NAME, + "release_attempted": evidence_lease_release_attempted, + "released": evidence_lease_released, + }, + "slots": list(SLOTS), + "approved_slots": list(approved_slots), + "operation_state": operation_state(), + "settings": { + "write_unrepaired": True, + "write_repaired": True, + "write_positive": True, + "repair_mode": RepairMode.HYBRID.value, + "positive_mode": PositiveColorMode.NIKON_EXACT.value, + "filename_pattern": FILENAME_PATTERN, + }, + "frames": frames, + "deep_acceptance": deep_batch, + "close": { + "iterator": { + "attempted": iterator_close_attempted, + "succeeded": iterator_close_succeeded, + }, + "roll": { + "attempted": close_attempted, + "succeeded": close_succeeded, + }, + }, + "telemetry_errors": list(telemetry_errors), + "telemetry_error_count": telemetry_error_count, + "telemetry_errors_truncated": (telemetry_error_count > len(telemetry_errors)), + "retry_count": 0, + "eject_requested": False, + } + if error is not None: + document["error"] = _error_payload(error) + return document + + def on_scan_progress(progress: object) -> None: + nonlocal last_progress + try: + event = _progress_event(progress) + except Exception as error: + record_telemetry_error("scan_progress", error) + return + last_progress = event + safe_emit(event) + + try: + safe_emit({"event": "run_started", "slots": list(SLOTS), "run_id": run_id}) + if request.confirm_live is not True: + raise ValueError("--confirm-live is required") + if not request.device_id.strip(): + raise ValueError("device id must be non-empty") + if _DIRECT_DEVICE_ID_RE.fullmatch(request.device_id) is None: + raise ValueError( + "device id must be the direct usb:BUS:ADDRESS form reported by " + "this app's own enumeration (for example usb:2:7); SANE-style " + f"identifiers are not accepted here, got {request.device_id!r}" + ) + output_dir, receipt_path, attempts_root = _preflight_paths(request) + + phase = "reserving_receipt" + receipt_reservation = receipt_reserver( + receipt_path, + receipt_document("in_progress"), + ) + receipt_reserved = True + receipt_inode = tuple(receipt_reservation.inode) + + phase = "reserving_output" + output_lease = output_lease_factory( + output_dir, + { + "schema": "negpy.ls5000-live-output-lock.v1", + "run_id": run_id, + "run_receipt": str(receipt_path), + "created_at": started_at, + }, + require_empty=True, + ) + lease_acquired = True + inventory_snapshot = output_lease.assert_inventory(()) + + phase = "reserving_capture_evidence" + evidence_lease = evidence_lease_factory( + attempts_root, + { + "schema": "negpy.ls5000-live-capture-evidence-lock.v1", + "run_id": run_id, + "run_receipt": str(receipt_path), + "created_at": started_at, + }, + require_empty=True, + ) + evidence_lease_acquired = True + phase = "output_reserved" + receipt_reservation.publish(receipt_document("in_progress")) + + phase = "validating_offline_inputs" + session = _load_session( + request.preview_session_path, + request.preview_session_sha256, + ) + review = review_loader(request, session) + reviewed_approval_slots = tuple(sorted(review.approvals)) + runtime = hybrid_runtime_loader(request.hybrid_runtime_manifest_path) + if runtime is None: + raise ValueError("the pinned Hybrid runtime is not installed") + runtime.validate_files() + receipt_reservation.assert_owned() + inventory_snapshot = output_lease.assert_inventory( + (), + previous=inventory_snapshot, + ) + + phase = "ready_to_open" + receipt_reservation.publish(receipt_document("in_progress")) + safe_emit( + { + "event": "preflight_passed", + "preview_session_sha256": session.sha256, + "reviewed_approval_sha256": review.sha256, + } + ) + + service = service_factory(hybrid_runtime=runtime) + service_constructed = True + receipt_reservation.assert_owned() + inventory_snapshot = output_lease.assert_inventory( + (), + previous=inventory_snapshot, + ) + phase = "opening_roll" + service.open_roll( + request.device_id, + attempts_root=attempts_root, + ) + roll_opened = True + safe_emit({"event": "roll_opened", "device_id": request.device_id}) + + phase = "restoring_review" + thumbnails = service.restore_preview_session(session.text, slots=SLOTS) + restored_slots.extend(getattr(thumbnail, "slot", None) for thumbnail in thumbnails) + if tuple(restored_slots) != SLOTS: + raise ValueError(f"restored slots are {tuple(restored_slots)}; expected exactly {SLOTS}") + validate_restored_thumbnails(thumbnails, review) + required_approval_slots.extend(slot for slot in SLOTS if service.needs_approval(slot)) + if tuple(required_approval_slots) != reviewed_approval_slots: + raise ValueError( + "restored session approval set does not match the reviewed " + f"evidence: session={tuple(required_approval_slots)}, " + f"review={reviewed_approval_slots}" + ) + safe_emit({"event": "preview_session_restored", "slots": list(restored_slots)}) + + phase = "applying_reviewed_approvals" + for slot in reviewed_approval_slots: + returned = service.approve(slot) + expected_payload = approval_payload(review.approvals[slot]) + if approval_payload(returned) != expected_payload: + raise ValueError(f"service approval for slot {slot} differs from reviewed evidence") + approved_slots.append(slot) + safe_emit({"event": "slot_approved", "slot": slot}) + receipt_reservation.publish(receipt_document("in_progress")) + + phase = "preparing_batch" + service.prepare_batch() + batch_prepared = True + receipt_reservation.assert_owned() + inventory_snapshot = output_lease.assert_inventory( + owned_files, + previous=inventory_snapshot, + ) + phase = "scanning" + receipt_reservation.publish(receipt_document("in_progress")) + iterator = iter( + service.scan_many( + SLOTS, + on_progress=on_scan_progress, + ) + ) + iterator_created = True + while True: + try: + frame = next(iterator) + except StopIteration: + batch_exhausted = True + break + actual_slot = getattr(frame, "slot", None) + yielded_slots.append(_event_value(actual_slot)) + if len(yielded_slots) > len(SLOTS): + raise ValueError("scanner yielded more than six frames") + expected_slot = SLOTS[len(yielded_slots) - 1] + processing_slot = expected_slot + if actual_slot != expected_slot: + raise ValueError(f"scanner yielded slot {actual_slot}; expected {expected_slot}") + receipt_reservation.assert_owned() + inventory_snapshot = output_lease.assert_inventory( + owned_files, + previous=inventory_snapshot, + ) + + def on_repair_progress( + fraction: object, + *, + slot: int = expected_slot, + ) -> None: + nonlocal last_progress + try: + event = { + "event": "repair_progress", + "slot": slot, + "fraction": _event_value(fraction), + } + except Exception as error: + record_telemetry_error("repair_progress", error) + return + last_progress = event + safe_emit(event) + + output = service.write_frame( + frame, + str(output_dir), + FILENAME_PATTERN, + write_unrepaired=True, + write_repaired=True, + write_positive=True, + repair_mode=RepairMode.HYBRID.value, + positive_mode=PositiveColorMode.NIKON_EXACT.value, + on_repair_progress=on_repair_progress, + ) + outputs.append(output) + returned_slot = _event_value(output.slot) + committed_slots.append(returned_slot) + frame_entry: dict[str, Any] = { + "slot": returned_slot, + "expected_slot": expected_slot, + "returned_output": { + field: getattr(output, field) + for field in ( + "rgb_path", + "ir_path", + "repaired_rgb_path", + "repaired_ir_path", + "positive_path", + "receipt_path", + "synthesis_mask_path", + "native_synthesis_mask_path", + "hybrid_receipt_path", + ) + }, + } + frames.append(frame_entry) + safe_emit({"event": "frame_committed", "slot": expected_slot}) + + frame_entry.update( + _validate_frame_output( + output, + expected_slot=expected_slot, + output_dir=output_dir, + ) + ) + referenced = frame_inventory_collector( + output, + output_dir=output_dir, + expected_slot=expected_slot, + ) + if type(referenced) is not list or not referenced or any(type(path) is not str for path in referenced): + raise ValueError(f"slot {expected_slot} publication inventory is invalid") + owned_files.update(Path(path) for path in referenced) + inventory_snapshot = output_lease.assert_inventory( + owned_files, + previous=inventory_snapshot, + ) + frame_entry["publication_inventory"] = list(referenced) + processing_slot = None + receipt_reservation.publish(receipt_document("in_progress")) + safe_emit({"event": "frame_checkpointed", "slot": expected_slot}) + + if tuple(yielded_slots) != SLOTS: + raise ValueError(f"scanner yielded {tuple(yielded_slots)}; expected exactly {SLOTS}") + if tuple(committed_slots) != SLOTS: + raise ValueError("not every yielded frame was durably committed") + phase = "batch_exhausted" + receipt_reservation.publish(receipt_document("in_progress")) + except ReservationConflict as error: + operation_error = error + except BaseException as error: + operation_error = error + finally: + if iterator is not None: + close_iterator = getattr(iterator, "close", None) + if callable(close_iterator): + iterator_close_attempted = True + try: + close_iterator() + except BaseException as iterator_error: + operation_error = _combine_error( + operation_error, + iterator_error, + label="scan iterator close failed", + ) + else: + iterator_close_succeeded = True + if service is not None: + close_attempted = True + try: + service.close() + except BaseException as close_error: + operation_error = _combine_error( + operation_error, + close_error, + label="scanner close failed", + ) + else: + close_succeeded = True + safe_emit({"event": "roll_closed"}) + + if iterator_created and not batch_exhausted: + transport_may_have_advanced_beyond_yielded = True + + if evidence_lease is not None: + try: + if roll_opened and not close_succeeded: + raise RuntimeError("capture evidence cannot be sealed while scanner ownership is uncertain") + if attempts_root is None: + raise RuntimeError("capture evidence path was not resolved") + evidence_snapshot = snapshot_capture_evidence(attempts_root) + evidence_inventory = evidence_lease.assert_inventory(evidence_snapshot.files) + capture_evidence = bind_capture_evidence_inventory( + evidence_snapshot, + evidence_inventory, + ) + capture_evidence["snapshot_error"] = None + except BaseException as evidence_error: + capture_evidence_error = _error_payload(evidence_error) + if operation_error is None: + operation_error = evidence_error + + if operation_error is None: + try: + phase = "validating_six_frame_batch" + if output_dir is None or output_lease is None or runtime is None: + raise RuntimeError("live acceptance state is incomplete") + if not close_succeeded: + raise RuntimeError("scanner was not closed before deep acceptance") + if evidence_snapshot is None or evidence_inventory is None or capture_evidence is None: + raise RuntimeError("capture evidence was not sealed before deep acceptance") + batch_binding = validate_six_frame_batch_capture_evidence(evidence_snapshot) + if review is None or (batch_binding.get("reviewed_fingerprint_sha256") != review.reviewed_fingerprint_sha256): + raise RuntimeError("capture evidence batch does not match the reviewed fingerprint") + capture_evidence["batch_binding"] = batch_binding + receipt_reservation.publish(receipt_document("in_progress")) + deep_batch = batch_validator( + outputs, + output_dir=output_dir, + allowed_output_lock_name=OUTPUT_LOCK_NAME, + hybrid_runtime=runtime, + ) + if review is None: + raise RuntimeError("reviewed approval disappeared before deep acceptance") + expected_manual_approval_bindings: list[dict[str, Any]] = [] + for slot in reviewed_approval_slots: + binding = approval_payload(review.approvals[slot]).get("binding_sha256") + if type(binding) is not str or _SHA256_RE.fullmatch(binding) is None: + raise ValueError(f"reviewed approval binding for slot {slot} is invalid") + expected_manual_approval_bindings.append( + { + "slot": slot, + "binding_sha256": binding, + } + ) + if ( + type(deep_batch) is not dict + or deep_batch.get("status") != "passed" + or deep_batch.get("slots") != list(SLOTS) + or deep_batch.get("approved_slots") != list(reviewed_approval_slots) + or deep_batch.get("device_id") != request.device_id + or deep_batch.get("device_model") != EXPECTED_DEVICE_MODEL + or deep_batch.get("reviewed_fingerprint_sha256") != review.reviewed_fingerprint_sha256 + or deep_batch.get("manual_approval_bindings") != expected_manual_approval_bindings + or type(deep_batch.get("referenced_files")) is not list + or any(type(path) is not str for path in deep_batch["referenced_files"]) + or type(deep_batch.get("frames")) is not list + or len(deep_batch["frames"]) != len(SLOTS) + ): + raise ValueError("six-frame deep acceptance returned an invalid result") + for expected_slot, frame_entry, deep_frame in zip( + SLOTS, + frames, + deep_batch["frames"], + strict=True, + ): + if ( + type(deep_frame) is not dict + or deep_frame.get("slot") != expected_slot + or type(deep_frame.get("referenced_files")) is not list + or any(type(path) is not str for path in deep_frame["referenced_files"]) + or set(deep_frame["referenced_files"]) != set(frame_entry["publication_inventory"]) + ): + raise ValueError(f"slot {expected_slot} deep inventory differs from its live checkpoint") + frame_entry["deep_acceptance"] = deep_frame + verified_slots.append(expected_slot) + safe_emit({"event": "frame_verified", "slot": expected_slot}) + batch_files = {Path(path) for path in deep_batch["referenced_files"]} + if batch_files != owned_files: + raise ValueError("six-frame deep acceptance inventory differs from frame checkpoints") + if tuple(verified_slots) != SLOTS: + raise ValueError("not every committed frame was deeply verified") + inventory_snapshot = output_lease.assert_inventory( + batch_files, + previous=inventory_snapshot, + ) + receipt_reservation.assert_owned() + phase = "six_frame_batch_verified" + receipt_reservation.publish(receipt_document("in_progress")) + safe_emit({"event": "six_frame_batch_verified", "slots": list(SLOTS)}) + except BaseException as validation_error: + operation_error = validation_error + + receipt: dict[str, Any] | None = None + failed_receipt_published = False + if operation_error is None: + if output_lease is None or inventory_snapshot is None: + operation_error = RuntimeError("output lease state is incomplete at finalization") + elif receipt_reservation is None: + operation_error = RuntimeError("run receipt was never reserved") + elif evidence_lease is None or evidence_snapshot is None or evidence_inventory is None: + operation_error = RuntimeError("capture evidence lease state is incomplete at finalization") + + if operation_error is None: + lease_release_attempted = True + phase = "finalizing_success" + safe_emit({"event": "run_finalizing", "status": "succeeded"}) + + def publish_success_while_output_directory_is_held() -> None: + nonlocal capture_evidence + nonlocal capture_evidence_error + nonlocal evidence_lease_release_attempted + nonlocal evidence_lease_released + nonlocal lease_released + nonlocal phase + nonlocal receipt + + evidence_lease_release_attempted = True + + def publish_success_while_both_directories_are_held() -> None: + nonlocal evidence_lease_released + nonlocal lease_released + nonlocal phase + nonlocal receipt + + # Both fixed locks have been unlinked, but both directory + # descriptors remain held. Each lease performs a second exact + # inventory check after this durable receipt publication. + evidence_lease_released = True + lease_released = True + phase = "succeeded" + safe_emit( + { + "event": "run_finished", + "status": "succeeded", + "run_receipt": str(receipt_path), + } + ) + receipt = receipt_document( + "succeeded", + finished_at=_utc_now(), + ) + receipt_reservation.publish(receipt) + + try: + evidence_lease.release_verified( + evidence_snapshot.files, + previous=evidence_inventory, + finalize=publish_success_while_both_directories_are_held, + ) + except BaseException as evidence_lease_error: + capture_evidence_error = _error_payload(evidence_lease_error) + if capture_evidence is not None: + capture_evidence = { + **capture_evidence, + "retained": False, + "snapshot_error": capture_evidence_error, + } + raise + finally: + evidence_lease_released = bool(evidence_lease.released) + + try: + output_lease.release_verified( + owned_files, + previous=inventory_snapshot, + finalize=publish_success_while_output_directory_is_held, + ) + except BaseException as lease_error: + operation_error = _combine_error( + operation_error, + lease_error, + label="verified output lease release failed", + ) + finally: + lease_released = bool(output_lease.released) + + if operation_error is not None: + if output_lease is not None and not output_lease.released: + lease_release_attempted = True + try: + output_lease.release() + except BaseException as lease_error: + operation_error = _combine_error( + operation_error, + lease_error, + label="output lease cleanup failed", + ) + finally: + lease_released = bool(output_lease.released) + phase = "failed" + safe_emit({"event": "run_finalizing", "status": "failed"}) + safe_emit( + { + "event": "run_finished", + "status": "failed", + "run_receipt": str(receipt_path), + } + ) + if evidence_lease is not None and not evidence_lease.released: + evidence_lease_release_attempted = True + if evidence_snapshot is not None and evidence_inventory is not None: + + def publish_failure_while_evidence_directory_is_held() -> None: + nonlocal evidence_lease_released + nonlocal failed_receipt_published + nonlocal receipt + + evidence_lease_released = True + receipt = receipt_document( + "failed", + finished_at=_utc_now(), + error=operation_error, + ) + if receipt_reservation is not None: + receipt_reservation.publish(receipt) + failed_receipt_published = True + + try: + evidence_lease.release_verified( + evidence_snapshot.files, + previous=evidence_inventory, + finalize=publish_failure_while_evidence_directory_is_held, + ) + except BaseException as evidence_lease_error: + operation_error = _combine_error( + operation_error, + evidence_lease_error, + label="verified capture evidence release failed", + ) + capture_evidence_error = _error_payload(evidence_lease_error) + if capture_evidence is not None: + capture_evidence = { + **capture_evidence, + "retained": False, + "snapshot_error": capture_evidence_error, + } + failed_receipt_published = False + finally: + evidence_lease_released = bool(evidence_lease.released) + else: + try: + evidence_lease.release() + except BaseException as evidence_lease_error: + operation_error = _combine_error( + operation_error, + evidence_lease_error, + label="capture evidence lease cleanup failed", + ) + finally: + evidence_lease_released = bool(evidence_lease.released) + if not failed_receipt_published: + receipt = receipt_document( + "failed", + finished_at=_utc_now(), + error=operation_error, + ) + + assert receipt is not None + if receipt_reservation is None: + message = _error_payload(operation_error or RuntimeError("run receipt reservation failed"))["message"] + raise LiveAcceptanceError(message, receipt=receipt) from operation_error + + if receipt.get("status") == "failed" and not failed_receipt_published: + try: + receipt_reservation.publish(receipt) + except BaseException as receipt_error: + failed_receipt = dict(receipt) + failed_receipt["status"] = "failed" + failed_receipt["phase"] = "failed" + failed_receipt["error"] = _error_payload(receipt_error) + raise LiveAcceptanceError( + f"could not publish the final run receipt: {_error_payload(receipt_error)['message']}", + receipt=failed_receipt, + ) from receipt_error + + try: + receipt_reservation.close() + except Exception: + # The document was already fsynced into the held inode. A close(2) + # housekeeping error must not replace that durable outcome. + pass + + if operation_error is not None: + raise LiveAcceptanceError( + _error_payload(operation_error)["message"], + receipt=receipt, + ) from operation_error + return receipt + + +def _parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser( + description=("Run the pinned six-frame LS-5000 live acceptance exactly once. The output directory must already exist and be empty.") + ) + parser.add_argument("--device-id", required=True) + parser.add_argument("--preview-session", required=True, type=Path) + parser.add_argument("--preview-session-sha256", required=True) + parser.add_argument("--reviewed-approval", required=True, type=Path) + parser.add_argument("--reviewed-approval-sha256", required=True) + parser.add_argument("--output-dir", required=True, type=Path) + parser.add_argument("--run-receipt", required=True, type=Path) + parser.add_argument("--attempts-root", required=True, type=Path) + parser.add_argument("--hybrid-runtime-manifest", type=Path) + parser.add_argument("--confirm-live", action="store_true") + return parser + + +class LiveAcceptanceInterrupted(BaseException): + """Raised at the interrupted call site when SIGINT/SIGTERM arrives. + + A BaseException on purpose: the acceptance's broad failure handling and + close/finalize path treat it exactly like any other fatal interruption, + producing a truthful failed receipt and a clean roll/service close. + """ + + +def install_interrupt_handlers() -> bool: + """Route SIGINT and SIGTERM through the truthful-failure path. + + The first signal raises LiveAcceptanceInterrupted at the interrupted call + site; later signals are ignored so close/finalize and receipt publication + cannot themselves be interrupted. Returns False outside the main thread, + where CPython forbids installing signal handlers. + """ + + if threading.current_thread() is not threading.main_thread(): + return False + + def _abort(signum: int, _frame: object) -> None: + signal.signal(signal.SIGINT, signal.SIG_IGN) + signal.signal(signal.SIGTERM, signal.SIG_IGN) + raise LiveAcceptanceInterrupted(f"received signal {signum}") + + signal.signal(signal.SIGINT, _abort) + signal.signal(signal.SIGTERM, _abort) + return True + + +def main(argv: list[str] | None = None) -> int: + install_interrupt_handlers() + args = _parser().parse_args(argv) + request = LiveAcceptanceRequest( + device_id=args.device_id, + preview_session_path=args.preview_session, + preview_session_sha256=args.preview_session_sha256, + reviewed_approval_path=args.reviewed_approval, + reviewed_approval_sha256=args.reviewed_approval_sha256, + output_dir=args.output_dir, + run_receipt_path=args.run_receipt, + attempts_root_path=args.attempts_root, + confirm_live=args.confirm_live, + hybrid_runtime_manifest_path=args.hybrid_runtime_manifest, + ) + try: + run_live_acceptance(request) + except LiveAcceptanceError: + return 1 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/negpy/services/roll/live_evidence.py b/negpy/services/roll/live_evidence.py new file mode 100644 index 00000000..ab1647b3 --- /dev/null +++ b/negpy/services/roll/live_evidence.py @@ -0,0 +1,658 @@ +"""Stable, receipt-ready inventory of retained Coolscan attempt evidence.""" + +from __future__ import annotations + +import dataclasses +import hashlib +import json +import os +import re +import stat +from pathlib import Path +from pathlib import PurePosixPath + +from negpy.services.roll.live_reservation import ( + OUTPUT_LOCK_NAME, + FileIdentity, + InventoryConflict, + InventorySnapshot, + is_ignorable_finder_metadata_file, +) + + +CAPTURE_EVIDENCE_MANIFEST_SCHEMA = "negpy.coolscan-attempts-manifest.v1" +_MAX_FILES = 4_096 +_MAX_TOTAL_BYTES = 64 * 1024 * 1024 * 1024 +_MAX_MANIFEST_BYTES = 128 * 1024 +_MAX_BATCH_JSON_BYTES = 256 * 1024 +_READ_CHUNK_BYTES = 4 * 1024 * 1024 +_SIX_FRAME_SLOTS = (1, 2, 3, 4, 5, 6) +_SIX_FRAME_BATCH_ROOT_RE = re.compile(r"batch-slot01-slot06-[^/]+") +_SHA256_RE = re.compile(r"[0-9a-f]{64}") +# The batch worker mints every parent-ACK nonce with secrets.token_hex(16). +_ACK_NONCE_RE = re.compile(r"[0-9a-f]{32}") +_FIRST_PLAN_NAME = "replay-first-rgbi4-plan.jsonl" +_CONTINUATION_PLAN_NAME = "replay-next-rgbi4-plan.json" +_CAPTURE_MANIFEST_NAME = "replay-first-rgbi4-manifest.json" +_FINE_READS = 2_980 +_FINE_BYTES = 619_458_560 +_METER_PASS_BYTES = 1_088_000 +_METER_BYTES = _METER_PASS_BYTES * 3 +_PREVIEW_BYTES = 6_250_496 +_BATCH_JOB_KEYS = { + "apply_all_boundary_offsets_before_first_frame", + "capture_plan_sha256", + "continuation_plan_sha256", + "expected_usb_address", + "expected_usb_bus", + "frames", + "parent_ack_required_after_every_frame", + "release_once_after_last_frame", + "reviewed_roll_fingerprint", + "schema_version", + "session_id", + "session_contract", +} +_BATCH_FRAME_KEYS = { + "ack", + "boundary_offset_rows", + "journal", + "manual_review_approval", + "output", + "slot", +} +_ACK_KEYS = { + "ack_nonce", + "action", + "frame_index", + "schema_version", + "session_id", + "slot", +} +_METER_LAYOUT = { + "passes": 3, + "rows_per_pass": 425, + "columns": 281, + "decoded_raster_channel_order": ["R", "G", "B", "IR"], + "wire_window_color_order": [9, 1, 2, 3], + "wire_color_to_controller_channel": { + "9": "IR", + "1": "R", + "2": "G", + "3": "B", + }, + "sample_byte_order": "big-endian-u16", + "row_core_bytes": 2_248, + "row_stride_bytes": 2_560, + "row_tail_bytes": 312, +} + + +@dataclasses.dataclass(frozen=True) +class CaptureEvidenceSnapshot: + root: Path + files: tuple[Path, ...] + identities: tuple[tuple[str, FileIdentity], ...] + manifest: dict[str, object] + + +def _identity(metadata: os.stat_result) -> FileIdentity: + return FileIdentity.from_stat(metadata) + + +def _stable_file_digest(path: Path) -> tuple[int, str, FileIdentity]: + before = path.lstat() + if stat.S_ISLNK(before.st_mode) or not stat.S_ISREG(before.st_mode) or before.st_nlink != 1: + raise InventoryConflict(f"capture evidence must be an exclusively owned regular file: {path}") + flags = os.O_RDONLY | getattr(os, "O_CLOEXEC", 0) | getattr(os, "O_NOFOLLOW", 0) | getattr(os, "O_NONBLOCK", 0) + descriptor = os.open(path, flags) + try: + opened = os.fstat(descriptor) + if not stat.S_ISREG(opened.st_mode) or _identity(opened) != _identity(before): + raise InventoryConflict(f"capture evidence changed while opening: {path}") + digest = hashlib.sha256() + total = 0 + while True: + chunk = os.read(descriptor, _READ_CHUNK_BYTES) + if not chunk: + break + digest.update(chunk) + total += len(chunk) + if total > _MAX_TOTAL_BYTES: + raise InventoryConflict("capture evidence exceeds its safe size limit") + after_read = os.fstat(descriptor) + finally: + os.close(descriptor) + after_path = path.lstat() + opened_identity = _identity(opened) + if total != opened.st_size or opened_identity != _identity(after_read) or opened_identity != _identity(after_path): + raise InventoryConflict(f"capture evidence changed while hashing: {path}") + return total, digest.hexdigest(), opened_identity + + +def _stable_file_bytes(path: Path, *, maximum_bytes: int) -> bytes: + """Read one small evidence file without following or racing a pathname.""" + + before = path.lstat() + if stat.S_ISLNK(before.st_mode) or not stat.S_ISREG(before.st_mode) or before.st_nlink != 1: + raise InventoryConflict(f"capture evidence must be an exclusively owned regular file: {path}") + if before.st_size > maximum_bytes: + raise InventoryConflict(f"capture evidence JSON is too large: {path}") + flags = os.O_RDONLY | getattr(os, "O_CLOEXEC", 0) | getattr(os, "O_NOFOLLOW", 0) | getattr(os, "O_NONBLOCK", 0) + descriptor = os.open(path, flags) + try: + opened = os.fstat(descriptor) + if not stat.S_ISREG(opened.st_mode) or _identity(opened) != _identity(before): + raise InventoryConflict(f"capture evidence changed while opening: {path}") + chunks: list[bytes] = [] + total = 0 + while True: + chunk = os.read(descriptor, min(_READ_CHUNK_BYTES, maximum_bytes + 1 - total)) + if not chunk: + break + chunks.append(chunk) + total += len(chunk) + if total > maximum_bytes: + raise InventoryConflict(f"capture evidence JSON is too large: {path}") + after_read = os.fstat(descriptor) + finally: + os.close(descriptor) + after_path = path.lstat() + opened_identity = _identity(opened) + if total != opened.st_size or opened_identity != _identity(after_read) or opened_identity != _identity(after_path): + raise InventoryConflict(f"capture evidence changed while reading: {path}") + return b"".join(chunks) + + +def snapshot_capture_evidence(root: Path) -> CaptureEvidenceSnapshot: + """Hash one retained attempts tree without following links or special files.""" + + canonical = root.resolve(strict=True) + linked_root = root.lstat() + if stat.S_ISLNK(linked_root.st_mode) or not stat.S_ISDIR(linked_root.st_mode): + raise InventoryConflict("capture evidence root must be an existing non-symlink directory") + + paths: list[Path] = [] + for current_text, directory_names, file_names in os.walk( + canonical, + topdown=True, + followlinks=False, + ): + current = Path(current_text) + current_metadata = current.lstat() + if stat.S_ISLNK(current_metadata.st_mode) or not stat.S_ISDIR(current_metadata.st_mode): + raise InventoryConflict(f"capture evidence entry is not a real directory: {current}") + for name in directory_names: + child = current / name + metadata = child.lstat() + if stat.S_ISLNK(metadata.st_mode) or not stat.S_ISDIR(metadata.st_mode): + raise InventoryConflict(f"capture evidence entry is not a real directory: {child}") + for name in file_names: + path = current / name + if current == canonical and name == OUTPUT_LOCK_NAME: + continue + metadata = path.lstat() + if is_ignorable_finder_metadata_file(name, metadata): + continue + paths.append(path) + if len(paths) > _MAX_FILES: + raise InventoryConflict("capture evidence contains too many files") + + rows: list[dict[str, object]] = [] + identities: list[tuple[str, FileIdentity]] = [] + total_bytes = 0 + ordered_paths = sorted(paths, key=lambda item: item.relative_to(canonical).as_posix()) + for path in ordered_paths: + relative = path.relative_to(canonical).as_posix() + size, digest, identity = _stable_file_digest(path) + total_bytes += size + if total_bytes > _MAX_TOTAL_BYTES: + raise InventoryConflict("capture evidence exceeds its safe size limit") + rows.append({"path": relative, "bytes": size, "sha256": digest}) + identities.append((relative, identity)) + + manifest_document = { + "schema": CAPTURE_EVIDENCE_MANIFEST_SCHEMA, + "files": rows, + } + manifest_payload = ( + json.dumps( + manifest_document, + sort_keys=True, + separators=(",", ":"), + ensure_ascii=True, + allow_nan=False, + ).encode("utf-8") + + b"\n" + ) + if len(manifest_payload) > _MAX_MANIFEST_BYTES: + raise InventoryConflict("capture evidence manifest exceeds its receipt-safe size limit") + manifest = { + **manifest_document, + "manifest_sha256": hashlib.sha256(manifest_payload).hexdigest(), + "file_count": len(rows), + "total_bytes": total_bytes, + } + return CaptureEvidenceSnapshot( + root=canonical, + files=tuple(ordered_paths), + identities=tuple(identities), + manifest=manifest, + ) + + +def _snapshot_json( + snapshot: CaptureEvidenceSnapshot, + relative: str, +) -> dict[str, object]: + rows = snapshot.manifest.get("files") + if not isinstance(rows, list): + raise InventoryConflict("capture evidence manifest has no file inventory") + matches = [row for row in rows if isinstance(row, dict) and row.get("path") == relative] + if len(matches) != 1 or not isinstance(matches[0].get("sha256"), str): + raise InventoryConflict(f"capture evidence manifest does not bind {relative}") + payload = _stable_file_bytes( + snapshot.root / Path(relative), + maximum_bytes=_MAX_BATCH_JSON_BYTES, + ) + if hashlib.sha256(payload).hexdigest() != matches[0]["sha256"]: + raise InventoryConflict(f"capture evidence changed after hashing: {relative}") + + def _reject_duplicate_keys(pairs: list[tuple[str, object]]) -> dict[str, object]: + merged: dict[str, object] = {} + for key, value in pairs: + if key in merged: + raise InventoryConflict(f"capture evidence JSON repeats key {key!r}: {relative}") + merged[key] = value + return merged + + try: + document = json.loads(payload, object_pairs_hook=_reject_duplicate_keys) + except (UnicodeDecodeError, json.JSONDecodeError) as error: + raise InventoryConflict(f"capture evidence JSON is invalid: {relative}") from error + if not isinstance(document, dict): + raise InventoryConflict(f"capture evidence JSON is not an object: {relative}") + return document + + +def _is_sha256(value: object) -> bool: + return isinstance(value, str) and _SHA256_RE.fullmatch(value) is not None + + +def _approval_binding(value: object) -> str | None: + if value is None: + return None + if not isinstance(value, dict): + raise InventoryConflict("accepted batch has a malformed manual approval") + binding = value.get("binding_sha256") + if not isinstance(binding, str) or not _is_sha256(binding): + raise InventoryConflict("accepted batch has a malformed manual approval") + return binding + + +def _usb_component(value: object, *, address: bool) -> int: + lower, upper = (1, 127) if address else (0, 999) + if isinstance(value, bool) or not isinstance(value, int) or not lower <= value <= upper: + label = "address" if address else "bus" + raise InventoryConflict(f"accepted batch USB {label} is malformed") + return value + + +def _artifact_row( + row_by_path: dict[str, dict[str, object]], + relative: str, +) -> dict[str, object]: + row = row_by_path.get(relative) + if not isinstance(row, dict) or type(row.get("bytes")) is not int or row["bytes"] < 0 or not _is_sha256(row.get("sha256")): + raise InventoryConflict(f"capture evidence manifest does not bind {relative}") + return row + + +def validate_six_frame_batch_capture_evidence( + snapshot: CaptureEvidenceSnapshot, +) -> dict[str, object]: + """Bind the durable post-finalization tree for one completed six-frame batch. + + Coolscan deletes each 619 MB ``capture.bin`` only after hashing, decoding, + and verifying its private completion outputs. The caller-owned tree keeps + the six journals, three-pass meter sidecars, parent ACKs, first-frame live + index artifacts, and sealed batch inputs. Those are the artifacts this + function requires and cross-binds; a deleted fine-stream scratch file is + deliberately not part of the retained contract. + """ + + rows = snapshot.manifest.get("files") + if not isinstance(rows, list): + raise InventoryConflict("capture evidence manifest has no file inventory") + typed_rows = [row for row in rows if isinstance(row, dict) and isinstance(row.get("path"), str)] + if len(typed_rows) != len(rows): + raise InventoryConflict("capture evidence manifest has a malformed file row") + relative_paths = {row["path"] for row in typed_rows} + if len(relative_paths) != len(typed_rows): + raise InventoryConflict("capture evidence manifest has duplicate file paths") + row_by_path = {row["path"]: row for row in typed_rows} + session_journals: list[PurePosixPath] = [] + for relative in relative_paths: + path = PurePosixPath(relative) + if len(path.parts) == 2 and path.name == "session-journal.json" and _SIX_FRAME_BATCH_ROOT_RE.fullmatch(path.parts[0]) is not None: + session_journals.append(path) + if len(session_journals) != 1: + raise InventoryConflict("capture evidence must contain exactly one slots-1-through-6 batch session") + + batch_root = session_journals[0].parent + if any(PurePosixPath(relative).parts[:1] != batch_root.parts for relative in relative_paths): + raise InventoryConflict("accepted batch capture evidence contains files outside the completed six-frame batch") + first_frame = batch_root / "frame-001" + required = { + str(batch_root / "batch-job.json"), + str(batch_root / _FIRST_PLAN_NAME), + str(batch_root / _CONTINUATION_PLAN_NAME), + str(batch_root / _CAPTURE_MANIFEST_NAME), + str(batch_root / "session-journal.json"), + str(batch_root / "stdout.txt"), + str(batch_root / "stderr.txt"), + str(first_frame / "capture-preview.bin"), + str(first_frame / "capture-008e.bin"), + str(first_frame / "capture-frame-map.json"), + *(str(batch_root / f"frame-{slot:03d}" / "journal.json") for slot in _SIX_FRAME_SLOTS), + *(str(batch_root / f"frame-{slot:03d}" / "capture-meter.bin") for slot in _SIX_FRAME_SLOTS), + *(str(batch_root / f"frame-{slot:03d}" / "parent-ack.json") for slot in _SIX_FRAME_SLOTS), + } + missing = sorted(required - relative_paths) + if missing: + raise InventoryConflict("accepted batch capture evidence is incomplete: " + ", ".join(missing)) + + job_path = str(batch_root / "batch-job.json") + session_path = str(batch_root / "session-journal.json") + job = _snapshot_json(snapshot, job_path) + session = _snapshot_json(snapshot, session_path) + if set(job) != _BATCH_JOB_KEYS: + raise InventoryConflict("accepted batch job has an unexpected schema") + + session_id = job.get("session_id") + if not isinstance(session_id, str) or session_id != batch_root.name: + raise InventoryConflict("accepted batch job session does not name its batch root") + if ( + job.get("schema_version") != 3 + or job.get("session_contract") != "one-process-one-reservation" + or job.get("apply_all_boundary_offsets_before_first_frame") is not True + or job.get("parent_ack_required_after_every_frame") is not True + or job.get("release_once_after_last_frame") is not True + ): + raise InventoryConflict("accepted batch job does not preserve the one-shot contract") + + expected_usb_bus = _usb_component(job.get("expected_usb_bus"), address=False) + expected_usb_address = _usb_component(job.get("expected_usb_address"), address=True) + reviewed_fingerprint = job.get("reviewed_roll_fingerprint") + reviewed_binding = reviewed_fingerprint.get("binding_sha256") if isinstance(reviewed_fingerprint, dict) else None + if not _is_sha256(reviewed_binding): + raise InventoryConflict("accepted batch job has no reviewed fingerprint") + + plan_path = str(batch_root / _FIRST_PLAN_NAME) + continuation_path = str(batch_root / _CONTINUATION_PLAN_NAME) + manifest_path = str(batch_root / _CAPTURE_MANIFEST_NAME) + plan_row = _artifact_row(row_by_path, plan_path) + continuation_row = _artifact_row(row_by_path, continuation_path) + plan_sha256 = job.get("capture_plan_sha256") + continuation_sha256 = job.get("continuation_plan_sha256") + if ( + not _is_sha256(plan_sha256) + or plan_sha256 != plan_row["sha256"] + or not _is_sha256(continuation_sha256) + or continuation_sha256 != continuation_row["sha256"] + ): + raise InventoryConflict("accepted batch job is not bound to its retained plans") + capture_manifest = _snapshot_json(snapshot, manifest_path) + if capture_manifest.get("plan_sha256") != plan_sha256: + raise InventoryConflict("accepted batch capture manifest is not bound to its plan") + + frames = job.get("frames") + if not isinstance(frames, list) or len(frames) != len(_SIX_FRAME_SLOTS): + raise InventoryConflict("accepted batch job does not contain exactly six frames") + frame_jobs: dict[int, dict[str, object]] = {} + approval_sha256_by_slot: dict[str, str | None] = {} + for expected_slot, frame in zip(_SIX_FRAME_SLOTS, frames, strict=True): + if not isinstance(frame, dict) or set(frame) != _BATCH_FRAME_KEYS: + raise InventoryConflict("accepted batch job has an invalid frame entry") + expected_directory = f"frame-{expected_slot:03d}" + offset = frame.get("boundary_offset_rows") + minimum_offset = 0 if expected_slot == 1 else -144 + if ( + frame.get("slot") != expected_slot + or frame.get("ack") != f"{expected_directory}/parent-ack.json" + or frame.get("journal") != f"{expected_directory}/journal.json" + or frame.get("output") != f"{expected_directory}/capture.bin" + or isinstance(offset, bool) + or not isinstance(offset, int) + or not minimum_offset <= offset <= 144 + ): + raise InventoryConflict("accepted batch job frame paths or order are invalid") + approval_sha256_by_slot[str(expected_slot)] = _approval_binding(frame.get("manual_review_approval")) + frame_jobs[expected_slot] = frame + + # session-journal.json and the frame journals are engine-owned documents: + # their full key sets belong to the pinned Coolscan engine, so they are + # deliberately not closed-world key-checked here (the NegPy-owned + # batch-job/frame/ACK documents above are). What pins their schema is the + # exact engine identity below — the session must carry the worker and + # bundle hashes of the coolscanpy actually imported by this process — plus + # duplicate-key rejection in _snapshot_json and strict per-field checks on + # every load-bearing key. + from coolscanpy.protocol.ls5000_single_pass.bundle import ( + CAPTURE_BUNDLE_SHA256, + CAPTURE_WORKER_SHA256, + ) + + job_sha256 = _artifact_row(row_by_path, job_path)["sha256"] + engine_sha256 = session.get("capture_engine_sha256") + bundle_sha256 = session.get("capture_bundle_sha256") + density_calibration = session.get("nikon_density_calibration") + if ( + session.get("status") != "complete" + or session.get("session_id") != session_id + or session.get("density_calibration_session_id") != session_id + or not isinstance(density_calibration, dict) + or density_calibration.get("session_id") != session_id + or session.get("batch_job_sha256") != job_sha256 + or session.get("selected_slots") != list(_SIX_FRAME_SLOTS) + or session.get("completed_slots") != list(_SIX_FRAME_SLOTS) + or session.get("plan_sha256") != plan_sha256 + or session.get("continuation_plan_sha256") != continuation_sha256 + or engine_sha256 != CAPTURE_WORKER_SHA256 + or bundle_sha256 != CAPTURE_BUNDLE_SHA256 + or session.get("manual_review_approval_sha256_by_slot") != approval_sha256_by_slot + or session.get("reviewed_roll_fingerprint_sha256") != reviewed_binding + or session.get("expected_usb_bus") != expected_usb_bus + or session.get("expected_usb_address") != expected_usb_address + or session.get("actual_usb_bus") != expected_usb_bus + or session.get("actual_usb_address") != expected_usb_address + or session.get("reservation_acquired") is not True + or session.get("unit_release_attempts") != 1 + or session.get("unit_released") is not True + or session.get("recovery_required") != "none" + or session.get("active_frame_index") is not None + or session.get("active_slot") is not None + ): + raise InventoryConflict("capture evidence is not bound to one completed six-frame batch") + + journal_sha256_by_slot: dict[str, object] = {} + output_sha256_by_slot: dict[str, object] = {} + meter_sha256_by_slot: dict[str, object] = {} + ack_sha256_by_slot: dict[str, object] = {} + first_journal: dict[str, object] | None = None + for frame_index, slot in enumerate(_SIX_FRAME_SLOTS, start=1): + frame_root = batch_root / f"frame-{slot:03d}" + journal_path = str(frame_root / "journal.json") + meter_path = str(frame_root / "capture-meter.bin") + ack_path = str(frame_root / "parent-ack.json") + journal = _snapshot_json(snapshot, journal_path) + frame_job = frame_jobs[slot] + batch_session = journal.get("batch_session") + output_sha256 = journal.get("output_sha256") + selection = journal.get("live_frame_selection") + boundary_offset = selection.get("boundary_offset") if isinstance(selection, dict) else None + roll_identity = selection.get("roll_identity") if isinstance(selection, dict) else None + selected_comparison = roll_identity.get("selected_slot_comparison") if isinstance(roll_identity, dict) else None + comparison = roll_identity.get("comparison") if isinstance(roll_identity, dict) else None + expected_output = str(snapshot.root / Path(str(batch_root / frame_job["output"]))) + if ( + journal.get("status") != "frame-complete" + or journal.get("requested_frame") != slot + or journal.get("requested_boundary_offset_rows") != frame_job["boundary_offset_rows"] + or journal.get("frame_complete") is not True + or journal.get("session_reservation_retained") is not True + or journal.get("unit_released") is not False + or journal.get("recovery_required") not in (None, "none") + or journal.get("capture_mode") != "full" + or journal.get("expected_reads") != _FINE_READS + or journal.get("completed_reads") != _FINE_READS + or journal.get("expected_bytes") != _FINE_BYTES + or journal.get("completed_bytes") != _FINE_BYTES + or journal.get("disk_bytes") != _FINE_BYTES + or journal.get("output") != expected_output + or not _is_sha256(output_sha256) + or journal.get("plan_sha256") != plan_sha256 + or journal.get("continuation_plan_sha256") != continuation_sha256 + or journal.get("capture_engine_sha256") != engine_sha256 + or journal.get("capture_bundle_sha256") != bundle_sha256 + or journal.get("manual_review_approval") != frame_job["manual_review_approval"] + or journal.get("reviewed_roll_fingerprint_sha256") != reviewed_binding + or journal.get("expected_usb_bus") != expected_usb_bus + or journal.get("expected_usb_address") != expected_usb_address + or journal.get("actual_usb_bus") != expected_usb_bus + or journal.get("actual_usb_address") != expected_usb_address + or journal.get("density_calibration_session_id") != session_id + or journal.get("nikon_density_calibration") != density_calibration + or not isinstance(batch_session, dict) + or batch_session.get("session_id") != session_id + or batch_session.get("frame_index") != frame_index + or batch_session.get("frame_total") != len(_SIX_FRAME_SLOTS) + or batch_session.get("selected_slots") != list(_SIX_FRAME_SLOTS) + or not isinstance(selection, dict) + or selection.get("frame") != slot + # Coolscan's engine-owned offset proof uses this nested layout. + or not isinstance(boundary_offset, dict) + or boundary_offset.get("requested_rows") != frame_job["boundary_offset_rows"] + or boundary_offset.get("applied_rows") != frame_job["boundary_offset_rows"] + or not isinstance(roll_identity, dict) + or roll_identity.get("reviewed_fingerprint_sha256") != reviewed_binding + or not _is_sha256(roll_identity.get("fresh_fingerprint_sha256")) + or not isinstance(comparison, dict) + or comparison.get("matches") is not True + or comparison.get("reason") != "matched" + or not isinstance(selected_comparison, dict) + or selected_comparison.get("matches") is not True + or selected_comparison.get("reason") != "matched" + or selected_comparison.get("slot") != slot + ): + raise InventoryConflict(f"capture evidence frame {slot} journal is not bound to the completed batch") + + meter_row = _artifact_row(row_by_path, meter_path) + meter_evidence = journal.get("meter_evidence") + if ( + meter_row["bytes"] != _METER_BYTES + or not isinstance(meter_evidence, dict) + or meter_evidence.get("path") != str(snapshot.root / Path(str(frame_root / "capture-meter.bin"))) + or meter_evidence.get("bytes") != _METER_BYTES + or meter_evidence.get("sha256") != meter_row["sha256"] + or meter_evidence.get("complete") is not True + or meter_evidence.get("durable_completed_passes") != 3 + or journal.get("meter_evidence_persisted_before_fine_arm") is not True + or journal.get("meter_group_bytes") != [_METER_PASS_BYTES] * 3 + or journal.get("meter_group_offsets") != [0, _METER_PASS_BYTES, _METER_PASS_BYTES * 2] + or journal.get("meter_completed_reads") != 15 + or journal.get("meter_completed_bytes") != _METER_BYTES + or journal.get("meter_layout") != _METER_LAYOUT + ): + raise InventoryConflict(f"capture evidence frame {slot} meter sidecar is not durably bound") + + ack = _snapshot_json(snapshot, ack_path) + nonce = journal.get("ack_nonce") + if ( + set(ack) != _ACK_KEYS + or not isinstance(nonce, str) + or _ACK_NONCE_RE.fullmatch(nonce) is None + or ack.get("ack_nonce") != nonce + or ack.get("action") != "continue" + or ack.get("frame_index") != frame_index + or ack.get("schema_version") != 1 + or ack.get("session_id") != session_id + or ack.get("slot") != slot + ): + raise InventoryConflict(f"capture evidence frame {slot} parent ACK is not bound to its journal") + + journal_sha256_by_slot[str(slot)] = _artifact_row(row_by_path, journal_path)["sha256"] + output_sha256_by_slot[str(slot)] = output_sha256 + meter_sha256_by_slot[str(slot)] = meter_row["sha256"] + ack_sha256_by_slot[str(slot)] = _artifact_row(row_by_path, ack_path)["sha256"] + if slot == 1: + first_journal = journal + + assert first_journal is not None + preview_path = str(first_frame / "capture-preview.bin") + table_path = str(first_frame / "capture-008e.bin") + mapping_path = str(first_frame / "capture-frame-map.json") + preview_row = _artifact_row(row_by_path, preview_path) + table_row = _artifact_row(row_by_path, table_path) + mapping = _snapshot_json(snapshot, mapping_path) + live_index_artifacts = first_journal.get("live_index_artifacts") + live_index_evidence = first_journal.get("live_index_evidence") + if ( + preview_row["bytes"] != _PREVIEW_BYTES + or not isinstance(live_index_artifacts, dict) + or live_index_artifacts + != { + "preview": str(snapshot.root / Path(preview_path)), + "table": str(snapshot.root / Path(table_path)), + "mapping": str(snapshot.root / Path(mapping_path)), + } + or not isinstance(live_index_evidence, dict) + or live_index_evidence.get("status") != "persisted-before-frame-detection" + or live_index_evidence.get("preview_bytes") != preview_row["bytes"] + or live_index_evidence.get("preview_sha256") != preview_row["sha256"] + or live_index_evidence.get("table_bytes") != table_row["bytes"] + or live_index_evidence.get("table_sha256") != table_row["sha256"] + or mapping != first_journal.get("live_frame_selection") + ): + raise InventoryConflict("capture evidence live preview, transport table, or frame map is not bound") + + return { + "batch_root": str(batch_root), + "session_id": session_id, + "batch_job_sha256": job_sha256, + "capture_plan_sha256": plan_sha256, + "continuation_plan_sha256": continuation_sha256, + "capture_engine_sha256": engine_sha256, + "capture_bundle_sha256": bundle_sha256, + "expected_usb_bus": expected_usb_bus, + "expected_usb_address": expected_usb_address, + "selected_slots": list(_SIX_FRAME_SLOTS), + "reviewed_fingerprint_sha256": reviewed_binding, + "first_frame_directory": str(first_frame), + "session_journal_sha256": _artifact_row(row_by_path, session_path)["sha256"], + "frame_journal_sha256_by_slot": journal_sha256_by_slot, + "frame_output_sha256_by_slot": output_sha256_by_slot, + "frame_meter_sha256_by_slot": meter_sha256_by_slot, + "frame_ack_sha256_by_slot": ack_sha256_by_slot, + "preview_sha256": preview_row["sha256"], + "transport_table_sha256": table_row["sha256"], + } + + +def bind_capture_evidence_inventory( + snapshot: CaptureEvidenceSnapshot, + inventory: InventorySnapshot, +) -> dict[str, object]: + """Bind file hashes to the lease's exact post-close directory inventory.""" + + actual = dict(inventory.files) + expected = dict(snapshot.identities) + actual.pop(OUTPUT_LOCK_NAME, None) + if actual != expected: + raise InventoryConflict("capture evidence changed between hashing and lease inventory") + return { + "path": str(snapshot.root), + "retained": True, + "directory_count": len(inventory.directories), + **snapshot.manifest, + } diff --git a/negpy/services/roll/live_reservation.py b/negpy/services/roll/live_reservation.py new file mode 100644 index 00000000..a2e22330 --- /dev/null +++ b/negpy/services/roll/live_reservation.py @@ -0,0 +1,767 @@ +"""Crash-visible, fail-closed ownership primitives for a live roll run. + +This module is deliberately hardware-inert. It owns only two filesystem +names: the caller-selected run receipt and a fixed lock inside an output +directory. A live runner can reserve both before opening a scanner, retain +their descriptors for the complete run, and refuse to overwrite names whose +ownership changes underneath it. +""" + +from __future__ import annotations + +import dataclasses +import json +import os +import stat +from collections.abc import Callable, Iterable, Mapping +from pathlib import Path +from typing import Any, Self + + +OUTPUT_LOCK_NAME = ".negpy-live-acceptance.lock" +_IGNORABLE_FINDER_METADATA_NAMES = frozenset({".DS_Store"}) +DEFAULT_RECEIPT_MAX_BYTES = 1024 * 1024 +DEFAULT_LOCK_MAX_BYTES = 16 * 1024 + + +class LiveReservationError(RuntimeError): + """A filesystem ownership boundary could not be established or retained.""" + + +class ReservationConflict(LiveReservationError): + """The requested receipt or fixed output lock already exists.""" + + +class OwnershipLost(LiveReservationError): + """A reserved pathname no longer identifies the object we opened.""" + + +class InventoryConflict(LiveReservationError): + """The output directory contains missing, changed, or unowned entries.""" + + +@dataclasses.dataclass(frozen=True) +class FileIdentity: + """Stable-enough checkpoint identity for one regular output file.""" + + device: int + inode: int + size: int + mtime_ns: int + ctime_ns: int + + @classmethod + def from_stat(cls, metadata: os.stat_result) -> Self: + return cls( + device=metadata.st_dev, + inode=metadata.st_ino, + size=metadata.st_size, + mtime_ns=metadata.st_mtime_ns, + ctime_ns=metadata.st_ctime_ns, + ) + + @property + def object_identity(self) -> tuple[int, int]: + return self.device, self.inode + + +@dataclasses.dataclass(frozen=True) +class InventorySnapshot: + """Exact file/directory names observed at an ownership checkpoint.""" + + files: tuple[tuple[str, FileIdentity], ...] + directories: tuple[str, ...] + + +def canonical_json_bytes( + document: Mapping[str, Any], + *, + maximum_bytes: int, + label: str, +) -> bytes: + """Encode one bounded canonical JSON object, including its final newline.""" + + if maximum_bytes <= 0: + raise ValueError("maximum_bytes must be positive") + if not isinstance(document, Mapping): + raise LiveReservationError(f"{label} must be a JSON object") + try: + payload = ( + json.dumps( + dict(document), + sort_keys=True, + separators=(",", ":"), + ensure_ascii=True, + allow_nan=False, + ).encode("utf-8") + + b"\n" + ) + except (TypeError, ValueError) as error: + raise LiveReservationError(f"{label} is not canonical JSON: {error}") from error + if len(payload) > maximum_bytes: + raise LiveReservationError(f"{label} exceeds its safe size limit ({len(payload)} > {maximum_bytes} bytes)") + return payload + + +def _require_posix_descriptor_operations() -> None: + missing: list[str] = [] + for name in ("O_DIRECTORY", "O_NOFOLLOW"): + if not hasattr(os, name): + missing.append(name) + for function, capability, name in ( + (os.open, os.supports_dir_fd, "open(dir_fd)"), + (os.stat, os.supports_dir_fd, "stat(dir_fd)"), + (os.stat, os.supports_follow_symlinks, "stat(follow_symlinks)"), + (os.unlink, os.supports_dir_fd, "unlink(dir_fd)"), + (os.listdir, os.supports_fd, "listdir(fd)"), + ): + if function not in capability: + missing.append(name) + if missing: + raise LiveReservationError("safe POSIX descriptor operations are unavailable: " + ", ".join(missing)) + + +def _directory_flags() -> int: + return os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW | getattr(os, "O_CLOEXEC", 0) + + +def _file_create_flags() -> int: + return os.O_CREAT | os.O_EXCL | os.O_RDWR | os.O_NOFOLLOW | getattr(os, "O_CLOEXEC", 0) + + +def _same_object(left: os.stat_result, right: os.stat_result) -> bool: + return (left.st_dev, left.st_ino) == (right.st_dev, right.st_ino) + + +def _directory_checkpoint(metadata: os.stat_result) -> tuple[int, int, int, int]: + return ( + metadata.st_dev, + metadata.st_ino, + metadata.st_mtime_ns, + metadata.st_ctime_ns, + ) + + +def is_ignorable_finder_metadata_file(name: str, metadata: os.stat_result) -> bool: + """Return whether ``name`` is macOS directory metadata, never an artifact. + + Finder creates a regular ``.DS_Store`` merely by browsing a directory. It + has no capture or output provenance, so it must not make a held receipt + fail. Keep the exception exact: a directory, symlink, or any other hidden + name remains an inventory conflict. + """ + + return name in _IGNORABLE_FINDER_METADATA_NAMES and stat.S_ISREG(metadata.st_mode) + + +def _write_all(descriptor: int, payload: bytes) -> None: + written = 0 + while written < len(payload): + count = os.write(descriptor, payload[written:]) + if count <= 0: + raise OSError("short write while publishing reserved JSON") + written += count + + +def _read_exact(descriptor: int, size: int) -> bytes: + chunks: list[bytes] = [] + offset = 0 + while offset < size: + chunk = os.pread(descriptor, size - offset, offset) + if not chunk: + break + chunks.append(chunk) + offset += len(chunk) + return b"".join(chunks) + + +def _replace_descriptor_payload(descriptor: int, payload: bytes) -> None: + os.lseek(descriptor, 0, os.SEEK_SET) + _write_all(descriptor, payload) + os.ftruncate(descriptor, len(payload)) + os.fsync(descriptor) + if _read_exact(descriptor, len(payload)) != payload: + raise OSError("reserved JSON did not verify after publication") + + +def _open_visible_directory(path: Path, *, label: str) -> tuple[Path, int]: + _require_posix_descriptor_operations() + canonical = path.resolve(strict=True) + descriptor = os.open(canonical, _directory_flags()) + try: + opened = os.fstat(descriptor) + visible = os.stat(canonical, follow_symlinks=False) + if not stat.S_ISDIR(opened.st_mode) or not stat.S_ISDIR(visible.st_mode) or not _same_object(opened, visible): + raise OwnershipLost(f"{label} changed while opening") + except BaseException: + os.close(descriptor) + raise + return canonical, descriptor + + +def _assert_visible_directory(path: Path, descriptor: int, *, label: str) -> None: + try: + opened = os.fstat(descriptor) + visible = os.stat(path, follow_symlinks=False) + except OSError as error: + raise OwnershipLost(f"{label} is no longer visible: {error}") from error + if not stat.S_ISDIR(opened.st_mode) or not stat.S_ISDIR(visible.st_mode) or not _same_object(opened, visible): + raise OwnershipLost(f"{label} pathname no longer identifies the opened directory") + + +def _child_metadata(directory_descriptor: int, name: str) -> os.stat_result: + return os.stat( + name, + dir_fd=directory_descriptor, + follow_symlinks=False, + ) + + +class ExclusiveReceiptReservation: + """An exact run-receipt pathname reserved before any live device open.""" + + def __init__( + self, + *, + path: Path, + parent: Path, + parent_descriptor: int, + descriptor: int, + maximum_bytes: int, + ) -> None: + self.path = path + self._parent = parent + self._parent_descriptor = parent_descriptor + self._descriptor = descriptor + self._maximum_bytes = maximum_bytes + self._closed = False + + @classmethod + def reserve( + cls, + path: Path, + initial_document: Mapping[str, Any], + *, + maximum_bytes: int = DEFAULT_RECEIPT_MAX_BYTES, + ) -> Self: + """Exclusively create and durably publish an in-progress receipt.""" + + if initial_document.get("status") != "in_progress": + raise LiveReservationError("initial run receipt status must be exactly 'in_progress'") + payload = canonical_json_bytes( + initial_document, + maximum_bytes=maximum_bytes, + label="run receipt", + ) + absolute = Path(os.path.abspath(os.fspath(path))) + if absolute.name in {"", ".", ".."}: + raise LiveReservationError("run receipt must name a file") + parent, parent_descriptor = _open_visible_directory( + absolute.parent, + label="run-receipt parent", + ) + canonical = parent / absolute.name + descriptor: int | None = None + try: + try: + descriptor = os.open( + absolute.name, + _file_create_flags(), + 0o600, + dir_fd=parent_descriptor, + ) + except FileExistsError as error: + raise ReservationConflict(f"run receipt already exists: {canonical}") from error + opened = os.fstat(descriptor) + visible = _child_metadata(parent_descriptor, absolute.name) + if not stat.S_ISREG(opened.st_mode) or opened.st_nlink != 1 or not _same_object(opened, visible): + raise OwnershipLost("new run receipt did not retain exclusive regular-file ownership") + _replace_descriptor_payload(descriptor, payload) + os.fsync(parent_descriptor) + reservation = cls( + path=canonical, + parent=parent, + parent_descriptor=parent_descriptor, + descriptor=descriptor, + maximum_bytes=maximum_bytes, + ) + reservation.assert_owned() + return reservation + except BaseException: + if descriptor is not None: + try: + opened = os.fstat(descriptor) + visible = _child_metadata(parent_descriptor, absolute.name) + if _same_object(opened, visible): + os.unlink(absolute.name, dir_fd=parent_descriptor) + os.fsync(parent_descriptor) + except OSError: + pass + os.close(descriptor) + os.close(parent_descriptor) + raise + + @property + def inode(self) -> tuple[int, int]: + self._require_open() + metadata = os.fstat(self._descriptor) + return metadata.st_dev, metadata.st_ino + + def _require_open(self) -> None: + if self._closed: + raise LiveReservationError("run-receipt reservation is closed") + + def assert_owned(self) -> None: + """Require the requested pathname to still name our held receipt inode.""" + + self._require_open() + _assert_visible_directory( + self._parent, + self._parent_descriptor, + label="run-receipt parent", + ) + try: + opened = os.fstat(self._descriptor) + visible = _child_metadata(self._parent_descriptor, self.path.name) + except OSError as error: + raise OwnershipLost(f"run receipt is no longer owned: {error}") from error + if ( + not stat.S_ISREG(opened.st_mode) + or not stat.S_ISREG(visible.st_mode) + or opened.st_nlink != 1 + or visible.st_nlink != 1 + or not _same_object(opened, visible) + ): + raise OwnershipLost("run-receipt pathname no longer identifies the reserved inode") + + def publish(self, document: Mapping[str, Any]) -> None: + """Publish new canonical JSON into, and only into, the reserved inode.""" + + payload = canonical_json_bytes( + document, + maximum_bytes=self._maximum_bytes, + label="run receipt", + ) + self.assert_owned() + _replace_descriptor_payload(self._descriptor, payload) + self.assert_owned() + os.fsync(self._parent_descriptor) + + def close(self) -> None: + """Release descriptors without deleting the durable receipt.""" + + if self._closed: + return + os.close(self._descriptor) + os.close(self._parent_descriptor) + self._closed = True + + def __enter__(self) -> Self: + self._require_open() + return self + + def __exit__(self, *_exc: object) -> None: + self.close() + + +class FixedOutputLease: + """A fixed, crash-visible output-directory lease held for one live run.""" + + def __init__( + self, + *, + output_dir: Path, + directory_descriptor: int, + lock_descriptor: int, + ) -> None: + self.output_dir = output_dir + self.lock_path = output_dir / OUTPUT_LOCK_NAME + self._directory_descriptor = directory_descriptor + self._lock_descriptor = lock_descriptor + self._released = False + + @classmethod + def acquire( + cls, + output_dir: Path, + lock_document: Mapping[str, Any], + *, + maximum_bytes: int = DEFAULT_LOCK_MAX_BYTES, + require_empty: bool = True, + ) -> Self: + """Exclusively own a real output directory through a fixed lock file.""" + + payload = canonical_json_bytes( + lock_document, + maximum_bytes=maximum_bytes, + label="output lock", + ) + absolute = Path(os.path.abspath(os.fspath(output_dir))) + try: + linked = absolute.lstat() + except OSError as error: + raise LiveReservationError(f"could not inspect output directory: {error}") from error + if stat.S_ISLNK(linked.st_mode) or not stat.S_ISDIR(linked.st_mode): + raise LiveReservationError("output directory must be an existing non-symlink directory") + canonical, directory_descriptor = _open_visible_directory( + absolute, + label="output directory", + ) + lock_descriptor: int | None = None + try: + try: + lock_descriptor = os.open( + OUTPUT_LOCK_NAME, + _file_create_flags(), + 0o600, + dir_fd=directory_descriptor, + ) + except FileExistsError as error: + raise ReservationConflict(f"output directory is already reserved: {canonical / OUTPUT_LOCK_NAME}") from error + opened = os.fstat(lock_descriptor) + visible = _child_metadata(directory_descriptor, OUTPUT_LOCK_NAME) + if not stat.S_ISREG(opened.st_mode) or opened.st_nlink != 1 or not _same_object(opened, visible): + raise OwnershipLost("new output lock did not retain exclusive regular-file ownership") + _replace_descriptor_payload(lock_descriptor, payload) + os.fsync(directory_descriptor) + lease = cls( + output_dir=canonical, + directory_descriptor=directory_descriptor, + lock_descriptor=lock_descriptor, + ) + lease.assert_owned() + if require_empty: + lease.assert_inventory(()) + return lease + except BaseException: + if lock_descriptor is not None: + try: + opened = os.fstat(lock_descriptor) + visible = _child_metadata(directory_descriptor, OUTPUT_LOCK_NAME) + if _same_object(opened, visible): + os.unlink(OUTPUT_LOCK_NAME, dir_fd=directory_descriptor) + os.fsync(directory_descriptor) + except OSError: + pass + os.close(lock_descriptor) + os.close(directory_descriptor) + raise + + @property + def released(self) -> bool: + return self._released + + def _require_active(self) -> None: + if self._released: + raise LiveReservationError("output lease is already released") + + def _lock_metadata(self) -> tuple[os.stat_result, os.stat_result]: + try: + opened = os.fstat(self._lock_descriptor) + visible = _child_metadata(self._directory_descriptor, OUTPUT_LOCK_NAME) + except OSError as error: + raise OwnershipLost(f"output lock is no longer owned: {error}") from error + return opened, visible + + def assert_owned(self) -> None: + """Require both the output pathname and fixed lock to remain ours.""" + + self._require_active() + _assert_visible_directory( + self.output_dir, + self._directory_descriptor, + label="output directory", + ) + opened, visible = self._lock_metadata() + if ( + not stat.S_ISREG(opened.st_mode) + or not stat.S_ISREG(visible.st_mode) + or opened.st_nlink != 1 + or visible.st_nlink != 1 + or not _same_object(opened, visible) + ): + raise OwnershipLost("fixed output-lock pathname no longer identifies the held inode") + + def _relative_entry(self, path: Path | str, *, label: str) -> str: + candidate = Path(path) + if candidate.is_absolute(): + absolute = Path(os.path.abspath(os.fspath(candidate))) + else: + absolute = Path(os.path.abspath(os.fspath(self.output_dir / candidate))) + try: + relative = absolute.relative_to(self.output_dir) + except ValueError as error: + raise InventoryConflict(f"{label} escapes the output directory: {path}") from error + if relative == Path(".") or any(part in {"", ".", ".."} for part in relative.parts): + raise InventoryConflict(f"{label} does not name a child entry: {path}") + name = os.fspath(relative) + if name == OUTPUT_LOCK_NAME: + raise InventoryConflict("the fixed output lock cannot be declared as an output") + if Path(name).name in _IGNORABLE_FINDER_METADATA_NAMES: + raise InventoryConflict("Finder metadata cannot be declared as an output") + return name + + def _scan_directory( + self, + descriptor: int, + *, + prefix: str, + files: dict[str, FileIdentity], + directories: set[str], + ) -> None: + before = os.fstat(descriptor) + if not stat.S_ISDIR(before.st_mode): + raise InventoryConflict("an opened output entry stopped being a directory") + try: + names = sorted(os.listdir(descriptor)) + except OSError as error: + raise InventoryConflict(f"could not list output directory: {error}") from error + for name in names: + relative = os.path.join(prefix, name) if prefix else name + try: + metadata = _child_metadata(descriptor, name) + except OSError as error: + raise InventoryConflict(f"output entry changed while inspecting {relative}: {error}") from error + if is_ignorable_finder_metadata_file(name, metadata): + continue + if stat.S_ISLNK(metadata.st_mode): + raise InventoryConflict(f"output entry is a symbolic link: {relative}") + if stat.S_ISREG(metadata.st_mode): + files[relative] = FileIdentity.from_stat(metadata) + continue + if not stat.S_ISDIR(metadata.st_mode): + raise InventoryConflict(f"output entry is not a regular file or directory: {relative}") + directories.add(relative) + try: + child_descriptor = os.open( + name, + _directory_flags(), + dir_fd=descriptor, + ) + except OSError as error: + raise InventoryConflict(f"could not open output directory {relative}: {error}") from error + try: + opened = os.fstat(child_descriptor) + if not _same_object(opened, metadata): + raise InventoryConflict(f"output directory changed while opening: {relative}") + self._scan_directory( + child_descriptor, + prefix=relative, + files=files, + directories=directories, + ) + finally: + os.close(child_descriptor) + after = os.fstat(descriptor) + if _directory_checkpoint(before) != _directory_checkpoint(after): + raise InventoryConflict("output directory changed while its inventory was inspected") + + @staticmethod + def _ancestor_directories(entries: Iterable[str]) -> set[str]: + ancestors: set[str] = set() + for entry in entries: + parent = os.path.dirname(entry) + while parent: + ancestors.add(parent) + parent = os.path.dirname(parent) + return ancestors + + def assert_inventory( + self, + owned_files: Iterable[Path | str], + *, + owned_directories: Iterable[Path | str] = (), + previous: InventorySnapshot | None = None, + ) -> InventorySnapshot: + """Require an exact, regular, non-symlink output inventory. + + ``owned_files`` and ``owned_directories`` exclude the fixed lease file; + it is included automatically. Passing the prior checkpoint also + refuses mutation or replacement of every previously owned file. + """ + + return self._assert_inventory( + owned_files, + owned_directories=owned_directories, + previous=previous, + include_lock=True, + ) + + def _assert_inventory( + self, + owned_files: Iterable[Path | str], + *, + owned_directories: Iterable[Path | str] = (), + previous: InventorySnapshot | None = None, + include_lock: bool, + ) -> InventorySnapshot: + """Check the exact held-directory inventory before or after unlock.""" + + self._require_active() + if include_lock: + self.assert_owned() + else: + _assert_visible_directory( + self.output_dir, + self._directory_descriptor, + label="output directory", + ) + expected_files = {self._relative_entry(path, label="owned file") for path in owned_files} + expected_directories = {self._relative_entry(path, label="owned directory") for path in owned_directories} + if include_lock: + expected_files.add(OUTPUT_LOCK_NAME) + expected_directories.update(self._ancestor_directories(expected_files)) + expected_directories.update(self._ancestor_directories(expected_directories)) + + actual_files: dict[str, FileIdentity] = {} + actual_directories: set[str] = set() + self._scan_directory( + self._directory_descriptor, + prefix="", + files=actual_files, + directories=actual_directories, + ) + if include_lock: + self.assert_owned() + else: + _assert_visible_directory( + self.output_dir, + self._directory_descriptor, + label="output directory", + ) + + missing_files = sorted(expected_files - actual_files.keys()) + unexpected_files = sorted(actual_files.keys() - expected_files) + missing_directories = sorted(expected_directories - actual_directories) + unexpected_directories = sorted(actual_directories - expected_directories) + differences: list[str] = [] + for label, entries in ( + ("missing files", missing_files), + ("unexpected files", unexpected_files), + ("missing directories", missing_directories), + ("unexpected directories", unexpected_directories), + ): + if entries: + differences.append(f"{label}: {', '.join(entries)}") + if differences: + raise InventoryConflict("output inventory conflict (" + "; ".join(differences) + ")") + + if previous is not None: + for name, prior_identity in previous.files: + if not include_lock and name == OUTPUT_LOCK_NAME: + continue + current_identity = actual_files.get(name) + if current_identity is None: + raise InventoryConflict(f"previously owned output disappeared: {name}") + if current_identity != prior_identity: + raise InventoryConflict(f"previously owned output changed: {name}") + + return InventorySnapshot( + files=tuple(sorted(actual_files.items())), + directories=tuple(sorted(actual_directories)), + ) + + def _unlink_owned_lock(self) -> None: + opened, visible = self._lock_metadata() + if ( + not stat.S_ISREG(opened.st_mode) + or not stat.S_ISREG(visible.st_mode) + or opened.st_nlink != 1 + or visible.st_nlink != 1 + or not _same_object(opened, visible) + ): + raise OwnershipLost("refusing to remove a fixed output lock that is no longer exclusively owned") + os.unlink(OUTPUT_LOCK_NAME, dir_fd=self._directory_descriptor) + os.fsync(self._directory_descriptor) + + def release_verified( + self, + owned_files: Iterable[Path | str], + *, + previous: InventorySnapshot, + finalize: Callable[[], None], + ) -> InventorySnapshot: + """Unlock and publish success inside two exact pathname checks. + + The fixed lock is removed only after the visible output pathname and + full inventory pass. ``finalize`` then publishes the success receipt + while the directory descriptor remains held. A second check catches + any pathname swap or artifact mutation during that publication. If + either check fails, the caller can overwrite a prematurely published + success receipt with a failed receipt before returning. + """ + + self._require_active() + expected_files = tuple(owned_files) + error: BaseException | None = None + result: InventorySnapshot | None = None + lock_unlinked = False + try: + self._assert_inventory( + expected_files, + previous=previous, + include_lock=True, + ) + self._unlink_owned_lock() + lock_unlinked = True + unlocked = self._assert_inventory( + expected_files, + previous=previous, + include_lock=False, + ) + finalize() + result = self._assert_inventory( + expected_files, + previous=unlocked, + include_lock=False, + ) + except BaseException as caught: + error = caught + finally: + if not lock_unlinked: + try: + self._unlink_owned_lock() + except BaseException as cleanup_error: + if error is None: + error = cleanup_error + else: + error = RuntimeError(f"{error}; output lock cleanup failed: {cleanup_error}") + try: + os.close(self._lock_descriptor) + except BaseException as close_error: + if error is None: + error = close_error + else: + error = RuntimeError(f"{error}; output lock descriptor close failed: {close_error}") + try: + os.close(self._directory_descriptor) + except BaseException as close_error: + if error is None: + error = close_error + else: + error = RuntimeError(f"{error}; output directory descriptor close failed: {close_error}") + self._released = True + if error is not None: + if isinstance(error, LiveReservationError): + raise error + raise LiveReservationError(f"could not verify and release fixed output lock: {error}") from error + assert result is not None + return result + + def release(self) -> None: + """Remove only the fixed lock inode opened by this lease, then fsync.""" + + self._require_active() + error: BaseException | None = None + try: + self._unlink_owned_lock() + except BaseException as caught: + error = caught + finally: + os.close(self._lock_descriptor) + os.close(self._directory_descriptor) + self._released = True + if error is not None: + if isinstance(error, LiveReservationError): + raise error + raise LiveReservationError(f"could not release fixed output lock: {error}") from error diff --git a/negpy/services/roll/live_review.py b/negpy/services/roll/live_review.py new file mode 100644 index 00000000..ca796ef0 --- /dev/null +++ b/negpy/services/roll/live_review.py @@ -0,0 +1,325 @@ +"""Pinned operator-review evidence for one LS-5000 live acceptance run.""" + +from __future__ import annotations + +import hashlib +import hmac +import io +import json +import os +import stat +from collections.abc import Callable, Mapping +from dataclasses import dataclass +from pathlib import Path +from types import MappingProxyType +from typing import Any, TypeGuard, cast + +import numpy as np +from PIL import Image + + +SCHEMA = "negpy.ls5000-reviewed-approval.v1" +REVIEW_BASIS = "visual-inspection-of-six-frame-contact-sheet-and-canonical-restored-thumbnails" +# Historical six-strip reviews used slots 1 and 6. A refeed can legitimately +# change which boundary has enough direct transport evidence, so production +# review must derive this list from the restored preview instead of treating +# the historical pair as a physical invariant. Keep the pair as a fallback +# for lightweight older test/session doubles that do not expose ``slots``. +APPROVED_SLOTS = (1, 6) +_SHA256_CHARS = frozenset("0123456789abcdef") +_MAX_REVIEW_BYTES = 64 * 1024 +_MAX_CONTACT_SHEET_BYTES = 64 * 1024 * 1024 + + +@dataclass(frozen=True) +class ValidatedReviewedApproval: + """One SHA-pinned visual review, rederived from the restored preview.""" + + sha256: str + byte_length: int + reviewed_fingerprint_sha256: str + contact_sheet_path: Path + contact_sheet_sha256: str + approvals: Mapping[int, object] + + def __post_init__(self) -> None: + object.__setattr__( + self, + "approvals", + MappingProxyType(dict(self.approvals)), + ) + + +def _is_sha256(value: object) -> TypeGuard[str]: + return type(value) is str and len(value) == 64 and not (set(value) - _SHA256_CHARS) + + +def _identity(value: os.stat_result) -> tuple[int, int, int, int, int]: + return ( + value.st_dev, + value.st_ino, + value.st_size, + value.st_mtime_ns, + value.st_ctime_ns, + ) + + +def _stable_regular_bytes( + path: Path, + *, + maximum_bytes: int, + label: str, +) -> bytes: + descriptor: int | None = None + try: + before = path.lstat() + if stat.S_ISLNK(before.st_mode) or not stat.S_ISREG(before.st_mode): + raise ValueError(f"{label} must be a regular non-symlink file") + if not 1 <= before.st_size <= maximum_bytes: + raise ValueError(f"{label} has an invalid byte length") + flags = os.O_RDONLY | getattr(os, "O_CLOEXEC", 0) | getattr(os, "O_NOFOLLOW", 0) | getattr(os, "O_NONBLOCK", 0) + descriptor = os.open(path, flags) + opened = os.fstat(descriptor) + if not stat.S_ISREG(opened.st_mode) or _identity(opened) != _identity(before): + raise ValueError(f"{label} changed while opening") + chunks: list[bytes] = [] + total = 0 + while chunk := os.read(descriptor, min(1024 * 1024, maximum_bytes + 1 - total)): + chunks.append(chunk) + total += len(chunk) + if total > maximum_bytes: + raise ValueError(f"{label} exceeds its safe size limit") + after = os.fstat(descriptor) + final = path.lstat() + except FileNotFoundError as error: + raise ValueError(f"{label} is missing: {path}") from error + except OSError as error: + raise ValueError(f"could not read {label}: {error}") from error + finally: + if descriptor is not None: + os.close(descriptor) + if _identity(opened) != _identity(after) or _identity(opened) != _identity(final): + raise ValueError(f"{label} changed while reading") + payload = b"".join(chunks) + if len(payload) != opened.st_size: + raise ValueError(f"{label} changed byte length while reading") + return payload + + +def _strict_json_object(payload: bytes, *, label: str) -> dict[str, Any]: + def unique(pairs: list[tuple[str, object]]) -> dict[str, object]: + result: dict[str, object] = {} + for key, value in pairs: + if key in result: + raise ValueError(f"duplicate key {key!r}") + result[key] = value + return result + + def reject_constant(value: str) -> object: + raise ValueError(f"non-finite constant {value!r}") + + try: + document = json.loads( + payload, + object_pairs_hook=unique, + parse_constant=reject_constant, + ) + except (UnicodeDecodeError, json.JSONDecodeError, ValueError) as error: + raise ValueError(f"{label} is invalid JSON: {error}") from error + if type(document) is not dict: + raise ValueError(f"{label} must contain a JSON object") + return document + + +def thumbnail_sha256(image: np.ndarray) -> str: + """Reproduce CoolscanPy's ManualFrameApproval thumbnail identity.""" + + thumbnail = np.asarray(image) + if thumbnail.dtype != np.uint16 or thumbnail.ndim != 3 or thumbnail.shape[2] != 3: + raise ValueError("reviewed thumbnail must be HxWx3 uint16") + digest = hashlib.sha256() + digest.update(str(thumbnail.shape).encode("ascii")) + digest.update(thumbnail.dtype.str.encode("ascii")) + digest.update(np.ascontiguousarray(thumbnail).tobytes()) + return digest.hexdigest() + + +def approval_payload(value: object) -> dict[str, Any]: + """Return a full ManualFrameApproval payload without trusting duck types.""" + + to_payload = getattr(value, "to_payload", None) + payload = to_payload() if callable(to_payload) else value + if type(payload) is not dict or any(type(key) is not str for key in payload): + raise ValueError("manual approval did not return a payload object") + return dict(cast(dict[str, Any], payload)) + + +def _default_session_loader(payload: str) -> object: + from coolscanpy.roll.preview_session import RollPreviewSession + + return RollPreviewSession.from_json(payload) + + +def _default_approval_parser(payload: object) -> object: + from coolscanpy.protocol.ls5000_single_pass.capture_process import ( + ManualFrameApproval, + ) + + return ManualFrameApproval.from_payload(payload) + + +def _required_approval_slots(session: object) -> tuple[int, ...]: + """Return the exact slots that this restored preview marks manual. + + The preview session is the authority for this list. Falling back only + preserves compatibility with deliberately minimal legacy test doubles; + real ``RollPreviewSession`` instances always expose immutable slots. + """ + + slots = getattr(session, "slots", None) + if not isinstance(slots, tuple): + return APPROVED_SLOTS + selected = getattr(session, "selected_slots", None) + if isinstance(selected, tuple) and selected and all(type(slot) is int for slot in selected): + selected_slots = frozenset(selected) + else: + selected_slots = None + required: list[int] = [] + for item in slots: + slot_id = getattr(item, "slot_id", None) + origin = getattr(item, "base_origin", None) + manual = getattr(origin, "manual_review", None) + if type(slot_id) is not int or type(manual) is not bool: + return APPROVED_SLOTS + if manual and (selected_slots is None or slot_id in selected_slots): + required.append(slot_id) + return tuple(required) + + +def load_reviewed_approval( + review_path: Path, + expected_sha256: str, + *, + preview_session_path: Path, + preview_session_payload: str, + preview_session_sha256: str, + session_loader: Callable[[str], object] = _default_session_loader, + approval_parser: Callable[[object], object] = _default_approval_parser, +) -> ValidatedReviewedApproval: + """Load, pin, and rederive the exact approvals the preview requires.""" + + if not _is_sha256(expected_sha256): + raise ValueError("reviewed-approval pin must be a lowercase SHA-256 digest") + if not _is_sha256(preview_session_sha256): + raise ValueError("preview-session SHA-256 is malformed") + review_bytes = _stable_regular_bytes( + review_path, + maximum_bytes=_MAX_REVIEW_BYTES, + label="reviewed approval", + ) + actual_sha256 = hashlib.sha256(review_bytes).hexdigest() + if not hmac.compare_digest(actual_sha256, expected_sha256): + raise ValueError(f"reviewed-approval SHA-256 mismatch: expected {expected_sha256}, got {actual_sha256}") + document = _strict_json_object(review_bytes, label="reviewed approval") + if set(document) != { + "approvals", + "contact_sheet", + "preview_session", + "review_basis", + "reviewed_fingerprint_sha256", + "schema", + }: + raise ValueError("reviewed approval fields are incomplete") + if document.get("schema") != SCHEMA or document.get("review_basis") != REVIEW_BASIS: + raise ValueError("reviewed approval schema or review basis is unsupported") + + session_row = document.get("preview_session") + if type(session_row) is not dict or set(session_row) != {"bytes", "path", "sha256"}: + raise ValueError("reviewed approval preview-session binding is malformed") + try: + session_path = Path(session_row["path"]).absolute() + except TypeError as error: + raise ValueError("reviewed approval preview-session path is malformed") from error + if ( + session_path != preview_session_path.absolute() + or session_row.get("sha256") != preview_session_sha256 + or session_row.get("bytes") != len(preview_session_payload.encode("utf-8")) + ): + raise ValueError("reviewed approval belongs to another preview session") + + contact_row = document.get("contact_sheet") + if type(contact_row) is not dict or set(contact_row) != {"bytes", "path", "sha256"}: + raise ValueError("reviewed approval contact-sheet binding is malformed") + if type(contact_row.get("path")) is not str or not _is_sha256(contact_row.get("sha256")): + raise ValueError("reviewed approval contact-sheet identity is malformed") + contact_path = Path(contact_row["path"]).absolute() + contact_bytes = _stable_regular_bytes( + contact_path, + maximum_bytes=_MAX_CONTACT_SHEET_BYTES, + label="reviewed contact sheet", + ) + if contact_row.get("bytes") != len(contact_bytes) or not hmac.compare_digest( + hashlib.sha256(contact_bytes).hexdigest(), + contact_row["sha256"], + ): + raise ValueError("reviewed contact-sheet content changed") + try: + with Image.open(io.BytesIO(contact_bytes)) as image: + if image.format != "PNG" or image.width < 6 or image.height < 1: + raise ValueError("reviewed contact sheet has invalid PNG geometry") + image.load() + except Exception as error: + raise ValueError(f"reviewed contact sheet is not a valid PNG: {error}") from error + + session = session_loader(preview_session_payload) + reviewed_fingerprint = getattr(session, "reviewed_fingerprint", None) + if not callable(reviewed_fingerprint): + raise ValueError("restored preview has no reviewed fingerprint") + fingerprint = getattr(reviewed_fingerprint(), "binding_sha256", None) + if not _is_sha256(fingerprint) or document.get("reviewed_fingerprint_sha256") != fingerprint: + raise ValueError("reviewed approval fingerprint does not match the preview") + + expected_slots = _required_approval_slots(session) + rows = document.get("approvals") + if type(rows) is not list or len(rows) != len(expected_slots): + raise ValueError("reviewed approval does not match the preview manual-review slots") + approvals: dict[int, object] = {} + for expected_slot, row in zip(expected_slots, rows, strict=True): + parsed = approval_parser(row) + parsed_payload = approval_payload(parsed) + if parsed_payload != row or parsed_payload.get("slot") != expected_slot: + raise ValueError("reviewed approval payload is non-canonical or out of order") + approve = getattr(session, "approve_manual_origin", None) + if not callable(approve): + raise ValueError("restored preview cannot rederive manual approvals") + derived = approve(expected_slot, parsed_payload["boundary_offset_rows"]) + if approval_payload(derived) != parsed_payload: + raise ValueError(f"reviewed approval for slot {expected_slot} does not match its thumbnail") + approvals[expected_slot] = parsed + + return ValidatedReviewedApproval( + sha256=actual_sha256, + byte_length=len(review_bytes), + reviewed_fingerprint_sha256=fingerprint, + contact_sheet_path=contact_path, + contact_sheet_sha256=contact_row["sha256"], + approvals=approvals, + ) + + +def validate_restored_thumbnails( + thumbnails: object, + review: ValidatedReviewedApproval, +) -> None: + """Cross-check returned thumbnails against the already-pinned review.""" + + if not isinstance(thumbnails, (list, tuple)): + raise ValueError("restored thumbnails must be an ordered sequence") + by_slot = {getattr(item, "slot", None): item for item in thumbnails} + if set(by_slot) != set(range(1, 7)): + raise ValueError("restored thumbnails are not exactly slots 1 through 6") + for slot, approval in review.approvals.items(): + image = getattr(by_slot[slot], "image", None) + expected = approval_payload(approval).get("thumbnail_sha256") + if thumbnail_sha256(np.asarray(image)) != expected: + raise ValueError(f"restored thumbnail for slot {slot} changed after review") diff --git a/negpy/services/roll/native_builder.py b/negpy/services/roll/native_builder.py new file mode 100644 index 00000000..e7d174e6 --- /dev/null +++ b/negpy/services/roll/native_builder.py @@ -0,0 +1,744 @@ +"""Fail-closed native LS-5000 prescan-to-pre-F builder.""" + +from __future__ import annotations + +import base64 +import hashlib +import json +import math +import re +import struct +from dataclasses import dataclass +from typing import Final, cast + +import numpy as np + +from negpy.services.roll import exact_color + + +CHANNELS: Final = ("r", "g", "b") +ANALYZER_SHAPE: Final = (425, 281, 3) +ANALYZER_RESOLUTION_DPI: Final = 285 +DENSITY_SOURCE_RESOLUTION_DPI: Final = 97 +# The scanner has two proven 97-dpi roll-preview geometries. The shorter +# variant is produced by the live 37-record full-roll table; accepting it is +# deliberately a closed whitelist, not a variable-length tolerance. +SUPPORTED_DENSITY_SOURCE_WIRE_BYTES: Final = frozenset((6_250_496, 5_804_032)) +DENSITY_ARITHMETIC: Final = "ls5000-md3-10088810-layout1-u16-proven-inputs-macos-binary64-exact-v6" +FRAME_OWNERSHIP_STATUS: Final = "proven-exact-reservation-preview-registration-and-transport" +DENSITY_PER_FRAME_BINDING_STATUS: Final = "requires-explicit-frame-ownership-receipt" +RESOURCE_SHA256: Final = exact_color.NATIVE_RESOURCE_SHA256 +PARAMETER_ALGORITHM_ID: Final = "ls5000-md3-100100d0-to-1000f470-v1" +CURVE_ALGORITHM_ID: Final = "ls5000-md3-10010c30-pref-v1" + +_SHA256 = re.compile(r"^[0-9a-f]{64}$") +_RESOURCE_B64 = "AAAAAAAADMCOBvAWSFALwPRsVn2utgnAAAAAAAAACMA/NV66SYwHwGb35GGhVgfAZapgVFInBsDBOSNKe4MFwCegibDh6QTAjgbwFkhQA8D0bFZ9rrYBwAAAAAAAAADAMzMzMzMz+79mZmZmZmb2v5qZmZmZmfG/mpmZmZmZ6b8AAAAAAADgv5qZmZmZmcm/mpmZmZmZuT+amZmZmZnZP2ZmZmZmZuY/AAAAAAAA8D/NzMzMzMz0P5qZmZmZmfk/ZmZmZmZm/j+amZmZmZkBQAAAAAAAAARAZmZmZmZmBkDNzMzMzMwIQDMzMzMzMwtAmpmZmZmZDUAAAAAAAAAQQNNNYhBYOdQ/001iEFg51D/TTWIQWDnUP9NNYhBYOdQ/001iEFg51D/TTWIQWDnUP5hMFYxK6tg/F0hQ/Bhz2z9t5/up8dLdPxSuR+F6FOI/c2iR7Xw/5T+kcD0K16PoPzEIrBxaZO0/30+Nl24S8T+mm8QgsHLzP23n+6nx0vU/MzMzMzMz+D/6fmq8dJP6P8HKoUW28/w/hxbZzvdT/z8nMQisHNoAQArXo3A9CgJA7nw/NV46A0DRItv5fmoEQLTIdr6fmgVAmG4Sg8DKBkB7FK5H4foHQF66SQwCKwlAQmDl0CJbCkAlBoGVQ4sLQAisHFpkuwxA7FG4HoXrDUAv3SQGgZXnPy/dJAaBlec/L90kBoGV5z8v3SQGgZXnPy/dJAaBlec/FK5H4XoU6D8dyeU/pN/qP/yp8dJNYuw/PnlYqDXN7T+1FfvL7snwP8zuycNCrfI/irDh6ZWy9D8r9pfdk4f3P807TtGRXPo/b4EExY8x/T+IY13cRgMAQFmGONbFbQFAKqkT0ETYAkD7y+7Jw0IEQMzuycNCrQVAnRGlvcEXB0BuNIC3QIIIQD9XW7G/7AlAEHo2qz5XC0DgnBGlvcEMQLG/7J48LA5AguLHmLuWD0CqglFJnYAQQBIUP8bcNRFAeqUsQxzrEUDjNhrAW6ASQEvIBz2bVRNAJzEIrBxa7D8nMQisHFrsPycxCKwcWuw/JzEIrBxa7D9JnYAmwobtP1D8GHPXEu4/+n5qvHST8D+cxCCwcmjxPxx8YTJVMPI/MCqpE9BE9D9F2PD0Sln2P/p+arx0k/g/GQRWDi2y+z83iUFg5dD+PyuHFtnO9wBAukkMAiuHAkBKDAIrhxYEQNnO91PjpQVAaJHtfD81B0D4U+Olm8QIQIcW2c73UwpAF9nO91PjC0Cmm8QgsHINQDVeukkMAg9AYhBYObRIEECq8dJNYhARQPLSTWIQ2BFAObTIdr6fEkCBlUOLbGcTQMl2vp8aLxRAEFg5tMj2FEBYObTIdr4VQA==" +_RESOURCE = base64.b64decode(_RESOURCE_B64) +if len(_RESOURCE) != 1_024 or hashlib.sha256(_RESOURCE).hexdigest() != RESOURCE_SHA256: + raise RuntimeError("packaged Nikon characteristic resource failed its hash pin") +_RESOURCE_TABLES = tuple(np.array(struct.unpack_from("<32d", _RESOURCE, index * 0x100), dtype=np.float64) for index in range(4)) + + +@dataclass(frozen=True) +class NativeBuilderEvidence: + """Distinct density, analyzer, and final-exposure evidence for one frame.""" + + session_id: str + capture_attempt_id: str + scan_identity: str + slot: int + density_source_wire_sha256: str + density_source_child_sha256: str + calibration_numerators: tuple[int, int, int] + density_f03_denominators: tuple[int, int, int] + densities: tuple[float, float, float] + density_arithmetic: str + frame_ownership_status: str + frame_ownership_receipt: bytes + frame_ownership_receipt_sha256: str + density_evidence_receipt: bytes + density_evidence_receipt_sha256: str + reservation_id: str + batch_session_id: str + preview_sha256: str + preview_identity_sha256: str + transport_table_sha256: str + transport_identity_sha256: str + reviewed_fingerprint_sha256: str + fresh_fingerprint_sha256: str + frame_index: int + frame_total: int + selected_slots: tuple[int, ...] + analyzer_rgb: np.ndarray + analyzer_rgb_sha256: str + analyzer_resolution_dpi: int + analyzer_rectangle: tuple[int, int, int, int] + final_f02_denominators: tuple[int, int, int] + + +@dataclass(frozen=True) +class _AnalyzerOutputs: + all_lower: tuple[float, float, float] + all_max: tuple[float, float, float] + gated_lower: tuple[float, float, float] + gated_max: tuple[float, float, float] + + +@dataclass(frozen=True) +class _BuilderArgs: + A: float + B: float + E: float + C: float + a: float + f: float + g: float + + +def adapt_native_builder_evidence(evidence: object) -> NativeBuilderEvidence: + """Convert Coolscanpy's exact neutral producer at the app boundary. + + The optional scanner dependency stays outside NegPy's builder core. Only + the exact Coolscanpy dataclass is accepted; similarly shaped objects are + deliberately rejected. ``build_native_builder_receipt`` subsequently + revalidates every copied identity, receipt digest, analyzer digest, and + numeric field under NegPy's own contract. + """ + + if type(evidence) is NativeBuilderEvidence: + return evidence + try: + from coolscanpy.protocol.ls5000_single_pass.density import ( + NikonExactBuilderEvidence, + ) + except ImportError as error: + raise exact_color.ExactColorUnavailable("frame native builder evidence has an invalid type") from error + if type(evidence) is not NikonExactBuilderEvidence: + raise exact_color.ExactColorUnavailable("frame native builder evidence has an invalid type") + return NativeBuilderEvidence( + session_id=evidence.session_id, + capture_attempt_id=evidence.capture_attempt_id, + scan_identity=evidence.scan_identity, + slot=evidence.slot, + density_source_wire_sha256=evidence.density_source_wire_sha256, + density_source_child_sha256=evidence.density_source_child_sha256, + calibration_numerators=evidence.calibration_numerators, + density_f03_denominators=evidence.density_f03_denominators, + densities=evidence.densities, + density_arithmetic=evidence.density_arithmetic, + frame_ownership_status=evidence.frame_ownership_status, + frame_ownership_receipt=evidence.frame_ownership_receipt, + frame_ownership_receipt_sha256=evidence.frame_ownership_receipt_sha256, + density_evidence_receipt=evidence.density_evidence_receipt, + density_evidence_receipt_sha256=evidence.density_evidence_receipt_sha256, + reservation_id=evidence.reservation_id, + batch_session_id=evidence.batch_session_id, + preview_sha256=evidence.preview_sha256, + preview_identity_sha256=evidence.preview_identity_sha256, + transport_table_sha256=evidence.transport_table_sha256, + transport_identity_sha256=evidence.transport_identity_sha256, + reviewed_fingerprint_sha256=evidence.reviewed_fingerprint_sha256, + fresh_fingerprint_sha256=evidence.fresh_fingerprint_sha256, + frame_index=evidence.frame_index, + frame_total=evidence.frame_total, + selected_slots=evidence.selected_slots, + analyzer_rgb=evidence.analyzer_rgb, + analyzer_rgb_sha256=evidence.analyzer_rgb_sha256, + analyzer_resolution_dpi=evidence.analyzer_resolution_dpi, + analyzer_rectangle=evidence.analyzer_rectangle, + final_f02_denominators=evidence.final_f02_denominators, + ) + + +def build_native_builder_receipt( + evidence: NativeBuilderEvidence, +) -> exact_color.NativeValidatedBuilderReceipt: + """Derive fresh pre-F LUTs only after every provenance gate closes.""" + + analyzer, analyzer_bytes = _validate_evidence(evidence) + r0, c0, r1, c1 = evidence.analyzer_rectangle + selected = analyzer[r0 : r1 + 1, c0 : c1 + 1].reshape(-1, 3) + exposure = ( + evidence.calibration_numerators[0] / evidence.final_f02_denominators[0], + evidence.calibration_numerators[1] / evidence.final_f02_denominators[1], + evidence.calibration_numerators[2] / evidence.final_f02_denominators[2], + ) + args, controls = _derive_parameters(selected, exposure, evidence.densities) + built_luts = [ + _build_lut(arg, controls[index][0], controls[index][1]).astype(" exact_color.NativeValidatedBuilderReceipt: + """Re-derive a retained snapshot instead of trusting its stored LUTs. + + A writable evidence directory is not an attestation boundary. The raw + analyzer and canonical acquisition receipts therefore reconstruct the + original ``NativeBuilderEvidence`` and run the pinned builder again. Only + byte-identical envelope, evidence, analyzer, and pre-F artifacts can be + promoted back into a trusted receipt. + """ + + try: + document = _parse_canonical_object( + evidence_payload, + label="retained native builder evidence", + ) + ownership = _parse_canonical_object( + frame_ownership_receipt, + label="retained frame ownership", + ) + _parse_canonical_object( + density_evidence_receipt, + label="retained density evidence", + ) + identity = document["identity"] + density = document["density_source"] + analyzer = document["analyzer_source"] + calibration = document["calibration"] + if not all(type(value) is dict for value in (identity, density, analyzer, calibration)): + raise TypeError("retained native builder sections are malformed") + identity_row = cast(dict[str, object], identity) + density_row = cast(dict[str, object], density) + analyzer_row = cast(dict[str, object], analyzer) + calibration_row = cast(dict[str, object], calibration) + if document.get("frame_ownership") != ownership: + raise ValueError("retained native builder evidence disagrees with frame ownership") + analyzer_array = np.frombuffer(analyzer_rgb, dtype=" tuple[np.ndarray, bytes]: + if type(evidence) is not NativeBuilderEvidence: + raise exact_color.ExactColorUnavailable("native builder evidence has an invalid type") + for label, value in ( + ("session_id", evidence.session_id), + ("capture_attempt_id", evidence.capture_attempt_id), + ("scan_identity", evidence.scan_identity), + ): + if type(value) is not str or not value: + raise exact_color.ExactColorUnavailable(f"native builder {label} is missing") + if type(evidence.slot) is not int or not 1 <= evidence.slot <= 40: + raise exact_color.ExactColorUnavailable("native builder slot must be an integer in 1..40") + for label, digest in ( + ("density source wire", evidence.density_source_wire_sha256), + ("density source child", evidence.density_source_child_sha256), + ("frame ownership", evidence.frame_ownership_receipt_sha256), + ("density evidence", evidence.density_evidence_receipt_sha256), + ("preview", evidence.preview_sha256), + ("preview identity", evidence.preview_identity_sha256), + ("transport table", evidence.transport_table_sha256), + ("transport identity", evidence.transport_identity_sha256), + ("reviewed fingerprint", evidence.reviewed_fingerprint_sha256), + ("fresh fingerprint", evidence.fresh_fingerprint_sha256), + ("analyzer RGB", evidence.analyzer_rgb_sha256), + ): + if type(digest) is not str or not _SHA256.fullmatch(digest): + raise exact_color.ExactColorUnavailable(f"native builder {label} SHA-256 is malformed") + if evidence.frame_ownership_status != FRAME_OWNERSHIP_STATUS: + raise exact_color.ExactColorUnavailable("97-dpi density evidence is not proven to belong to this frame") + ownership = _parse_canonical_object(evidence.frame_ownership_receipt, label="frame ownership") + density_receipt = _parse_canonical_object(evidence.density_evidence_receipt, label="density evidence") + if hashlib.sha256(evidence.frame_ownership_receipt).hexdigest() != evidence.frame_ownership_receipt_sha256: + raise exact_color.ExactColorUnavailable("frame ownership receipt does not match its SHA-256") + if hashlib.sha256(evidence.density_evidence_receipt).hexdigest() != evidence.density_evidence_receipt_sha256: + raise exact_color.ExactColorUnavailable("density evidence receipt does not match its SHA-256") + _validate_ownership_binding(evidence, ownership, density_receipt) + if evidence.density_arithmetic != DENSITY_ARITHMETIC: + raise exact_color.ExactColorUnavailable("density arithmetic is not x87 bit-certified") + for label, values in ( + ("calibration numerators", evidence.calibration_numerators), + ("density f03 denominators", evidence.density_f03_denominators), + ("final f02 denominators", evidence.final_f02_denominators), + ): + if type(values) is not tuple or len(values) != 3 or any(type(value) is not int or not 1 <= value <= 0xFFFFFFFF for value in values): + raise exact_color.ExactColorUnavailable(f"native builder {label} must contain three nonzero uint32 values") + if evidence.density_f03_denominators == evidence.final_f02_denominators: + raise exact_color.ExactColorUnavailable("density f03 and final f02 exposure triplets were conflated") + if ( + type(evidence.densities) is not tuple + or len(evidence.densities) != 3 + or any(type(value) is not float or not math.isfinite(value) for value in evidence.densities) + ): + raise exact_color.ExactColorUnavailable("native builder densities must contain three finite doubles") + if evidence.analyzer_resolution_dpi != ANALYZER_RESOLUTION_DPI: + raise exact_color.ExactColorUnavailable("native builder analyzer source must be the 285-dpi pass") + analyzer = np.asarray(evidence.analyzer_rgb) + if analyzer.dtype != np.uint16 or analyzer.shape != ANALYZER_SHAPE: + raise exact_color.ExactColorUnavailable(f"native builder analyzer RGB must be uint16 with shape {ANALYZER_SHAPE}") + snapshot = np.array(analyzer, dtype=" dict[str, object]: + return { + "batch_session_id": evidence.batch_session_id, + "capture_attempt_id": evidence.capture_attempt_id, + "preview_identity_sha256": evidence.preview_identity_sha256, + "preview_sha256": evidence.preview_sha256, + "reservation_id": evidence.reservation_id, + "scan_identity": evidence.scan_identity, + "session_id": evidence.session_id, + "slot": evidence.slot, + } + + +def _evidence_payload(evidence: NativeBuilderEvidence, analyzer: np.ndarray) -> dict[str, object]: + return { + "algorithm": { + "curve": CURVE_ALGORITHM_ID, + "parameter_derivation": PARAMETER_ALGORITHM_ID, + }, + "analyzer_source": { + "final_f02_denominators_raw_10ns_rgb": list(evidence.final_f02_denominators), + "geometry": list(analyzer.shape), + "rectangle_inclusive": list(evidence.analyzer_rectangle), + "resolution_dpi": evidence.analyzer_resolution_dpi, + "rgb_bytes": len(analyzer.tobytes()), + "rgb_sha256": evidence.analyzer_rgb_sha256, + }, + "calibration": {"read8c_numerators_rgb": list(evidence.calibration_numerators)}, + "density_source": { + "arithmetic": evidence.density_arithmetic, + "child_buffer_sha256": evidence.density_source_child_sha256, + "densities_binary64_hex_rgb": [float(value).hex() for value in evidence.densities], + "f03_denominators_raw_10ns_rgb": list(evidence.density_f03_denominators), + "resolution_dpi": DENSITY_SOURCE_RESOLUTION_DPI, + "wire_sha256": evidence.density_source_wire_sha256, + }, + "density_evidence": { + "receipt_sha256": evidence.density_evidence_receipt_sha256, + "scope": "reservation-preview", + }, + "frame_ownership": json.loads(evidence.frame_ownership_receipt), + "identity": _identity_payload(evidence), + "resource": {"bytes": len(_RESOURCE), "sha256": RESOURCE_SHA256}, + "schema": "negpy.native-stage1-builder-evidence", + "version": 1, + } + + +def _canonical_json(value: dict[str, object]) -> bytes: + return json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=False, allow_nan=False).encode("utf-8") + + +def _parse_canonical_object(payload: bytes, *, label: str) -> dict[str, object]: + if type(payload) is not bytes or not payload: + raise exact_color.ExactColorUnavailable(f"native builder {label} receipt is missing") + try: + parsed = json.loads(payload) + except (UnicodeDecodeError, ValueError, RecursionError) as error: + raise exact_color.ExactColorUnavailable(f"native builder {label} receipt is invalid JSON") from error + if type(parsed) is not dict or _canonical_json(parsed) != payload: + raise exact_color.ExactColorUnavailable(f"native builder {label} receipt is not canonical") + return parsed + + +def _validate_ownership_binding( + evidence: NativeBuilderEvidence, + ownership: dict[str, object], + density_receipt: dict[str, object], +) -> None: + expected = { + "schema_version": 1, + "scope": "reservation-preview-frame", + "binding_status": FRAME_OWNERSHIP_STATUS, + "session_reservation_retained": True, + "reservation_id": evidence.reservation_id, + "batch_session_id": evidence.batch_session_id, + "preview_sha256": evidence.preview_sha256, + "preview_identity_sha256": evidence.preview_identity_sha256, + "transport_table_sha256": evidence.transport_table_sha256, + "transport_identity_sha256": evidence.transport_identity_sha256, + "reviewed_fingerprint_sha256": evidence.reviewed_fingerprint_sha256, + "fresh_fingerprint_sha256": evidence.fresh_fingerprint_sha256, + "frame_capture_attempt_id": evidence.capture_attempt_id, + "frame_index": evidence.frame_index, + "frame_total": evidence.frame_total, + "selected_slots": list(evidence.selected_slots), + "selected_slot": evidence.slot, + } + if ownership != expected: + raise exact_color.ExactColorUnavailable("frame ownership receipt does not bind the native builder acquisition") + transport_material: dict[str, object] = { + "reservation_id": evidence.reservation_id, + "batch_session_id": evidence.batch_session_id, + "preview_sha256": evidence.preview_sha256, + "preview_identity_sha256": evidence.preview_identity_sha256, + "transport_table_sha256": evidence.transport_table_sha256, + "reviewed_fingerprint_sha256": evidence.reviewed_fingerprint_sha256, + "fresh_fingerprint_sha256": evidence.fresh_fingerprint_sha256, + "selected_slots": list(evidence.selected_slots), + } + if hashlib.sha256(_canonical_json(transport_material)).hexdigest() != evidence.transport_identity_sha256: + raise exact_color.ExactColorUnavailable("frame ownership transport identity digest is invalid") + if ( + type(evidence.reservation_id) is not str + or not evidence.reservation_id + or type(evidence.batch_session_id) is not str + or not evidence.batch_session_id + or evidence.session_id != evidence.reservation_id + or evidence.reservation_id != evidence.batch_session_id + or type(evidence.frame_index) is not int + or type(evidence.frame_total) is not int + or not 1 <= evidence.frame_index <= evidence.frame_total + or type(evidence.selected_slots) is not tuple + or len(evidence.selected_slots) != evidence.frame_total + or evidence.slot not in evidence.selected_slots + or evidence.selected_slots[evidence.frame_index - 1] != evidence.slot + or any(type(slot) is not int or not 1 <= slot <= 40 for slot in evidence.selected_slots) + or len(set(evidence.selected_slots)) != len(evidence.selected_slots) + ): + raise exact_color.ExactColorUnavailable("frame ownership batch identity is malformed") + _validate_density_receipt_binding(evidence, density_receipt) + + +def _validate_density_receipt_binding( + evidence: NativeBuilderEvidence, + density_receipt: dict[str, object], +) -> None: + if set(density_receipt) != { + "schema_version", + "scope", + "per_frame_binding_status", + "preview_identity_sha256", + "source_payload_bytes", + "calibration_binding", + "source_binding", + "exposure_binding", + "result", + }: + raise exact_color.ExactColorUnavailable("density evidence receipt fields are incomplete") + calibration_binding = density_receipt.get("calibration_binding") + source_binding = density_receipt.get("source_binding") + exposure_binding = density_receipt.get("exposure_binding") + result = density_receipt.get("result") + if not all(type(value) is dict for value in (calibration_binding, source_binding, exposure_binding, result)): + raise exact_color.ExactColorUnavailable("density evidence bindings are malformed") + calibration_binding = cast(dict[str, object], calibration_binding) + source_binding = cast(dict[str, object], source_binding) + exposure_binding = cast(dict[str, object], exposure_binding) + result = cast(dict[str, object], result) + calibration = calibration_binding.get("calibration") + if type(calibration) is not dict: + raise exact_color.ExactColorUnavailable("density calibration receipt is malformed") + calibration = cast(dict[str, object], calibration) + binding_identities = tuple( + (binding.get("session_id"), binding.get("capture_attempt_id"), binding.get("scan_identity")) + for binding in (source_binding, exposure_binding, result) + ) + calibration_identity = ( + calibration.get("session_id"), + calibration_binding.get("capture_attempt_id"), + calibration_binding.get("scan_identity"), + ) + density_hex = [struct.pack(">d", value).hex() for value in evidence.densities] + if ( + density_receipt.get("schema_version") != 1 + or density_receipt.get("scope") != "reservation-preview" + or density_receipt.get("per_frame_binding_status") != DENSITY_PER_FRAME_BINDING_STATUS + or density_receipt.get("preview_identity_sha256") != evidence.preview_identity_sha256 + or density_receipt.get("source_payload_bytes") + not in SUPPORTED_DENSITY_SOURCE_WIRE_BYTES + or any(identity != calibration_identity for identity in binding_identities) + or calibration_identity[0] != evidence.reservation_id + or calibration_identity[2] != evidence.scan_identity + or source_binding.get("resolution_dpi") != DENSITY_SOURCE_RESOLUTION_DPI + or source_binding.get("wire_sha256") != evidence.preview_sha256 + or source_binding.get("wire_sha256") != evidence.density_source_wire_sha256 + or source_binding.get("child_buffer_sha256") != evidence.density_source_child_sha256 + or calibration.get("numerators_rgb") != list(evidence.calibration_numerators) + or exposure_binding.get("density_f03_exposures_raw_10ns_rgb") != list(evidence.density_f03_denominators) + or result.get("algorithm_id") != evidence.density_arithmetic + or result.get("promotable") is not True + or result.get("source_wire_sha256") != evidence.density_source_wire_sha256 + or result.get("source_child_buffer_sha256") != evidence.density_source_child_sha256 + or result.get("numerators_rgb") != list(evidence.calibration_numerators) + or result.get("density_f03_denominators_raw_10ns_rgb") != list(evidence.density_f03_denominators) + or result.get("densities_rgb") != list(evidence.densities) + or result.get("density_binary64_be_hex_rgb") != density_hex + ): + raise exact_color.ExactColorUnavailable("density evidence belongs to a different preview or reservation") + + +def _curve(xt: np.ndarray, yt: np.ndarray, value: np.ndarray) -> np.ndarray: + index = np.full(value.shape, 32, dtype=np.int64) + for candidate in range(31, 0, -1): + index = np.where(value < xt[candidate], candidate, index) + hit = index < 32 + bounded = np.clip(index, 1, 31) + x1, x0, y1, y0 = xt[bounded], xt[bounded - 1], yt[bounded], yt[bounded - 1] + denominator = x1 - x0 + interpolated = np.where( + denominator != 0, + y0 + (value - x0) * (y1 - y0) / np.where(denominator == 0, 1, denominator), + y0, + ) + return np.where(hit, interpolated, yt[31]) + + +def _histogram(values: np.ndarray) -> np.ndarray: + return (np.bincount(values.astype(np.int64), minlength=65_536) % 65_536).astype(np.int64) + + +def _lower_cutoff(histogram: np.ndarray, population: int) -> float: + threshold = math.trunc(0.0042 * population) + hit = np.flatnonzero(np.cumsum(histogram) >= threshold) + return float(hit[0]) if hit.size else 0.0 + + +def _maximum(histogram: np.ndarray) -> float: + hit = np.flatnonzero(histogram) + return float(hit[-1]) if hit.size else 0.0 + + +def _analyse(pixels: np.ndarray, exposure: tuple[float, float, float], densities: tuple[float, float, float]) -> _AnalyzerOutputs: + count = pixels.shape[0] + scaled = pixels.astype(np.float64) * np.asarray(exposure, dtype=np.float64) + density = np.asarray(densities, dtype=np.float64) + with np.errstate(divide="ignore", invalid="ignore"): + ratio01 = scaled[:, 0] / (10.0 ** (density[1] - density[0]) * scaled[:, 1]) + ratio02 = scaled[:, 0] / (10.0 ** (density[2] - density[0]) * scaled[:, 2]) + ratio12 = scaled[:, 1] / (10.0 ** (density[2] - density[1]) * scaled[:, 2]) + + def good(ratio: np.ndarray) -> np.ndarray: + return (ratio > 1.0 / 3.0) & (ratio < 3.0) + + good01, good02, good12 = good(ratio01), good(ratio02), good(ratio12) + passes = (good01 | good02, good01 | good12, good02 | good12) + all_lower: list[float] = [] + all_max: list[float] = [] + gated_lower: list[float] = [] + gated_max: list[float] = [] + for channel in range(3): + values = pixels[:, channel] + histogram = _histogram(values) + all_lower.append(_lower_cutoff(histogram, count)) + all_max.append(_maximum(histogram)) + keep = passes[channel].copy() + failing = np.flatnonzero(~keep) + if failing.size: + budget = math.ceil(0.9 * count) + if failing.size > budget: + keep[failing[budget:]] = True + gated = values[keep] + histogram = _histogram(gated) + gated_lower.append(_lower_cutoff(histogram, gated.size)) + gated_max.append(_maximum(histogram)) + return _AnalyzerOutputs( + (all_lower[0], all_lower[1], all_lower[2]), + (all_max[0], all_max[1], all_max[2]), + (gated_lower[0], gated_lower[1], gated_lower[2]), + (gated_max[0], gated_max[1], gated_max[2]), + ) + + +def _scale(outputs: _AnalyzerOutputs, exposure: tuple[float, float, float]) -> tuple[tuple[float, ...], ...]: + zlo: list[float] = [] + zhi: list[float] = [] + Zlo: list[float] = [] + Zhi: list[float] = [] + for channel in range(3): + all_lo = max(1.0, outputs.all_lower[channel] * exposure[channel]) + all_hi = max(1.0, outputs.all_max[channel] * exposure[channel]) + gate_lo = outputs.gated_lower[channel] * exposure[channel] + gate_hi = outputs.gated_max[channel] * exposure[channel] + zlo.append(math.log10(65_535.0 / gate_lo)) + zhi.append(math.log10(65_535.0 / gate_hi)) + Zlo.append(math.log10(65_535.0 / all_lo)) + Zhi.append(math.log10(65_535.0 / all_hi)) + return tuple(zlo), tuple(zhi), tuple(Zlo), tuple(Zhi) + + +def _derive_parameters( + pixels: np.ndarray, + exposure: tuple[float, float, float], + densities: tuple[float, float, float], +) -> tuple[list[_BuilderArgs], list[tuple[np.ndarray, np.ndarray]]]: + outputs = _analyse(pixels, exposure, densities) + zlo, zhi, Zlo, Zhi = _scale(outputs, exposure) + D = tuple(min(densities[channel], zhi[channel]) for channel in range(3)) + X = [table + (D[channel] - table[0]) for channel, table in enumerate(_RESOURCE_TABLES[1:])] + master = _RESOURCE_TABLES[0] + + def first(channel: int, value: float) -> float: + return float(_curve(X[channel], master, np.array([value], dtype=np.float64))[0]) + + reference = [first(channel, D[channel] + 0.15) for channel in range(3)] + Lo = [first(channel, zlo[channel]) for channel in range(3)] + Hi = [first(channel, zhi[channel]) for channel in range(3)] + lo = [Lo[channel] - reference[channel] for channel in range(3)] + hi = [Hi[channel] - reference[channel] for channel in range(3)] + wh0 = 10.0 ** (-abs(hi[0] - hi[1])) + wh2 = 10.0 ** (-abs(hi[2] - hi[1])) + ws0 = 3.0 ** (-abs((lo[0] - hi[0]) - (lo[1] - hi[1]))) + ws2 = 3.0 ** (-abs((lo[2] - hi[2]) - (lo[1] - hi[1]))) + tone = min(1.0, 10.0 ** (max(lo) - 0.8)) + p = tone * ((1.0 - wh0) + wh0 * ws0) + q = tone * ((1.0 - wh2) + wh2 * ws2) + offset0 = p * (Lo[0] - Lo[1]) + wh0 * (1.0 - p) * (Hi[0] - Hi[1]) + offset1 = q * (Lo[1] - Lo[2]) + wh2 * (1.0 - q) * (Hi[1] - Hi[2]) + Y2 = [master - offset0, master.copy(), master + offset1] + + def second(channel: int, value: float) -> float: + return float(_curve(X[channel], Y2[channel], np.array([value], dtype=np.float64))[0]) + + reference2 = [second(channel, D[channel] + 0.15) for channel in range(3)] + vlo = [second(channel, zlo[channel]) - reference2[channel] for channel in range(3)] + vhi = [second(channel, zhi[channel]) - reference2[channel] for channel in range(3)] + E0 = min(vhi) + a = [min(1.0, 10.0 ** (-(vhi[channel] + 0.23) * 4.0)) for channel in range(3)] + m = max(a) + L = max(vlo) + A0 = max(second(channel, min(Zlo[channel], X[channel][31])) for channel in range(3)) + C = [second(channel, max(Zhi[channel], X[channel][0])) for channel in range(3)] + B0 = min(C) + difference0 = A0 - B0 + if difference0 >= 0.6: + B = B0 + A1 = A0 + 0.05 + else: + delta = 0.6 - difference0 + B = B0 - (1.0 - m) * delta + A1 = max(B + 0.6, A0 + 0.05) + difference = A1 - B + te = min(1.9, 1.9 * 0.4 ** (E0 + 0.23)) + E = min(1.9, max(difference, te)) + A = A1 + (difference / E) * (1.2 - L) if L < 1.2 else A1 + f = [float(_curve(Y2[channel], X[channel], np.array([A], dtype=np.float64))[0]) for channel in range(3)] + g = [float(_curve(Y2[channel], X[channel], np.array([B], dtype=np.float64))[0]) for channel in range(3)] + args = [_BuilderArgs(A, B, E, C[channel], a[channel], f[channel], g[channel]) for channel in range(3)] + return args, list(zip(X, Y2, strict=True)) + + +def _build_lut(arg: _BuilderArgs, xt: np.ndarray, yt: np.ndarray) -> np.ndarray: + slope = float(min(max((arg.A - ((1.0 - arg.a) * arg.B + arg.a * arg.C)) / arg.E, 0.01), 4.0)) + clip = max(0, min(int(10.0 ** (-(arg.f - arg.g)) * 65_535.0), 65_535)) + lut = np.zeros(65_536, dtype=np.uint16) + lut[: clip + 1] = 0xFFFF + if clip + 1 < 65_536: + index = np.arange(clip + 1, 65_536, dtype=np.float64) + tone = _curve(xt, yt, np.log10(65_535.0 / index) + arg.g) + output = 65_535.0 * np.power(10.0, (arg.A - tone) * (-1.0 / slope)) + lut[clip + 1 :] = (output.astype(np.int64) & 0xFFFF).astype(np.uint16) + return lut + + +__all__ = [ + "ANALYZER_RESOLUTION_DPI", + "DENSITY_SOURCE_RESOLUTION_DPI", + "NativeBuilderEvidence", + "adapt_native_builder_evidence", + "build_native_builder_receipt", + "rebuild_retained_native_builder_receipt", +] diff --git a/negpy/services/roll/nikon_icc.py b/negpy/services/roll/nikon_icc.py new file mode 100644 index 00000000..c126993a --- /dev/null +++ b/negpy/services/roll/nikon_icc.py @@ -0,0 +1,65 @@ +"""Package-safe Nikon Adobe RGB profile used by exact C-41 positives. + +Nikon Scan's LS-5000 C-41 CMS-on TIFFs embed this exact 492-byte profile. +Keeping the tiny payload in source avoids a filesystem-relative runtime asset +that can disappear from a wheel or frozen desktop app. The decoded bytes are +validated before use; exact output must fail closed if packaging or source +corruption changes either their size or identity. +""" + +from __future__ import annotations + +import base64 +import hashlib +from functools import lru_cache +from typing import Final + + +NIKON_ADOBE_RGB_PROFILE_NAME: Final = "Nikon Adobe RGB 4.0.0.3000" +NIKON_ADOBE_RGB_PROFILE_BYTES: Final = 492 +NIKON_ADOBE_RGB_PROFILE_SHA256: Final = "a8d0d753bd6129357cc2647435ce675e8637a679eb526fa180fba460874ce1d3" + +_NIKON_ADOBE_RGB_PROFILE_BASE64: Final = ( + "AAAB7E5LT04CIAAAbW50clJHQiBYWVogB88ADAAHABIAOwAWYWNzcEFQUEwAAAAA" + "bm9uZQAAAAEAAAAAAAAAAAAAAAAAAPbWAAEAAAAA0y0AAAAAAAAAAAAAAAAAAAAA" + "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAJZGVzYwAAAPAAAABN" + "clhZWgAAAUAAAAAUZ1hZWgAAAVQAAAAUYlhZWgAAAWgAAAAUclRSQwAAAXwAAAAO" + "Z1RSQwAAAYwAAAAOYlRSQwAAAZwAAAAOd3RwdAAAAawAAAAUY3BydAAAAcAAAAAs" + "ZGVzYwAAAAAAAAAbTmlrb24gQWRvYmUgUkdCIDQuMC4wLjMwMDAAAAAAAAAAAAAA" + "ABtOaWtvbiBBZG9iZSBSR0IgNC4wLjAuMzAwMAAAAABYWVogAAAAAAAAnBkAAE+m" + "AAAE/FhZWiAAAAAAAAA0iwAAoCsAAA+VWFlaIAAAAAAAACYyAAAQLwAAvqBjdXJ2" + "AAAAAAAAAAECMwAAY3VydgAAAAAAAAABAjMAAGN1cnYAAAAAAAAAAQIzAABYWVog" + "AAAAAAAA81QAAQAAAAEWz3RleHQAAAAATmlrb24gSW5jLiAmIE5pa29uIENvcnBv" + "cmF0aW9uIDIwMDEA" +) + + +class NikonICCProfileError(RuntimeError): + """The source-embedded exact-output profile failed identity checks.""" + + +@lru_cache(maxsize=1) +def nikon_adobe_rgb_profile() -> bytes: + """Return the immutable, identity-checked NKAdobe ICC payload.""" + + try: + profile = base64.b64decode(_NIKON_ADOBE_RGB_PROFILE_BASE64, validate=True) + except ValueError as error: + raise NikonICCProfileError("embedded Nikon Adobe RGB profile is not valid base64") from error + if len(profile) != NIKON_ADOBE_RGB_PROFILE_BYTES: + raise NikonICCProfileError(f"embedded Nikon Adobe RGB profile is {len(profile)} bytes, expected {NIKON_ADOBE_RGB_PROFILE_BYTES}") + digest = hashlib.sha256(profile).hexdigest() + if digest != NIKON_ADOBE_RGB_PROFILE_SHA256: + raise NikonICCProfileError("embedded Nikon Adobe RGB profile does not match its pinned SHA-256") + return profile + + +def profile_receipt_binding() -> dict[str, str | int]: + """Receipt fields for the profile after reusing the validation boundary.""" + + nikon_adobe_rgb_profile() + return { + "name": NIKON_ADOBE_RGB_PROFILE_NAME, + "bytes": NIKON_ADOBE_RGB_PROFILE_BYTES, + "sha256": NIKON_ADOBE_RGB_PROFILE_SHA256, + } diff --git a/negpy/services/roll/portable_builder.py b/negpy/services/roll/portable_builder.py new file mode 100644 index 00000000..b145899b --- /dev/null +++ b/negpy/services/roll/portable_builder.py @@ -0,0 +1,194 @@ +"""Portable application of validated Nikon Stage-1 builder artifacts. + +The pre-F LUTs come from either the validated Stage-3 replay bridge or the +identity-bound native builder. This applicator composes each plane in the +recovered order ``F[B_c(i)]`` with the pinned LS5000.md3 fixed output LUT, +then applies the three composed u16 LUTs to repaired RGB in bounded chunks. +""" + +from __future__ import annotations + +import hashlib +import json +import os +import stat +from pathlib import Path +from typing import Final + +import numpy as np + +from negpy.services.roll import exact_color +from negpy.services.roll.exact_color import ( + BuilderReceipt, + ExactColorIntegrityError, + ExactColorUnavailable, + Stage1BuilderResult, + VerifiedBuilderApplicationReceipt, +) + + +ALGORITHM_ID: Final = "ls5000-md3-pref-fixed-postf-v1" +FIXED_LUT_FILENAME: Final = "ls5000-fixed-output-lut-u16le.bin" +FIXED_LUT_SHA256: Final = exact_color.FIXED_COMPOSITION_SHA256 +FIXED_LUT_BYTES: Final = 65_536 * 2 +DEFAULT_CHUNK_PIXELS: Final = 65_536 +MAX_CHUNK_PIXELS: Final = 262_144 + + +class PortableStage1Builder: + """Verified, chunked pre-F → fixed post-F Stage-1 input builder.""" + + def __init__( + self, + *, + assets_dir: Path | None = None, + chunk_pixels: int = DEFAULT_CHUNK_PIXELS, + ) -> None: + if type(chunk_pixels) is not int or not 1 <= chunk_pixels <= MAX_CHUNK_PIXELS: + raise ExactColorUnavailable(f"chunk_pixels must be an integer in 1..{MAX_CHUNK_PIXELS}") + self._chunk_pixels = chunk_pixels + self.assets_dir = ( + Path(__file__).resolve().parents[2] / "assets" / "portable_builder" + if assets_dir is None + else Path(assets_dir).expanduser().resolve() + ) + fixed_payload = _read_verified_file( + self.assets_dir / FIXED_LUT_FILENAME, + expected_sha256=FIXED_LUT_SHA256, + expected_bytes=FIXED_LUT_BYTES, + ) + self._fixed_lut = np.frombuffer(fixed_payload, dtype=" int: + return self._chunk_pixels + + def apply( + self, + rgb: np.ndarray, + *, + builder_receipt: BuilderReceipt, + ) -> Stage1BuilderResult: + exact_color.builder_receipt_payload(builder_receipt) + source_hash = exact_color.rgb16_content_sha256(rgb) + source = np.array(rgb, dtype=np.uint16, order="C", copy=True) + source.setflags(write=False) + flat_source = source.reshape(-1, 3) + + pre_f = [np.frombuffer(blob, dtype=" bytes: + try: + before = path.lstat() + if stat.S_ISLNK(before.st_mode) or not stat.S_ISREG(before.st_mode): + raise OSError("not a regular non-symlink file") + with path.open("rb") as stream: + opened = os.fstat(stream.fileno()) + payload = stream.read() + after = os.fstat(stream.fileno()) + except OSError as error: + raise ExactColorUnavailable(f"fixed composition LUT is unavailable: {path}: {error}") from error + if not (_identity(before) == _identity(opened) == _identity(after)): + raise ExactColorIntegrityError(f"fixed composition LUT changed while being read: {path}") + if len(payload) != expected_bytes: + raise ExactColorIntegrityError(f"fixed composition LUT byte size mismatch: {len(payload)} != {expected_bytes}") + actual = hashlib.sha256(payload).hexdigest() + if actual != expected_sha256: + raise ExactColorIntegrityError(f"fixed composition LUT hash mismatch: {path}: {actual} != {expected_sha256}") + return payload + + +def _identity(value: os.stat_result) -> tuple[int, int, int, int, int]: + return ( + value.st_dev, + value.st_ino, + value.st_size, + value.st_mtime_ns, + value.st_ctime_ns, + ) + + +def _canonical_json(value: dict[str, object]) -> bytes: + return json.dumps( + value, + sort_keys=True, + separators=(",", ":"), + ensure_ascii=False, + allow_nan=False, + ).encode("utf-8") + + +__all__ = [ + "ALGORITHM_ID", + "DEFAULT_CHUNK_PIXELS", + "FIXED_LUT_BYTES", + "FIXED_LUT_FILENAME", + "FIXED_LUT_SHA256", + "MAX_CHUNK_PIXELS", + "PortableStage1Builder", +] diff --git a/negpy/services/roll/portable_cms.py b/negpy/services/roll/portable_cms.py new file mode 100644 index 00000000..ca730292 --- /dev/null +++ b/negpy/services/roll/portable_cms.py @@ -0,0 +1,480 @@ +"""Production adapter for the verified DLL-free two-stage CML4 evaluator. + +The integer implementation is reused byte-for-byte from +'portable_oracle_evaluator.py'. This wrapper owns production concerns: +stable/hash-checked asset loading, strict validation of the packaged 12-event +zero-mismatch receipt, bounded full-frame chunking, validated builder-evidence +binding, and an immutable CMS receipt. + +Scope is intentionally narrow. It evaluates only the captured Stage 1 then +Stage 2 CML transforms. It does not implement or invent the upstream +per-frame builder; callers must first apply the separately verified Stage-1 +builder and present that computed RGB at this boundary. +""" + +from __future__ import annotations + +import hashlib +import json +import os +import stat +import sys +from pathlib import Path +from types import MappingProxyType, ModuleType +from typing import Final, Mapping, cast + +import numpy as np + +from negpy.services.roll import exact_color +from negpy.services.roll.exact_color import ( + ExactColorIntegrityError, + ExactColorResult, + ExactColorUnavailable, + BuilderReceipt, +) + +ORACLE_SOURCE_SHA256: Final = exact_color.CMS_ORACLE_SOURCE_SHA256 +ORACLE_VALIDATION_RECEIPT_SHA256: Final = exact_color.CMS_VALIDATION_RECEIPT_SHA256 +VALIDATION_RECEIPT_FILENAME: Final = "portable-oracle-receipt.json" +VALIDATION_RECEIPT_BYTES: Final = 4_014 +VALIDATION_RECEIPT_SHA256: Final = ORACLE_VALIDATION_RECEIPT_SHA256 +ALGORITHM_ID: Final = exact_color.CMS_ALGORITHM_ID +DEFAULT_CHUNK_PIXELS: Final = 65_536 +MAX_CHUNK_PIXELS: Final = 262_144 +PACKAGED_ASSET_BYTES: Final = 2_506_760 +ASSET_SHA256: Mapping[str, str] = MappingProxyType(dict(exact_color.CMS_ASSET_SHA256)) +_ORACLE_MODULE_NAME = "negpy.services.roll.portable_oracle_evaluator" + +_VALIDATION_EVENTS = { + "stage1": { + "30": (32_448, 32_448), + "34": (16_848, 32_448), + "38": (22_464, 22_464), + "42": (11_664, 22_464), + "47": (32_448, 32_448), + "52": (16_848, 32_448), + }, + "stage2": { + "31": (32_448, 32_448), + "35": (16_848, 32_448), + "39": (22_464, 22_464), + "43": (11_664, 22_464), + "48": (32_448, 32_448), + "53": (16_848, 32_448), + }, +} + +_ASSET_LAYOUT = { + "lch-atan-u16le.bin": (" None: + arrays: dict[str, np.ndarray] = {} + for name, (dtype, shape) in _ASSET_LAYOUT.items(): + value = np.frombuffer(payloads[name], dtype=dtype).reshape(shape) + value.setflags(write=False) + arrays[name] = value + self.atan = arrays["lch-atan-u16le.bin"] + self.sincos = arrays["lch-sincos-i16le.bin"] + self.reciprocal = arrays["lch-reciprocal-u16le.bin"] + self.s1_clut = arrays["cml4-stage1-clut0.bin"] + self.s1_input = arrays["cml4-stage1-input-lut0.bin"] + self.s1_output = arrays["cml4-stage1-output-lut0.bin"] + self.s2_clut = arrays["cml4-stage2-clut0.bin"] + self.s2_input = arrays["cml4-stage2-input-lut0.bin"] + self.s2_output = arrays["cml4-stage2-output-lut0.bin"] + self._lab_to_lch_codes = oracle.lab_to_lch_codes + self._lch_to_lab_codes = oracle.lch_to_lab_codes + self._lookup_three = oracle.lookup_three + self._optimized_trilinear = oracle.optimized_trilinear + + def stage1(self, source: np.ndarray) -> np.ndarray: + working = self._lookup_three(self.s1_input, source) + working = self._optimized_trilinear(self.s1_clut, working) + working = self._lab_to_lch_codes(working, self.atan) + return self._lookup_three(self.s1_output, working) + + def stage2(self, source: np.ndarray) -> np.ndarray: + # Preserve the recovered hue bypass: Stage-2 input LUT plane 2 is not + # used by the active CML4 path. + working = source.copy() + working[:, 0] = self.s2_input[0, working[:, 0]] + working[:, 1] = self.s2_input[1, working[:, 1]] + working = self._lch_to_lab_codes(working, self.sincos, self.reciprocal) + working = self._optimized_trilinear(self.s2_clut, working) + return self._lookup_three(self.s2_output, working) + + +class PortableCMSOnEvaluator: + """Chunked implementation of VerifiedPortableCMSEvaluator. + + Construction verifies the byte-identical oracle source and reads every + required transform asset plus the exact validation receipt once through a + stable-file check. No DLL, Wine process, scanner, VM, or external + repository is used at runtime. + """ + + def __init__( + self, + *, + assets_dir: Path | None = None, + chunk_pixels: int = DEFAULT_CHUNK_PIXELS, + ) -> None: + if type(chunk_pixels) is not int or not 1 <= chunk_pixels <= MAX_CHUNK_PIXELS: + raise ExactColorUnavailable(f"chunk_pixels must be an integer in 1..{MAX_CHUNK_PIXELS}") + self._chunk_pixels = chunk_pixels + self.assets_dir = ( + Path(__file__).resolve().parents[2] / "assets" / "portable_cms" + if assets_dir is None + else Path(assets_dir).expanduser().resolve() + ) + validation_payload = _read_verified_payload( + self.assets_dir / VALIDATION_RECEIPT_FILENAME, + label="portable CMS validation receipt", + expected_bytes=VALIDATION_RECEIPT_BYTES, + ) + validation_digest = hashlib.sha256(validation_payload).hexdigest() + if validation_digest != VALIDATION_RECEIPT_SHA256: + raise ExactColorIntegrityError( + f"portable CMS validation receipt hash mismatch: {validation_digest} != {VALIDATION_RECEIPT_SHA256}" + ) + self._validation_summary = MappingProxyType(_validate_validation_receipt(validation_payload)) + oracle = _load_verified_oracle() + payloads = _read_verified_assets(self.assets_dir) + self._transforms = _LoadedTransforms(payloads, oracle) + + @property + def chunk_pixels(self) -> int: + return self._chunk_pixels + + def evaluate( + self, + rgb: np.ndarray, + *, + builder_receipt: BuilderReceipt, + ) -> ExactColorResult: + input_hash = exact_color.rgb16_content_sha256(rgb) + exact_color.builder_receipt_payload(builder_receipt) + + source = np.array(rgb, dtype=np.uint16, order="C", copy=True) + source.setflags(write=False) + flat_source = source.reshape(-1, 3) + output = np.empty_like(source) + flat_output = output.reshape(-1, 3) + for start in range(0, flat_source.shape[0], self.chunk_pixels): + stop = min(start + self.chunk_pixels, flat_source.shape[0]) + stage1 = self._transforms.stage1(flat_source[start:stop]) + flat_output[start:stop] = self._transforms.stage2(stage1) + + if exact_color.rgb16_content_sha256(source) != input_hash: + raise ExactColorIntegrityError("portable CMS evaluator mutated its Stage-1 input") + output_hash = exact_color.rgb16_content_sha256(output) + output.setflags(write=False) + cms_payload = _canonical_json( + { + "algorithm": ALGORITHM_ID, + "assets": dict(sorted(ASSET_SHA256.items())), + "builder_receipt_sha256": builder_receipt.sha256, + "chunk_pixels": self.chunk_pixels, + "dll_free": True, + "input_rgb_sha256": input_hash, + "kind": exact_color.CMS_RECEIPT_KIND, + "oracle_source": { + "path": "portable_oracle_evaluator.py", + "sha256": ORACLE_SOURCE_SHA256, + }, + "output_rgb_sha256": output_hash, + "scope": exact_color.CMS_SCOPE, + "stage_order": ["stage1", "stage2"], + "upstream_builder_included": False, + "validation": { + "events": self._validation_summary["events"], + "full_payload_mismatched_bytes": self._validation_summary["full_payload_mismatched_bytes"], + "full_payload_total_bytes": self._validation_summary["full_payload_total_bytes"], + "mismatched_u16": self._validation_summary["mismatched_u16"], + "receipt_sha256": ORACLE_VALIDATION_RECEIPT_SHA256, + "total_u16": self._validation_summary["total_u16"], + }, + "version": exact_color.CMS_RECEIPT_VERSION, + } + ) + cms_receipt = exact_color._issue_verified_cms_receipt(cms_payload) # noqa: SLF001 - trusted adapter boundary + return ExactColorResult( + rgb=output, + input_rgb_sha256=input_hash, + output_rgb_sha256=output_hash, + builder_receipt=builder_receipt, + cms_receipt=cms_receipt, + ) + + +def _load_verified_oracle(source_path: Path | None = None) -> ModuleType: + if _ORACLE_MODULE_NAME in sys.modules: + raise ExactColorIntegrityError("portable CMS oracle module is preloaded; verified isolated loading is required") + path = Path(__file__).with_name("portable_oracle_evaluator.py") if source_path is None else Path(source_path) + payload = _read_verified_payload(path, label="portable CMS oracle source") + if _ORACLE_MODULE_NAME in sys.modules: + raise ExactColorIntegrityError("portable CMS oracle module was substituted during verification") + actual = hashlib.sha256(payload).hexdigest() + if actual != ORACLE_SOURCE_SHA256: + raise ExactColorIntegrityError(f"portable CMS oracle source hash mismatch: {actual} != {ORACLE_SOURCE_SHA256}") + isolated_name = f"_negpy_verified_portable_oracle_{actual[:16]}" + oracle = ModuleType(isolated_name) + oracle.__file__ = str(path) + oracle.__package__ = "negpy.services.roll" + try: + code = compile(payload, str(path), "exec") + exec(code, oracle.__dict__) + except Exception as error: + raise ExactColorIntegrityError(f"verified portable CMS oracle could not execute: {error}") from error + if _ORACLE_MODULE_NAME in sys.modules: + raise ExactColorIntegrityError("portable CMS oracle module was substituted during isolated execution") + exports = ("lab_to_lch_codes", "lch_to_lab_codes", "lookup_three", "optimized_trilinear") + if oracle.__dict__.get("ASSET_SHA256") != dict(ASSET_SHA256) or any(not callable(oracle.__dict__.get(name)) for name in exports): + raise ExactColorIntegrityError("verified portable CMS oracle exports do not match the production contract") + return oracle + + +def _read_verified_assets(directory: Path) -> Mapping[str, bytes]: + payloads: dict[str, bytes] = {} + for name, expected in ASSET_SHA256.items(): + path = directory / name + dtype, shape = _ASSET_LAYOUT[name] + expected_bytes = int(np.prod(shape)) * np.dtype(dtype).itemsize + payload = _read_verified_payload( + path, + label="portable CMS asset", + expected_bytes=expected_bytes, + ) + actual = hashlib.sha256(payload).hexdigest() + if actual != expected: + raise ExactColorIntegrityError(f"portable CMS asset hash mismatch: {path}: {actual} != {expected}") + payloads[name] = payload + if sum(len(payload) for payload in payloads.values()) != PACKAGED_ASSET_BYTES: + raise ExactColorIntegrityError("portable CMS asset byte total is inconsistent") + return MappingProxyType(payloads) + + +def _read_verified_payload( + path: Path, + *, + label: str, + expected_bytes: int | None = None, +) -> bytes: + descriptor: int | None = None + try: + before = path.lstat() + if stat.S_ISLNK(before.st_mode) or not stat.S_ISREG(before.st_mode): + raise OSError("not a regular non-symlink file") + if expected_bytes is not None and before.st_size != expected_bytes: + raise ExactColorIntegrityError(f"{label} byte size mismatch: {before.st_size} != {expected_bytes}") + flags = os.O_RDONLY | getattr(os, "O_CLOEXEC", 0) | getattr(os, "O_NOFOLLOW", 0) + descriptor = os.open(path, flags) + opened = os.fstat(descriptor) + chunks: list[bytes] = [] + while chunk := os.read(descriptor, 1024 * 1024): + chunks.append(chunk) + after = os.fstat(descriptor) + final = path.lstat() + except OSError as error: + raise ExactColorUnavailable(f"{label} is unavailable: {path}: {error}") from error + finally: + if descriptor is not None: + os.close(descriptor) + if len({_identity(item) for item in (before, opened, after, final)}) != 1: + raise ExactColorIntegrityError(f"{label} changed while being read: {path}") + payload = b"".join(chunks) + if len(payload) != before.st_size: + raise ExactColorIntegrityError(f"{label} changed byte length while being read: {path}") + return payload + + +def _validate_validation_receipt(payload: bytes) -> dict[str, int]: + """Close the packaged zero-mismatch claim over its exact 12-event receipt.""" + + try: + document = json.loads( + payload, + object_pairs_hook=_reject_duplicate_keys, + parse_constant=_reject_json_constant, + ) + except (UnicodeDecodeError, json.JSONDecodeError, ValueError) as error: + raise ExactColorIntegrityError(f"portable CMS validation receipt is invalid JSON: {error}") from error + if type(document) is not dict or set(document) != { + "all_12_events", + "all_12_full_payloads", + "stage1", + "stage2", + }: + raise ExactColorIntegrityError("portable CMS validation receipt top-level schema is incomplete") + + stage_totals: list[int] = [] + stage_full_totals: list[int] = [] + for stage_name, expected_events in _VALIDATION_EVENTS.items(): + stage = document.get(stage_name) + if type(stage) is not dict or set(stage) != { + "full_payload_total", + "per_event", + "total", + }: + raise ExactColorIntegrityError(f"portable CMS validation receipt {stage_name} schema is incomplete") + per_event = stage.get("per_event") + if type(per_event) is not dict or set(per_event) != set(expected_events): + raise ExactColorIntegrityError(f"portable CMS validation receipt {stage_name} event inventory is incomplete") + event_total = 0 + event_full_total = 0 + for event, (expected_total, expected_full_total) in expected_events.items(): + row = per_event.get(event) + if type(row) is not dict or set(row) != { + "full_payload", + "mae", + "max_abs", + "mismatched_u16", + "total_u16", + }: + raise ExactColorIntegrityError(f"portable CMS validation receipt event {event} schema is incomplete") + _require_zero_metrics( + {key: row[key] for key in ("mae", "max_abs", "mismatched_u16", "total_u16")}, + expected_total=expected_total, + label=f"event {event}", + ) + full_payload = row.get("full_payload") + _require_zero_metrics( + full_payload, + expected_total=expected_full_total, + label=f"event {event} full payload", + ) + event_total += expected_total + event_full_total += expected_full_total + _require_zero_metrics( + stage.get("total"), + expected_total=event_total, + label=f"{stage_name} total", + ) + _require_zero_metrics( + stage.get("full_payload_total"), + expected_total=event_full_total, + label=f"{stage_name} full-payload total", + ) + stage_totals.append(event_total) + stage_full_totals.append(event_full_total) + + total_u16 = sum(stage_totals) + full_total_u16 = sum(stage_full_totals) + _require_zero_metrics( + document.get("all_12_events"), + expected_total=total_u16, + label="all-12-event total", + ) + full_payload = document.get("all_12_full_payloads") + if type(full_payload) is not dict or set(full_payload) != { + "mae", + "max_abs", + "mismatched_bytes", + "mismatched_u16", + "total_bytes", + "total_u16", + }: + raise ExactColorIntegrityError("portable CMS validation receipt full-payload schema is incomplete") + _require_zero_metrics( + {key: full_payload[key] for key in ("mae", "max_abs", "mismatched_u16", "total_u16")}, + expected_total=full_total_u16, + label="all-12 full-payload total", + ) + if ( + type(full_payload.get("total_bytes")) is not int + or full_payload.get("total_bytes") != full_total_u16 * 2 + or type(full_payload.get("mismatched_bytes")) is not int + or full_payload.get("mismatched_bytes") != 0 + ): + raise ExactColorIntegrityError("portable CMS validation receipt byte totals are inconsistent") + return { + "events": sum(len(events) for events in _VALIDATION_EVENTS.values()), + "mismatched_u16": 0, + "total_u16": total_u16, + "full_payload_mismatched_bytes": 0, + "full_payload_total_bytes": full_total_u16 * 2, + } + + +def _require_zero_metrics( + value: object, + *, + expected_total: int, + label: str, +) -> None: + if type(value) is not dict: + raise ExactColorIntegrityError(f"portable CMS validation receipt {label} is not an exact zero-mismatch result") + metrics = cast(dict[str, object], value) + if ( + set(metrics) != {"mae", "max_abs", "mismatched_u16", "total_u16"} + or type(metrics.get("mismatched_u16")) is not int + or metrics.get("mismatched_u16") != 0 + or type(metrics.get("total_u16")) is not int + or metrics.get("total_u16") != expected_total + or type(metrics.get("max_abs")) is not int + or metrics.get("max_abs") != 0 + or type(metrics.get("mae")) is not float + or metrics.get("mae") != 0.0 + ): + raise ExactColorIntegrityError(f"portable CMS validation receipt {label} is not an exact zero-mismatch result") + + +def _reject_duplicate_keys(pairs: list[tuple[str, object]]) -> dict[str, object]: + parsed: dict[str, object] = {} + for key, value in pairs: + if key in parsed: + raise ValueError(f"duplicate key {key!r}") + parsed[key] = value + return parsed + + +def _reject_json_constant(value: str) -> None: + raise ValueError(f"non-standard JSON constant {value!r}") + + +def _identity(value: os.stat_result) -> tuple[int, int, int, int, int]: + return ( + value.st_dev, + value.st_ino, + value.st_size, + value.st_mtime_ns, + value.st_ctime_ns, + ) + + +def _canonical_json(value: dict) -> bytes: + return json.dumps( + value, + sort_keys=True, + separators=(",", ":"), + ensure_ascii=False, + allow_nan=False, + ).encode("utf-8") + + +__all__ = [ + "ALGORITHM_ID", + "ASSET_SHA256", + "DEFAULT_CHUNK_PIXELS", + "MAX_CHUNK_PIXELS", + "ORACLE_SOURCE_SHA256", + "ORACLE_VALIDATION_RECEIPT_SHA256", + "PACKAGED_ASSET_BYTES", + "PortableCMSOnEvaluator", + "VALIDATION_RECEIPT_BYTES", + "VALIDATION_RECEIPT_FILENAME", + "VALIDATION_RECEIPT_SHA256", +] diff --git a/negpy/services/roll/portable_oracle_evaluator.py b/negpy/services/roll/portable_oracle_evaluator.py new file mode 100644 index 00000000..5fb11410 --- /dev/null +++ b/negpy/services/roll/portable_oracle_evaluator.py @@ -0,0 +1,354 @@ +#!/usr/bin/env python3 +"""DLL-free evaluator for the two captured CML4 optimized transforms. + +This program implements the integer runtime around two vendor-derived 32^3 +CLUTs. It intentionally treats the dumped CLUTs and 65,536-entry LUTs as +oracle data, verifies every asset hash before use, and never loads CML4.dll. + +The evaluator covers the twelve stored events: + +* Stage 1: NKLS5000_N + ramp8to16 -> NKLch (30,34,38,42,47,52) +* Stage 2: NKLch -> NKAdobe, no merged LUT (31,35,39,43,48,53) +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import math +from pathlib import Path + +import numpy as np + + +HERE = Path(__file__).resolve().parent +PROJECT = HERE.parents[3] +DEFAULT_LAB = PROJECT / "reverse_engineering/.work-cml-replay/lab" + +STAGE1_EVENTS = (30, 34, 38, 42, 47, 52) +STAGE2_EVENTS = (31, 35, 39, 43, 48, 53) +GEOMETRY = { + 30: (104, 104), 31: (104, 104), 34: (54, 104), 35: (54, 104), + 38: (104, 72), 39: (104, 72), 42: (54, 72), 43: (54, 72), + 47: (104, 104), 48: (104, 104), 52: (54, 104), 53: (54, 104), +} +ROW_PIXELS = 104 + +ASSET_SHA256 = { + "lch-atan-u16le.bin": + "56b8ac82456941a0a8aad6d7de2c79b21785529037b504582731ce6abcd143b1", + "lch-sincos-i16le.bin": + "dc8e71681bc46e60e33448171865fc96770dc17d3cfd0db835f6173e6fed7a35", + "lch-reciprocal-u16le.bin": + "6959ef7deeb57dc96eb4653fe13b4d37fcb4250c28be15654b8e8dd236849042", + "cml4-stage1-clut0.bin": + "a2abbc1e76dc037e6b364a58b483ddf01b9dde7c0e072f85885cb1f8c9dcbf1c", + "cml4-stage1-input-lut0.bin": + "1a487c024ceaf83018c8ab0e405e9c12b25a0500a2ae6e3465e20b29638729d7", + "cml4-stage1-output-lut0.bin": + "fe87ce159ec126597f9fb605b57cebec9b0264da1ca16d1725efe34db2e4fd2b", + "cml4-stage2-clut0.bin": + "d14b7c76091552bf03899327ca6a7c74c712a0423e429de8f8cc8203d5c98da3", + "cml4-stage2-input-lut0.bin": + "60fa510492f2adad2be9b00107b3d7ed7188a798f29748c3401560b70be3a248", + "cml4-stage2-output-lut0.bin": + "b88574377d1a0cf47fe8806335641db2ca35197817cd055a4cd55141708c8291", +} + + +def verify_assets(directory: Path) -> None: + for name, expected in ASSET_SHA256.items(): + path = directory / name + actual = hashlib.sha256(path.read_bytes()).hexdigest() + if actual != expected: + raise RuntimeError(f"asset hash mismatch: {path}: {actual} != {expected}") + + +def load_event(directory: Path, event: int, kind: str) -> np.ndarray: + width, height = GEOMETRY[event] + return load_event_payload(directory, event, kind)[:, :width, :].reshape(-1, 3) + + +def load_event_payload(directory: Path, event: int, kind: str) -> np.ndarray: + _, height = GEOMETRY[event] + path = directory / f"event{event}-{kind}.bin" + raw = np.fromfile(path, dtype=" np.ndarray: + return np.column_stack([lut[c, pixels[:, c]] for c in range(3)]).astype(np.uint16) + + +def optimized_trilinear(clut: np.ndarray, pixels: np.ndarray) -> np.ndarray: + """Exact scalar expression of optimized kernel CML4.dll+0x11e30. + + Nodes are four padded u16s. The MMX kernel halves every node, performs + three q15 linear interpolations with arithmetic shifts, then doubles the + result. Coordinate multiplication by 0x1f001f maps u16 endpoints onto a + 32-point grid and yields a 15-bit fraction. + """ + source = np.asarray(pixels, dtype=np.uint16).reshape(-1, 3) + packed = source.astype(np.uint64) * np.uint64(0x1F001F) + np.uint64(0x10000) + index = (packed >> np.uint64(32)).astype(np.int64) + fraction = ((packed & np.uint64(0xFFFFFFFF)) >> np.uint64(17)).astype(np.int64) + ix, iy, iz = index.T + fx, fy, fz = fraction.T + + def node(dx: int, dy: int, dz: int) -> np.ndarray: + # At the top endpoint the fraction is zero. The DLL may read a padded + # neighbor, but that value is multiplied by zero; clamping is exactly + # equivalent and avoids an out-of-array access in the portable model. + return ( + clut[ + np.minimum(ix + dx, 31), + np.minimum(iy + dy, 31), + np.minimum(iz + dz, 31), + ].astype(np.int64) + >> 1 + ) + + def lerp(a: np.ndarray, b: np.ndarray, q15: np.ndarray) -> np.ndarray: + return a + (((b - a) * q15[:, None]) >> 15) + + z0 = lerp( + lerp(node(0, 0, 0), node(1, 0, 0), fx), + lerp(node(0, 1, 0), node(1, 1, 0), fx), + fy, + ) + z1 = lerp( + lerp(node(0, 0, 1), node(1, 0, 1), fx), + lerp(node(0, 1, 1), node(1, 1, 1), fx), + fy, + ) + result = (lerp(z0, z1, fz) << 1) & 0xFFFF + return result[:, :3].astype(np.uint16) + + +def trunc_q15(product: np.ndarray) -> np.ndarray: + """Signed division by 32768, truncating toward zero (0x100041d0).""" + return np.where(product < 0, -((-product) >> 15), product >> 15) + + +def lch_to_lab_codes( + pixels: np.ndarray, + sincos: np.ndarray, + reciprocal: np.ndarray, +) -> np.ndarray: + """Exact pre-CLUT Lch special case at CML4.dll+0x100041d0.""" + source = np.asarray(pixels, dtype=np.uint16).reshape(-1, 3) + chroma = source[:, 1].astype(np.int64) + hue = source[:, 2].astype(np.int64) + used = np.minimum(chroma, reciprocal[hue].astype(np.int64)) + # Captured sincos records are [sin, cos]. + a = trunc_q15(sincos[hue, 1].astype(np.int64) * used) + b = trunc_q15(sincos[hue, 0].astype(np.int64) * used) + return np.column_stack( + [ + source[:, 0], + (32768 + a).astype(np.uint16), + (32768 + b).astype(np.uint16), + ] + ) + + +def hue_from_signed_ab(a: np.ndarray, b: np.ndarray, atan: np.ndarray) -> np.ndarray: + """Exact ratio table, axes, and quadrant reconstruction in 0x10004290.""" + a = np.asarray(a, dtype=np.int64) + b = np.asarray(b, dtype=np.int64) + aa, bb = np.abs(a), np.abs(b) + high, low = np.maximum(aa, bb), np.minimum(aa, bb) + ratio = np.zeros_like(high) + nz = high != 0 + ratio[nz] = (low[nz] << 13) // high[nz] + angle = atan[ratio].astype(np.int64) + hue = np.zeros_like(a) + a_major = aa >= bb + q1 = (a > 0) & (b > 0) + q2 = (a < 0) & (b > 0) + q3 = (a < 0) & (b < 0) + q4 = (a > 0) & (b < 0) + hue[q1] = np.where(a_major[q1], angle[q1], 16384 - angle[q1]) + hue[q2] = np.where(a_major[q2], 32768 - angle[q2], 16384 + angle[q2]) + hue[q3] = np.where(a_major[q3], 32768 + angle[q3], 49152 - angle[q3]) + hue[q4] = np.where(a_major[q4], 65536 - angle[q4], 49152 + angle[q4]) + hue[(a == 0) & (b > 0)] = 16384 + hue[(a < 0) & (b == 0)] = 32768 + hue[(a == 0) & (b < 0)] = 49152 + return (hue & 0xFFFF).astype(np.uint16) + + +def rounded_integer_hypot(a: np.ndarray, b: np.ndarray) -> np.ndarray: + """Round sqrt(a*a+b*b) to nearest, without host floating-point drift.""" + squared = a.astype(np.int64) ** 2 + b.astype(np.int64) ** 2 + floor_root = np.fromiter( + (math.isqrt(int(value)) for value in squared), + dtype=np.int64, + count=squared.size, + ) + # sqrt(integer) can never be exactly k+0.5, so no tie rule is needed. + return floor_root + ((squared - floor_root * floor_root) > floor_root) + + +def lab_to_lch_codes(pixels: np.ndarray, atan: np.ndarray) -> np.ndarray: + """Exact post-CLUT Lab special case at CML4.dll+0x10004290.""" + source = np.asarray(pixels, dtype=np.uint16).reshape(-1, 3) + + def rescale(code: np.ndarray) -> np.ndarray: + value = (code.astype(np.uint64) - np.uint64(8192)) * np.uint64(65535) + product = value * np.uint64(87384) # ceil(2**32 / 49151) + return ( + (product >> np.uint64(32)) + + ((product & np.uint64(0xFFFFFFFF)) >> np.uint64(31)) + ).astype(np.int64) + + a = rescale(source[:, 1]) - 32768 + b = rescale(source[:, 2]) - 32768 + chroma = rounded_integer_hypot(a, b) + hue = hue_from_signed_ab(a, b, atan) + return np.column_stack([source[:, 0], chroma, hue]).astype(np.uint16) + + +class CapturedTransforms: + def __init__(self, assets: Path): + verify_assets(assets) + self.atan = np.fromfile(assets / "lch-atan-u16le.bin", dtype=" np.ndarray: + working = lookup_three(self.s1_input, source) + working = optimized_trilinear(self.s1_clut, working) + working = lab_to_lch_codes(working, self.atan) + return lookup_three(self.s1_output, working) + + def stage2(self, source: np.ndarray) -> np.ndarray: + # CML4's Lch special-case deliberately bypasses the generated input + # LUT for hue. Its dumped plane 2 maps the top half down by one; using + # it causes 4,729 mismatches across these events. + working = source.copy() + working[:, 0] = self.s2_input[0, working[:, 0]] + working[:, 1] = self.s2_input[1, working[:, 1]] + working = lch_to_lab_codes(working, self.sincos, self.reciprocal) + working = optimized_trilinear(self.s2_clut, working) + return lookup_three(self.s2_output, working) + + +def metric(predicted: np.ndarray, expected: np.ndarray) -> dict[str, int | float]: + difference = predicted.astype(np.int64) - expected.astype(np.int64) + absolute = np.abs(difference) + return { + "mismatched_u16": int(np.count_nonzero(difference)), + "total_u16": int(difference.size), + "max_abs": int(absolute.max(initial=0)), + "mae": float(absolute.mean()), + } + + +def evaluate(model: CapturedTransforms, lab: Path) -> dict: + result: dict[str, dict] = {} + every_predicted_active = [] + every_expected_active = [] + every_predicted_payload = [] + every_expected_payload = [] + for label, events, function in ( + ("stage1", STAGE1_EVENTS, model.stage1), + ("stage2", STAGE2_EVENTS, model.stage2), + ): + predicted_all, expected_all = [], [] + predicted_payload_all, expected_payload_all = [], [] + per_event = {} + for event in events: + width, height = GEOMETRY[event] + source = load_event(lab, event, "source") + expected = load_event(lab, event, "expected") + predicted = function(source) + expected_payload = load_event_payload(lab, event, "expected") + predicted_payload = np.zeros_like(expected_payload) + predicted_payload[:, :width, :] = predicted.reshape(height, width, 3) + per_event[str(event)] = { + **metric(predicted, expected), + "full_payload": metric(predicted_payload, expected_payload), + } + predicted_all.append(predicted) + expected_all.append(expected) + predicted_payload_all.append(predicted_payload) + expected_payload_all.append(expected_payload) + result[label] = { + "per_event": per_event, + "total": metric(np.concatenate(predicted_all), np.concatenate(expected_all)), + "full_payload_total": metric( + np.concatenate(predicted_payload_all), + np.concatenate(expected_payload_all), + ), + } + every_predicted_active.extend(predicted_all) + every_expected_active.extend(expected_all) + every_predicted_payload.extend(predicted_payload_all) + every_expected_payload.extend(expected_payload_all) + result["all_12_events"] = metric( + np.concatenate(every_predicted_active), np.concatenate(every_expected_active) + ) + result["all_12_full_payloads"] = { + **metric( + np.concatenate(every_predicted_payload), + np.concatenate(every_expected_payload), + ), + "total_bytes": int(sum(value.nbytes for value in every_expected_payload)), + "mismatched_bytes": int( + np.count_nonzero( + np.concatenate(every_predicted_payload).view(np.uint8) + != np.concatenate(every_expected_payload).view(np.uint8) + ) + ), + } + return result + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--assets", type=Path, default=HERE) + parser.add_argument("--lab", type=Path, default=DEFAULT_LAB) + parser.add_argument("--json-out", type=Path) + args = parser.parse_args() + result = evaluate(CapturedTransforms(args.assets), args.lab) + print(json.dumps(result, indent=2)) + if args.json_out is not None: + args.json_out.write_text(json.dumps(result, indent=2) + "\n", encoding="utf-8") + return ( + 0 + if result["all_12_events"]["mismatched_u16"] == 0 + and result["all_12_full_payloads"]["mismatched_u16"] == 0 + and result["all_12_full_payloads"]["mismatched_bytes"] == 0 + else 1 + ) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/negpy/services/roll/positive.py b/negpy/services/roll/positive.py new file mode 100644 index 00000000..e076b95c --- /dev/null +++ b/negpy/services/roll/positive.py @@ -0,0 +1,108 @@ +"""Tier-3 positive rendering: NegPy's own negative-to-positive pipeline, +called on an in-memory Tier-2 buffer instead of introducing a second one. + +`DarkroomEngine.process()` (`negpy.services.rendering.engine`) is how any +imported negative reaches a rendered positive in NegPy today -- it is what +runs when a scan is first opened, what the live canvas re-runs on every +edit, and what the export pipeline (`ImageProcessor.process_export`) calls +after decoding a source file. `ImageProcessor.run_pipeline` is that same +entry point one layer up, taking an in-memory buffer directly rather than a +file path -- it is what `process_export` itself calls after its own decode +step. A roll-scanned frame is already an in-memory buffer, so this module +calls `run_pipeline` straight, skipping the file-decode and the ICC export +encoding `process_export` also does, neither of which apply to a Tier 2 +buffer that never touches disk to get here (see `service.write_frame`'s +"never derive an upper tier from a lower one" -- the positive comes from the +same in-memory array Tier 2 optionally writes, not by reading a Tier 2 file +back). +""" + +from __future__ import annotations + +import uuid +from dataclasses import dataclass + +import numpy as np + +import negpy +from negpy.domain.models import WorkspaceConfig +from negpy.kernel.image.logic import float_to_uint16, uint16_to_float32 +from negpy.kernel.system.config import APP_CONFIG +from negpy.services.rendering.image_processor import ImageProcessor + + +@dataclass(frozen=True) +class PositiveResult: + """One frame's Tier-3 RGB, plus what the receipt needs to audit or + reproduce it.""" + + rgb: np.ndarray + render_intent: str + process_mode: str + auto_exposure: bool + negpy_version: str + + +def available() -> bool: + """True if the inversion path can be used. + + Unlike coolscanpy or a Tier-2 repair engine, NegPy's rendering pipeline + is not an optional external dependency -- it ships with this + application, so this always returns True today. It exists as a seam + with the same shape as `coolscanpy_roll.available()` and + `repair.available()`, so `service.write_frame` has one place to check + before rendering (and a test simulating "inversion unavailable" has one + place to patch) instead of a bare `try/except` being the only signal. + """ + return True + + +def render_positive(rgb_u16: np.ndarray, *, processor: ImageProcessor) -> PositiveResult: + """Invert a scanner-linear RGB buffer (Tier 2, or Tier 1 if Tier 2 could + not be produced -- callers decide what to pass) to a positive. + + Uses a stock, unedited `WorkspaceConfig`: process mode C41 (this + adapter only ever opens a roll as color negative -- see + `coolscanpy_roll.open_roll`), render intent PRINT (NegPy's default + photographic-paper conversion, as opposed to the flat digital- + intermediate master), and auto exposure on. That is exactly what a user + would see on first opening this frame's negative in NegPy without + touching a single slider -- the ordinary default conversion, not a + custom or maximal-latitude one, since this tier is a regenerable view + rather than an edit a person has invested time in. + + `processor` is injected rather than constructed here so a caller can + hold one `ImageProcessor` across a whole batch scan (its constructor + probes for GPU acceleration, which is worth doing once, not per frame) + even though rendering itself always runs on the CPU engine below. + """ + settings = WorkspaceConfig() + img = uint16_to_float32(np.ascontiguousarray(rgb_u16, dtype=np.uint16)) + + rendered, _metrics = processor.run_pipeline( + img, + settings, + # A fresh id per call, deliberately not a stable hash of the frame: the + # engine's stage cache reuses a prior render whenever both source_hash + # and the settings hash match, and every Tier-3 render here shares the + # same stock settings -- a stable/reused hash across two different + # frames (or two different slots re-scanned in one session) would + # silently hand back a *previous frame's* rendered pixels instead of + # this one's. A fresh id guarantees a real render every time; it has + # no bearing on reproducibility, since the pipeline math itself is a + # pure function of (image, settings) and this cache is a same-process + # performance optimization, not a determinism mechanism. + source_hash=uuid.uuid4().hex, + render_size_ref=float(APP_CONFIG.preview_render_size), + prefer_gpu=False, # CPU engine: deterministic, and this is a batch worker, not the live canvas + wants_uv_grid=False, + skip_flatfield=True, # no flat-field calibration exists for a roll-scanner buffer + ) + + return PositiveResult( + rgb=float_to_uint16(rendered), + render_intent=str(settings.exposure.render_intent), + process_mode=str(settings.process.process_mode), + auto_exposure=settings.exposure.auto_exposure, + negpy_version=negpy.__version__, + ) diff --git a/negpy/services/roll/service.py b/negpy/services/roll/service.py new file mode 100644 index 00000000..90424ffa --- /dev/null +++ b/negpy/services/roll/service.py @@ -0,0 +1,2722 @@ +"""Roll-scanning service backed by the optional `coolscanpy` package. + +Sibling of `negpy.services.capture.service.CaptureService`: one class that +orchestrates the hardware workflow and writes results to disk in NegPy's +conventional layout, on top of an infrastructure-layer adapter for an +optional dependency (`negpy.infrastructure.roll.coolscanpy_roll`, mirroring +`negpy.infrastructure.capture.gphoto`). It also mirrors the older, simpler +`negpy.services.scanning.service.ScannerService`: where that wraps a single +ad-hoc `Device.scan()`, this wraps coolscanpy's whole-roll workflow +(preview -> approve -> batch fine-scan). + +Entirely inert if `coolscanpy` is not installed: `available()` is a cheap +presence check re-exported from that module, and every other method only +reaches coolscanpy (transitively, through `coolscanpy_roll`) once actually +called. + +`write_frame` writes a captured frame across up to three output tiers -- +unrepaired, repaired, positive -- each independently selectable and each +derived from the one before it. See its docstring for the tier definitions, +the write order, and how a tier that cannot be produced degrades instead of +losing the tiers that still can be. Tier 2 (repaired) is backed by +`negpy.infrastructure.roll.repair`; the bundled `fauxice_bridge` registers +the portable engine and reports it unavailable if that import fails. Tier 3 +(positive) defaults to the receipt-bound portable Nikon builder/CMS path, +with `negpy.services.roll.positive` retained as an explicitly selected +approximate renderer. +""" + +from __future__ import annotations + +import dataclasses +import functools +import hashlib +import io +import json +import os +import shutil +import stat +import tempfile +import threading +from contextvars import ContextVar +from datetime import date as _date +from typing import TYPE_CHECKING, Callable, Iterable, Iterator, cast + +import numpy as np +import tifffile +from PIL import Image + +try: + import fcntl +except ImportError: # pragma: no cover - exercised by Windows packaging smoke + fcntl = None # type: ignore[assignment] + +from negpy.infrastructure.roll import coolscanpy_roll +from negpy.infrastructure.roll import fauxice_bridge as _fauxice_bridge # noqa: F401 — registers engine on import +from negpy.infrastructure.roll import repair as roll_repair +from negpy.infrastructure.roll.repair import RepairMode +from negpy.kernel.system.logging import get_logger +from negpy.services.repair.fauxice_hybrid_runner import HybridRuntimeConfig +from negpy.services.rendering.image_processor import ImageProcessor +from negpy.services.roll import exact_color as roll_exact_color +from negpy.services.roll import native_builder as roll_native_builder +from negpy.services.roll import nikon_icc as roll_nikon_icc +from negpy.services.roll import positive as roll_positive +from negpy.services.scanning.templating import render_scan_filename + +if TYPE_CHECKING: + import coolscanpy + from coolscanpy.protocol.ls5000_single_pass.capture_process import ( + ManualFrameApproval as CoolscanManualFrameApproval, + ) + from coolscanpy.types import ProgressCallback as CoolscanProgressCallback + +logger = get_logger(__name__) + +_RETAINED_EVIDENCE_DIRECTORIES = frozenset( + { + ".negpy-dice-hybrid", + ".negpy-dice-acquisition", + ".negpy-native-builder", + ".negpy-stage3-replay", + } +) +_MAX_RECEIPT_BYTES = 16 * 1024 * 1024 +_MAX_SHARED_EVIDENCE_BYTES = 64 * 1024 * 1024 + + +def _strict_json_loads(payload: bytes | str) -> object: + def unique_object(pairs: list[tuple[str, object]]) -> dict[str, object]: + result: dict[str, object] = {} + for key, value in pairs: + if key in result: + raise ValueError(f"duplicate JSON object key {key!r}") + result[key] = value + return result + + def reject_constant(value: str) -> object: + raise ValueError(f"non-finite JSON constant {value!r}") + + return json.loads( + payload, + object_pairs_hook=unique_object, + parse_constant=reject_constant, + ) + + +def _stable_regular_bytes( + path: str, + *, + maximum_bytes: int, + label: str, +) -> bytes: + before = os.lstat(path) + if stat.S_ISLNK(before.st_mode) or not stat.S_ISREG(before.st_mode): + raise OSError(f"{label} must be a regular non-symlink file") + if before.st_size > maximum_bytes: + raise OSError(f"{label} exceeds its safe size limit") + flags = os.O_RDONLY + if hasattr(os, "O_CLOEXEC"): + flags |= os.O_CLOEXEC + if hasattr(os, "O_NOFOLLOW"): + flags |= os.O_NOFOLLOW + if hasattr(os, "O_NONBLOCK"): + flags |= os.O_NONBLOCK + descriptor = os.open(path, flags) + try: + opened = os.fstat(descriptor) + + def identity(metadata: os.stat_result) -> tuple[int, int, int, int, int]: + return ( + metadata.st_dev, + metadata.st_ino, + metadata.st_size, + metadata.st_mtime_ns, + metadata.st_ctime_ns, + ) + + if not stat.S_ISREG(opened.st_mode) or identity(opened) != identity(before): + raise OSError(f"{label} changed while it was opened") + chunks: list[bytes] = [] + total = 0 + while True: + chunk = os.read(descriptor, min(1024 * 1024, maximum_bytes + 1 - total)) + if not chunk: + break + chunks.append(chunk) + total += len(chunk) + if total > maximum_bytes: + raise OSError(f"{label} exceeds its safe size limit") + after_read = os.fstat(descriptor) + finally: + os.close(descriptor) + after_path = os.lstat(path) + if identity(after_read) != identity(opened) or identity(after_path) != identity(opened): + raise OSError(f"{label} changed while it was read") + return b"".join(chunks) + + +def _load_bounded_receipt(path: str) -> dict: + payload = _stable_regular_bytes( + path, + maximum_bytes=_MAX_RECEIPT_BYTES, + label="frame receipt", + ) + try: + document = _strict_json_loads(payload) + except (UnicodeDecodeError, ValueError, TypeError) as error: + raise OSError(f"frame receipt JSON is invalid: {error}") from error + if not isinstance(document, dict): + raise OSError("frame receipt must contain a JSON object") + return document + + +def _fsync_directory(path: str) -> None: + flags = os.O_RDONLY + if hasattr(os, "O_DIRECTORY"): + flags |= os.O_DIRECTORY + descriptor = os.open(path or ".", flags) + try: + os.fsync(descriptor) + finally: + os.close(descriptor) + + +# Re-exported so callers only need this module: `from negpy.services.roll +# .service import available`, matching how the plain Scan sidebar checks +# `_sane_available()` before showing its device combo. +available = coolscanpy_roll.available + + +class RollScanningError(RuntimeError): + """Raised for a roll-scanning failure that originates in this service's + own orchestration (lifecycle misuse), as opposed to one translated from + coolscanpy itself. Mirrors `negpy.services.capture.service.CaptureError`.""" + + +class OutputRollbackError(RollScanningError): + """Publication failed and at least one prior file needs recovery.""" + + def __init__(self, recovery_path: str, errors: list[str]) -> None: + self.recovery_path = recovery_path + self.errors = tuple(errors) + super().__init__( + f"frame publication failed and rollback was incomplete; prior files are retained at {recovery_path}: {'; '.join(errors)}" + ) + + +@dataclasses.dataclass(frozen=True) +class RollFrameOutput: + """Where one scanned frame's files landed on disk, one pair of fields per + tier. A tier that was not selected, or could not be produced, leaves its + field(s) `None` -- the receipt at `receipt_path` records which of those + two it was and, for a tier that did write, its provenance.""" + + slot: int + rgb_path: str | None # Tier 1 (unrepaired): RGB plane + ir_path: str | None # Tier 1 (unrepaired): infrared plane + repaired_rgb_path: str | None # Tier 2 (repaired): RGB plane + repaired_ir_path: str | None # Tier 2 (repaired): infrared plane (Tier 1's own, retained unchanged) + positive_path: str | None # Tier 3 (positive) + receipt_path: str + synthesis_mask_path: str | None + native_synthesis_mask_path: str | None + hybrid_receipt_path: str | None + + +_ACTIVE_OUTPUT_TRANSACTION: ContextVar[_OutputTransaction | None] = ContextVar( + "negpy_roll_output_transaction", + default=None, +) + + +class _OutputTransaction: + """Stage one frame's files, publish its receipt last, and roll back. + + The rollback guarantee covers process-level write/rename failures. File + and directory fsync ordering narrows power-loss exposure, but a set of + multiple POSIX pathnames is not one power-fail-atomic filesystem unit. + """ + + def __init__(self, output_folder: str) -> None: + self.output_folder = os.path.realpath(output_folder) + self._root = tempfile.mkdtemp( + prefix=".negpy-frame-stage-", + dir=self.output_folder, + ) + self._staged: dict[str, str] = {} + self._removals: set[str] = set() + self._finished = False + self._preserve_staging = False + self._lock_descriptor: int | None = None + + def _validate_output_path(self, final_path: str) -> str: + final = os.path.abspath(final_path) + try: + inside = os.path.commonpath((self.output_folder, final)) == self.output_folder + except ValueError: + inside = False + if not inside: + raise OSError("frame output escapes its selected output folder") + + parent = os.path.dirname(final) + if os.path.commonpath((self.output_folder, os.path.realpath(parent))) != self.output_folder: + raise OSError("frame output parent escapes through a symbolic link") + relative_parent = os.path.relpath(parent, self.output_folder) + current = self.output_folder + if relative_parent != ".": + for component in relative_parent.split(os.sep): + current = os.path.join(current, component) + try: + metadata = os.lstat(current) + except FileNotFoundError: + break + if stat.S_ISLNK(metadata.st_mode) or not stat.S_ISDIR(metadata.st_mode): + raise OSError("frame output parent is not a real directory") + return final + + def stage_path(self, final_path: str) -> str: + final = self._validate_output_path(final_path) + staged = self._staged.get(final) + if staged is None: + token = hashlib.sha256(final.encode("utf-8")).hexdigest() + staged = os.path.join(self._root, f"{token}-{os.path.basename(final)}") + self._staged[final] = staged + return staged + + def staged_path(self, final_path: str) -> str: + final = os.path.abspath(final_path) + try: + return self._staged[final] + except KeyError as error: + raise OSError(f"output was not staged: {final_path}") from error + + def discard(self, final_path: str) -> None: + staged = self._staged.pop(os.path.abspath(final_path), None) + if staged is not None: + try: + os.unlink(staged) + except FileNotFoundError: + pass + + def checkpoint(self) -> frozenset[str]: + """Remember the outputs already staged by lower tiers.""" + + return frozenset(self._staged) + + def discard_after(self, checkpoint: frozenset[str]) -> None: + """Discard staged outputs added after ``checkpoint``.""" + + for final_path in tuple(self._staged): + if final_path not in checkpoint: + self.discard(final_path) + + def schedule_remove(self, final_path: str) -> None: + final = self._validate_output_path(final_path) + self._removals.add(final) + + def _existing_receipt_paths(self, receipt_path: str) -> set[str]: + try: + os.lstat(receipt_path) + except FileNotFoundError: + return set() + document = _load_bounded_receipt(receipt_path) + return _receipt_output_paths(document, output_folder=self.output_folder) + + def _acquire_frame_lock(self, receipt_path: str) -> None: + if self._lock_descriptor is not None: + return + if fcntl is None: + raise OSError("safe cross-process frame locking is unavailable on this platform") + lock_directory = os.path.join(self.output_folder, ".negpy-locks") + try: + metadata = os.lstat(lock_directory) + except FileNotFoundError: + try: + os.mkdir(lock_directory, 0o700) + except FileExistsError: + pass + metadata = os.lstat(lock_directory) + if stat.S_ISLNK(metadata.st_mode) or not stat.S_ISDIR(metadata.st_mode): + raise OSError("frame lock directory is not a real directory") + lock_name = hashlib.sha256(os.path.abspath(receipt_path).encode("utf-8")).hexdigest() + ".lock" + flags = os.O_CREAT | os.O_RDWR + if hasattr(os, "O_CLOEXEC"): + flags |= os.O_CLOEXEC + if hasattr(os, "O_NOFOLLOW"): + flags |= os.O_NOFOLLOW + descriptor = os.open( + os.path.join(lock_directory, lock_name), + flags, + 0o600, + ) + if not stat.S_ISREG(os.fstat(descriptor).st_mode): + os.close(descriptor) + raise OSError("frame lock is not a regular file") + try: + # A crashed or wedged peer must not freeze the scanner worker/UI. + # Publication fails closed immediately; the caller may retry once + # the other process has completed or been recovered. + fcntl.flock(descriptor, fcntl.LOCK_EX | fcntl.LOCK_NB) + except BlockingIOError as error: + os.close(descriptor) + raise OSError("frame output is busy in another NegPy process") from error + except BaseException: + os.close(descriptor) + raise + self._lock_descriptor = descriptor + + def _release_frame_lock(self) -> None: + if self._lock_descriptor is None: + return + try: + assert fcntl is not None + fcntl.flock(self._lock_descriptor, fcntl.LOCK_UN) + finally: + os.close(self._lock_descriptor) + self._lock_descriptor = None + + def schedule_stale_outputs(self, receipt_path: str) -> None: + self._acquire_frame_lock(receipt_path) + staged_receipt = self._staged.get(os.path.abspath(receipt_path)) + if staged_receipt is None: + raise OSError("frame receipt was not staged") + new_document = _load_bounded_receipt(staged_receipt) + new_paths = _receipt_output_paths( + new_document, + output_folder=self.output_folder, + ) + excluded_receipt = os.path.abspath(receipt_path) + for final_path in tuple(self._staged): + if final_path == excluded_receipt: + continue + if not _other_receipt_references( + final_path, + output_folder=self.output_folder, + excluded_receipt=excluded_receipt, + ): + continue + if _is_retained_evidence_path(final_path, self.output_folder): + try: + staged_bytes = _stable_regular_bytes( + self._staged[final_path], + maximum_bytes=_MAX_SHARED_EVIDENCE_BYTES, + label="staged retained evidence", + ) + existing_bytes = _stable_regular_bytes( + final_path, + maximum_bytes=_MAX_SHARED_EVIDENCE_BYTES, + label="shared retained evidence", + ) + except OSError: + pass + else: + if staged_bytes == existing_bytes: + self.discard(final_path) + continue + raise OSError(f"frame output is already owned by another receipt: {final_path}") + staged_outputs = set(self._staged) - {os.path.abspath(receipt_path)} + unbound = sorted(staged_outputs - new_paths) + if unbound: + raise OSError("frame transaction contains outputs absent from its receipt: " + ", ".join(unbound)) + base_path = receipt_path[: -len("_receipt.json")] + for derived in ( + base_path + ".tif", + base_path + "_IR.tif", + base_path + "_repaired.tif", + base_path + "_repaired_IR.tif", + base_path + "_repaired_SYNTH.png", + base_path + "_positive.tif", + ): + if os.path.abspath(derived) not in new_paths and not _other_receipt_references( + derived, + output_folder=self.output_folder, + excluded_receipt=excluded_receipt, + ): + self.schedule_remove(derived) + old_paths = self._existing_receipt_paths(receipt_path) + for old_path in old_paths - new_paths: + if not _is_retained_evidence_path(old_path, self.output_folder): + continue + if not _other_receipt_references( + old_path, + output_folder=self.output_folder, + excluded_receipt=excluded_receipt, + ): + self.schedule_remove(old_path) + + def commit(self, *, receipt_path: str) -> None: + receipt = os.path.abspath(receipt_path) + if receipt not in self._staged: + raise OSError("frame receipt was not staged") + writes = sorted(path for path in self._staged if path != receipt) + removals = sorted(self._removals - set(self._staged)) + backup_root = os.path.join(self._root, "backups") + os.makedirs(backup_root, exist_ok=True) + backups: dict[str, str] = {} + installed: list[str] = [] + + def backup(path: str) -> None: + self._validate_output_path(path) + if not os.path.lexists(path): + return + if os.path.islink(path) or not os.path.isfile(path): + raise OSError(f"refusing to replace non-regular output {path}") + backup_path = os.path.join( + backup_root, + hashlib.sha256(path.encode("utf-8")).hexdigest(), + ) + os.replace(path, backup_path) + backups[path] = backup_path + + try: + for final in removals: + backup(final) + for final in writes: + backup(final) + self._validate_output_path(final) + os.makedirs(os.path.dirname(final) or ".", exist_ok=True) + self._validate_output_path(final) + os.replace(self._staged[final], final) + installed.append(final) + for directory in sorted({os.path.dirname(path) or "." for path in (*writes, *removals)}): + _fsync_directory(directory) + # Keep the prior receipt in place until every referenced artifact + # has landed. The receipt rename is the transaction's commit mark. + backup(receipt) + self._validate_output_path(receipt) + os.makedirs(os.path.dirname(receipt) or ".", exist_ok=True) + self._validate_output_path(receipt) + os.replace(self._staged[receipt], receipt) + installed.append(receipt) + _fsync_directory(os.path.dirname(receipt) or ".") + except BaseException as publication_error: + rollback_errors: list[str] = [] + for final in reversed(installed): + try: + if os.path.isfile(final) and not os.path.islink(final): + os.unlink(final) + elif os.path.lexists(final): + rollback_errors.append(f"refused to remove replacement {final}") + except OSError as error: + rollback_errors.append(f"could not remove replacement {final}: {error}") + for final, backup_path in reversed(tuple(backups.items())): + try: + os.makedirs(os.path.dirname(final) or ".", exist_ok=True) + os.replace(backup_path, final) + except OSError as error: + rollback_errors.append(f"could not restore {final} from {backup_path}: {error}") + if rollback_errors: + recovery_path = self._retain_recovery( + publication_error=publication_error, + rollback_errors=rollback_errors, + backups=backups, + ) + raise OutputRollbackError(recovery_path, rollback_errors) from publication_error + raise + else: + self._finished = True + self._cleanup_empty_evidence_directories(removals) + finally: + self._cleanup_staging() + + def _retain_recovery( + self, + *, + publication_error: BaseException, + rollback_errors: list[str], + backups: dict[str, str], + ) -> str: + self._preserve_staging = True + prior_root = self._root + recovery_name = os.path.basename(self._root).replace( + ".negpy-frame-stage-", + ".negpy-recovery-", + 1, + ) + recovery_path = os.path.join(os.path.dirname(self._root), recovery_name) + try: + os.rename(self._root, recovery_path) + except OSError as error: + recovery_path = self._root + rollback_errors.append(f"could not rename recovery directory: {error}") + else: + self._root = recovery_path + + def relocated(path: str) -> str: + return os.path.join(recovery_path, os.path.relpath(path, prior_root)) + + manifest = { + "publication_error": repr(publication_error), + "recovery_path": recovery_path, + "rollback_errors": rollback_errors, + "unrestored_backups": { + final: os.path.relpath(relocated(backup), recovery_path) + for final, backup in backups.items() + if os.path.exists(relocated(backup)) + }, + } + try: + with open( + os.path.join(recovery_path, "RECOVERY.json"), + "w", + encoding="utf-8", + ) as stream: + json.dump(manifest, stream, indent=2, sort_keys=True) + stream.write("\n") + stream.flush() + os.fsync(stream.fileno()) + except OSError as error: + rollback_errors.append(f"could not write recovery manifest: {error}") + return recovery_path + + def _cleanup_staging(self) -> None: + if not self._preserve_staging and os.path.isdir(self._root): + try: + shutil.rmtree(self._root) + except OSError as error: + logger.warning("Could not remove frame staging directory %s: %s", self._root, error) + self._release_frame_lock() + + def _cleanup_empty_evidence_directories(self, removals: list[str]) -> None: + for removed in removals: + evidence_root = _retained_evidence_root( + removed, + self.output_folder, + ) + if evidence_root is None: + continue + parent = os.path.dirname(removed) + while True: + try: + inside = os.path.commonpath((evidence_root, parent)) == evidence_root + except ValueError: + inside = False + if not inside: + break + try: + os.rmdir(parent) + except OSError: + break + if parent == evidence_root: + break + parent = os.path.dirname(parent) + + def abort(self) -> None: + if not self._finished: + self._cleanup_staging() + + +def _transactional_frame_output(method): + @functools.wraps(method) + def wrapped(self, frame, output_folder, *args, **kwargs): + os.makedirs(output_folder, exist_ok=True) + transaction = _OutputTransaction(output_folder) + token = _ACTIVE_OUTPUT_TRANSACTION.set(transaction) + try: + result = method( + self, + frame, + transaction.output_folder, + *args, + **kwargs, + ) + transaction.schedule_stale_outputs(result.receipt_path) + transaction.commit(receipt_path=result.receipt_path) + return result + finally: + _ACTIVE_OUTPUT_TRANSACTION.reset(token) + transaction.abort() + + return wrapped + + +def _receipt_output_paths( + document: object, + *, + output_folder: str, +) -> set[str]: + if not isinstance(document, dict): + return set() + outputs = document.get("outputs") + found: set[str] = set() + + def visit(value: object, key: str | None = None) -> None: + if isinstance(value, dict): + for child_key, child in value.items(): + visit(child, child_key) + elif isinstance(value, list): + for child in value: + visit(child, key) + elif isinstance(value, str) and key is not None and (key == "path" or key.endswith("_path")): + candidate = os.path.abspath(value) + try: + inside = os.path.commonpath((output_folder, candidate)) == output_folder + except ValueError: + inside = False + if inside: + found.add(candidate) + + visit(outputs) + return found + + +def _retained_evidence_root(path: str, output_folder: str) -> str | None: + candidate = os.path.abspath(path) + root = os.path.abspath(output_folder) + try: + if os.path.commonpath((root, candidate)) != root: + return None + except ValueError: + return None + relative_parts = os.path.relpath(candidate, root).split(os.sep) + for index, component in enumerate(relative_parts): + if component in _RETAINED_EVIDENCE_DIRECTORIES: + return os.path.join(root, *relative_parts[: index + 1]) + return None + + +def _is_retained_evidence_path(path: str, output_folder: str) -> bool: + evidence_root = _retained_evidence_root(path, output_folder) + return evidence_root is not None and os.path.abspath(path) != evidence_root + + +def _other_receipt_references( + path: str, + *, + output_folder: str, + excluded_receipt: str, +) -> bool: + try: + + def fail_walk(error: OSError) -> None: + raise error + + walker = os.walk( + output_folder, + topdown=True, + onerror=fail_walk, + followlinks=False, + ) + for directory, subdirectories, names in walker: + retained_subdirectories: list[str] = [] + for name in subdirectories: + child = os.path.join(directory, name) + if ( + name in _RETAINED_EVIDENCE_DIRECTORIES + or name.startswith(".negpy-frame-stage-") + or name.startswith(".negpy-recovery-") + or os.path.islink(child) + ): + continue + retained_subdirectories.append(name) + subdirectories[:] = retained_subdirectories + for name in names: + if not name.endswith("_receipt.json"): + continue + receipt = os.path.abspath(os.path.join(directory, name)) + if receipt == excluded_receipt: + continue + try: + document = _load_bounded_receipt(receipt) + except OSError: + # An unsafe/unreadable sibling may still own this path. + # Preserve rather than risk deleting or overwriting it. + return True + if os.path.abspath(path) in _receipt_output_paths( + document, + output_folder=output_folder, + ): + return True + except OSError: + return True + return False + + +def _prepare_evidence_directory(path: str) -> None: + if _ACTIVE_OUTPUT_TRANSACTION.get() is None: + os.makedirs(path, exist_ok=True) + + +class RollScanningService: + """Orchestrates coolscanpy device/roll lifecycle and output writing. + + One roll open at a time, matching `coolscanpy.Device.roll()`'s own + single-reservation lock -- call `close()` (or use this as a context + manager) before opening another. + """ + + def __init__( + self, + *, + exact_color_builder: roll_exact_color.VerifiedStage1Builder | None = None, + exact_color_evaluator: roll_exact_color.VerifiedPortableCMSEvaluator | None = None, + hybrid_runtime: HybridRuntimeConfig | None = None, + ) -> None: + self._roll: coolscanpy_roll.RollHandle | None = None + # Explicitly injected: NegPy never treats its generic renderer as a + # Nikon-exact substitute when either verified stage is absent. + self._exact_color_builder = exact_color_builder + self._exact_color_evaluator = exact_color_evaluator + self._hybrid_runtime = hybrid_runtime + self._repair_cancel = threading.Event() + # Built lazily, on the first frame that actually needs Tier 3 -- see + # `_get_image_processor()`. Most batches never touch it. + self._image_processor: ImageProcessor | None = None + + # -- device / roll lifecycle ----------------------------------------- + + def list_devices(self) -> "list[coolscanpy.DeviceInfo]": + return coolscanpy_roll.list_devices() + + def open_roll( + self, + device_id: str | None = None, + *, + material: "coolscanpy.Material | None" = None, + attempts_root: str | os.PathLike[str] | None = None, + ) -> None: + """Open a device and its roll extension. Call `close()` when done.""" + if self._roll is not None: + raise RollScanningError("a roll is already open on this service; call close() first") + self._roll = coolscanpy_roll.open_roll( + device_id, + material=material, + attempts_root=attempts_root, + ) + + def close(self) -> None: + """Idempotent. Ends the roll reservation and releases the device.""" + if self._roll is not None: + self._roll.close() + self._roll = None + + def eject(self, device_id: str) -> bool: + """Eject a released direct-USB roll through the proven SANE action.""" + + if self._roll is not None: + raise RollScanningError("close the roll reservation before ejecting") + return coolscanpy_roll.eject(device_id) + + def __enter__(self) -> "RollScanningService": + return self + + def __exit__(self, *exc: object) -> None: + self.close() + + # -- preview / approval ------------------------------------------------ + + def preview( + self, slots: Iterable[int] | None = None, *, on_progress: "CoolscanProgressCallback | None" = None + ) -> "list[coolscanpy.Thumbnail]": + return self._require_roll().preview(slots, on_progress=on_progress) + + def restore_preview_session( + self, + payload: str, + slots: Iterable[int] | None = None, + ) -> "list[coolscanpy.Thumbnail]": + """Restore a saved, content-verified review on the open roll.""" + + return self._require_roll().restore_preview_session(payload, slots) + + def set_spacing_offset(self, slot: int, offset_rows: int) -> None: + self._require_roll().set_spacing_offset(slot, offset_rows) + + def approve(self, slot: int) -> "CoolscanManualFrameApproval": + """Approve one reviewed slot and return its content-bound receipt.""" + + return self._require_roll().approve(slot) + + def needs_approval(self, slot: int) -> bool: + return self._require_roll().needs_approval(slot) + + # -- scanning ---------------------------------------------------------- + + def prepare_batch(self) -> None: + """Arm a new batch before its queued worker request is exposed.""" + + self._repair_cancel.clear() + + def scan_many(self, slots: Iterable[int], *, on_progress: "CoolscanProgressCallback | None" = None) -> Iterator["coolscanpy.Frame"]: + if self._repair_cancel.is_set(): + raise roll_repair.RepairCancelled("roll batch cancelled before scanning started") + yield from self._require_roll().scan_many(slots, on_progress=on_progress) + + def safe_stop(self) -> None: + self._repair_cancel.set() + if self._roll is not None: + self._roll.safe_stop() + + # -- writing ------------------------------------------------------------- + + @_transactional_frame_output + def write_frame( + self, + frame: "coolscanpy.Frame", + output_folder: str, + filename_pattern: str, + *, + write_unrepaired: bool = True, + write_repaired: bool = False, + write_positive: bool = False, + repair_mode: str = RepairMode.EXACT.value, + positive_mode: str = roll_exact_color.PositiveColorMode.NIKON_EXACT.value, + builder_receipt: roll_exact_color.BuilderReceipt | None = None, + hybrid_runtime: HybridRuntimeConfig | None = None, + on_repair_progress: Callable[[float], None] | None = None, + ) -> RollFrameOutput: + """Write one scanned `Frame` to disk across up to three tiers, plus a + `_receipt.json` sidecar. Each tier is independently selectable; any + combination of the three flags is valid. + + Tier 1, unrepaired (`write_unrepaired`): the frame exactly as + captured -- a 16-bit RGB TIFF plus an `_IR` sidecar when the frame + carries an infrared plane, matching `writer.write_tiff_16bit`'s own + `_IR.tif` convention. This is the archival master: the + only tier the scanner itself can reproduce. A failure writing it is + allowed to raise, unlike Tier 2/3 below. + + Tier 2, repaired (`write_repaired`): Tier 1 with infrared-guided + dust/scratch repair applied through `negpy.infrastructure.roll + .repair`, written as `_repaired.tif` plus a + `_repaired_IR.tif` sidecar. That sidecar is Tier 1's own + infrared plane, unchanged -- repair consumes infrared to find + defects, it does not produce a repaired version of it, and keeping + the original lets a later re-repair under a different mode start + from the same evidence Tier 1 captured. + + Tier 3, positive (`write_positive`): Tier 2 inverted through either + receipt-bound portable Nikon Stage-1 and CMS stages (`positive_mode= + "nikon-exact", the fail-closed default) or NegPy's explicitly selected + approximate renderer (`"negpy-approximate"`). Exact mode accepts an explicit Stage-3 replay + receipt or automatically builds a native receipt when the frame carries + proven per-frame density and analyzer evidence. + Requested independently of `write_repaired`, but always derived from + Tier 2's in-memory result -- repair runs whenever either flag needs + it, so a positive is never silently missing the repair pass a caller + asked for, and never from a Tier 1 or Tier 2 file already on disk + (an upper tier is never derived from a lower tier's file, only from + the in-memory data the lower tier is also optionally written from). + + Tier 2 and Tier 3 degrade instead of raising when they cannot be + produced -- no repair engine registered, no infrared plane to guide + one, the engine itself fails, or the render fails -- recording why in + the receipt and leaving the corresponding `RollFrameOutput` path(s) + `None`. Tier 1 always writes regardless (or Tier 2, for a Tier-3 + failure): losing the archival capture to a problem in a derived, + frame-bound tier is exactly what this must not do. When Tier 1 has an + infrared plane, its transaction also retains the scanner-native + prepass, IR-validity map, and acquisition binding required to replay + repair after the physical media has moved. + + When a frame carries the native per-acquisition Nikon builder + evidence, that evidence is validated and retained independently of + Tier 3 selection or success. The frame receipt binds the retained + artifacts under ``outputs.native_color_evidence`` so a later exact + render does not depend on the transient in-memory frame object. + + `filename_pattern` is the same Jinja2 template the plain scan path + uses (variables: `date`, `seq`), but `seq` is seeded from the frame's + physical slot number rather than probed for the next free name: a + roll slot is already a stable identity, so re-scanning slot 5 + replaces slot 5's old files instead of piling up a `..._002` beside + them, for every tier. + + `frame.ir_validity` is bound with the two scanner-native RGBI captures + and consumed by repair. Invalid IR pixels retain their original RGB. + """ + try: + resolved_repair_mode = RepairMode(repair_mode) + except ValueError as error: + raise ValueError(f"unknown repair mode {repair_mode!r}") from error + + os.makedirs(output_folder, exist_ok=True) + date_str = _date.today().strftime("%Y%m%d") + basename = render_scan_filename(filename_pattern, date_str, frame.slot) + base_path = os.path.join(output_folder, basename) + + outputs: dict = {} + + # -- Tier 1: unrepaired ----------------------------------------------- + rgb_path = ir_path = None + if write_unrepaired: + rgb_path = _atomic_write_tiff(base_path + ".tif", frame.rgb, photometric="rgb") + if frame.ir is not None: + ir_path = _atomic_write_tiff(base_path + "_IR.tif", frame.ir, photometric="minisblack") + outputs["unrepaired"] = {"written": True, "rgb_path": rgb_path, "ir_path": ir_path} + else: + outputs["unrepaired"] = {"written": False, "status": "not selected"} + + # Tier-1 RGB/IR TIFFs are lossless, but by themselves omit the meter + # prepass, per-pixel IR validity, and frame identities needed to + # reconstruct the scanner-native RepairAcquisition after media moves. + # Retain that small unique evidence independently of immediate Tier-2 + # availability or success. A retention failure is disclosed without + # sacrificing the irreplaceable Tier-1 capture. + prepared_repair_acquisition: roll_repair.RepairAcquisition | None = None + if write_unrepaired and frame.ir is not None: + transaction = _ACTIVE_OUTPUT_TRANSACTION.get() + acquisition_checkpoint = transaction.checkpoint() if transaction is not None else frozenset() + try: + prepared_repair_acquisition = _repair_acquisition_from_frame(frame) + retained_repair_acquisition = _retain_repair_acquisition_evidence( + base_path, + prepared_repair_acquisition, + storage_rgb=frame.rgb, + storage_ir=frame.ir, + rgb_path=rgb_path, + ir_path=ir_path, + ) + except Exception as error: + if transaction is not None: + transaction.discard_after(acquisition_checkpoint) + outputs["repair_acquisition_evidence"] = { + "replayable": False, + "retained": False, + "status": f"unavailable: {error}", + } + else: + outputs["repair_acquisition_evidence"] = { + **retained_repair_acquisition, + "retained": True, + } + + # Preserve the scanner-native color inputs as an acquisition artifact, + # not merely as a side effect of a successful Tier-3 render. A TIFF or + # CMS failure must never discard the only evidence from which that + # exact render can be retried later. + native_receipt_from_frame: roll_exact_color.NativeValidatedBuilderReceipt | None = None + retained_native_evidence: dict[str, object] | None = None + native_receipt_error: Exception | None = None + if getattr(frame, "nikon_exact_builder_evidence", None) is not None: + transaction = _ACTIVE_OUTPUT_TRANSACTION.get() + evidence_checkpoint = transaction.checkpoint() if transaction is not None else frozenset() + try: + native_receipt_from_frame = _build_native_receipt_from_frame(frame) + retained_native_evidence = _retain_native_builder_evidence( + base_path, + native_receipt_from_frame, + ) + except Exception as error: + if transaction is not None: + transaction.discard_after(evidence_checkpoint) + native_receipt_from_frame = None + retained_native_evidence = None + native_receipt_error = error + outputs["native_color_evidence"] = { + "native_per_acquisition_builder": True, + "retained": False, + "scope": roll_exact_color.NATIVE_BUILDER_SCOPE, + "status": f"unavailable: {error}", + } + else: + outputs["native_color_evidence"] = { + "builder_receipt": roll_exact_color.receipt_payload(native_receipt_from_frame), + "builder_receipt_sha256": native_receipt_from_frame.sha256, + "native_per_acquisition_builder": True, + "retained": True, + "retained_builder_evidence": retained_native_evidence, + "scope": roll_exact_color.NATIVE_BUILDER_SCOPE, + } + + # -- Tier 2: repaired (also feeds Tier 3, even when not itself written) -- + outputs["repaired"] = {"written": False, "status": "not selected"} + repair_result: roll_repair.RepairResult | None = None + repaired_rgb_path = repaired_ir_path = None + synthesis_mask_path = None + native_synthesis_mask_path = None + hybrid_receipt_path = None + if write_repaired or write_positive: + if frame.ir is None: + outputs["repaired"] = {"written": False, "status": "unavailable: frame has no infrared plane to guide repair"} + elif not roll_repair.available(): + outputs["repaired"] = {"written": False, "status": "unavailable: no dust-repair engine registered"} + else: + try: + acquisition = ( + prepared_repair_acquisition if prepared_repair_acquisition is not None else _repair_acquisition_from_frame(frame) + ) + repair_result = roll_repair.repair( + acquisition, + resolved_repair_mode, + hybrid_runtime=hybrid_runtime or self._hybrid_runtime, + progress=on_repair_progress, + cancel=self._repair_cancel, + ) + except roll_repair.RepairCancelled: + raise + except Exception as error: + outputs["repaired"] = {"written": False, "status": f"repair failed: {error}"} + else: + transaction = _ACTIVE_OUTPUT_TRANSACTION.get() + repair_checkpoint = transaction.checkpoint() if transaction is not None else frozenset() + try: + source_rgb = np.asarray(repair_result.rgb) + rgb_snapshot = np.array( + source_rgb, + order="C", + copy=True, + ) + rgb_snapshot.setflags(write=False) + routing_snapshot = ( + dict(repair_result.routing_counts) + if isinstance(repair_result.routing_counts, dict) + else repair_result.routing_counts + ) + repair_result = dataclasses.replace( + repair_result, + rgb=rgb_snapshot, + routing_counts=routing_snapshot, + ) + _validate_repair_result_binding( + acquisition, + repair_result, + requested_mode=resolved_repair_mode, + ) + hybrid_evidence = None + if repair_result.mode_resolved is RepairMode.HYBRID: + synthesis_mask_path = _atomic_write_bytes( + base_path + "_repaired_SYNTH.png", + repair_result.storage_synthesis_mask_png, + ) + hybrid_evidence = _retain_hybrid_repair_evidence( + base_path, + acquisition, + repair_result, + ) + native_synthesis_mask_path = hybrid_evidence["native_mask"]["path"] + hybrid_receipt_path = hybrid_evidence["receipt"]["path"] + except Exception as error: + if transaction is not None: + transaction.discard_after(repair_checkpoint) + repair_result = None + synthesis_mask_path = None + native_synthesis_mask_path = None + hybrid_receipt_path = None + outputs["repaired"] = { + "written": False, + "status": f"repair evidence failed: {error}", + } + hybrid_evidence = None + if repair_result is None: + pass + else: + acquisition_entry = { + "acquisition_id": acquisition.acquisition_id, + "slot": acquisition.slot, + "reservation_id": acquisition.reservation_id, + "capture_attempt_id": acquisition.capture_attempt_id, + "evidence_sha256": acquisition.evidence_sha256, + "storage_transform": acquisition.storage_transform, + "main_rgbi_sha256": acquisition.main_rgbi_sha256, + "prepass_rgbi_sha256": acquisition.prepass_rgbi_sha256, + "ir_validity_sha256": acquisition.ir_validity_sha256, + } + entry: dict[str, object] = { + "engine": repair_result.engine, + "engine_version": repair_result.engine_version, + "mode_requested": str(repair_result.mode_requested), + "mode_resolved": str(repair_result.mode_resolved), + "degraded": repair_result.degraded, + "reason": repair_result.reason, + "acquisition": acquisition_entry, + # Kept at top level for older receipt consumers. + "acquisition_id": repair_result.acquisition_id, + "slot": repair_result.slot, + "reservation_id": repair_result.reservation_id, + "evidence_sha256": repair_result.evidence_sha256, + "backend_requested": repair_result.backend_requested, + "backend_used": repair_result.backend_used, + "backend_selection_reason": (repair_result.backend_selection_reason), + "native_output_rgb_sha256": (repair_result.native_output_rgb_sha256), + "storage_output_rgb_sha256": (repair_result.storage_output_rgb_sha256), + } + if hybrid_evidence is not None: + applied_pixels = int(round(repair_result.synthesis_fraction * acquisition.ir_validity.size)) + entry["disclosure_mask"] = { + "applied_final": { + "fraction": repair_result.synthesis_fraction, + "native": hybrid_evidence["native_mask"], + "pixel_count": applied_pixels, + "storage": { + "bytes": len(repair_result.storage_synthesis_mask_png), + "path": synthesis_mask_path, + "sha256": (repair_result.storage_synthesis_mask_sha256), + "shape": list(repair_result.storage_synthesis_mask_shape), + }, + "transform": (repair_result.synthesis_mask_transform), + }, + "routed_raw": { + "native": hybrid_evidence["routed_native_mask"], + "routing_counts": (repair_result.routing_counts), + }, + } + entry["hybrid_receipt"] = { + **hybrid_evidence["receipt"], + "provenance_class": (repair_result.hybrid_provenance_class), + "verified_output_rgb_sha256": (repair_result.hybrid_receipt_output_rgb_sha256), + } + entry["hybrid_evidence_binding"] = hybrid_evidence["binding"] + if write_repaired: + repaired_rgb_path = _atomic_write_tiff(base_path + "_repaired.tif", repair_result.rgb, photometric="rgb") + repaired_ir_path = _atomic_write_tiff(base_path + "_repaired_IR.tif", frame.ir, photometric="minisblack") + entry.update(written=True, rgb_path=repaired_rgb_path, ir_path=repaired_ir_path) + else: + entry.update(written=False, status="not selected (computed in memory for the positive)") + outputs["repaired"] = entry + + # -- Tier 3: positive ----------------------------------------------- + outputs["positive"] = {"written": False, "status": "not selected"} + positive_path = None + if write_positive: + if repair_result is None: + tier2_status = outputs["repaired"].get("status", "unavailable") + outputs["positive"] = { + "written": False, + "status": f"unavailable: Tier 2 (repaired) could not be produced ({tier2_status})", + "color_mode": positive_mode, + } + elif positive_mode == roll_exact_color.PositiveColorMode.NIKON_EXACT.value: + transaction = _ACTIVE_OUTPUT_TRANSACTION.get() + exact_checkpoint = transaction.checkpoint() if transaction is not None else frozenset() + try: + exact_icc_profile = roll_nikon_icc.nikon_adobe_rgb_profile() + active_receipt = builder_receipt + active_builder = self._exact_color_builder + active_evaluator = self._exact_color_evaluator + if active_receipt is None: + if native_receipt_from_frame is not None: + active_receipt = native_receipt_from_frame + elif native_receipt_error is not None: + raise roll_exact_color.ExactColorUnavailable(str(native_receipt_error)) from native_receipt_error + else: + active_receipt = _build_native_receipt_from_frame(frame) + if active_builder is None: + from negpy.services.roll.portable_builder import PortableStage1Builder + + active_builder = PortableStage1Builder() + if active_evaluator is None: + from negpy.services.roll.portable_cms import PortableCMSOnEvaluator + + active_evaluator = PortableCMSOnEvaluator() + result = roll_exact_color.evaluate_exact_color( + repair_result.rgb, + builder_receipt=active_receipt, + builder=active_builder, + evaluator=active_evaluator, + ) + builder_application_receipt = result.builder_application_receipt + if builder_application_receipt is None: + raise roll_exact_color.ExactColorIntegrityError("exact-color result is missing its builder application receipt") + native_builder = type(result.builder_receipt) is roll_exact_color.NativeValidatedBuilderReceipt + positive_path = _atomic_write_tiff( + base_path + "_positive.tif", + result.rgb, + photometric="rgb", + iccprofile=exact_icc_profile, + ) + exact_tiff_artifact = _verify_exact_positive_tiff( + positive_path, + expected_rgb=result.rgb, + expected_icc=exact_icc_profile, + ) + # Retain replay evidence only after the TIFF itself has + # passed verification. If retention then fails, the + # transaction checkpoint below discards both together. + if ( + retained_native_evidence is not None + and native_receipt_from_frame is not None + and result.builder_receipt.sha256 == native_receipt_from_frame.sha256 + ): + retained_evidence = retained_native_evidence + else: + retained_evidence = _retain_builder_evidence( + base_path, + result.builder_receipt, + ) + except Exception as error: + if transaction is not None: + transaction.discard_after(exact_checkpoint) + positive_path = None + outputs["positive"] = { + "written": False, + "status": f"unavailable: exact Nikon color: {error}", + "color_mode": roll_exact_color.PositiveColorMode.NIKON_EXACT.value, + } + else: + outputs["positive"] = { + "written": True, + "rgb_path": positive_path, + "color_mode": roll_exact_color.PositiveColorMode.NIKON_EXACT.value, + "exact_nikon_color": True, + "inversion_path": ( + "native-per-acquisition-builder-and-verified-portable-cms" + if native_builder + else "stage3-evidence-replay-bridge-and-verified-portable-cms" + ), + "native_per_acquisition_builder": native_builder, + "repaired_input_rgb_sha256": result.source_rgb_sha256, + "input_rgb_sha256": result.input_rgb_sha256, + "stage1_input_rgb_sha256": result.input_rgb_sha256, + "output_rgb_sha256": result.output_rgb_sha256, + "builder_receipt": roll_exact_color.receipt_payload(result.builder_receipt), + "builder_receipt_sha256": result.builder_receipt.sha256, + "builder_validated": True, + "builder_application_receipt": roll_exact_color.receipt_payload(builder_application_receipt), + "builder_application_receipt_sha256": (builder_application_receipt.sha256), + "cms_receipt": roll_exact_color.receipt_payload(result.cms_receipt), + "cms_receipt_sha256": result.cms_receipt.sha256, + "cms_verified": True, + "icc_profile": roll_nikon_icc.profile_receipt_binding(), + "tiff_artifact": exact_tiff_artifact, + "retained_builder_evidence": retained_evidence, + "repair_engine": repair_result.engine, + "repair_engine_version": repair_result.engine_version, + "repair_mode": str(repair_result.mode_resolved), + } + if native_builder: + outputs["positive"]["native_builder_scope"] = roll_exact_color.NATIVE_BUILDER_SCOPE + else: + outputs["positive"]["replay_bridge_scope"] = roll_exact_color.STAGE3_REPLAY_SCOPE + elif positive_mode != roll_exact_color.PositiveColorMode.NEGPY_APPROXIMATE.value: + outputs["positive"] = { + "written": False, + "status": f"unavailable: unknown positive color mode {positive_mode!r}", + "color_mode": positive_mode, + } + elif not roll_positive.available(): + outputs["positive"] = { + "written": False, + "status": "unavailable: inversion path not available", + "color_mode": roll_exact_color.PositiveColorMode.NEGPY_APPROXIMATE.value, + } + else: + try: + result = roll_positive.render_positive(repair_result.rgb, processor=self._get_image_processor()) + except Exception as error: + outputs["positive"] = { + "written": False, + "status": f"inversion failed: {error}", + "color_mode": roll_exact_color.PositiveColorMode.NEGPY_APPROXIMATE.value, + } + else: + positive_path = _atomic_write_tiff(base_path + "_positive.tif", result.rgb, photometric="rgb") + outputs["positive"] = { + "written": True, + "rgb_path": positive_path, + "color_mode": roll_exact_color.PositiveColorMode.NEGPY_APPROXIMATE.value, + "exact_nikon_color": False, + "inversion_path": "negpy.services.rendering.image_processor.ImageProcessor.run_pipeline", + "render_intent": result.render_intent, + "process_mode": result.process_mode, + "auto_exposure": result.auto_exposure, + "negpy_version": result.negpy_version, + "repair_engine": repair_result.engine, + "repair_engine_version": repair_result.engine_version, + "repair_mode": str(repair_result.mode_resolved), + } + + receipt_path = base_path + "_receipt.json" + receipt_payload = dataclasses.asdict(frame.receipt) + receipt_artifacts = getattr(frame.receipt, "artifacts", None) + if receipt_artifacts is not None: + receipt_payload["artifacts"] = {name: dataclasses.asdict(artifact) for name, artifact in receipt_artifacts.items()} + receipt_payload["outputs"] = outputs + _atomic_write_json(receipt_path, receipt_payload) + + return RollFrameOutput( + slot=frame.slot, + rgb_path=rgb_path, + ir_path=ir_path, + repaired_rgb_path=repaired_rgb_path, + repaired_ir_path=repaired_ir_path, + positive_path=positive_path, + receipt_path=receipt_path, + synthesis_mask_path=synthesis_mask_path, + native_synthesis_mask_path=native_synthesis_mask_path, + hybrid_receipt_path=hybrid_receipt_path, + ) + + def _get_image_processor(self) -> ImageProcessor: + """Built on first use and reused across a batch: its constructor + probes for GPU acceleration, worth paying once per service instance + rather than once per frame, even though Tier-3 rendering itself + always runs on the CPU engine (see `positive.render_positive`).""" + if self._image_processor is None: + self._image_processor = ImageProcessor() + return self._image_processor + + # -- internals ----------------------------------------------------------- + + def _require_roll(self) -> coolscanpy_roll.RollHandle: + if self._roll is None: + raise RollScanningError("no roll is open; call open_roll() first") + return self._roll + + +def _repair_acquisition_from_frame(frame: object) -> roll_repair.RepairAcquisition: + prepare = getattr(frame, "prepare_digital_ice", None) + if not callable(prepare): + raise ValueError("frame has no bound scanner-native Digital ICE acquisition") + source = prepare() + if isinstance(source, roll_repair.RepairAcquisition): + if source.slot != getattr(frame, "slot", None): + raise ValueError("Digital ICE acquisition belongs to another frame slot") + _validate_digital_ice_producer_binding(source) + return source + required = ( + "acquisition_id", + "slot", + "reservation_id", + "capture_attempt_id", + "storage_transform", + "evidence_sha256", + "main_rgbi_sha256", + "meter_rgbi_sha256", + "ir_validity_sha256", + "main_rgbi", + "meter_rgbi", + "ir_validity", + ) + missing = [name for name in required if not hasattr(source, name)] + if missing: + raise ValueError("Digital ICE acquisition producer omitted: " + ", ".join(missing)) + if source.slot != getattr(frame, "slot", None): + raise ValueError("Digital ICE acquisition belongs to another frame slot") + acquisition = roll_repair.RepairAcquisition( + acquisition_id=source.acquisition_id, + slot=source.slot, + reservation_id=source.reservation_id, + capture_attempt_id=source.capture_attempt_id, + storage_transform=source.storage_transform, + evidence_sha256=source.evidence_sha256, + main_rgbi_sha256=source.main_rgbi_sha256, + prepass_rgbi_sha256=source.meter_rgbi_sha256, + ir_validity_sha256=source.ir_validity_sha256, + main_rgbi=source.main_rgbi, + prepass_rgbi=source.meter_rgbi, + ir_validity=source.ir_validity, + ) + _validate_digital_ice_producer_binding(acquisition) + return acquisition + + +def _npy_payload(array: np.ndarray, *, dtype: np.dtype) -> bytes: + canonical = np.array(array, dtype=dtype, order="C", copy=True) + stream = io.BytesIO() + np.save(stream, canonical, allow_pickle=False) + return stream.getvalue() + + +def _derive_digital_ice_producer_binding( + *, + slot: int, + reservation_id: str, + capture_attempt_id: str, + main_rgbi: np.ndarray, + prepass_rgbi: np.ndarray, + ir_validity: np.ndarray, +) -> tuple[str, str]: + """Reproduce Coolscanpy's v1 DigitalIceAcquisitionEvidence hashes.""" + + identity_document = { + "capture_attempt_id": capture_attempt_id, + "kind": "coolscanpy.digital-ice-acquisition-identity", + "reservation_id": reservation_id, + "slot": slot, + "version": 1, + } + identity_bytes = json.dumps( + identity_document, + sort_keys=True, + separators=(",", ":"), + ensure_ascii=False, + allow_nan=False, + ).encode("utf-8") + acquisition_id = "dice-" + hashlib.sha256(identity_bytes).hexdigest() + + native_main = np.array(main_rgbi, dtype=" dict[str, object]: + canonical = np.array(array, dtype=dtype, order="C", copy=True) + payload = memoryview(canonical).cast("B") + return { + "byte_length": payload.nbytes, + "dtype": canonical.dtype.str, + "sha256": hashlib.sha256(payload).hexdigest(), + "shape": list(canonical.shape), + } + + evidence_document = { + "acquisition_id": acquisition_id, + "artifacts": { + "scanner_native_ir_validity": artifact( + native_validity, + dtype=np.dtype(np.bool_), + ), + "scanner_native_main_rgbi": artifact( + native_main, + dtype=np.dtype(" None: + expected_acquisition_id, expected_evidence_sha256 = _derive_digital_ice_producer_binding( + slot=acquisition.slot, + reservation_id=acquisition.reservation_id, + capture_attempt_id=acquisition.capture_attempt_id, + main_rgbi=acquisition.main_rgbi, + prepass_rgbi=acquisition.prepass_rgbi, + ir_validity=acquisition.ir_validity, + ) + if acquisition.acquisition_id != expected_acquisition_id: + raise ValueError("Digital ICE acquisition identity is not canonical") + if acquisition.evidence_sha256 != expected_evidence_sha256: + raise ValueError("Digital ICE producer evidence SHA-256 changed") + + +def _retain_repair_acquisition_evidence( + base_path: str, + acquisition: roll_repair.RepairAcquisition, + *, + storage_rgb: np.ndarray, + storage_ir: np.ndarray, + rgb_path: str | None, + ir_path: str | None, +) -> dict[str, object]: + """Retain the unique inputs needed to replay repair after media moves.""" + + if rgb_path is None or ir_path is None: + raise ValueError("Tier-1 RGB and IR paths are required for repair replay") + rgb = np.asarray(storage_rgb) + infrared = np.asarray(storage_ir) + expected_storage_shape = ( + acquisition.main_rgbi.shape[1], + acquisition.main_rgbi.shape[0], + ) + if ( + rgb.dtype != np.uint16 + or rgb.shape != (*expected_storage_shape, 3) + or infrared.dtype != np.uint16 + or infrared.shape != expected_storage_shape + ): + raise ValueError("Tier-1 planes do not match the Digital ICE acquisition") + storage_rgbi = np.dstack((rgb, infrared)) + reconstructed_native = np.ascontiguousarray(np.rot90(storage_rgbi, k=-1, axes=(0, 1))) + if _rgb16_sha256(reconstructed_native) != acquisition.main_rgbi_sha256: + raise ValueError("Tier-1 planes do not reconstruct the captured main RGBI") + _validate_digital_ice_producer_binding(acquisition) + + evidence_root = os.path.join( + os.path.dirname(base_path) or ".", + ".negpy-dice-acquisition", + ) + evidence_token = hashlib.sha256((acquisition.acquisition_id + "\0" + os.path.basename(base_path)).encode("utf-8")).hexdigest() + evidence_directory = os.path.join(evidence_root, evidence_token) + if any(os.path.lexists(path) and os.path.islink(path) for path in (evidence_root, evidence_directory)): + raise OSError("repair acquisition evidence path is a symlink") + _prepare_evidence_directory(evidence_directory) + + prepass_payload = _npy_payload( + acquisition.prepass_rgbi, + dtype=np.dtype(" dict[str, object]: + row: dict[str, object] = { + "dtype": dtype, + "path": path, + "raw_sha256": raw_sha256, + "relative_path": os.path.relpath(path, evidence_directory), + "shape": list(shape), + } + if payload is not None: + row.update( + bytes=len(payload), + file_sha256=hashlib.sha256(payload).hexdigest(), + ) + return row + + sources = { + "storage_ir_tiff": artifact( + path=ir_path, + payload=None, + raw_sha256=_rgb16_sha256(infrared), + shape=infrared.shape, + dtype=" np.ndarray: + before = os.lstat(path) + if stat.S_ISLNK(before.st_mode) or not stat.S_ISREG(before.st_mode): + raise ValueError(f"{label} must be a regular non-symlink file") + flags = os.O_RDONLY + if hasattr(os, "O_CLOEXEC"): + flags |= os.O_CLOEXEC + if hasattr(os, "O_NOFOLLOW"): + flags |= os.O_NOFOLLOW + if hasattr(os, "O_NONBLOCK"): + flags |= os.O_NONBLOCK + descriptor = os.open(path, flags) + + def identity(metadata: os.stat_result) -> tuple[int, int, int, int, int]: + return ( + metadata.st_dev, + metadata.st_ino, + metadata.st_size, + metadata.st_mtime_ns, + metadata.st_ctime_ns, + ) + + try: + opened = os.fstat(descriptor) + if not stat.S_ISREG(opened.st_mode) or identity(opened) != identity(before): + raise ValueError(f"{label} changed while it was opened") + try: + with os.fdopen(os.dup(descriptor), "rb") as stream: + with tifffile.TiffFile( + stream, + name=os.path.basename(path), + ) as image: + if len(image.pages) != 1: + raise ValueError(f"{label} must contain exactly one page") + page = image.pages[0] + if tuple(page.shape) != expected_shape or page.dtype != np.uint16: + raise ValueError(f"{label} geometry or dtype changed") + decoded = np.array(page.asarray(), order="C", copy=True) + except ValueError: + raise + except Exception as error: + raise ValueError(f"cannot decode {label}: {error}") from error + after_decode = os.fstat(descriptor) + finally: + os.close(descriptor) + after_path = os.lstat(path) + if identity(after_decode) != identity(opened) or identity(after_path) != identity(opened): + raise ValueError(f"{label} changed while it was decoded") + return decoded + + +def load_repair_acquisition_evidence( + binding_path: str | os.PathLike[str], +) -> roll_repair.RepairAcquisition: + """Rebuild a hash-verified RepairAcquisition from a Tier-1 archive.""" + + binding_file = os.path.abspath(os.fspath(binding_path)) + payload = _stable_regular_bytes( + binding_file, + maximum_bytes=_MAX_RECEIPT_BYTES, + label="repair acquisition binding", + ) + try: + document = _strict_json_loads(payload) + except (UnicodeDecodeError, json.JSONDecodeError) as error: + raise ValueError(f"repair acquisition binding is invalid JSON: {error}") from error + if not isinstance(document, dict): + raise ValueError("repair acquisition binding must contain an object") + canonical = ( + json.dumps( + document, + sort_keys=True, + separators=(",", ":"), + ensure_ascii=True, + allow_nan=False, + ).encode("utf-8") + + b"\n" + ) + if canonical != payload or document.get("schema") != "negpy.dice-acquisition-replay-v1": + raise ValueError("repair acquisition binding is not canonical or supported") + acquisition = document.get("acquisition") + artifacts = document.get("artifacts") + sources = document.get("sources") + expected_replay_contract = { + "authenticity": "integrity-bound-not-signed", + "complete": True, + "reconstruction": "stack-upright-storage-rgb-ir-then-rot90-k-1", + "requires": [ + "storage_rgb_tiff", + "storage_ir_tiff", + "prepass_rgbi", + "ir_validity", + "acquisition_provenance", + ], + "storage_orientation": "upright-storage", + } + if not all(isinstance(value, dict) for value in (acquisition, artifacts, sources)): + raise ValueError("repair acquisition binding sections are malformed") + if document.get("replay") != expected_replay_contract: + raise ValueError("repair acquisition replay requirements changed") + + evidence_directory = os.path.dirname(binding_file) + archive_root = os.path.realpath(os.path.dirname(os.path.dirname(evidence_directory))) + + def row_path(row: object, *, label: str) -> tuple[str, dict[str, object]]: + if not isinstance(row, dict): + raise ValueError(f"repair acquisition {label} artifact is malformed") + relative = row.get("relative_path") + if type(relative) is not str or os.path.isabs(relative): + raise ValueError(f"repair acquisition {label} relative path is invalid") + candidate = os.path.abspath(os.path.join(evidence_directory, relative)) + resolved_parent = os.path.realpath(os.path.dirname(candidate)) + resolved_candidate = os.path.join( + resolved_parent, + os.path.basename(candidate), + ) + try: + inside = os.path.commonpath((archive_root, resolved_candidate)) == archive_root + except ValueError: + inside = False + if not inside: + raise ValueError(f"repair acquisition {label} path escapes its archive") + return candidate, cast(dict[str, object], row) + + def shape(row: dict[str, object], *, label: str) -> tuple[int, ...]: + values = row.get("shape") + if ( + not isinstance(values, list) + or not values + or len(values) > 3 + or any(type(value) is not int or not 1 <= value <= 10_000 for value in values) + ): + raise ValueError(f"repair acquisition {label} shape is invalid") + result = tuple(cast(list[int], values)) + element_count = 1 + for dimension in result: + element_count *= dimension + if element_count > 150_000_000: + raise ValueError(f"repair acquisition {label} shape is too large") + return result + + def load_npy( + row: object, + *, + label: str, + expected_dtype: np.dtype, + ) -> np.ndarray: + path, artifact_row = row_path(row, label=label) + expected_shape = shape(artifact_row, label=label) + if (label == "prepass RGBI" and (len(expected_shape) != 3 or expected_shape[-1] != 4)) or ( + label == "IR validity" and len(expected_shape) != 2 + ): + raise ValueError(f"repair acquisition {label} shape is invalid") + if artifact_row.get("dtype") != expected_dtype.str: + raise ValueError(f"repair acquisition {label} dtype declaration changed") + encoded = _stable_regular_bytes( + path, + maximum_bytes=_MAX_SHARED_EVIDENCE_BYTES, + label=f"repair acquisition {label}", + ) + if artifact_row.get("file_sha256") != hashlib.sha256(encoded).hexdigest(): + raise ValueError(f"repair acquisition {label} file SHA-256 changed") + try: + stream = io.BytesIO(encoded) + version = np.lib.format.read_magic(stream) + if version == (1, 0): + header_shape, fortran_order, header_dtype = np.lib.format.read_array_header_1_0(stream) + elif version == (2, 0): + header_shape, fortran_order, header_dtype = np.lib.format.read_array_header_2_0(stream) + else: + raise ValueError(f"unsupported NPY version {version!r}") + if tuple(header_shape) != expected_shape or header_dtype != expected_dtype or fortran_order: + raise ValueError("NPY header geometry or dtype changed") + expected_bytes = int(np.prod(expected_shape, dtype=np.int64)) * expected_dtype.itemsize + if len(encoded) - stream.tell() != expected_bytes: + raise ValueError("NPY byte length changed") + stream.seek(0) + decoded = np.load(stream, allow_pickle=False) + except Exception as error: + raise ValueError(f"repair acquisition {label} NPY is invalid: {error}") from error + if not isinstance(decoded, np.ndarray) or decoded.dtype != expected_dtype or decoded.shape != expected_shape: + raise ValueError(f"repair acquisition {label} array changed") + canonical = np.array(decoded, dtype=expected_dtype, order="C", copy=True) + digest_dtype = np.dtype(np.bool_) if expected_dtype == np.dtype(np.bool_) else np.dtype(" str: + canonical = np.array(array, dtype=" np.ndarray: + try: + with Image.open(io.BytesIO(payload)) as image: + if image.format != "PNG": + raise ValueError("not a PNG") + if image.size != (expected_shape[1], expected_shape[0]): + raise ValueError("geometry changed") + decoded = np.asarray(image.convert("L")) + except Exception as error: + raise ValueError(f"{label} is not a valid PNG: {error}") from error + if not np.all(np.isin(np.unique(decoded), np.array([0, 255], dtype=np.uint8))): + raise ValueError(f"{label} is not binary") + return np.ascontiguousarray(decoded != 0) + + +def _validate_repair_result_binding( + acquisition: roll_repair.RepairAcquisition, + result: roll_repair.RepairResult, + *, + requested_mode: RepairMode, +) -> None: + """Fail closed before any Tier-2 result or evidence is published.""" + + for label, actual, expected in ( + ("acquisition ID", result.acquisition_id, acquisition.acquisition_id), + ("slot", result.slot, acquisition.slot), + ("reservation ID", result.reservation_id, acquisition.reservation_id), + ("evidence SHA-256", result.evidence_sha256, acquisition.evidence_sha256), + ("requested mode", result.mode_requested, requested_mode), + ): + if actual != expected: + raise ValueError(f"repair result {label} disagrees with its acquisition") + if requested_mode is RepairMode.EXACT and result.mode_resolved is RepairMode.HYBRID: + raise ValueError("exact repair request cannot resolve to hybrid") + if result.mode_resolved not in (RepairMode.EXACT, RepairMode.HYBRID): + raise ValueError("repair result resolved mode is invalid") + storage_shape = ( + acquisition.main_rgbi.shape[1], + acquisition.main_rgbi.shape[0], + 3, + ) + rgb = np.asarray(result.rgb) + if rgb.dtype != np.uint16 or rgb.shape != storage_shape or not rgb.flags.c_contiguous: + raise ValueError("repair result has invalid storage-oriented RGB geometry") + storage_hash = _rgb16_sha256(rgb) + if result.storage_output_rgb_sha256 != storage_hash: + raise ValueError("repair result storage RGB SHA-256 changed") + native_rgb = np.ascontiguousarray(np.rot90(rgb, k=-1, axes=(0, 1))) + native_hash = _rgb16_sha256(native_rgb) + if result.native_output_rgb_sha256 != native_hash: + raise ValueError("repair result scanner-native RGB SHA-256 changed") + + if result.mode_resolved is not RepairMode.HYBRID: + forbidden = ( + result.native_synthesis_mask_png, + result.native_synthesis_mask_sha256, + result.native_synthesis_mask_shape, + result.routed_native_synthesis_mask_png, + result.routed_native_synthesis_mask_sha256, + result.routed_native_synthesis_mask_shape, + result.storage_synthesis_mask_png, + result.storage_synthesis_mask_sha256, + result.storage_synthesis_mask_shape, + result.synthesis_mask_transform, + result.synthesis_fraction, + result.routing_counts, + result.hybrid_receipt, + result.hybrid_receipt_sha256, + result.hybrid_provenance_class, + result.hybrid_receipt_output_rgb_sha256, + ) + if any(value is not None for value in forbidden): + raise ValueError("non-hybrid repair result carries hybrid evidence") + return + + required = ( + result.native_synthesis_mask_png, + result.native_synthesis_mask_sha256, + result.native_synthesis_mask_shape, + result.routed_native_synthesis_mask_png, + result.routed_native_synthesis_mask_sha256, + result.routed_native_synthesis_mask_shape, + result.storage_synthesis_mask_png, + result.storage_synthesis_mask_sha256, + result.storage_synthesis_mask_shape, + result.synthesis_mask_transform, + result.synthesis_fraction, + result.routing_counts, + result.hybrid_receipt, + result.hybrid_receipt_sha256, + result.hybrid_receipt_output_rgb_sha256, + result.hybrid_provenance_class, + ) + if any(value is None for value in required): + raise ValueError("hybrid repair result omitted disclosure evidence") + native_png = result.native_synthesis_mask_png + routed_native_png = result.routed_native_synthesis_mask_png + storage_png = result.storage_synthesis_mask_png + receipt = result.hybrid_receipt + if not all(type(value) is bytes for value in (native_png, routed_native_png, storage_png, receipt)): + raise ValueError("hybrid disclosure evidence must be immutable bytes") + if hashlib.sha256(native_png).hexdigest() != result.native_synthesis_mask_sha256: + raise ValueError("scanner-native disclosure mask SHA-256 changed") + if hashlib.sha256(storage_png).hexdigest() != result.storage_synthesis_mask_sha256: + raise ValueError("storage disclosure mask SHA-256 changed") + if hashlib.sha256(routed_native_png).hexdigest() != result.routed_native_synthesis_mask_sha256: + raise ValueError("routed scanner-native mask SHA-256 changed") + if hashlib.sha256(receipt).hexdigest() != result.hybrid_receipt_sha256: + raise ValueError("hybrid receipt SHA-256 changed") + if len(result.hybrid_receipt_output_rgb_sha256) != 64 or any( + character not in "0123456789abcdef" for character in result.hybrid_receipt_output_rgb_sha256 + ): + raise ValueError("hybrid receipt output RGB SHA-256 is malformed") + try: + receipt_document = _strict_json_loads(receipt) + except (UnicodeDecodeError, ValueError, TypeError) as error: + raise ValueError(f"hybrid receipt JSON is invalid: {error}") from error + if not isinstance(receipt_document, dict): + raise ValueError("hybrid receipt must contain a JSON object") + try: + canonical_receipt = ( + json.dumps( + receipt_document, + sort_keys=True, + separators=(",", ":"), + ensure_ascii=True, + allow_nan=False, + ).encode("utf-8") + + b"\n" + ) + except (TypeError, ValueError, RecursionError) as error: + raise ValueError(f"hybrid receipt JSON is not canonical: {error}") from error + if canonical_receipt != receipt: + raise ValueError("hybrid receipt JSON is not canonical") + if receipt_document.get("schema") != "fauxce-hybrid-receipt-v2" or result.hybrid_provenance_class != "caller_asserted_bare_npy": + raise ValueError("hybrid receipt schema or provenance class changed") + native_mask = _decode_binary_mask( + native_png, + expected_shape=acquisition.main_rgbi.shape[:2], + label="final scanner-native disclosure mask", + ) + routed_native_mask = _decode_binary_mask( + routed_native_png, + expected_shape=acquisition.main_rgbi.shape[:2], + label="routed scanner-native disclosure mask", + ) + storage_mask = _decode_binary_mask( + storage_png, + expected_shape=( + acquisition.main_rgbi.shape[1], + acquisition.main_rgbi.shape[0], + ), + label="storage disclosure mask", + ) + if ( + tuple(native_mask.shape) != result.native_synthesis_mask_shape + or native_mask.shape != acquisition.main_rgbi.shape[:2] + or tuple(routed_native_mask.shape) != result.routed_native_synthesis_mask_shape + or routed_native_mask.shape != acquisition.main_rgbi.shape[:2] + or not np.array_equal( + native_mask, + routed_native_mask & acquisition.ir_validity, + ) + ): + raise ValueError("scanner-native disclosure mask geometry changed") + expected_storage = acquisition.storage_mask(native_mask) + if ( + tuple(storage_mask.shape) != result.storage_synthesis_mask_shape + or not np.array_equal(storage_mask, expected_storage) + or result.synthesis_mask_transform != acquisition.storage_transform + ): + raise ValueError("storage disclosure mask transform binding changed") + synthesis_pixels = int(np.count_nonzero(native_mask)) + routed_pixels = int(np.count_nonzero(routed_native_mask)) + frame_pixels = int(native_mask.size) + if result.synthesis_fraction != synthesis_pixels / frame_pixels: + raise ValueError("hybrid synthesis fraction disagrees with disclosure mask") + counts = result.routing_counts + routing = receipt_document.get("routing") + receipt_counts = routing.get("counts") if isinstance(routing, dict) else None + count_keys = ( + "final_regions", + "synthesis_pixels", + "frame_pixels", + "at_floor_pixels", + ) + if ( + not isinstance(counts, dict) + or not isinstance(receipt_counts, dict) + or any(type(counts.get(key)) is not int for key in count_keys) + or any(type(receipt_counts.get(key)) is not int for key in count_keys) + or any(counts[key] != receipt_counts[key] for key in count_keys) + or any(counts[key] < 0 for key in count_keys) + or counts.get("synthesis_pixels") != routed_pixels + or counts.get("frame_pixels") != frame_pixels + or counts["at_floor_pixels"] > frame_pixels + or counts["final_regions"] > counts["synthesis_pixels"] + or (counts["final_regions"] == 0) != (routed_pixels == 0) + ): + raise ValueError("hybrid routing counts disagree with disclosure mask") + synthesis = receipt_document.get("synthesis") + if ( + not isinstance(synthesis, dict) + or synthesis.get("pixel_count") != routed_pixels + or synthesis.get("frame_pixel_count") != frame_pixels + or synthesis.get("fraction") != routed_pixels / frame_pixels + or synthesis.get("within_budget") is not True + ): + raise ValueError("hybrid receipt synthesis accounting changed") + artifacts = receipt_document.get("artifacts") + output_artifacts = ( + [artifact for artifact in artifacts if isinstance(artifact, dict) and artifact.get("role") == "hybrid_output_rgb16"] + if isinstance(artifacts, list) + else [] + ) + composite = receipt_document.get("composite") + if ( + len(output_artifacts) != 1 + or output_artifacts[0].get("raw_sha256") != result.hybrid_receipt_output_rgb_sha256 + or not isinstance(composite, dict) + or composite.get("hybrid_rgb16_raw_sha256") != result.hybrid_receipt_output_rgb_sha256 + ): + raise ValueError("hybrid receipt output binding changed") + mask_artifacts = ( + [artifact for artifact in artifacts if isinstance(artifact, dict) and artifact.get("role") == "synthesis_mask_png"] + if isinstance(artifacts, list) + else [] + ) + routed_u8 = np.ascontiguousarray(routed_native_mask.astype(np.uint8) * np.uint8(255)) + if ( + len(mask_artifacts) != 1 + or mask_artifacts[0].get("file_sha256") != result.routed_native_synthesis_mask_sha256 + or mask_artifacts[0].get("raw_sha256") != hashlib.sha256(memoryview(routed_u8).cast("B")).hexdigest() + or mask_artifacts[0].get("shape") != list(routed_u8.shape) + or mask_artifacts[0].get("dtype") != "|u1" + ): + raise ValueError("hybrid receipt routed mask binding changed") + + +def _retain_hybrid_repair_evidence( + base_path: str, + acquisition: roll_repair.RepairAcquisition, + result: roll_repair.RepairResult, +) -> dict[str, dict[str, object]]: + """Retain verbatim verified native evidence beside user-facing output.""" + + evidence_root = os.path.join( + os.path.dirname(base_path) or ".", + ".negpy-dice-hybrid", + ) + evidence_directory = os.path.join(evidence_root, result.hybrid_receipt_sha256) + if any(os.path.lexists(path) and os.path.islink(path) for path in (evidence_root, evidence_directory)): + raise OSError("hybrid evidence path is a symlink") + _prepare_evidence_directory(evidence_directory) + receipt_path = _atomic_write_bytes( + os.path.join(evidence_directory, "hybrid-receipt.json"), + result.hybrid_receipt, + ) + native_mask_path = _atomic_write_bytes( + os.path.join( + evidence_directory, + "synth-mask-applied-scanner-native.png", + ), + result.native_synthesis_mask_png, + ) + routed_native_mask_path = _atomic_write_bytes( + os.path.join( + evidence_directory, + "synth-mask-routed-scanner-native.png", + ), + result.routed_native_synthesis_mask_png, + ) + binding = { + "acquisition": { + "acquisition_id": acquisition.acquisition_id, + "capture_attempt_id": acquisition.capture_attempt_id, + "evidence_sha256": acquisition.evidence_sha256, + "ir_validity_sha256": acquisition.ir_validity_sha256, + "main_rgbi_sha256": acquisition.main_rgbi_sha256, + "prepass_rgbi_sha256": acquisition.prepass_rgbi_sha256, + "reservation_id": acquisition.reservation_id, + "slot": acquisition.slot, + "storage_transform": acquisition.storage_transform, + }, + "hybrid_receipt_sha256": result.hybrid_receipt_sha256, + "hybrid_receipt_output_rgb_sha256": (result.hybrid_receipt_output_rgb_sha256), + "native_output_rgb_sha256": result.native_output_rgb_sha256, + "native_synthesis_mask_sha256": result.native_synthesis_mask_sha256, + "routed_native_synthesis_mask_sha256": (result.routed_native_synthesis_mask_sha256), + "provenance_class": result.hybrid_provenance_class, + "schema": "negpy.dice-hybrid-retained-evidence-v2", + "storage_output_rgb_sha256": result.storage_output_rgb_sha256, + "storage_synthesis_mask_sha256": result.storage_synthesis_mask_sha256, + "synthesis_mask_transform": result.synthesis_mask_transform, + } + binding_bytes = json.dumps( + binding, + sort_keys=True, + separators=(",", ":"), + ensure_ascii=False, + allow_nan=False, + ).encode("utf-8") + binding_path = _atomic_write_bytes( + os.path.join(evidence_directory, "negpy-binding.json"), + binding_bytes, + ) + return { + "binding": { + "bytes": len(binding_bytes), + "path": binding_path, + "sha256": hashlib.sha256(binding_bytes).hexdigest(), + }, + "native_mask": { + "bytes": len(result.native_synthesis_mask_png), + "path": native_mask_path, + "sha256": result.native_synthesis_mask_sha256, + "shape": list(result.native_synthesis_mask_shape), + }, + "routed_native_mask": { + "bytes": len(result.routed_native_synthesis_mask_png), + "path": routed_native_mask_path, + "sha256": result.routed_native_synthesis_mask_sha256, + "shape": list(result.routed_native_synthesis_mask_shape), + }, + "receipt": { + "bytes": len(result.hybrid_receipt), + "path": receipt_path, + "sha256": result.hybrid_receipt_sha256, + }, + } + + +def _ensure_uint16(array: np.ndarray) -> np.ndarray: + return array if array.dtype == np.uint16 else array.astype(np.uint16) + + +def _atomic_write_tiff(path: str, array: np.ndarray, *, photometric: str, iccprofile: bytes | None = None) -> str: + """Write `array` to `path` via a temp file + rename, matching the + atomic-write convention `negpy.services.scanning.writer` already uses + for the plain scan path.""" + final_path = path + transaction = _ACTIVE_OUTPUT_TRANSACTION.get() + if transaction is not None: + path = transaction.stage_path(path) + fd, tmp_path = tempfile.mkstemp(suffix=".tif", dir=os.path.dirname(path) or ".") + os.close(fd) + try: + if iccprofile is None: + tifffile.imwrite(tmp_path, _ensure_uint16(array), photometric=photometric, compression="lzw") + else: + tifffile.imwrite( + tmp_path, + _ensure_uint16(array), + photometric=photometric, + compression="lzw", + iccprofile=iccprofile, + ) + sync_descriptor = os.open(tmp_path, os.O_RDONLY) + try: + os.fsync(sync_descriptor) + finally: + os.close(sync_descriptor) + os.replace(tmp_path, path) + except Exception: + if os.path.exists(tmp_path): + os.unlink(tmp_path) + raise + return final_path + + +def _atomic_write_json(path: str, payload: dict) -> str: + final_path = path + transaction = _ACTIVE_OUTPUT_TRANSACTION.get() + if transaction is not None: + path = transaction.stage_path(path) + tmp_path = None + try: + with tempfile.NamedTemporaryFile("w", dir=os.path.dirname(path) or ".", delete=False, suffix=".part", encoding="utf-8") as tmp: + tmp_path = tmp.name + json.dump(payload, tmp, indent=2, allow_nan=False) + tmp.flush() + os.fsync(tmp.fileno()) + os.replace(tmp_path, path) + except Exception: + if tmp_path is not None and os.path.exists(tmp_path): + os.unlink(tmp_path) + raise + return final_path + + +def _atomic_write_bytes(path: str, payload: bytes) -> str: + final_path = path + transaction = _ACTIVE_OUTPUT_TRANSACTION.get() + if transaction is not None: + path = transaction.stage_path(path) + fd, tmp_path = tempfile.mkstemp(suffix=".part", dir=os.path.dirname(path) or ".") + try: + with os.fdopen(fd, "wb") as stream: + stream.write(payload) + stream.flush() + os.fsync(stream.fileno()) + os.replace(tmp_path, path) + except Exception: + if os.path.exists(tmp_path): + os.unlink(tmp_path) + raise + return final_path + + +def _verify_exact_positive_tiff( + path: str, + *, + expected_rgb: np.ndarray, + expected_icc: bytes, +) -> dict[str, object]: + """Reopen the staged exact-positive TIFF and bind its real container.""" + + transaction = _ACTIVE_OUTPUT_TRANSACTION.get() + actual_path = transaction.staged_path(path) if transaction is not None else path + before = os.lstat(actual_path) + if stat.S_ISLNK(before.st_mode) or not stat.S_ISREG(before.st_mode): + raise roll_exact_color.ExactColorIntegrityError("exact-positive TIFF is not a regular file") + flags = os.O_RDONLY + if hasattr(os, "O_CLOEXEC"): + flags |= os.O_CLOEXEC + if hasattr(os, "O_NOFOLLOW"): + flags |= os.O_NOFOLLOW + if hasattr(os, "O_NONBLOCK"): + flags |= os.O_NONBLOCK + descriptor = os.open(actual_path, flags) + file_digest = hashlib.sha256() + byte_count = 0 + try: + opened = os.fstat(descriptor) + + def identity(metadata: os.stat_result) -> tuple[int, int, int, int, int]: + return ( + metadata.st_dev, + metadata.st_ino, + metadata.st_size, + metadata.st_mtime_ns, + metadata.st_ctime_ns, + ) + + if not stat.S_ISREG(opened.st_mode) or identity(opened) != identity(before): + raise roll_exact_color.ExactColorIntegrityError("exact-positive TIFF changed while it was opened") + while True: + chunk = os.read(descriptor, 1024 * 1024) + if not chunk: + break + file_digest.update(chunk) + byte_count += len(chunk) + after_read = os.fstat(descriptor) + if identity(after_read) != identity(opened) or byte_count != opened.st_size: + raise roll_exact_color.ExactColorIntegrityError("exact-positive TIFF size changed while it was verified") + + expected = np.asarray(expected_rgb) + expected_pixel_sha256 = roll_exact_color.rgb16_content_sha256(expected) + try: + # Decode through a duplicate of the descriptor whose bytes were + # just hashed. Reopening the pathname here would let a swap bind + # one file's hash to another file's pixels and ICC profile. + os.lseek(descriptor, 0, os.SEEK_SET) + with os.fdopen(os.dup(descriptor), "rb") as stream: + # ``fdopen`` exposes the integer descriptor as ``.name``; + # tifffile normalizes that as a path unless given a benign + # display hint. The stream remains the descriptor-pinned + # source of every decoded byte. + with tifffile.TiffFile( + stream, + name=os.path.basename(actual_path), + ) as image: + if len(image.pages) != 1: + raise roll_exact_color.ExactColorIntegrityError("exact-positive TIFF must contain exactly one page") + page = image.pages[0] + decoded = np.asarray(page.asarray()) + samples_per_pixel = int(page.samplesperpixel) + bits_value = page.tags[258].value + if isinstance(bits_value, (tuple, list)): + bits = tuple(int(value) for value in bits_value) + else: + bits = (int(bits_value),) * samples_per_pixel + photometric = int(page.photometric) + planar = int(page.planarconfig) + orientation_tag = page.tags.get(274) + orientation = 1 if orientation_tag is None else int(orientation_tag.value) + icc_tag = page.tags.get(34675) + icc = None if icc_tag is None else bytes(icc_tag.value) + except roll_exact_color.ExactColorIntegrityError: + raise + except Exception as error: + raise roll_exact_color.ExactColorIntegrityError(f"cannot verify exact-positive TIFF: {error}") from error + after_decode = os.fstat(descriptor) + finally: + os.close(descriptor) + after_path = os.lstat(actual_path) + if identity(after_decode) != identity(opened) or identity(after_path) != identity(opened) or not stat.S_ISREG(after_path.st_mode): + raise roll_exact_color.ExactColorIntegrityError("exact-positive TIFF changed while it was verified") + if ( + decoded.dtype != np.uint16 + or decoded.shape != expected.shape + or expected.shape[-1:] != (3,) + or samples_per_pixel != 3 + or photometric != 2 + or planar != 1 + or orientation != 1 + or bits != (16, 16, 16) + ): + raise roll_exact_color.ExactColorIntegrityError("exact-positive TIFF layout is not one contiguous 16-bit RGB image") + pixel_sha256 = roll_exact_color.rgb16_content_sha256(decoded) + if pixel_sha256 != expected_pixel_sha256 or not np.array_equal(decoded, expected): + raise roll_exact_color.ExactColorIntegrityError("exact-positive TIFF pixels changed during encoding") + if ( + icc is None + or len(icc) != len(expected_icc) + or icc != expected_icc + or hashlib.sha256(icc).hexdigest() != hashlib.sha256(expected_icc).hexdigest() + ): + raise roll_exact_color.ExactColorIntegrityError("exact-positive TIFF Nikon ICC tag is missing or changed") + return { + "bits_per_sample": list(bits), + "bytes": byte_count, + "dtype": str(decoded.dtype), + "file_sha256": file_digest.hexdigest(), + "icc_bytes": len(icc), + "icc_sha256": hashlib.sha256(icc).hexdigest(), + "page_count": 1, + "orientation": "top-left", + "photometric": "rgb", + "planar_config": "contiguous", + "pixel_sha256": pixel_sha256, + "samples_per_pixel": 3, + "shape": list(decoded.shape), + } + + +def _build_native_receipt_from_frame( + frame: object, +) -> roll_exact_color.NativeValidatedBuilderReceipt: + evidence = getattr(frame, "nikon_exact_builder_evidence", None) + if evidence is None: + if getattr(frame, "nikon_density_evidence", None) is not None or getattr(frame, "nikon_density_ownership", None) is not None: + raise roll_exact_color.ExactColorUnavailable("frame-bound Nikon density evidence has no native builder evidence") + raise roll_exact_color.ExactColorUnavailable( + "validated Stage-3 builder receipt is not supplied and frame has no native builder evidence" + ) + evidence = roll_native_builder.adapt_native_builder_evidence(evidence) + if evidence.slot != getattr(frame, "slot", None): + raise roll_exact_color.ExactColorUnavailable("frame native builder evidence belongs to a different slot") + public_receipt = getattr(frame, "receipt", None) + ownership = getattr(frame, "nikon_density_ownership", None) + receipt_ownership = None if public_receipt is None else getattr(public_receipt, "nikon_density_ownership", None) + if ownership is None: + raise roll_exact_color.ExactColorUnavailable("Nikon density frame ownership receipt is missing") + if receipt_ownership is None: + raise roll_exact_color.ExactColorUnavailable("public frame receipt has no Nikon density ownership") + frame_density_evidence = getattr(frame, "nikon_density_evidence", None) + if frame_density_evidence is None: + raise roll_exact_color.ExactColorUnavailable("frame-bound Nikon density evidence is missing") + ownership_payload = _canonical_component_payload(ownership, label="Nikon density frame ownership") + receipt_ownership_payload = _canonical_component_payload( + receipt_ownership, + label="public-frame Nikon density ownership", + ) + if receipt_ownership_payload != ownership_payload: + raise roll_exact_color.ExactColorUnavailable("frame and public receipt disagree on Nikon density ownership") + density_payload = _canonical_component_payload(frame_density_evidence, label="frame-bound Nikon density evidence") + if ownership_payload != evidence.frame_ownership_receipt: + raise roll_exact_color.ExactColorUnavailable("Nikon density frame ownership does not match the native builder evidence") + if density_payload != evidence.density_evidence_receipt: + raise roll_exact_color.ExactColorUnavailable("frame-bound Nikon density evidence does not match the native builder evidence") + validator = getattr(ownership, "validate_evidence", None) + if callable(validator): + try: + validator(frame_density_evidence) + except (TypeError, ValueError) as error: + raise roll_exact_color.ExactColorUnavailable(f"Nikon density frame ownership is invalid: {error}") from error + return roll_native_builder.build_native_builder_receipt(evidence) + + +def _canonical_component_payload(component: object, *, label: str) -> bytes: + if type(component) is dict: + payload = component + else: + producer = getattr(component, "to_dict", None) + if not callable(producer): + producer = getattr(component, "to_payload", None) + if callable(producer): + payload = producer() + else: + raise roll_exact_color.ExactColorUnavailable(f"{label} has no canonical receipt payload") + if type(payload) is not dict: + raise roll_exact_color.ExactColorUnavailable(f"{label} payload is malformed") + try: + return json.dumps( + payload, + sort_keys=True, + separators=(",", ":"), + ensure_ascii=False, + allow_nan=False, + ).encode("utf-8") + except (TypeError, ValueError, RecursionError) as error: + raise roll_exact_color.ExactColorUnavailable(f"{label} payload is not canonical JSON: {error}") from error + + +def _retain_builder_evidence( + base_path: str, + receipt: roll_exact_color.BuilderReceipt, +) -> dict[str, object]: + if isinstance(receipt, roll_exact_color.ValidatedBuilderReceipt): + return _retain_stage3_replay_evidence(base_path, receipt) + if isinstance(receipt, roll_exact_color.NativeValidatedBuilderReceipt): + return _retain_native_builder_evidence(base_path, receipt) + raise roll_exact_color.ExactColorIntegrityError("builder receipt has an invalid type") + + +def _retain_stage3_replay_evidence( + base_path: str, + receipt: roll_exact_color.ValidatedBuilderReceipt, +) -> dict[str, object]: + """Atomically retain the raw replay inputs consumed by exact Tier 3.""" + + roll_exact_color.builder_receipt_payload(receipt) + evidence_root = os.path.join(os.path.dirname(base_path) or ".", ".negpy-stage3-replay") + evidence_directory = os.path.join(evidence_root, receipt.sha256) + try: + if any(os.path.lexists(path) and os.path.islink(path) for path in (evidence_root, evidence_directory)): + raise OSError("evidence path is a symlink") + _prepare_evidence_directory(evidence_directory) + report_path = _atomic_write_bytes( + os.path.join(evidence_directory, "stage3-validation.json"), + receipt.stage3_receipt, + ) + except OSError as error: + raise roll_exact_color.ExactColorUnavailable(f"cannot retain Stage-3 replay evidence: {error}") from error + report_row = { + "bytes": len(receipt.stage3_receipt), + "path": report_path, + "sha256": receipt.stage3_receipt_sha256, + } + lut_rows = [] + for channel, blob, digest in zip( + ("r", "g", "b"), + receipt.pre_f_luts, + receipt.pre_f_lut_sha256, + strict=True, + ): + try: + path = _atomic_write_bytes(os.path.join(evidence_directory, f"builder-preF-{channel}.bin"), blob) + except OSError as error: + raise roll_exact_color.ExactColorUnavailable(f"cannot retain Stage-3 replay evidence: {error}") from error + lut_rows.append( + { + "bytes": len(blob), + "channel": channel, + "path": path, + "sha256": digest, + } + ) + return { + "native_per_acquisition_builder": False, + "pre_f_luts": lut_rows, + "scope": roll_exact_color.STAGE3_REPLAY_SCOPE, + "stage3_report": report_row, + } + + +def _retain_native_builder_evidence( + base_path: str, + receipt: roll_exact_color.NativeValidatedBuilderReceipt, +) -> dict[str, object]: + """Atomically retain the native evidence snapshot and derived pre-F LUTs.""" + + roll_exact_color.builder_receipt_payload(receipt) + evidence_root = os.path.join(os.path.dirname(base_path) or ".", ".negpy-native-builder") + evidence_directory = os.path.join(evidence_root, receipt.sha256) + try: + if any(os.path.lexists(path) and os.path.islink(path) for path in (evidence_root, evidence_directory)): + raise OSError("evidence path is a symlink") + _prepare_evidence_directory(evidence_directory) + builder_receipt_path = _atomic_write_bytes( + os.path.join(evidence_directory, "native-builder-receipt.json"), + receipt.payload, + ) + evidence_path = _atomic_write_bytes( + os.path.join(evidence_directory, "native-builder-evidence.json"), + receipt.evidence_payload, + ) + ownership_path = _atomic_write_bytes( + os.path.join(evidence_directory, "nikon-density-frame-ownership.json"), + receipt.frame_ownership_receipt, + ) + density_path = _atomic_write_bytes( + os.path.join(evidence_directory, "nikon-density-evidence.json"), + receipt.density_evidence_receipt, + ) + analyzer_path = _atomic_write_bytes( + os.path.join(evidence_directory, "analyzer-rgb-u16le.bin"), + receipt.analyzer_rgb, + ) + except OSError as error: + raise roll_exact_color.ExactColorUnavailable(f"cannot retain native builder evidence: {error}") from error + lut_rows = [] + for channel, blob, digest in zip(("r", "g", "b"), receipt.pre_f_luts, receipt.pre_f_lut_sha256, strict=True): + try: + path = _atomic_write_bytes(os.path.join(evidence_directory, f"builder-preF-{channel}.bin"), blob) + except OSError as error: + raise roll_exact_color.ExactColorUnavailable(f"cannot retain native builder evidence: {error}") from error + lut_rows.append({"bytes": len(blob), "channel": channel, "path": path, "sha256": digest}) + return { + "builder_receipt": { + "bytes": len(receipt.payload), + "path": builder_receipt_path, + "sha256": receipt.sha256, + }, + "analyzer_rgb": { + "bytes": len(receipt.analyzer_rgb), + "path": analyzer_path, + "sha256": receipt.analyzer_rgb_sha256, + "shape": list(receipt.analyzer_shape), + }, + "evidence_receipt": { + "bytes": len(receipt.evidence_payload), + "path": evidence_path, + "sha256": receipt.evidence_sha256, + }, + "frame_ownership_receipt": { + "bytes": len(receipt.frame_ownership_receipt), + "path": ownership_path, + "sha256": receipt.frame_ownership_receipt_sha256, + }, + "density_evidence_receipt": { + "bytes": len(receipt.density_evidence_receipt), + "path": density_path, + "sha256": receipt.density_evidence_receipt_sha256, + }, + "native_per_acquisition_builder": True, + "pre_f_luts": lut_rows, + "scope": roll_exact_color.NATIVE_BUILDER_SCOPE, + } diff --git a/pyproject.toml b/pyproject.toml index 368661ee..b7819c9f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -55,6 +55,19 @@ camera = [ # Camera Scanning tab with them — are macOS and Linux only. "gphoto2>=2.5 ; sys_platform != 'win32'", ] +coolscan-roll = [ + # Direct-USB roll scanning on a Nikon Coolscan LS-5000 with an SA-21/SA-30 feeder. + # Pin the reviewed color-negative density/DICE/direct-USB implementation + # until its first tagged release. + # No SANE dependency for this workflow; see docs/COOLSCANPY_ROLL_SCANNING.md. + "coolscanpy @ git+https://github.com/rohanpandula/coolscanpy.git@970d18e", +] +fauxice = [ + "portable-digital-ice @ https://github.com/rohanpandula/digital-fauxice/releases/download/v0.3.1/portable_digital_ice-0.3.1-py3-none-any.whl", +] +fauxice-hybrid = [ + "fauxce-hybrid @ https://github.com/rohanpandula/digital-fauxice/releases/download/v0.3.1/fauxce_hybrid-0.3.1-py3-none-any.whl", +] [project.urls] Homepage = "https://github.com/marcinz606/NegPy" @@ -66,8 +79,15 @@ Issues = "https://github.com/marcinz606/NegPy/issues" requires = ["setuptools~=80.10.2"] build-backend = "setuptools.build_meta" -[tool.setuptools] -packages = ["negpy"] +[tool.setuptools.packages.find] +include = ["negpy*"] + +[tool.setuptools.package-data] +negpy = [ + "assets/portable_builder/*.bin", + "assets/portable_cms/*.bin", + "assets/portable_cms/*.json", +] [tool.setuptools.dynamic] version = { file = "VERSION" } @@ -97,6 +117,8 @@ exclude = [ "dist", "desktop/bin", "build", + # Vendored byte-identical verified oracle; linting would change its pinned SHA. + "negpy/services/roll/portable_oracle_evaluator.py", ] [tool.coverage.run] diff --git a/tests/repair/__init__.py b/tests/repair/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/repair/test_fauxice_hybrid_runner.py b/tests/repair/test_fauxice_hybrid_runner.py new file mode 100644 index 00000000..13dfe90e --- /dev/null +++ b/tests/repair/test_fauxice_hybrid_runner.py @@ -0,0 +1,1086 @@ +"""Tests for the fauxce-hybrid subprocess bridge, with a fake CLI runner. + +No real ``fauxce-hybrid`` install is required: ``run_hybrid_repair`` never +imports it, it only shells out to the console script named on +``HybridRuntimeConfig.executable``. Every test here supplies its own fake +``runner`` callable in place of ``subprocess.run`` and writes the output +files the real CLI would have written, so these tests exercise the argv +construction and output-reading contract without a pinned IOPaint runtime. +""" + +import hashlib +import io +import json +import os +import subprocess +import sys +import threading +import time +import types +from pathlib import Path + +import numpy as np +import pytest +from PIL import Image + +from negpy.services.repair import fauxice_hybrid_runner as hybrid_runner +from negpy.services.repair.fauxice_hybrid_runner import ( + HybridRunCancelled, + HybridRunError, + HybridRuntimeConfig, + run_hybrid_repair, +) + +EXPECTED_CORE_SOURCE_SHA256 = "c" * 64 +EXPECTED_HYBRID_SOURCE_SHA256 = "d" * 64 + + +def _runtime_config(tmp_path: Path) -> HybridRuntimeConfig: + hybrid_python = tmp_path / "hybrid-venv" / "bin" / "python" + executable = tmp_path / "hybrid-venv" / "bin" / "fauxce-hybrid" + iopaint_python = tmp_path / "iopaint-venv" / "bin" / "python" + iopaint_executable = tmp_path / "iopaint-venv" / "bin" / "iopaint" + model_dir = tmp_path / "lama-models" + model_weights = model_dir / "torch" / "hub" / "checkpoints" / "big-lama.pt" + for path in (hybrid_python, executable, iopaint_python, iopaint_executable): + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(b"#!/bin/sh\nexit 0\n") + path.chmod(0o700) + model_weights.parent.mkdir(parents=True, exist_ok=True) + model_weights.write_bytes(b"big-lama-weights") + return HybridRuntimeConfig( + hybrid_python=hybrid_python, + executable=executable, + core_source_manifest_sha256=EXPECTED_CORE_SOURCE_SHA256, + hybrid_source_manifest_sha256=EXPECTED_HYBRID_SOURCE_SHA256, + iopaint_python=iopaint_python, + iopaint_executable=iopaint_executable, + iopaint_source_manifest_sha256=hashlib.sha256(b"iopaint-source").hexdigest(), + model_dir=model_dir, + model_weights=model_weights, + model_weights_sha256=hashlib.sha256(b"big-lama-weights").hexdigest(), + ) + + +def _rgbi(height: int, width: int, base: int) -> np.ndarray: + return np.full((height, width, 4), base, dtype=np.uint16) + + +def _receipt( + *, + requested: str = "cpu", + used: str = "cpu", + fraction: float = 0.01, + at_floor_pixels: int | None = None, +) -> dict: + return { + "at_floor_pixels": at_floor_pixels, + "fraction": fraction, + "requested": requested, + "used": used, + } + + +def _arg(argv: list[str], flag: str) -> str: + return argv[argv.index(flag) + 1] + + +def _raw_sha256(array: np.ndarray) -> str: + canonical = np.array(array, dtype=" None: + out_dir.mkdir(parents=True) + np.save(out_dir / "output-hybrid.rgb16.npy", hybrid_rgb16, allow_pickle=False) + main = np.load(_arg(argv, "--main"), allow_pickle=False) + prepass = np.load(_arg(argv, "--prepass"), allow_pickle=False) + manifest_bytes = Path(_arg(argv, "--acquisition-manifest")).read_bytes() + requested_fraction = float(receipt_options["fraction"]) + synthesis_pixels = min( + int(round(requested_fraction * main.shape[0] * main.shape[1])), + main.shape[0] * main.shape[1], + ) + mask = np.zeros(main.shape[:2], dtype=np.uint8) + mask.reshape(-1)[:synthesis_pixels] = 255 + mask_path = out_dir / "synth-mask.png" + Image.fromarray(mask, mode="L").save(mask_path, format="PNG") + mask_bytes = mask_path.read_bytes() + fraction = synthesis_pixels / mask.size + output_hash = _raw_sha256(hybrid_rgb16) + receipt = { + "artifacts": [ + { + "dtype": " None: + self._hybrid_rgb16 = hybrid_rgb16 + self._receipt = receipt + self._returncode = returncode + self.calls: list[list[str]] = [] + + def __call__(self, argv, **kwargs) -> subprocess.CompletedProcess: + self.calls.append(list(argv)) + out_index = argv.index("--out") + out_dir = Path(argv[out_index + 1]) + if self._returncode == 0: + _write_success_outputs( + out_dir, + hybrid_rgb16=self._hybrid_rgb16, + receipt_options=self._receipt, + argv=list(argv), + ) + return subprocess.CompletedProcess(argv, 0, stdout="ok", stderr="") + return subprocess.CompletedProcess(argv, self._returncode, stdout="", stderr="synthetic failure") + + +class TestRunHybridRepairSuccess: + def test_reports_hybrid_and_verification_progress(self, tmp_path: Path) -> None: + runner = _RecordingRunner( + hybrid_rgb16=np.zeros((2, 2, 3), dtype=np.uint16), + receipt=_receipt(), + ) + seen: list[float] = [] + + run_hybrid_repair( + _rgbi(2, 2, 1000), + _rgbi(2, 2, 500), + same_frame_id="progress-bound", + backend="cpu", + runtime=_runtime_config(tmp_path), + scratch_dir=tmp_path, + runner=runner, + receipt_verifier=_test_receipt_verifier, + progress=seen.append, + ) + + assert seen == [0.0, 0.8, 0.9, 1.0] + + @pytest.mark.parametrize("boundary", [0.9, 1.0]) + def test_cancellation_at_late_progress_boundary_never_returns_a_result( + self, + tmp_path: Path, + boundary: float, + ) -> None: + runner = _RecordingRunner( + hybrid_rgb16=np.zeros((2, 2, 3), dtype=np.uint16), + receipt=_receipt(), + ) + cancel = threading.Event() + + def progress(value: float) -> None: + if value == boundary: + cancel.set() + + with pytest.raises(HybridRunCancelled, match="cancelled"): + run_hybrid_repair( + _rgbi(2, 2, 1000), + _rgbi(2, 2, 500), + same_frame_id=f"late-cancel-{boundary}", + backend="cpu", + runtime=_runtime_config(tmp_path), + scratch_dir=tmp_path, + runner=runner, + receipt_verifier=_test_receipt_verifier, + progress=progress, + cancel=cancel, + ) + + def test_subprocess_environment_ignores_python_injection_variables( + self, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + ) -> None: + monkeypatch.setenv("PYTHONHOME", "/attacker/python-home") + monkeypatch.setenv("PYTHONPATH", "/attacker/imports") + monkeypatch.setenv("DYLD_INSERT_LIBRARIES", "/attacker/inject.dylib") + monkeypatch.setenv("LD_PRELOAD", "/attacker/inject.so") + seen_environment: dict[str, str] = {} + hybrid_rgb16 = np.zeros((2, 2, 3), dtype=np.uint16) + + def runner(argv, **kwargs): + seen_environment.update(kwargs["env"]) + out_dir = Path(_arg(list(argv), "--out")) + _write_success_outputs( + out_dir, + hybrid_rgb16=hybrid_rgb16, + receipt_options=_receipt(), + argv=list(argv), + ) + return subprocess.CompletedProcess(argv, 0, stdout="", stderr="") + + run_hybrid_repair( + _rgbi(2, 2, 1000), + _rgbi(2, 2, 500), + same_frame_id="sanitized-environment", + backend="cpu", + runtime=_runtime_config(tmp_path), + scratch_dir=tmp_path, + runner=runner, + receipt_verifier=_test_receipt_verifier, + ) + + assert "PYTHONHOME" not in seen_environment + assert "PYTHONPATH" not in seen_environment + assert "DYLD_INSERT_LIBRARIES" not in seen_environment + assert "LD_PRELOAD" not in seen_environment + assert seen_environment["PYTHONHASHSEED"] == "0" + assert seen_environment["PYTHONNOUSERSITE"] == "1" + assert seen_environment["LC_ALL"] == "C" + + def test_resolves_scratch_to_link_free_real_path_before_cli(self, tmp_path: Path) -> None: + real_scratch = tmp_path / "real-scratch" + real_scratch.mkdir() + linked_scratch = tmp_path / "linked-scratch" + linked_scratch.symlink_to(real_scratch, target_is_directory=True) + hybrid_rgb16 = np.zeros((2, 2, 3), dtype=np.uint16) + + def runner(argv, **kwargs): + out_dir = Path(_arg(list(argv), "--out")) + assert out_dir.parent.parent == real_scratch.resolve(strict=True) + assert out_dir.parent.name.startswith("negpy-hybrid-run-") + _write_success_outputs( + out_dir, + hybrid_rgb16=hybrid_rgb16, + receipt_options=_receipt(), + argv=list(argv), + ) + return subprocess.CompletedProcess(argv, 0, stdout="", stderr="") + + run_hybrid_repair( + _rgbi(2, 2, 1000), + _rgbi(2, 2, 500), + same_frame_id="link-free-scratch", + backend="cpu", + runtime=_runtime_config(tmp_path / "runtime"), + scratch_dir=linked_scratch, + runner=runner, + receipt_verifier=_test_receipt_verifier, + ) + + def test_default_receipt_verification_runs_in_external_hybrid_python(self, tmp_path: Path) -> None: + hybrid_rgb16 = np.full((3, 4, 3), 4242, dtype=np.uint16) + runner = _RecordingRunner(hybrid_rgb16=hybrid_rgb16, receipt=_receipt()) + verification_calls: list[list[str]] = [] + + def verification_runner(argv, **kwargs): + verification_calls.append(list(argv)) + receipt_path = Path(argv[-3]) + attestation = { + "model_weights_rehashed": True, + "receipt_sha256": hashlib.sha256(receipt_path.read_bytes()).hexdigest(), + "schema": "negpy.external-fauxce-receipt-verification-v1", + } + return subprocess.CompletedProcess( + argv, + 0, + stdout=json.dumps(attestation, sort_keys=True, separators=(",", ":")), + stderr="", + ) + + runtime = _runtime_config(tmp_path) + result = run_hybrid_repair( + _rgbi(3, 4, 1000), + _rgbi(2, 2, 500), + same_frame_id="frame-external-verifier", + backend="cpu", + runtime=runtime, + scratch_dir=tmp_path, + runner=runner, + verification_runner=verification_runner, + ) + + assert result.receipt_sha256 + assert len(verification_calls) == 1 + assert verification_calls[0][0] == str(runtime.hybrid_python) + assert "fauxce_hybrid.receipts" in verification_calls[0][3] + + def test_reads_hybrid_output_and_mask(self, tmp_path: Path) -> None: + hybrid_rgb16 = np.full((4, 4, 3), 4242, dtype=np.uint16) + runner = _RecordingRunner(hybrid_rgb16=hybrid_rgb16, receipt=_receipt()) + + result = run_hybrid_repair( + _rgbi(4, 4, 1000), + _rgbi(4, 4, 500), + same_frame_id="frame-001", + backend="cpu", + runtime=_runtime_config(tmp_path), + scratch_dir=tmp_path, + runner=runner, + receipt_verifier=_test_receipt_verifier, + ) + + np.testing.assert_array_equal(result.hybrid_rgb16, hybrid_rgb16) + assert result.synth_mask_png.startswith(b"\x89PNG\r\n\x1a\n") + assert result.synth_mask_sha256 == hashlib.sha256(result.synth_mask_png).hexdigest() + assert result.receipt_sha256 == hashlib.sha256(result.receipt).hexdigest() + assert result.provenance_class == "caller_asserted_bare_npy" + + def test_receipt_fields_are_extracted(self, tmp_path: Path) -> None: + hybrid_rgb16 = np.zeros((10, 10, 3), dtype=np.uint16) + receipt = _receipt(requested="auto", used="cpu-fast", fraction=0.01) + runner = _RecordingRunner(hybrid_rgb16=hybrid_rgb16, receipt=receipt) + + result = run_hybrid_repair( + _rgbi(10, 10, 1000), + _rgbi(5, 5, 500), + same_frame_id="frame-002", + backend="auto", + runtime=_runtime_config(tmp_path), + scratch_dir=tmp_path, + runner=runner, + receipt_verifier=_test_receipt_verifier, + ) + + assert result.engine_version == "0.3.0" + assert result.backend_requested == "auto" + assert result.backend_used == "cpu-fast" + assert result.synthesis_fraction == 0.01 + assert result.routing_counts == { + "final_regions": 1, + "synthesis_pixels": 1, + "frame_pixels": 100, + "at_floor_pixels": 1, + } + + def test_accepts_synthesis_context_outside_at_floor_evidence( + self, + tmp_path: Path, + ) -> None: + hybrid_rgb16 = np.zeros((2, 2, 3), dtype=np.uint16) + runner = _RecordingRunner( + hybrid_rgb16=hybrid_rgb16, + receipt=_receipt(fraction=0.5, at_floor_pixels=1), + ) + + result = run_hybrid_repair( + _rgbi(2, 2, 1000), + _rgbi(2, 2, 500), + same_frame_id="context-halo", + backend="cpu", + runtime=_runtime_config(tmp_path), + scratch_dir=tmp_path, + runner=runner, + receipt_verifier=_test_receipt_verifier, + ) + + assert result.routing_counts == { + "final_regions": 1, + "synthesis_pixels": 2, + "frame_pixels": 4, + "at_floor_pixels": 1, + } + + def test_rejects_receipt_that_omits_routing_counts(self, tmp_path: Path) -> None: + hybrid_rgb16 = np.zeros((2, 2, 3), dtype=np.uint16) + base_runner = _RecordingRunner(hybrid_rgb16=hybrid_rgb16, receipt=_receipt()) + + def runner(argv, **kwargs): + completed = base_runner(argv, **kwargs) + receipt_path = Path(argv[argv.index("--out") + 1]) / "hybrid-receipt.json" + document = json.loads(receipt_path.read_text(encoding="utf-8")) + del document["routing"]["counts"]["frame_pixels"] + receipt_path.write_bytes(json.dumps(document, sort_keys=True, separators=(",", ":")).encode() + b"\n") + return completed + + with pytest.raises(HybridRunError, match="routing counts"): + run_hybrid_repair( + _rgbi(2, 2, 1000), + _rgbi(2, 2, 500), + same_frame_id="frame-003", + backend="cpu", + runtime=_runtime_config(tmp_path), + scratch_dir=tmp_path, + runner=runner, + receipt_verifier=_test_receipt_verifier, + ) + + def test_argv_carries_the_documented_cli_flags(self, tmp_path: Path) -> None: + hybrid_rgb16 = np.zeros((2, 2, 3), dtype=np.uint16) + runner = _RecordingRunner(hybrid_rgb16=hybrid_rgb16, receipt=_receipt()) + runtime = _runtime_config(tmp_path) + + run_hybrid_repair( + _rgbi(2, 2, 1000), + _rgbi(2, 2, 500), + same_frame_id="frame-004", + backend="cpu", + runtime=runtime, + scratch_dir=tmp_path, + runner=runner, + receipt_verifier=_test_receipt_verifier, + ) + + assert len(runner.calls) == 1 + argv = runner.calls[0] + assert argv[0] == str(runtime.executable) + for flag, value in ( + ("--same-frame-id", "frame-004"), + ("--backend", "cpu"), + ("--max-synth-fraction", "0.02"), + ("--iopaint-python", str(runtime.iopaint_python)), + ("--iopaint-executable", str(runtime.iopaint_executable)), + ("--iopaint-source-manifest-sha256", runtime.iopaint_source_manifest_sha256), + ("--model-dir", str(runtime.model_dir)), + ("--model-weights", str(runtime.model_weights)), + ("--model-weights-sha256", runtime.model_weights_sha256), + ("--inpaint-device", "cpu"), + ("--inpaint-threads", "1"), + ("--inpaint-seed", "0"), + ): + assert flag in argv, f"missing {flag}" + assert argv[argv.index(flag) + 1] == value + assert "--assert-focus-exposure-locked" in argv + # A real hybrid run must not pass --no-inpaint, or the CLI would + # never touch the model and every hybrid call would silently + # degrade to routing-only. + assert "--no-inpaint" not in argv + + def test_scratch_dir_out_subdirectory_does_not_preexist(self, tmp_path: Path) -> None: + """The CLI refuses a pre-existing --out directory; this module must create it fresh.""" + hybrid_rgb16 = np.zeros((2, 2, 3), dtype=np.uint16) + + def runner(argv, **kwargs) -> subprocess.CompletedProcess: + out_dir = Path(argv[argv.index("--out") + 1]) + assert not out_dir.exists() + _write_success_outputs( + out_dir, + hybrid_rgb16=hybrid_rgb16, + receipt_options=_receipt(), + argv=list(argv), + ) + return subprocess.CompletedProcess(argv, 0, stdout="", stderr="") + + scratch = tmp_path / "scratch" + scratch.mkdir() + run_hybrid_repair( + _rgbi(2, 2, 1000), + _rgbi(2, 2, 500), + same_frame_id="frame-005", + backend="cpu", + runtime=_runtime_config(tmp_path), + scratch_dir=scratch, + runner=runner, + receipt_verifier=_test_receipt_verifier, + ) + + +class TestRunHybridRepairFailure: + @pytest.mark.skipif(not hasattr(os, "mkfifo"), reason="POSIX FIFO test") + def test_model_hash_regular_to_fifo_swap_never_blocks( + self, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + ) -> None: + weights = tmp_path / "weights.bin" + weights.write_bytes(b"weights") + real_open = hybrid_runner.os.open + swapped = False + + def swap_before_open(path, flags, *args): + nonlocal swapped + if not swapped and Path(path) == weights: + swapped = True + weights.unlink() + os.mkfifo(weights) + return real_open(path, flags, *args) + + monkeypatch.setattr(hybrid_runner.os, "open", swap_before_open) + started = time.monotonic() + with pytest.raises(ValueError, match="changed while it was opened"): + hybrid_runner._stable_regular_sha256(weights, label="model weights") + assert time.monotonic() - started < 1.0 + + @pytest.mark.skipif(not hasattr(os, "mkfifo"), reason="POSIX FIFO test") + def test_result_regular_to_fifo_swap_never_blocks( + self, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + ) -> None: + artifact = tmp_path / "artifact.bin" + artifact.write_bytes(b"artifact") + real_open = hybrid_runner.os.open + swapped = False + + def swap_before_open(path, flags, *args): + nonlocal swapped + if not swapped and Path(path) == artifact: + swapped = True + artifact.unlink() + os.mkfifo(artifact) + return real_open(path, flags, *args) + + monkeypatch.setattr(hybrid_runner.os, "open", swap_before_open) + started = time.monotonic() + with pytest.raises(HybridRunError, match="changed while being opened"): + hybrid_runner._stable_regular_bytes( + artifact, + maximum_bytes=1024, + label="hybrid artifact", + ) + assert time.monotonic() - started < 1.0 + + def test_npy_header_geometry_is_rejected_before_array_allocation( + self, + monkeypatch: pytest.MonkeyPatch, + ) -> None: + stream = io.BytesIO() + np.lib.format.write_array_header_1_0( + stream, + { + "descr": " None: + class HugeHeader: + format = "PNG" + size = (1_000_000, 1_000_000) + + def __enter__(self): + return self + + def __exit__(self, *_args): + return False + + def convert(self, _mode): + pytest.fail("oversized PNG pixels must not be decompressed") + + monkeypatch.setattr( + hybrid_runner.Image, + "open", + lambda *_args, **_kwargs: HugeHeader(), + ) + + with pytest.raises(HybridRunError, match="geometry"): + hybrid_runner._decode_mask_png( + b"synthetic-png-header", + expected_shape=(2, 2), + ) + + def test_external_process_logs_are_bounded(self, tmp_path: Path) -> None: + runtime = _runtime_config(tmp_path) + runtime.executable.write_text( + "#!/bin/sh\nhead -c 17825792 /dev/zero\n", + encoding="utf-8", + ) + runtime.executable.chmod(0o700) + + with pytest.raises(HybridRunError, match="output exceeded"): + run_hybrid_repair( + _rgbi(2, 2, 1000), + _rgbi(2, 2, 500), + same_frame_id="bounded-log", + backend="cpu", + runtime=runtime, + scratch_dir=tmp_path, + timeout_seconds=10, + ) + + def test_cancellation_terminates_blocking_external_process_group(self, tmp_path: Path) -> None: + runtime = _runtime_config(tmp_path) + runtime.executable.write_text("#!/bin/sh\nsleep 30\n", encoding="utf-8") + runtime.executable.chmod(0o700) + cancel = threading.Event() + timer = threading.Timer(0.2, cancel.set) + started = time.monotonic() + timer.start() + try: + with pytest.raises(HybridRunCancelled, match="cancelled"): + run_hybrid_repair( + _rgbi(2, 2, 1000), + _rgbi(2, 2, 500), + same_frame_id="cancel-blocking-process", + backend="cpu", + runtime=runtime, + scratch_dir=tmp_path, + cancel=cancel, + ) + finally: + timer.cancel() + + assert time.monotonic() - started < 5.0 + assert not (tmp_path / "out" / "hybrid-receipt.json").exists() + + def test_cancellation_kills_child_that_ignores_term_after_leader_exits( + self, + tmp_path: Path, + ) -> None: + runtime = _runtime_config(tmp_path) + child_pid_path = tmp_path / "child.pid" + child_program = tmp_path / "ignore-term-child.py" + child_program.write_text( + "\n".join( + ( + "import os", + "from pathlib import Path", + "import signal", + "import sys", + "import time", + "signal.signal(signal.SIGTERM, signal.SIG_IGN)", + "Path(sys.argv[1]).write_text(str(os.getpid()), encoding='ascii')", + "while True:", + " time.sleep(0.1)", + ) + ) + + "\n", + encoding="utf-8", + ) + runtime.executable.write_text( + "\n".join( + ( + "#!/bin/sh", + f"{sys.executable!s} {child_program!s} {child_pid_path!s} &", + "trap 'exit 0' TERM", + "while :; do sleep 1; done", + ) + ) + + "\n", + encoding="utf-8", + ) + runtime.executable.chmod(0o700) + cancel = threading.Event() + + def cancel_after_child_starts() -> None: + deadline = time.monotonic() + 5.0 + while not child_pid_path.exists() and time.monotonic() < deadline: + time.sleep(0.01) + cancel.set() + + canceller = threading.Thread(target=cancel_after_child_starts, daemon=True) + canceller.start() + with pytest.raises(HybridRunCancelled, match="cancelled"): + run_hybrid_repair( + _rgbi(2, 2, 1000), + _rgbi(2, 2, 500), + same_frame_id="cancel-orphan-process", + backend="cpu", + runtime=runtime, + scratch_dir=tmp_path, + cancel=cancel, + ) + canceller.join(timeout=1.0) + assert child_pid_path.exists() + child_pid = int(child_pid_path.read_text(encoding="ascii")) + deadline = time.monotonic() + 3.0 + while time.monotonic() < deadline: + try: + os.kill(child_pid, 0) + except ProcessLookupError: + break + time.sleep(0.05) + else: + pytest.fail(f"hybrid child process {child_pid} survived cancellation") + + @pytest.mark.parametrize( + ("section", "field", "match"), + [ + ("core", "source_manifest_sha256", "core source manifest"), + ( + "generation", + "hybrid_source_manifest_sha256", + "hybrid source manifest", + ), + ], + ) + def test_rejects_independent_source_manifest_mismatch( + self, + tmp_path: Path, + section: str, + field: str, + match: str, + ) -> None: + base = _RecordingRunner( + hybrid_rgb16=np.zeros((2, 2, 3), dtype=np.uint16), + receipt=_receipt(), + ) + + def runner(argv, **kwargs): + completed = base(argv, **kwargs) + receipt_path = Path(_arg(list(argv), "--out")) / "hybrid-receipt.json" + document = json.loads(receipt_path.read_bytes()) + document[section][field] = "0" * 64 + receipt_path.write_bytes(json.dumps(document, sort_keys=True, separators=(",", ":")).encode() + b"\n") + return completed + + with pytest.raises(HybridRunError, match=match): + run_hybrid_repair( + _rgbi(2, 2, 1000), + _rgbi(2, 2, 500), + same_frame_id="source-manifest-mismatch", + backend="cpu", + runtime=_runtime_config(tmp_path), + scratch_dir=tmp_path, + runner=runner, + receipt_verifier=_attest_only, + ) + + def test_rejects_malformed_receipt_json(self, tmp_path: Path) -> None: + base = _RecordingRunner( + hybrid_rgb16=np.zeros((2, 2, 3), dtype=np.uint16), + receipt=_receipt(), + ) + + def runner(argv, **kwargs): + completed = base(argv, **kwargs) + receipt_path = Path(_arg(list(argv), "--out")) / "hybrid-receipt.json" + receipt_path.write_bytes(b"{") + return completed + + with pytest.raises(HybridRunError, match="receipt JSON is invalid"): + run_hybrid_repair( + _rgbi(2, 2, 1000), + _rgbi(2, 2, 500), + same_frame_id="malformed-receipt", + backend="cpu", + runtime=_runtime_config(tmp_path), + scratch_dir=tmp_path, + runner=runner, + receipt_verifier=_attest_only, + ) + + def test_rejects_receipt_input_hash_swap(self, tmp_path: Path) -> None: + base = _RecordingRunner( + hybrid_rgb16=np.zeros((2, 2, 3), dtype=np.uint16), + receipt=_receipt(), + ) + + def runner(argv, **kwargs): + completed = base(argv, **kwargs) + receipt_path = Path(_arg(list(argv), "--out")) / "hybrid-receipt.json" + document = json.loads(receipt_path.read_bytes()) + document["inputs"]["main"]["raw_sha256"] = "0" * 64 + receipt_path.write_bytes(json.dumps(document, sort_keys=True, separators=(",", ":")).encode() + b"\n") + return completed + + with pytest.raises(HybridRunError, match="main SHA-256 changed"): + run_hybrid_repair( + _rgbi(2, 2, 1000), + _rgbi(2, 2, 500), + same_frame_id="input-hash-swap", + backend="cpu", + runtime=_runtime_config(tmp_path), + scratch_dir=tmp_path, + runner=runner, + receipt_verifier=_attest_only, + ) + + def test_rejects_invalid_disclosure_png_even_when_file_hash_matches_receipt(self, tmp_path: Path) -> None: + base = _RecordingRunner( + hybrid_rgb16=np.zeros((2, 2, 3), dtype=np.uint16), + receipt=_receipt(), + ) + + def runner(argv, **kwargs): + completed = base(argv, **kwargs) + out_dir = Path(_arg(list(argv), "--out")) + bad_png = b"not-a-png" + (out_dir / "synth-mask.png").write_bytes(bad_png) + receipt_path = out_dir / "hybrid-receipt.json" + document = json.loads(receipt_path.read_bytes()) + mask_artifact = next(row for row in document["artifacts"] if row["role"] == "synthesis_mask_png") + mask_artifact["file_sha256"] = hashlib.sha256(bad_png).hexdigest() + receipt_path.write_bytes(json.dumps(document, sort_keys=True, separators=(",", ":")).encode() + b"\n") + return completed + + with pytest.raises(HybridRunError, match="mask PNG is invalid"): + run_hybrid_repair( + _rgbi(2, 2, 1000), + _rgbi(2, 2, 500), + same_frame_id="bad-mask-png", + backend="cpu", + runtime=_runtime_config(tmp_path), + scratch_dir=tmp_path, + runner=runner, + receipt_verifier=_attest_only, + ) + + def test_rejects_symlinked_result_artifact(self, tmp_path: Path) -> None: + base = _RecordingRunner( + hybrid_rgb16=np.zeros((2, 2, 3), dtype=np.uint16), + receipt=_receipt(), + ) + + def runner(argv, **kwargs): + completed = base(argv, **kwargs) + out_dir = Path(_arg(list(argv), "--out")) + mask_path = out_dir / "synth-mask.png" + target = out_dir / "attacker-mask.png" + target.write_bytes(mask_path.read_bytes()) + mask_path.unlink() + mask_path.symlink_to(target) + return completed + + with pytest.raises(HybridRunError, match="non-symlink"): + run_hybrid_repair( + _rgbi(2, 2, 1000), + _rgbi(2, 2, 500), + same_frame_id="symlink-mask", + backend="cpu", + runtime=_runtime_config(tmp_path), + scratch_dir=tmp_path, + runner=runner, + receipt_verifier=_attest_only, + ) + + def test_rejects_external_verifier_receipt_hash_mismatch(self, tmp_path: Path) -> None: + runner = _RecordingRunner( + hybrid_rgb16=np.zeros((2, 2, 3), dtype=np.uint16), + receipt=_receipt(), + ) + + def verifier(_path: Path, _runtime: HybridRuntimeConfig): + return types.SimpleNamespace( + receipt_sha256="0" * 64, + model_weights_rehashed=True, + ) + + with pytest.raises(HybridRunError, match="changed after verification"): + run_hybrid_repair( + _rgbi(2, 2, 1000), + _rgbi(2, 2, 500), + same_frame_id="receipt-hash-mismatch", + backend="cpu", + runtime=_runtime_config(tmp_path), + scratch_dir=tmp_path, + runner=runner, + receipt_verifier=verifier, + ) + + def test_rejects_a_verifier_result_with_wrong_output_dtype_and_geometry(self, tmp_path: Path) -> None: + main = _rgbi(4, 5, 1000) + prepass = _rgbi(2, 3, 500) + + def runner(argv, **kwargs) -> subprocess.CompletedProcess: + out_dir = Path(argv[argv.index("--out") + 1]) + out_dir.mkdir(parents=True) + np.save( + out_dir / "output-hybrid.rgb16.npy", + np.zeros((1, 2), dtype=np.float32), + allow_pickle=False, + ) + (out_dir / "synth-mask.png").write_bytes(b"not-a-png") + (out_dir / "hybrid-receipt.json").write_text("{}\n", encoding="utf-8") + return subprocess.CompletedProcess(argv, 0, stdout="", stderr="") + + def verifier(_path: Path, _runtime: HybridRuntimeConfig): + return types.SimpleNamespace( + document={}, + receipt_sha256=hashlib.sha256(_path.read_bytes()).hexdigest(), + hybrid_output_rgb16=np.zeros((1, 2), dtype=np.float32), + synthesis_mask=np.zeros((1, 2), dtype=np.bool_), + model_weights_rehashed=True, + ) + + with pytest.raises(HybridRunError, match="HxWx3 uint16"): + run_hybrid_repair( + main, + prepass, + same_frame_id="frame-malformed-output", + backend="cpu", + runtime=_runtime_config(tmp_path), + scratch_dir=tmp_path, + runner=runner, + receipt_verifier=verifier, + ) + + def test_nonzero_exit_raises_hybrid_run_error(self, tmp_path: Path) -> None: + runner = _RecordingRunner( + hybrid_rgb16=np.zeros((2, 2, 3), dtype=np.uint16), + receipt=_receipt(), + returncode=2, + ) + + with pytest.raises(HybridRunError, match="synthetic failure"): + run_hybrid_repair( + _rgbi(2, 2, 1000), + _rgbi(2, 2, 500), + same_frame_id="frame-006", + backend="cpu", + runtime=_runtime_config(tmp_path), + scratch_dir=tmp_path, + runner=runner, + ) + + def test_missing_output_file_raises_hybrid_run_error(self, tmp_path: Path) -> None: + def runner(argv, **kwargs) -> subprocess.CompletedProcess: + out_dir = Path(argv[argv.index("--out") + 1]) + out_dir.mkdir(parents=True) + # Deliberately omit synth-mask.png and hybrid-receipt.json. + np.save(out_dir / "output-hybrid.rgb16.npy", np.zeros((2, 2, 3), dtype=np.uint16)) + return subprocess.CompletedProcess(argv, 0, stdout="", stderr="") + + with pytest.raises(HybridRunError, match="missing"): + run_hybrid_repair( + _rgbi(2, 2, 1000), + _rgbi(2, 2, 500), + same_frame_id="frame-007", + backend="cpu", + runtime=_runtime_config(tmp_path), + scratch_dir=tmp_path, + runner=runner, + ) + + +def test_runtime_config_rejects_relative_process_or_artifact_paths() -> None: + with pytest.raises(ValueError, match="absolute"): + HybridRuntimeConfig( + hybrid_python=Path("hybrid/bin/python"), + executable=Path("hybrid/bin/fauxce-hybrid"), + core_source_manifest_sha256=EXPECTED_CORE_SOURCE_SHA256, + hybrid_source_manifest_sha256=EXPECTED_HYBRID_SOURCE_SHA256, + iopaint_python=Path("iopaint/bin/python"), + iopaint_executable=Path("iopaint/bin/iopaint"), + iopaint_source_manifest_sha256="a" * 64, + model_dir=Path("models"), + model_weights=Path("models/big-lama.pt"), + model_weights_sha256="b" * 64, + ) + + +def test_launch_oserror_raises_hybrid_run_error(tmp_path: Path) -> None: + def runner(argv, **kwargs): + raise OSError("fauxce-hybrid executable not found") + + with pytest.raises(HybridRunError, match="could not launch"): + run_hybrid_repair( + _rgbi(2, 2, 1000), + _rgbi(2, 2, 500), + same_frame_id="frame-008", + backend="cpu", + runtime=_runtime_config(tmp_path), + scratch_dir=tmp_path, + runner=runner, + ) diff --git a/tests/repair/test_fauxice_ir_repair.py b/tests/repair/test_fauxice_ir_repair.py new file mode 100644 index 00000000..e80d8e9f --- /dev/null +++ b/tests/repair/test_fauxice_ir_repair.py @@ -0,0 +1,845 @@ +"""Tests for the fauxice IR repair adapter, with stub engine/hybrid modules. + +Neither ``portable_digital_ice`` nor ``fauxce_hybrid`` is installed in this +environment (they are optional dependencies distributed from GitHub +releases, not PyPI), so every test that needs "the engine is installed" +installs a small fake module into ``sys.modules`` first via +``_install_stub_engine`` below, rather than requiring a real install. +``importlib.util.find_spec`` resolves an already-imported module through +``sys.modules``, so this is enough to make ``engine_available()`` (and a +real ``from portable_digital_ice import ...``) see the stub. + +The stub engine's ``process()`` inverts the 16-bit main RGB plane +(``65535 - value``) as its "repair": deterministic, trivially distinguished +from the untouched input, and safely within uint16 range for the small +fixture values used here. +""" + +from __future__ import annotations + +import importlib.machinery +import hashlib +import json +import subprocess +import sys +import threading +import types +from dataclasses import dataclass +from pathlib import Path +from typing import Callable + +import numpy as np +import pytest +import tifffile + +from negpy.services.repair.fauxice_hybrid_runner import ( + HybridRunResult, + HybridRuntimeConfig, +) +from negpy.services.repair.fauxice_ir_repair import ( + ENGINE_IMPORT_NAME, + HYBRID_IMPORT_NAME, + FauxiceRepairCancelled, + FauxiceRepairConfig, + RepairMode, + RepairStatus, + engine_available, + hybrid_available, + repair_frame_files, + repair_ir_dust, +) + +INVERT = np.uint16(65535) + + +def _install_stub_engine(monkeypatch: pytest.MonkeyPatch, **process_kwargs: object) -> types.ModuleType: + """Register a minimal fake ``portable_digital_ice`` in ``sys.modules``.""" + + module = types.ModuleType(ENGINE_IMPORT_NAME) + module.__spec__ = importlib.machinery.ModuleSpec(ENGINE_IMPORT_NAME, loader=None) + + class AcquisitionEpoch: + PREPASS = "prepass" + MAIN = "main" + + class ComputeBackend(str): + def __new__(cls, value: str) -> "ComputeBackend": + return str.__new__(cls, value) + + @property + def value(self) -> str: + return str(self) + + class ProcessingMode: + NORMAL = "normal" + + class ScannerModel: + NIKON_SUPER_COOLSCAN_5000_ED = "nikon-ls5000" + + class ProcessingCancelled(RuntimeError): + pass + + @dataclass + class RGBI16Frame: + pixels: np.ndarray + epoch: str + resolution_dpi: int + evidence_id: str + + @dataclass + class DualRGBIAcquisition: + prepass: RGBI16Frame + main: RGBI16Frame + same_frame_id: str + + @dataclass + class ProcessingJob: + acquisition: DualRGBIAcquisition + scanner_model: str + mode: str + selector: int + resolution_metric: int + bit_depth: int + focus_exposure_locked: bool + + @dataclass + class _Progress: + completed: int + total: int + + @dataclass + class _Selection: + requested: object + used: object + reason: str + + @dataclass + class _Result: + output_rgb16: np.ndarray + + @dataclass + class _Routed: + result: _Result + selection: _Selection + + calls_made = process_kwargs.get("calls", None) + + def process( + job: ProcessingJob, + *, + backend, + output_rgb16=None, + progress=None, + cancelled=None, + export_diagnostics: bool = False, + ) -> _Routed: + if calls_made is not None: + calls_made.append(job) + if progress is not None: + progress(_Progress(completed=0, total=2)) + if cancelled is not None and cancelled(): + raise ProcessingCancelled("stub engine cancelled before completion") + if progress is not None: + progress(_Progress(completed=2, total=2)) + main_rgb = job.acquisition.main.pixels[:, :, :3] + output = (INVERT - main_rgb.astype(np.int64)).astype(np.uint16) + backend_value = ComputeBackend(backend) + return _Routed( + result=_Result(output_rgb16=output), + selection=_Selection(requested=backend_value, used=backend_value, reason="stub selection"), + ) + + module.AcquisitionEpoch = AcquisitionEpoch + module.ComputeBackend = ComputeBackend + module.DualRGBIAcquisition = DualRGBIAcquisition + module.ProcessingJob = ProcessingJob + module.ProcessingMode = ProcessingMode + module.RGBI16Frame = RGBI16Frame + module.ScannerModel = ScannerModel + module.ProcessingCancelled = ProcessingCancelled + module.process = process + + monkeypatch.setitem(sys.modules, ENGINE_IMPORT_NAME, module) + return module + + +def _install_stub_hybrid(monkeypatch: pytest.MonkeyPatch) -> types.ModuleType: + """Register a bare-presence fake ``fauxce_hybrid`` (only ``find_spec`` needs it).""" + + module = types.ModuleType(HYBRID_IMPORT_NAME) + module.__spec__ = importlib.machinery.ModuleSpec(HYBRID_IMPORT_NAME, loader=None) + monkeypatch.setitem(sys.modules, HYBRID_IMPORT_NAME, module) + return module + + +class _FlippingCancelEvent: + """``threading.Event`` look-alike: unset for the first N calls, set after.""" + + def __init__(self, set_after_calls: int) -> None: + self._calls = 0 + self._set_after = set_after_calls + + def is_set(self) -> bool: + self._calls += 1 + return self._calls > self._set_after + + +def _rgb(height: int, width: int, base: int) -> np.ndarray: + return np.full((height, width, 3), base, dtype=np.uint16) + + +def _ir(height: int, width: int, base: int) -> np.ndarray: + return np.full((height, width), base, dtype=np.uint16) + + +def _prepass(height: int, width: int) -> np.ndarray: + return np.full((height, width, 4), 2000, dtype=np.uint16) + + +def _hybrid_runtime(tmp_path: Path) -> HybridRuntimeConfig: + hybrid_python = tmp_path / "hybrid" / "bin" / "python" + executable = tmp_path / "hybrid" / "bin" / "fauxce-hybrid" + iopaint_python = tmp_path / "iopaint" / "bin" / "python" + iopaint_executable = tmp_path / "iopaint" / "bin" / "iopaint" + model_dir = tmp_path / "models" + model_weights = model_dir / "torch" / "hub" / "checkpoints" / "big-lama.pt" + for path in (hybrid_python, executable, iopaint_python, iopaint_executable): + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(b"#!/bin/sh\nexit 0\n") + path.chmod(0o700) + model_weights.parent.mkdir(parents=True, exist_ok=True) + model_weights.write_bytes(b"stub-model") + return HybridRuntimeConfig( + hybrid_python=hybrid_python, + executable=executable, + core_source_manifest_sha256="c" * 64, + hybrid_source_manifest_sha256="d" * 64, + iopaint_python=iopaint_python, + iopaint_executable=iopaint_executable, + iopaint_source_manifest_sha256="a" * 64, + model_dir=model_dir, + model_weights=model_weights, + model_weights_sha256=hashlib.sha256(b"stub-model").hexdigest(), + ) + + +def _hybrid_receipt(*, fraction: float = 0.02) -> dict: + return { + "core": { + "version": "0.3.0", + "backend": {"requested": "cpu", "used": "cpu", "reason": "stub"}, + }, + "synthesis": {"fraction": fraction}, + "routing": { + "counts": { + "final_regions": 2, + "synthesis_pixels": 40, + "frame_pixels": 400, + "at_floor_pixels": 60, + } + }, + } + + +def _stub_hybrid_runner(hybrid_rgb16: np.ndarray, *, receipt: dict | None = None) -> Callable: + def runner(argv, **kwargs) -> subprocess.CompletedProcess: + out_dir = Path(argv[argv.index("--out") + 1]) + out_dir.mkdir(parents=True) + np.save(out_dir / "output-hybrid.rgb16.npy", hybrid_rgb16, allow_pickle=False) + (out_dir / "synth-mask.png").write_bytes(b"mask-bytes") + (out_dir / "hybrid-receipt.json").write_text(json.dumps(receipt or _hybrid_receipt()), encoding="utf-8") + return subprocess.CompletedProcess(argv, 0, stdout="", stderr="") + + return runner + + +def _hybrid_outcome(hybrid_rgb16: np.ndarray) -> HybridRunResult: + mask = np.zeros(hybrid_rgb16.shape[:2], dtype=np.bool_) + receipt = b'{"schema":"fauxce-hybrid-receipt-v2"}' + mask_bytes = b"mask-bytes" + return HybridRunResult( + hybrid_rgb16=np.ascontiguousarray(hybrid_rgb16), + synth_mask_png=mask_bytes, + synth_mask_sha256=hashlib.sha256(mask_bytes).hexdigest(), + synth_mask=mask, + receipt=receipt, + receipt_sha256=hashlib.sha256(receipt).hexdigest(), + acquisition_manifest_sha256="1" * 64, + main_rgbi_sha256="2" * 64, + prepass_rgbi_sha256="3" * 64, + output_rgb16_sha256=hashlib.sha256(hybrid_rgb16.astype(" None: + monkeypatch.setattr( + "negpy.services.repair.fauxice_ir_repair.importlib.util.find_spec", + lambda _name: None, + ) + assert engine_available() is False + + def test_hybrid_unavailable_by_default(self) -> None: + assert hybrid_available() is False + + def test_engine_available_with_stub_installed(self, monkeypatch: pytest.MonkeyPatch) -> None: + _install_stub_engine(monkeypatch) + assert engine_available() is True + + def test_hybrid_available_with_explicit_external_runtime(self, tmp_path: Path) -> None: + assert hybrid_available(_hybrid_runtime(tmp_path)) is True + + def test_engine_and_hybrid_availability_are_independent(self, monkeypatch: pytest.MonkeyPatch) -> None: + _install_stub_engine(monkeypatch) + assert engine_available() is True + assert hybrid_available() is False + + +# --------------------------------------------------------------------------- +# Disabled / unavailable / no-prepass short circuits +# --------------------------------------------------------------------------- + + +class TestShortCircuits: + def test_disabled_config_skips_without_checking_engine(self) -> None: + config = FauxiceRepairConfig(enabled=False) + result = repair_ir_dust(_rgb(4, 4, 1000), _ir(4, 4, 500), same_frame_id="f1", config=config, prepass_rgbi=_prepass(4, 4)) + assert result.status is RepairStatus.SKIPPED + assert "disabled" in result.reason + + def test_engine_unavailable_reports_unavailable_status(self, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr( + "negpy.services.repair.fauxice_ir_repair.importlib.util.find_spec", + lambda _name: None, + ) + config = FauxiceRepairConfig(enabled=True) + result = repair_ir_dust(_rgb(4, 4, 1000), _ir(4, 4, 500), same_frame_id="f1", config=config, prepass_rgbi=_prepass(4, 4)) + assert result.status is RepairStatus.UNAVAILABLE + assert ENGINE_IMPORT_NAME in result.reason + + def test_missing_prepass_skips(self, monkeypatch: pytest.MonkeyPatch) -> None: + _install_stub_engine(monkeypatch) + config = FauxiceRepairConfig(enabled=True) + result = repair_ir_dust(_rgb(4, 4, 1000), _ir(4, 4, 500), same_frame_id="f1", config=config) + assert result.status is RepairStatus.SKIPPED + assert "prepass" in result.reason + + +# --------------------------------------------------------------------------- +# Validation +# --------------------------------------------------------------------------- + + +class TestValidation: + def test_malformed_rgb_shape_raises(self, monkeypatch: pytest.MonkeyPatch) -> None: + _install_stub_engine(monkeypatch) + config = FauxiceRepairConfig(enabled=True) + with pytest.raises(ValueError, match="rgb"): + repair_ir_dust( + np.zeros((4, 4), dtype=np.uint16), + _ir(4, 4, 500), + same_frame_id="f1", + config=config, + prepass_rgbi=_prepass(4, 4), + ) + + def test_mismatched_ir_shape_raises(self, monkeypatch: pytest.MonkeyPatch) -> None: + _install_stub_engine(monkeypatch) + config = FauxiceRepairConfig(enabled=True) + with pytest.raises(ValueError, match="shape"): + repair_ir_dust( + _rgb(4, 4, 1000), + _ir(8, 8, 500), + same_frame_id="f1", + config=config, + prepass_rgbi=_prepass(4, 4), + ) + + def test_empty_same_frame_id_raises(self, monkeypatch: pytest.MonkeyPatch) -> None: + _install_stub_engine(monkeypatch) + config = FauxiceRepairConfig(enabled=True) + with pytest.raises(ValueError, match="same_frame_id"): + repair_ir_dust( + _rgb(4, 4, 1000), + _ir(4, 4, 500), + same_frame_id=" ", + config=config, + prepass_rgbi=_prepass(4, 4), + ) + + def test_malformed_prepass_shape_raises(self, monkeypatch: pytest.MonkeyPatch) -> None: + _install_stub_engine(monkeypatch) + config = FauxiceRepairConfig(enabled=True) + with pytest.raises(ValueError, match="prepass_rgbi"): + repair_ir_dust( + _rgb(4, 4, 1000), + _ir(4, 4, 500), + same_frame_id="f1", + config=config, + prepass_rgbi=np.zeros((4, 4, 3), dtype=np.uint16), + ) + + +# --------------------------------------------------------------------------- +# Mode selection: exact +# --------------------------------------------------------------------------- + + +class TestExactMode: + def test_default_mode_is_exact(self) -> None: + assert FauxiceRepairConfig().mode is RepairMode.EXACT + + def test_exact_mode_applies_via_engine(self, monkeypatch: pytest.MonkeyPatch) -> None: + _install_stub_engine(monkeypatch) + config = FauxiceRepairConfig(enabled=True, mode=RepairMode.EXACT) + rgb = _rgb(4, 4, 1000) + result = repair_ir_dust(rgb, _ir(4, 4, 500), same_frame_id="f1", config=config, prepass_rgbi=_prepass(4, 4)) + assert result.status is RepairStatus.APPLIED + assert result.mode_requested is RepairMode.EXACT + assert result.mode_resolved is RepairMode.EXACT + assert result.repaired_rgb16 is not None + np.testing.assert_array_equal(result.repaired_rgb16, INVERT - rgb.astype(np.int64)) + assert result.backend_used == "auto" + + def test_exact_mode_honours_explicit_backend(self, monkeypatch: pytest.MonkeyPatch) -> None: + _install_stub_engine(monkeypatch) + config = FauxiceRepairConfig(enabled=True, mode=RepairMode.EXACT, backend="cpu") + result = repair_ir_dust(_rgb(4, 4, 1000), _ir(4, 4, 500), same_frame_id="f1", config=config, prepass_rgbi=_prepass(4, 4)) + assert result.backend_requested == "cpu" + assert result.backend_used == "cpu" + + def test_validity_mask_restricts_repair_to_valid_region(self, monkeypatch: pytest.MonkeyPatch) -> None: + _install_stub_engine(monkeypatch) + config = FauxiceRepairConfig(enabled=True) + rgb = _rgb(4, 4, 1000) + mask = np.zeros((4, 4), dtype=bool) + mask[1:3, 1:3] = True + + result = repair_ir_dust( + rgb, + _ir(4, 4, 500), + same_frame_id="f1", + config=config, + prepass_rgbi=_prepass(4, 4), + validity_mask=mask, + ) + + repaired = result.repaired_rgb16 + assert repaired is not None + # Inside the mask: repaired (inverted) value. + np.testing.assert_array_equal(repaired[1:3, 1:3], INVERT - rgb[1:3, 1:3].astype(np.int64)) + # Outside the mask: untouched original. + np.testing.assert_array_equal(repaired[0, :], rgb[0, :]) + np.testing.assert_array_equal(repaired[3, :], rgb[3, :]) + + def test_engine_rejection_reports_skipped(self, monkeypatch: pytest.MonkeyPatch) -> None: + module = _install_stub_engine(monkeypatch) + + def failing_process(*args, **kwargs): + raise ValueError("synthetic profile rejection") + + module.process = failing_process + config = FauxiceRepairConfig(enabled=True) + result = repair_ir_dust(_rgb(4, 4, 1000), _ir(4, 4, 500), same_frame_id="f1", config=config, prepass_rgbi=_prepass(4, 4)) + assert result.status is RepairStatus.SKIPPED + assert "rejected the acquisition" in result.reason + assert "synthetic profile rejection" in result.reason + + +# --------------------------------------------------------------------------- +# Progress and cancellation (exact path) +# --------------------------------------------------------------------------- + + +class TestProgressAndCancellation: + def test_progress_callback_receives_fractional_updates(self, monkeypatch: pytest.MonkeyPatch) -> None: + _install_stub_engine(monkeypatch) + config = FauxiceRepairConfig(enabled=True) + observed: list[float] = [] + + result = repair_ir_dust( + _rgb(4, 4, 1000), + _ir(4, 4, 500), + same_frame_id="f1", + config=config, + prepass_rgbi=_prepass(4, 4), + progress=observed.append, + ) + + assert result.status is RepairStatus.APPLIED + assert observed == [0.0, 1.0] + + def test_cancel_before_start_skips_without_calling_engine(self, monkeypatch: pytest.MonkeyPatch) -> None: + calls: list[object] = [] + _install_stub_engine(monkeypatch, calls=calls) + config = FauxiceRepairConfig(enabled=True) + cancel = threading.Event() + cancel.set() + + with pytest.raises(FauxiceRepairCancelled, match="cancelled"): + repair_ir_dust( + _rgb(4, 4, 1000), + _ir(4, 4, 500), + same_frame_id="f1", + config=config, + prepass_rgbi=_prepass(4, 4), + cancel=cancel, + ) + + assert calls == [] # the engine was never invoked + + def test_cancel_during_exact_run_skips_with_clear_reason(self, monkeypatch: pytest.MonkeyPatch) -> None: + _install_stub_engine(monkeypatch) + config = FauxiceRepairConfig(enabled=True) + # Unset for the adapter's pre-flight check, set by the time the stub + # engine polls it mid-run. + cancel = _FlippingCancelEvent(set_after_calls=1) + + with pytest.raises( + FauxiceRepairCancelled, + match="cancelled before completion", + ): + repair_ir_dust( + _rgb(4, 4, 1000), + _ir(4, 4, 500), + same_frame_id="f1", + config=config, + prepass_rgbi=_prepass(4, 4), + cancel=cancel, + ) + + +# --------------------------------------------------------------------------- +# Mode selection: hybrid, and hybrid-selected-but-absent degradation +# --------------------------------------------------------------------------- + + +class TestHybridMode: + def test_hybrid_mode_applies_via_hybrid_runner(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + _install_stub_engine(monkeypatch) + _install_stub_hybrid(monkeypatch) + config = FauxiceRepairConfig(enabled=True, mode=RepairMode.HYBRID) + hybrid_output = np.full((4, 4, 3), 9999, dtype=np.uint16) + monkeypatch.setattr( + "negpy.services.repair.fauxice_ir_repair.run_hybrid_repair", + lambda *args, **kwargs: _hybrid_outcome(hybrid_output), + ) + + result = repair_ir_dust( + _rgb(4, 4, 1000), + _ir(4, 4, 500), + same_frame_id="f1", + config=config, + prepass_rgbi=_prepass(4, 4), + hybrid_runtime=_hybrid_runtime(tmp_path), + hybrid_subprocess_runner=_stub_hybrid_runner(hybrid_output), + ) + + assert result.status is RepairStatus.APPLIED + assert result.mode_requested is RepairMode.HYBRID + assert result.mode_resolved is RepairMode.HYBRID + np.testing.assert_array_equal(result.repaired_rgb16, hybrid_output) + assert result.hybrid_synthesis_fraction == 0.02 + assert result.hybrid_routing_counts == { + "final_regions": 2, + "synthesis_pixels": 40, + "frame_pixels": 400, + "at_floor_pixels": 60, + } + assert "2 region(s) routed" in result.reason + + def test_hybrid_validity_composite_has_its_own_final_output_hash(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + _install_stub_engine(monkeypatch) + hybrid_output = np.full((4, 4, 3), 9999, dtype=np.uint16) + outcome = _hybrid_outcome(hybrid_output) + monkeypatch.setattr( + "negpy.services.repair.fauxice_ir_repair.run_hybrid_repair", + lambda *args, **kwargs: outcome, + ) + source = _rgb(4, 4, 1000) + validity = np.ones((4, 4), dtype=np.bool_) + validity[1, 2] = False + + result = repair_ir_dust( + source, + _ir(4, 4, 500), + same_frame_id="validity-bound", + config=FauxiceRepairConfig(enabled=True, mode=RepairMode.HYBRID), + prepass_rgbi=_prepass(4, 4), + validity_mask=validity, + hybrid_runtime=_hybrid_runtime(tmp_path), + ) + + expected = hybrid_output.copy() + expected[1, 2] = source[1, 2] + np.testing.assert_array_equal(result.repaired_rgb16, expected) + assert result.native_output_rgb_sha256 == hashlib.sha256(expected.astype(" None: + _install_stub_engine(monkeypatch) + config = FauxiceRepairConfig(enabled=True, mode=RepairMode.HYBRID) + + result = repair_ir_dust(_rgb(4, 4, 1000), _ir(4, 4, 500), same_frame_id="f1", config=config, prepass_rgbi=_prepass(4, 4)) + + assert result.status is RepairStatus.APPLIED + assert result.mode_requested is RepairMode.HYBRID + assert result.mode_resolved is RepairMode.EXACT + assert "no hybrid runtime is configured" in result.reason + assert "degraded to exact" in result.reason + + def test_hybrid_run_failure_degrades_to_exact_never_fails(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + _install_stub_engine(monkeypatch) + _install_stub_hybrid(monkeypatch) + config = FauxiceRepairConfig(enabled=True, mode=RepairMode.HYBRID) + + def broken_runner(argv, **kwargs): + return subprocess.CompletedProcess(argv, 2, stdout="", stderr="model runtime missing") + + result = repair_ir_dust( + _rgb(4, 4, 1000), + _ir(4, 4, 500), + same_frame_id="f1", + config=config, + prepass_rgbi=_prepass(4, 4), + hybrid_runtime=_hybrid_runtime(tmp_path), + hybrid_subprocess_runner=broken_runner, + ) + + assert result.status is RepairStatus.APPLIED + assert result.mode_resolved is RepairMode.EXACT + assert "fauxce-hybrid run failed" in result.reason + assert result.repaired_rgb16 is not None + + def test_hybrid_mode_cancelled_before_start_skips_entirely(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + _install_stub_engine(monkeypatch) + _install_stub_hybrid(monkeypatch) + config = FauxiceRepairConfig(enabled=True, mode=RepairMode.HYBRID) + cancel = threading.Event() + cancel.set() + + def unreachable_runner(argv, **kwargs): + raise AssertionError("hybrid subprocess must not run once cancelled") + + # The top-level pre-flight check (shared with exact mode) fires + # before the mode is even inspected, so a cancellation requested up + # front skips the whole call, hybrid included, without touching the + # subprocess runner. + with pytest.raises( + FauxiceRepairCancelled, + match="cancelled by caller before repair started", + ): + repair_ir_dust( + _rgb(4, 4, 1000), + _ir(4, 4, 500), + same_frame_id="f1", + config=config, + prepass_rgbi=_prepass(4, 4), + hybrid_runtime=_hybrid_runtime(tmp_path), + hybrid_subprocess_runner=unreachable_runner, + cancel=cancel, + ) + + def test_hybrid_mode_cancelled_between_preflight_and_hybrid_start(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + """Covers the hybrid-specific cancel check, distinct from the top-level one. + + The top-level pre-flight check and the engine's own cooperative + cancellation both read ``cancel.is_set()``; a stateful fake flips + from unset to set between calls to simulate another thread calling + ``cancel.set()`` in that narrow window, the scenario the extra + hybrid-branch check exists for. + """ + _install_stub_engine(monkeypatch) + _install_stub_hybrid(monkeypatch) + config = FauxiceRepairConfig(enabled=True, mode=RepairMode.HYBRID) + # Call 1: the top-level pre-flight check (unset). Call 2: the + # hybrid-branch check (set). Call 3: the exact fallback's + # cooperative check inside the stub engine (stays set). + cancel = _FlippingCancelEvent(set_after_calls=1) + + def unreachable_runner(argv, **kwargs): + raise AssertionError("hybrid subprocess must not run once cancelled") + + with pytest.raises( + FauxiceRepairCancelled, + match="cancelled before the hybrid run started", + ): + repair_ir_dust( + _rgb(4, 4, 1000), + _ir(4, 4, 500), + same_frame_id="f1", + config=config, + prepass_rgbi=_prepass(4, 4), + hybrid_runtime=_hybrid_runtime(tmp_path), + hybrid_subprocess_runner=unreachable_runner, + cancel=cancel, + ) + + +# --------------------------------------------------------------------------- +# repair_frame_files: tmp-file invocation, sidecars, IR immutability +# --------------------------------------------------------------------------- + + +class TestRepairFrameFiles: + def _write_frame(self, tmp_path: Path, *, rgb_base: int = 1000, ir_base: int = 500) -> tuple[Path, Path]: + rgb_path = tmp_path / "frame001.tif" + ir_path = tmp_path / "frame001_IR.tif" + tifffile.imwrite(rgb_path, _rgb(4, 4, rgb_base), photometric="rgb") + tifffile.imwrite(ir_path, _ir(4, 4, ir_base), photometric="minisblack") + return rgb_path, ir_path + + def test_writes_repaired_output_and_sidecar(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + _install_stub_engine(monkeypatch) + rgb_path, ir_path = self._write_frame(tmp_path) + prepass_path = tmp_path / "frame001_prepass.npy" + np.save(prepass_path, _prepass(4, 4), allow_pickle=False) + config = FauxiceRepairConfig(enabled=True) + + result = repair_frame_files(rgb_path, config=config, prepass_path=prepass_path) + + assert result.status is RepairStatus.APPLIED + output_path = tmp_path / "frame001_FAUXICE.tif" + sidecar_path = tmp_path / "frame001_FAUXICE.json" + assert output_path.is_file() + assert sidecar_path.is_file() + + written = np.asarray(tifffile.imread(output_path)) + np.testing.assert_array_equal(written, result.repaired_rgb16) + + payload = json.loads(sidecar_path.read_text(encoding="utf-8")) + assert payload["status"] == "applied" + assert payload["engine"]["package"] == ENGINE_IMPORT_NAME + assert payload["source"] == {"rgb": rgb_path.name, "ir": ir_path.name} + assert payload["output"]["repaired_rgb"] == output_path.name + + def test_missing_ir_companion_skips_without_writing_repaired_output(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + _install_stub_engine(monkeypatch) + rgb_path = tmp_path / "frame002.tif" + tifffile.imwrite(rgb_path, _rgb(4, 4, 1000), photometric="rgb") + config = FauxiceRepairConfig(enabled=True) + + result = repair_frame_files(rgb_path, config=config) + + assert result.status is RepairStatus.SKIPPED + assert "no IR companion" in result.reason + assert not (tmp_path / "frame002_FAUXICE.tif").exists() + assert (tmp_path / "frame002_FAUXICE.json").is_file() + + def test_disabled_config_writes_sidecar_only(self, tmp_path: Path) -> None: + rgb_path, _ = self._write_frame(tmp_path) + config = FauxiceRepairConfig(enabled=False) + + result = repair_frame_files(rgb_path, config=config) + + assert result.status is RepairStatus.SKIPPED + sidecar_path = tmp_path / "frame001_FAUXICE.json" + assert sidecar_path.is_file() + assert not (tmp_path / "frame001_FAUXICE.tif").exists() + + def test_ir_file_untouched_on_disk_after_repair(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + _install_stub_engine(monkeypatch) + rgb_path, ir_path = self._write_frame(tmp_path) + prepass_path = tmp_path / "frame001_prepass.npy" + np.save(prepass_path, _prepass(4, 4), allow_pickle=False) + before = ir_path.read_bytes() + config = FauxiceRepairConfig(enabled=True) + + result = repair_frame_files(rgb_path, config=config, prepass_path=prepass_path) + + assert result.status is RepairStatus.APPLIED + after = ir_path.read_bytes() + assert before == after + + def test_ir_array_not_mutated_in_memory(self, monkeypatch: pytest.MonkeyPatch) -> None: + _install_stub_engine(monkeypatch) + config = FauxiceRepairConfig(enabled=True) + ir = _ir(4, 4, 500) + ir_before = ir.copy() + + result = repair_ir_dust(_rgb(4, 4, 1000), ir, same_frame_id="f1", config=config, prepass_rgbi=_prepass(4, 4)) + + assert result.status is RepairStatus.APPLIED + np.testing.assert_array_equal(ir, ir_before) + + def test_provenance_sidecar_written_even_when_unavailable(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr( + "negpy.services.repair.fauxice_ir_repair.importlib.util.find_spec", + lambda _name: None, + ) + rgb_path, _ = self._write_frame(tmp_path) + config = FauxiceRepairConfig(enabled=True) + + result = repair_frame_files(rgb_path, config=config) + + assert result.status is RepairStatus.UNAVAILABLE + sidecar_path = tmp_path / "frame001_FAUXICE.json" + payload = json.loads(sidecar_path.read_text(encoding="utf-8")) + assert payload["status"] == "unavailable" + assert "output" not in payload + + def test_hybrid_provenance_sidecar_records_disclosure_mask_and_routing(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + _install_stub_engine(monkeypatch) + _install_stub_hybrid(monkeypatch) + rgb_path, _ = self._write_frame(tmp_path) + prepass_path = tmp_path / "frame001_prepass.npy" + np.save(prepass_path, _prepass(4, 4), allow_pickle=False) + config = FauxiceRepairConfig(enabled=True, mode=RepairMode.HYBRID) + hybrid_output = np.full((4, 4, 3), 9999, dtype=np.uint16) + monkeypatch.setattr( + "negpy.services.repair.fauxice_ir_repair.run_hybrid_repair", + lambda *args, **kwargs: _hybrid_outcome(hybrid_output), + ) + + result = repair_frame_files( + rgb_path, + config=config, + prepass_path=prepass_path, + hybrid_runtime=_hybrid_runtime(tmp_path), + hybrid_subprocess_runner=_stub_hybrid_runner(hybrid_output), + ) + + assert result.status is RepairStatus.APPLIED + assert result.mode_resolved is RepairMode.HYBRID + mask_path = tmp_path / "frame001_FAUXICE_SYNTH.png" + assert mask_path.is_file() + assert mask_path.read_bytes() == b"mask-bytes" + + sidecar_path = tmp_path / "frame001_FAUXICE.json" + payload = json.loads(sidecar_path.read_text(encoding="utf-8")) + assert payload["hybrid"]["routing_counts"]["final_regions"] == 2 + assert payload["output"]["disclosure_mask"] == mask_path.name + + def test_no_overwrite_of_existing_ir_when_ir_path_explicit(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + """A caller-supplied ir_path is also never opened for writing.""" + _install_stub_engine(monkeypatch) + rgb_path = tmp_path / "custom.tif" + ir_path = tmp_path / "custom_infrared.tif" + tifffile.imwrite(rgb_path, _rgb(4, 4, 1000), photometric="rgb") + tifffile.imwrite(ir_path, _ir(4, 4, 500), photometric="minisblack") + prepass_path = tmp_path / "prepass.npy" + np.save(prepass_path, _prepass(4, 4), allow_pickle=False) + before = ir_path.read_bytes() + config = FauxiceRepairConfig(enabled=True) + + result = repair_frame_files(rgb_path, config=config, ir_path=ir_path, prepass_path=prepass_path) + + assert result.status is RepairStatus.APPLIED + assert ir_path.read_bytes() == before diff --git a/tests/repair/test_hybrid_runtime_manifest.py b/tests/repair/test_hybrid_runtime_manifest.py new file mode 100644 index 00000000..f5400be9 --- /dev/null +++ b/tests/repair/test_hybrid_runtime_manifest.py @@ -0,0 +1,136 @@ +from __future__ import annotations + +import hashlib +import json +import os +from pathlib import Path + +import pytest + +from negpy.services.repair import hybrid_runtime_manifest as runtime_manifest +from negpy.services.repair.hybrid_runtime_manifest import ( + HybridRuntimeManifestError, + RUNTIME_MANIFEST_SCHEMA, + canonical_runtime_manifest_bytes, + load_default_hybrid_runtime_manifest, + load_hybrid_runtime_manifest, + runtime_manifest_pin_path, +) + + +def _document(root: Path) -> dict[str, object]: + return { + "core_source_manifest_sha256": "1" * 64, + "executable": str(root / "hybrid" / "bin" / "fauxce-hybrid"), + "hybrid_python": str(root / "hybrid" / "bin" / "python"), + "hybrid_source_manifest_sha256": "2" * 64, + "inpaint_device": "cpu", + "inpaint_seed": 0, + "inpaint_threads": 1, + "iopaint_executable": str(root / "iopaint" / "bin" / "iopaint"), + "iopaint_python": str(root / "iopaint" / "bin" / "python"), + "iopaint_source_manifest_sha256": "3" * 64, + "max_synthesis_fraction": 0.1, + "model_dir": str(root / "models"), + "model_weights": str(root / "models" / "torch" / "hub" / "checkpoints" / "big-lama.pt"), + "model_weights_sha256": "4" * 64, + "schema": RUNTIME_MANIFEST_SCHEMA, + } + + +def _write_pair(path: Path, document: dict[str, object]) -> str: + payload = canonical_runtime_manifest_bytes(document) + path.write_bytes(payload) + digest = hashlib.sha256(payload).hexdigest() + runtime_manifest_pin_path(path).write_text(digest + "\n", encoding="ascii") + return digest + + +def test_loads_canonical_hash_pinned_runtime(tmp_path: Path) -> None: + path = tmp_path / "fauxce-hybrid-runtime.json" + digest = _write_pair(path, _document(tmp_path)) + + runtime = load_hybrid_runtime_manifest(path, expected_sha256=digest) + + assert runtime.hybrid_python == tmp_path / "hybrid" / "bin" / "python" + assert runtime.core_source_manifest_sha256 == "1" * 64 + assert runtime.hybrid_source_manifest_sha256 == "2" * 64 + assert runtime.max_synthesis_fraction == 0.1 + + +def test_default_loader_requires_manifest_and_independent_pin(tmp_path: Path) -> None: + path = tmp_path / "fauxce-hybrid-runtime.json" + assert load_default_hybrid_runtime_manifest(path) is None + + path.write_bytes(canonical_runtime_manifest_bytes(_document(tmp_path))) + with pytest.raises(HybridRuntimeManifestError, match="must both exist"): + load_default_hybrid_runtime_manifest(path) + + +def test_default_loader_rejects_tampering_after_pin(tmp_path: Path) -> None: + path = tmp_path / "fauxce-hybrid-runtime.json" + document = _document(tmp_path) + _write_pair(path, document) + document["inpaint_threads"] = 2 + path.write_bytes(canonical_runtime_manifest_bytes(document)) + + with pytest.raises(HybridRuntimeManifestError, match="SHA-256 mismatch"): + load_default_hybrid_runtime_manifest(path) + + +def test_loader_rejects_noncanonical_or_extended_manifest(tmp_path: Path) -> None: + path = tmp_path / "fauxce-hybrid-runtime.json" + document = _document(tmp_path) + document["unexpected"] = True + payload = (json.dumps(document, indent=2) + "\n").encode() + path.write_bytes(payload) + + with pytest.raises(HybridRuntimeManifestError, match="not canonical"): + load_hybrid_runtime_manifest( + path, + expected_sha256=hashlib.sha256(payload).hexdigest(), + ) + + +def test_loader_rejects_symlinked_manifest(tmp_path: Path) -> None: + target = tmp_path / "target.json" + payload = canonical_runtime_manifest_bytes(_document(tmp_path)) + target.write_bytes(payload) + path = tmp_path / "fauxce-hybrid-runtime.json" + path.symlink_to(target) + + with pytest.raises(HybridRuntimeManifestError, match="non-symlink"): + load_hybrid_runtime_manifest( + path, + expected_sha256=hashlib.sha256(payload).hexdigest(), + ) + + +def test_loader_does_not_block_if_manifest_is_swapped_to_fifo( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + path = tmp_path / "fauxce-hybrid-runtime.json" + payload = canonical_runtime_manifest_bytes(_document(tmp_path)) + path.write_bytes(payload) + real_open = runtime_manifest.os.open + swapped = False + + def swap_before_open(candidate, flags, *args, **kwargs): + nonlocal swapped + if os.fspath(candidate) == os.fspath(path) and not swapped: + swapped = True + assert flags & os.O_NONBLOCK + path.unlink() + os.mkfifo(path) + return real_open(candidate, flags, *args, **kwargs) + + monkeypatch.setattr(runtime_manifest.os, "open", swap_before_open) + + with pytest.raises(HybridRuntimeManifestError, match="changed while opening"): + load_hybrid_runtime_manifest( + path, + expected_sha256=hashlib.sha256(payload).hexdigest(), + ) + + assert swapped is True diff --git a/tests/roll/__init__.py b/tests/roll/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/roll/_exact_fixtures.py b/tests/roll/_exact_fixtures.py new file mode 100644 index 00000000..5d0739ce --- /dev/null +++ b/tests/roll/_exact_fixtures.py @@ -0,0 +1,143 @@ +"""Explicit test-only files for the production exact-color trust boundary.""" + +from __future__ import annotations + +import json +import tempfile +from hashlib import sha256 +from pathlib import Path + +import numpy as np + +from negpy.services.roll import exact_color + + +CHANNELS = ("r", "g", "b") +PRE_F_BYTES = 65_536 * 2 +FIXED_ARTIFACT_BYTES = { + "analyzer-desc.bin": 96, + "analyzer-pixels.bin": 281 * 425 * 3 * 2, + "builder-args.bin": 204, + **{f"builder-control-{axis}-{channel}.bin": 256 for channel in CHANNELS for axis in ("x", "y")}, +} + + +def write_stage3_replay_fixture( + root: Path, + pre_f_arrays: tuple[np.ndarray, np.ndarray, np.ndarray], +) -> Path: + """Write a complete real-shaped PASS JSON plus the three consumed LUTs.""" + + root.mkdir(parents=True, exist_ok=True) + pre_f_luts = tuple(np.asarray(array, dtype=" exact_color.ValidatedBuilderReceipt: + with tempfile.TemporaryDirectory(prefix="negpy-stage3-test-") as directory: + report_path = write_stage3_replay_fixture(Path(directory), pre_f_arrays) + return exact_color.load_stage3_replay_builder_receipt(report_path) + + +def production_cms_payload( + *, + builder_receipt_sha256: str, + input_rgb_sha256: str, + output_rgb_sha256: str, + chunk_pixels: int = 17, +) -> dict[str, object]: + return { + "algorithm": exact_color.CMS_ALGORITHM_ID, + "assets": dict(exact_color.CMS_ASSET_SHA256), + "builder_receipt_sha256": builder_receipt_sha256, + "chunk_pixels": chunk_pixels, + "dll_free": True, + "input_rgb_sha256": input_rgb_sha256, + "kind": exact_color.CMS_RECEIPT_KIND, + "oracle_source": { + "path": "portable_oracle_evaluator.py", + "sha256": exact_color.CMS_ORACLE_SOURCE_SHA256, + }, + "output_rgb_sha256": output_rgb_sha256, + "scope": exact_color.CMS_SCOPE, + "stage_order": ["stage1", "stage2"], + "upstream_builder_included": False, + "validation": { + "events": 12, + "full_payload_mismatched_bytes": 0, + "full_payload_total_bytes": 698_880, + "mismatched_u16": 0, + "receipt_sha256": exact_color.CMS_VALIDATION_RECEIPT_SHA256, + "total_u16": 265_440, + }, + "version": exact_color.CMS_RECEIPT_VERSION, + } diff --git a/tests/roll/conftest.py b/tests/roll/conftest.py new file mode 100644 index 00000000..ffdf88c6 --- /dev/null +++ b/tests/roll/conftest.py @@ -0,0 +1,301 @@ +"""Shared fixtures for tests/roll/: a fake `coolscanpy` module. + +`negpy.infrastructure.roll.coolscanpy_roll` and `negpy.services.roll.service` +treat coolscanpy as optional and import it lazily, so these tests never need +the real package installed -- `fake_coolscanpy` injects a minimal stand-in +module into `sys.modules` instead, exercising the same `import coolscanpy` +path the production code takes. +""" + +from __future__ import annotations + +import dataclasses +import hashlib +import importlib.machinery +import sys +import types +from typing import Any + +import numpy as np +import pytest + +from negpy.infrastructure.roll import repair as roll_repair + + +class FakePyCoolscanError(Exception): + """Stand-in for coolscanpy.PyCoolscanError.""" + + +class FakeDeviceNotFound(FakePyCoolscanError): + pass + + +class FakeManualReviewRequired(FakePyCoolscanError): + def __init__(self, message: str, *, slot: int) -> None: + super().__init__(message) + self.slot = slot + + +class FakeSafeStopRequested(FakePyCoolscanError): + pass + + +class FakeMaterial: + COLOR_NEGATIVE = "color-negative" + BLACK_AND_WHITE_NEGATIVE = "black-and-white-negative" + + +@dataclasses.dataclass(frozen=True) +class FakeThumbnail: + slot: int + image: Any + boundary_rows: tuple + spacing_offset: int + needs_approval: bool + warnings: tuple = () + + +@dataclasses.dataclass(frozen=True) +class FakeReceipt: + """Deliberately smaller than the real coolscanpy Receipt -- write_frame() + only needs *a* dataclass instance to round-trip through dataclasses.asdict().""" + + version: int + slot: int + dpi: int + depth: int + device_id: str + transport_smear_verdict: str + + +@dataclasses.dataclass(frozen=True) +class FakeFrame: + slot: int + rgb: Any + ir: Any + ir_validity: Any + receipt: Any + meter_rgbi: Any = None + digital_ice_acquisition: Any = None + + def prepare_digital_ice(self): + if self.digital_ice_acquisition is None: + raise ValueError("frame has no bound Digital ICE acquisition") + return self.digital_ice_acquisition + + +@dataclasses.dataclass(frozen=True) +class FakeProgress: + """Stand-in for coolscanpy.types.Progress, same field names.""" + + stage: str + slot: Any + index: int + total: int + fraction: float + message: str + + +class FakeRoll: + """Scripted in-memory stand-in for `coolscanpy.Roll`.""" + + def __init__(self, thumbnails=None, frames=None, raise_on=None) -> None: + self._thumbnails = list(thumbnails or []) + self._frames = {frame.slot: frame for frame in (frames or [])} + self._raise_on = raise_on or {} + self.approved: list[int] = [] + self.restore_preview_session_calls: list[tuple[str, tuple[int, ...] | None]] = [] + self.spacing_offsets: dict[int, int] = {} + self.safe_stop_called = False + self.closed = False + + def preview(self, slots=None, *, on_progress=None): + if "preview" in self._raise_on: + raise self._raise_on["preview"] + if on_progress is not None: + on_progress( + FakeProgress(stage="preview", slot=None, index=0, total=1, fraction=0.0, message="reading whole-roll transport index") + ) + wanted = None if slots is None else set(slots) + result = [t for t in self._thumbnails if wanted is None or t.slot in wanted] + if on_progress is not None: + on_progress(FakeProgress(stage="preview", slot=None, index=1, total=1, fraction=1.0, message="preview complete")) + return result + + def restore_preview_session(self, payload, slots=None): + if "restore_preview_session" in self._raise_on: + raise self._raise_on["restore_preview_session"] + requested = None if slots is None else tuple(slots) + self.restore_preview_session_calls.append((payload, requested)) + wanted = None if requested is None else set(requested) + return [thumbnail for thumbnail in self._thumbnails if wanted is None or thumbnail.slot in wanted] + + def set_spacing_offset(self, slot, offset_rows) -> None: + self.spacing_offsets[slot] = offset_rows + + def approve(self, slot) -> None: + if "approve" in self._raise_on: + raise self._raise_on["approve"] + self.approved.append(slot) + + def needs_approval(self, slot) -> bool: + return slot in self._raise_on.get("needs_approval_slots", ()) + + def scan_many(self, slots, *, on_progress=None): + ordered = list(slots) + for i, slot in enumerate(ordered): + error = self._raise_on.get("scan_many_slots", {}).get(slot) + if error is not None: + raise error + if on_progress is not None: + on_progress( + FakeProgress(stage="fine-scan", slot=slot, index=i, total=len(ordered), fraction=1.0, message=f"slot {slot} complete") + ) + yield self._frames[slot] + + def safe_stop(self) -> None: + self.safe_stop_called = True + + def close(self) -> None: + self.closed = True + + +class FakeDevice: + def __init__(self, roll) -> None: + self._roll = roll + self.closed = False + self.roll_called_with = None + + def roll(self, *, material=None): + self.roll_called_with = material + return self._roll + + def close(self) -> None: + self.closed = True + + +@pytest.fixture +def fake_coolscanpy(monkeypatch): + """Install a fake `coolscanpy` module; returns an object to script it. + + Usage:: + + fake_coolscanpy.state["open_device"] = fake_coolscanpy.Device(fake_coolscanpy.Roll()) + handle = coolscanpy_roll.open_roll() + """ + module = types.ModuleType("coolscanpy") + # importlib.util.find_spec() raises ValueError on a sys.modules entry + # with no __spec__, so give the fake one like a real imported module has. + module.__spec__ = importlib.machinery.ModuleSpec("coolscanpy", loader=None) + + state: dict[str, Any] = {"devices": [], "open_device": None, "open_error": None} + + def fake_get_devices(local_only: bool = False): + return state["devices"] + + def fake_open(devname: str): + if state["open_error"] is not None: + raise state["open_error"] + return state["open_device"] + + module.get_devices = fake_get_devices + module.open = fake_open + module.Material = FakeMaterial + module.PyCoolscanError = FakePyCoolscanError + module.DeviceNotFound = FakeDeviceNotFound + module.ManualReviewRequired = FakeManualReviewRequired + module.SafeStopRequested = FakeSafeStopRequested + + monkeypatch.setitem(sys.modules, "coolscanpy", module) + + return types.SimpleNamespace( + module=module, + state=state, + Device=FakeDevice, + Roll=FakeRoll, + Thumbnail=FakeThumbnail, + Frame=FakeFrame, + Receipt=FakeReceipt, + ) + + +class FakeRepairEngine: + """Scripted stand-in for `negpy.infrastructure.roll.repair.RepairEngine`. + + By default `repair()` is the identity transform (returns `rgb` unchanged, + tagged with this engine's name/version/mode) -- set `.transform` to a + function to script a detectable change, or `.raise_error` to an exception + instance to script a failure. `.calls` records every `(rgb, ir, mode)` + and `.prepasses` the matching prepass the service passed in, so a test + can assert *what* was repaired (e.g. + the Tier-1 array, not something already touched) without needing the + repair to actually alter pixels. + """ + + def __init__(self, *, engine: str = "test-repair-engine", engine_version: str = "0.0.1-test") -> None: + self.engine = engine + self.engine_version = engine_version + self.transform = lambda rgb: rgb + self.raise_error: Exception | None = None + self.calls: list[tuple[Any, roll_repair.RepairMode, Any]] = [] + self.prepasses: list[Any] = [] + + def repair( + self, + acquisition: Any, + mode: roll_repair.RepairMode, + *, + hybrid_runtime: Any = None, + progress: Any = None, + cancel: Any = None, + ) -> roll_repair.RepairResult: + self.calls.append((acquisition, mode, hybrid_runtime)) + self.prepasses.append(acquisition.prepass_rgbi) + if progress is not None: + progress(1.0) + if self.raise_error is not None: + raise self.raise_error + native_rgb = self.transform(acquisition.main_rgbi[..., :3]) + storage_rgb = acquisition.storage_rgb(native_rgb) + resolved_mode = roll_repair.RepairMode.EXACT if mode is roll_repair.RepairMode.HYBRID and hybrid_runtime is None else mode + return roll_repair.RepairResult( + rgb=storage_rgb, + engine=self.engine, + engine_version=self.engine_version, + mode_requested=mode, + mode_resolved=resolved_mode, + reason=( + "hybrid mode requested but no hybrid runtime is configured; degraded to exact repair" + if resolved_mode is not mode + else "test repair applied" + ), + acquisition_id=acquisition.acquisition_id, + slot=acquisition.slot, + reservation_id=acquisition.reservation_id, + evidence_sha256=acquisition.evidence_sha256, + native_output_rgb_sha256=hashlib.sha256(np.asarray(native_rgb, dtype=" None: + monkeypatch.delitem(sys.modules, "coolscanpy", raising=False) + # The optional group may intentionally be installed by this test run, + # so model an absent package at the availability seam rather than + # assuming anything about the interpreter's installed extras. + monkeypatch.setattr(coolscanpy_roll.importlib.util, "find_spec", lambda _name: None) + assert coolscanpy_roll.available() is False + + def test_true_when_fake_module_present(self, fake_coolscanpy) -> None: + assert coolscanpy_roll.available() is True + + +class TestListDevices: + def test_passthrough(self, fake_coolscanpy) -> None: + fake_coolscanpy.state["devices"] = ["device-a", "device-b"] + assert coolscanpy_roll.list_devices() == ["device-a", "device-b"] + + +class TestOpenRoll: + def test_opens_named_device_with_default_material(self, fake_coolscanpy) -> None: + roll = fake_coolscanpy.Roll() + device = fake_coolscanpy.Device(roll) + fake_coolscanpy.state["open_device"] = device + + handle = coolscanpy_roll.open_roll("ls5000-usb-001") + + assert device.roll_called_with == fake_coolscanpy.module.Material.COLOR_NEGATIVE + handle.close() + assert roll.closed is True + assert device.closed is True + + def test_explicit_material_forwarded(self, fake_coolscanpy) -> None: + roll = fake_coolscanpy.Roll() + device = fake_coolscanpy.Device(roll) + fake_coolscanpy.state["open_device"] = device + + coolscanpy_roll.open_roll(material=fake_coolscanpy.module.Material.BLACK_AND_WHITE_NEGATIVE) + + assert device.roll_called_with == fake_coolscanpy.module.Material.BLACK_AND_WHITE_NEGATIVE + + def test_caller_owned_attempts_root_is_forwarded( + self, + fake_coolscanpy, + tmp_path: Path, + ) -> None: + roll = fake_coolscanpy.Roll() + + class EvidenceDevice(fake_coolscanpy.Device): + def __init__(self) -> None: + super().__init__(roll) + self.attempts_root = None + + def roll(self, *, material=None, attempts_root=None): + self.roll_called_with = material + self.attempts_root = attempts_root + return self._roll + + device = EvidenceDevice() + fake_coolscanpy.state["open_device"] = device + evidence = tmp_path / "scanner-attempts" + + handle = coolscanpy_roll.open_roll( + "ls5000-usb-001", + attempts_root=evidence, + ) + + assert device.attempts_root == evidence + handle.close() + + def test_device_not_found_translated_to_runtime_error(self, fake_coolscanpy) -> None: + fake_coolscanpy.state["open_error"] = fake_coolscanpy.module.DeviceNotFound("no Coolscan LS-5000 unit is attached") + + with pytest.raises(RuntimeError, match="no Coolscan LS-5000 unit is attached"): + coolscanpy_roll.open_roll() + + def test_roll_open_failure_closes_device_and_translates(self, fake_coolscanpy) -> None: + class ExplodingDevice(fake_coolscanpy.Device): + def roll(self, *, material=None): + raise ValueError("no roll adapter is attached/detected on this device") + + device = ExplodingDevice(None) + fake_coolscanpy.state["open_device"] = device + + with pytest.raises(RuntimeError, match="no roll adapter is attached"): + coolscanpy_roll.open_roll() + assert device.closed is True + + +class TestRollHandle: + @staticmethod + def _handle(fake_coolscanpy, roll=None): + roll = roll if roll is not None else fake_coolscanpy.Roll() + device = fake_coolscanpy.Device(roll) + fake_coolscanpy.state["open_device"] = device + return coolscanpy_roll.open_roll(), roll, device + + def test_preview_passthrough(self, fake_coolscanpy) -> None: + thumb = fake_coolscanpy.Thumbnail(slot=1, image=np.zeros((4, 4, 3)), boundary_rows=(0, 4), spacing_offset=0, needs_approval=False) + handle, _roll, _device = self._handle(fake_coolscanpy, fake_coolscanpy.Roll(thumbnails=[thumb])) + + assert handle.preview() == [thumb] + + def test_preview_filters_by_requested_slots(self, fake_coolscanpy) -> None: + thumbs = [ + fake_coolscanpy.Thumbnail(slot=i, image=np.zeros((2, 2, 3)), boundary_rows=(0, 2), spacing_offset=0, needs_approval=False) + for i in (1, 2, 3) + ] + handle, _roll, _device = self._handle(fake_coolscanpy, fake_coolscanpy.Roll(thumbnails=thumbs)) + + result = handle.preview([2]) + + assert [t.slot for t in result] == [2] + + def test_preview_exception_translated(self, fake_coolscanpy) -> None: + handle, _roll, _device = self._handle( + fake_coolscanpy, + fake_coolscanpy.Roll(raise_on={"preview": fake_coolscanpy.module.PyCoolscanError("fingerprint mismatch")}), + ) + with pytest.raises(RuntimeError, match="fingerprint mismatch"): + handle.preview() + + def test_restore_preview_session_passthrough(self, fake_coolscanpy) -> None: + thumbnails = [ + fake_coolscanpy.Thumbnail( + slot=slot, + image=np.zeros((2, 2, 3)), + boundary_rows=(0, 2), + spacing_offset=0, + needs_approval=False, + ) + for slot in (1, 2, 3) + ] + handle, roll, _device = self._handle( + fake_coolscanpy, + fake_coolscanpy.Roll(thumbnails=thumbnails), + ) + + result = handle.restore_preview_session("saved-session", [1, 3]) + + assert [thumbnail.slot for thumbnail in result] == [1, 3] + assert roll.restore_preview_session_calls == [("saved-session", (1, 3))] + + def test_restore_preview_session_exception_translated(self, fake_coolscanpy) -> None: + error = fake_coolscanpy.module.PyCoolscanError("saved preview hash mismatch") + handle, _roll, _device = self._handle( + fake_coolscanpy, + fake_coolscanpy.Roll(raise_on={"restore_preview_session": error}), + ) + + with pytest.raises(RuntimeError, match="saved preview hash mismatch") as excinfo: + handle.restore_preview_session("saved-session") + + assert excinfo.value.__cause__ is error + + def test_approve_returns_underlying_content_bound_receipt(self, fake_coolscanpy) -> None: + approval = object() + + class ReturningApprovalRoll(fake_coolscanpy.Roll): + def approve(self, slot): + super().approve(slot) + return approval + + handle, roll, _device = self._handle( + fake_coolscanpy, + ReturningApprovalRoll(), + ) + + result = handle.approve(3) + + assert result is approval + assert roll.approved == [3] + + def test_manual_review_required_translated(self, fake_coolscanpy) -> None: + handle, _roll, _device = self._handle( + fake_coolscanpy, + fake_coolscanpy.Roll(raise_on={"approve": fake_coolscanpy.module.ManualReviewRequired("slot 3 needs review", slot=3)}), + ) + with pytest.raises(RuntimeError, match="slot 3 needs review"): + handle.approve(3) + + def test_set_spacing_offset_records_value(self, fake_coolscanpy) -> None: + handle, roll, _device = self._handle(fake_coolscanpy) + handle.set_spacing_offset(5, -12) + assert roll.spacing_offsets[5] == -12 + + def test_scan_many_yields_frames(self, fake_coolscanpy) -> None: + frame = fake_coolscanpy.Frame(slot=2, rgb=np.zeros((4, 4, 3), dtype=np.uint16), ir=None, ir_validity=None, receipt=None) + handle, _roll, _device = self._handle(fake_coolscanpy, fake_coolscanpy.Roll(frames=[frame])) + + assert list(handle.scan_many([2])) == [frame] + + def test_scan_many_exception_translated(self, fake_coolscanpy) -> None: + error = fake_coolscanpy.module.SafeStopRequested("safe stop requested; 0 of 1 requested frames completed") + handle, _roll, _device = self._handle(fake_coolscanpy, fake_coolscanpy.Roll(raise_on={"scan_many_slots": {5: error}})) + + with pytest.raises(RuntimeError, match="safe stop requested") as excinfo: + list(handle.scan_many([5])) + # __cause__ preserves the original typed exception for a caller that + # wants to distinguish a deliberate stop from a genuine failure. + assert isinstance(excinfo.value.__cause__, fake_coolscanpy.module.SafeStopRequested) + + def test_safe_stop_forwarded(self, fake_coolscanpy) -> None: + handle, roll, _device = self._handle(fake_coolscanpy) + handle.safe_stop() + assert roll.safe_stop_called is True + + def test_close_releases_roll_then_device(self, fake_coolscanpy) -> None: + handle, roll, device = self._handle(fake_coolscanpy) + handle.close() + assert roll.closed is True + assert device.closed is True + + def test_close_retains_device_when_roll_ownership_is_uncertain(self, fake_coolscanpy) -> None: + ownership_error = RuntimeError("USB ownership is retained") + + class UncertainRoll(fake_coolscanpy.Roll): + def close(self) -> None: + raise ownership_error + + handle, _roll, device = self._handle(fake_coolscanpy, UncertainRoll()) + + with pytest.raises(RuntimeError) as raised: + handle.close() + + assert raised.value is ownership_error + assert device.closed is False + + def test_context_manager_closes_on_exit(self, fake_coolscanpy) -> None: + roll = fake_coolscanpy.Roll() + device = fake_coolscanpy.Device(roll) + fake_coolscanpy.state["open_device"] = device + + with coolscanpy_roll.open_roll() as handle: + assert handle is not None + assert roll.closed is False + + assert roll.closed is True diff --git a/tests/roll/test_deep_acceptance.py b/tests/roll/test_deep_acceptance.py new file mode 100644 index 00000000..e21a5ba1 --- /dev/null +++ b/tests/roll/test_deep_acceptance.py @@ -0,0 +1,1196 @@ +from __future__ import annotations + +import hashlib +import json +import os +import fcntl +from dataclasses import replace +from pathlib import Path +from types import SimpleNamespace + +import numpy as np +import pytest + +from coolscanpy.protocol.ls5000_single_pass.density import ( + NikonDensityFrameOwnershipReceipt, +) +from coolscanpy.types import ( + ArtifactEvidence, + ClippingTelemetry, + ExposureVector, + FocusDetailTelemetry, + Receipt, + TransportSmearAssessment, +) + +from negpy.infrastructure.roll import repair as roll_repair +from negpy.services.roll import deep_acceptance +from negpy.services.roll import service as roll_service +from negpy.services.roll.service import RollFrameOutput, RollScanningService +from tests.roll.test_native_builder import ( + FRESH_FINGERPRINT_SHA256, + RESERVATION_ID, + REVIEWED_FINGERPRINT_SHA256, + _PayloadReceipt, + _density_document, + _ownership_document, + _synthetic_native_evidence, +) +from tests.roll.test_service import _valid_hybrid_result + + +class _Runtime: + core_source_manifest_sha256 = "1" * 64 + hybrid_source_manifest_sha256 = "2" * 64 + iopaint_source_manifest_sha256 = "3" * 64 + model_weights_sha256 = "4" * 64 + inpaint_device = "cpu" + inpaint_threads = 1 + inpaint_seed = 0 + + def __init__(self) -> None: + self.validated = False + + def validate_files(self) -> None: + self.validated = True + + +def _runtime_bound_hybrid_result( + acquisition: roll_repair.RepairAcquisition, + runtime: _Runtime, +) -> roll_repair.RepairResult: + base = _valid_hybrid_result(acquisition) + document = json.loads(base.hybrid_receipt) + assertion = { + "assertions": { + "focus_exposure_locked": True, + "same_frame_id": acquisition.acquisition_id, + }, + "inputs": { + "main": {"raw_sha256": acquisition.main_rgbi_sha256}, + "prepass": {"raw_sha256": acquisition.prepass_rgbi_sha256}, + }, + "provenance_class": "caller_asserted_bare_npy", + "schema": "negpy.fauxce-hybrid-acquisition-assertion-v1", + } + document.update( + core={ + "backend": { + "reason": base.backend_selection_reason, + "requested": base.backend_requested, + "used": base.backend_used, + }, + "source_manifest_sha256": runtime.core_source_manifest_sha256, + "version": base.engine_version, + }, + generation={"hybrid_source_manifest_sha256": (runtime.hybrid_source_manifest_sha256)}, + inpainting={"invoked": False}, + inputs={ + "geometry": { + "mask_shape": list(acquisition.main_rgbi.shape[:2]), + "output_shape": [*acquisition.main_rgbi.shape[:2], 3], + }, + "main": { + "canonical_encoding": "uint16_little_endian_c_order", + "raw_sha256": acquisition.main_rgbi_sha256, + "shape": list(acquisition.main_rgbi.shape), + }, + "prepass": { + "canonical_encoding": "uint16_little_endian_c_order", + "raw_sha256": acquisition.prepass_rgbi_sha256, + "shape": list(acquisition.prepass_rgbi.shape), + }, + "provenance": { + "basis": "caller_asserted", + "source_manifest_sha256": hashlib.sha256(deep_acceptance._canonical_json(assertion)).hexdigest(), + }, + }, + ) + payload = ( + json.dumps( + document, + sort_keys=True, + separators=(",", ":"), + ensure_ascii=True, + allow_nan=False, + ).encode() + + b"\n" + ) + return replace( + base, + hybrid_receipt=payload, + hybrid_receipt_sha256=hashlib.sha256(payload).hexdigest(), + ) + + +def _production_frame() -> tuple[SimpleNamespace, roll_repair.RepairAcquisition]: + slot = 5 + storage_rgb = np.arange(4 * 6 * 3, dtype=np.uint16).reshape(4, 6, 3) + storage_rgb += np.uint16(10_000) + storage_ir = np.arange(4 * 6, dtype=np.uint16).reshape(4, 6) + storage_ir += np.uint16(2_000) + storage_validity = np.ones((4, 6), dtype=np.bool_) + meter = np.arange(2 * 3 * 4, dtype=np.uint16).reshape(2, 3, 4) + native_rgbi = np.ascontiguousarray( + np.rot90( + np.dstack((storage_rgb, storage_ir)), + k=-1, + axes=(0, 1), + ) + ) + native_validity = np.ascontiguousarray(np.rot90(storage_validity, k=-1, axes=(0, 1))) + acquisition_id, evidence_sha256 = roll_service._derive_digital_ice_producer_binding( + slot=slot, + reservation_id=RESERVATION_ID, + capture_attempt_id="capture-d", + main_rgbi=native_rgbi, + prepass_rgbi=meter, + ir_validity=native_validity, + ) + acquisition = roll_repair.RepairAcquisition.from_arrays( + acquisition_id=acquisition_id, + slot=slot, + reservation_id=RESERVATION_ID, + capture_attempt_id="capture-d", + storage_transform=roll_repair.DIGITAL_ICE_STORAGE_TRANSFORM, + evidence_sha256=evidence_sha256, + main_rgbi=native_rgbi, + prepass_rgbi=meter, + ir_validity=native_validity, + ) + + def artifact(array: np.ndarray) -> ArtifactEvidence: + contiguous = np.ascontiguousarray(array) + return ArtifactEvidence( + sha256=hashlib.sha256(memoryview(contiguous).cast("B")).hexdigest(), + byte_length=contiguous.nbytes, + shape=contiguous.shape, + dtype=str(contiguous.dtype), + ) + + ownership_document = _ownership_document() + ownership = NikonDensityFrameOwnershipReceipt.from_dict(ownership_document) + receipt = Receipt( + version=1, + slot=slot, + spacing_offset=0, + dpi=4_000, + depth=16, + device_id="usb:2:7", + device_model="Nikon LS-5000 ED 1.03", + reviewed_fingerprint_sha256=REVIEWED_FINGERPRINT_SHA256, + fresh_fingerprint_sha256=FRESH_FINGERPRINT_SHA256, + manual_approval=None, + exposure=ExposureVector(1, 1.0, 1.0, 1.0, 1.0), + split_alignment=None, + clipping=ClippingTelemetry((0.0, 0.0, 0.0), 65_535.0, 0.01, False), + focus_detail=FocusDetailTelemetry("laplacian", "measured", 1.0, 1.0), + transport_smear=TransportSmearAssessment( + "clean", + None, + 0, + 0, + None, + None, + None, + None, + "clean", + ), + artifacts={"rgb": artifact(storage_rgb), "ir": artifact(storage_ir)}, + nikon_density_ownership=ownership, + ) + frame = SimpleNamespace( + slot=slot, + rgb=storage_rgb, + ir=storage_ir, + ir_validity=storage_validity, + meter_rgbi=meter, + receipt=receipt, + nikon_density_evidence=_PayloadReceipt(_density_document()), + nikon_density_ownership=_PayloadReceipt(ownership_document), + nikon_exact_builder_evidence=_synthetic_native_evidence(), + prepare_digital_ice=lambda: acquisition, + ) + return frame, acquisition + + +def _digest(text: str) -> str: + return hashlib.sha256(text.encode()).hexdigest() + + +def _output(root: Path, slot: int) -> RollFrameOutput: + base = root / f"acceptance_slot{slot:02d}" + return RollFrameOutput( + slot=slot, + rgb_path=str(base.with_suffix(".tif")), + ir_path=str(base.with_name(base.name + "_IR.tif")), + repaired_rgb_path=str(base.with_name(base.name + "_repaired.tif")), + repaired_ir_path=str(base.with_name(base.name + "_repaired_IR.tif")), + positive_path=str(base.with_name(base.name + "_positive.tif")), + receipt_path=str(base.with_name(base.name + "_receipt.json")), + synthesis_mask_path=str(base.with_name(base.name + "_repaired_SYNTH.png")), + native_synthesis_mask_path=str(root / ".negpy-dice-hybrid" / f"slot-{slot}" / "native.png"), + hybrid_receipt_path=str(root / ".negpy-dice-hybrid" / f"slot-{slot}" / "hybrid-receipt.json"), + ) + + +def _completed_receipt_fixture( + root: Path, + *, + slot: int = 1, +) -> tuple[RollFrameOutput, dict[str, object]]: + output = _output(root, slot) + + def touch(path: Path) -> str: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(f"slot-{slot}:{path.name}".encode()) + return str(path) + + for value in ( + output.rgb_path, + output.ir_path, + output.repaired_rgb_path, + output.repaired_ir_path, + output.positive_path, + output.synthesis_mask_path, + output.native_synthesis_mask_path, + output.hybrid_receipt_path, + ): + touch(Path(value)) + + dice_dir = root / ".negpy-dice-acquisition" / f"slot-{slot}" + dice_binding = dice_dir / "acquisition-binding.json" + dice_prepass = dice_dir / "prepass.rgbi16.npy" + dice_validity = dice_dir / "ir-validity.npy" + for path in (dice_binding, dice_prepass, dice_validity): + touch(path) + + hybrid_dir = Path(output.hybrid_receipt_path).parent + routed_mask = hybrid_dir / "routed.png" + hybrid_binding = hybrid_dir / "binding.json" + for path in (routed_mask, hybrid_binding): + touch(path) + + native_dir = root / ".negpy-native-builder" / f"slot-{slot}" + native_paths = { + "builder_receipt": native_dir / "native-builder-receipt.json", + "analyzer_rgb": native_dir / "analyzer-rgb-u16le.bin", + "evidence_receipt": native_dir / "native-builder-evidence.json", + "frame_ownership_receipt": (native_dir / "nikon-density-frame-ownership.json"), + "density_evidence_receipt": native_dir / "nikon-density-evidence.json", + } + lut_paths = [native_dir / f"builder-preF-{channel}.bin" for channel in "rgb"] + for path in (*native_paths.values(), *lut_paths): + touch(path) + + retained = { + **{key: {"path": str(path)} for key, path in native_paths.items()}, + "native_per_acquisition_builder": True, + "pre_f_luts": [{"path": str(path)} for path in lut_paths], + } + document: dict[str, object] = { + "artifacts": {"ir": {}, "rgb": {}}, + "depth": 16, + "device_id": "usb:2:7", + "device_model": "Nikon LS-5000 ED 1.03", + "dpi": 4_000, + "outputs": { + "native_color_evidence": { + "native_per_acquisition_builder": True, + "retained": True, + "retained_builder_evidence": retained, + }, + "positive": { + "builder_validated": True, + "cms_verified": True, + "color_mode": "nikon-exact", + "exact_nikon_color": True, + "native_per_acquisition_builder": True, + "retained_builder_evidence": retained, + "rgb_path": output.positive_path, + "written": True, + }, + "repair_acquisition_evidence": { + "artifacts": { + "ir_validity": {"path": str(dice_validity)}, + "prepass_rgbi": {"path": str(dice_prepass)}, + }, + "binding": {"path": str(dice_binding)}, + "replayable": True, + "retained": True, + "sources": { + "storage_ir_tiff": {"path": output.ir_path}, + "storage_rgb_tiff": {"path": output.rgb_path}, + }, + }, + "repaired": { + "degraded": False, + "disclosure_mask": { + "applied_final": { + "native": {"path": output.native_synthesis_mask_path}, + "storage": {"path": output.synthesis_mask_path}, + }, + "routed_raw": {"native": {"path": str(routed_mask)}}, + }, + "hybrid_evidence_binding": {"path": str(hybrid_binding)}, + "hybrid_receipt": {"path": output.hybrid_receipt_path}, + "ir_path": output.repaired_ir_path, + "mode_requested": "hybrid", + "mode_resolved": "hybrid", + "rgb_path": output.repaired_rgb_path, + "written": True, + }, + "unrepaired": { + "ir_path": output.ir_path, + "rgb_path": output.rgb_path, + "written": True, + }, + }, + "slot": slot, + "version": 1, + } + receipt_path = Path(output.receipt_path) + receipt_path.write_text( + json.dumps(document, indent=2, ensure_ascii=True, allow_nan=False), + encoding="utf-8", + ) + lock_dir = root / ".negpy-locks" + lock_dir.mkdir() + lock_path = lock_dir / (hashlib.sha256(str(receipt_path).encode()).hexdigest() + ".lock") + lock_path.write_bytes(b"") + return output, document + + +def _approval(slot: int, reviewed: str) -> dict[str, object]: + return { + "reviewed_fingerprint_sha256": reviewed, + "slot": slot, + "spacing_offset": 0, + "thumbnail_sha256": _digest(f"thumbnail-{slot}"), + "reviewed_lookup_row": (slot - 1) * 143, + "reviewed_native_origin": (slot - 1) * 5_959, + "review_reasons": ["strip boundary requires review"], + } + + +def _fake_audits(root: Path) -> tuple[list[RollFrameOutput], list[deep_acceptance._FrameAudit]]: + reservation = "reservation-six-frame" + preview = _digest("preview") + preview_identity = _digest("preview-identity") + transport_table = _digest("transport-table") + reviewed = _digest("reviewed") + fresh = _digest("fresh") + transport_material = { + "reservation_id": reservation, + "batch_session_id": reservation, + "preview_sha256": preview, + "preview_identity_sha256": preview_identity, + "transport_table_sha256": transport_table, + "reviewed_fingerprint_sha256": reviewed, + "fresh_fingerprint_sha256": fresh, + "selected_slots": list(deep_acceptance.SLOTS), + } + transport_identity = hashlib.sha256( + json.dumps( + transport_material, + sort_keys=True, + separators=(",", ":"), + ).encode() + ).hexdigest() + outputs: list[RollFrameOutput] = [] + audits: list[deep_acceptance._FrameAudit] = [] + for slot in deep_acceptance.SLOTS: + output = _output(root, slot) + outputs.append(output) + visible = { + Path(output.rgb_path), + Path(output.ir_path), + Path(output.repaired_rgb_path), + Path(output.repaired_ir_path), + Path(output.positive_path), + Path(output.receipt_path), + Path(output.synthesis_mask_path), + } + for path in visible: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(f"slot-{slot}:{path.name}".encode()) + ownership = SimpleNamespace( + reservation_id=reservation, + batch_session_id=reservation, + preview_sha256=preview, + preview_identity_sha256=preview_identity, + transport_table_sha256=transport_table, + transport_identity_sha256=transport_identity, + reviewed_fingerprint_sha256=reviewed, + fresh_fingerprint_sha256=fresh, + frame_capture_attempt_id=f"attempt-{slot}", + frame_index=slot, + frame_total=6, + selected_slots=deep_acceptance.SLOTS, + selected_slot=slot, + ) + receipt = { + "device_id": "usb:2:7", + "device_model": "Nikon LS-5000 ED 1.03", + "spacing_offset": 0, + "manual_approval": (_approval(slot, reviewed) if slot in deep_acceptance.APPROVED_SLOTS else None), + } + receipt_path = Path(output.receipt_path) + receipt_sha = hashlib.sha256(receipt_path.read_bytes()).hexdigest() + lock_dir = root / ".negpy-locks" + lock_dir.mkdir(exist_ok=True) + lock_path = lock_dir / (hashlib.sha256(str(receipt_path).encode()).hexdigest() + ".lock") + lock_path.write_bytes(b"") + visible.add(lock_path) + audits.append( + deep_acceptance._FrameAudit( + summary={ + "slot": slot, + "frame_receipt": { + "path": str(receipt_path), + "bytes": receipt_path.stat().st_size, + "sha256": receipt_sha, + }, + "referenced_file_count": len(visible), + "referenced_files": sorted(str(path) for path in visible), + }, + referenced_files=frozenset(visible), + receipt_path=receipt_path, + receipt=receipt, + ownership=ownership, + builder_receipt=SimpleNamespace(density_evidence_receipt_sha256=_digest("density")), + output_artifacts={field: getattr(output, field) for field in deep_acceptance._OUTPUT_FIELDS}, + ) + ) + return outputs, audits + + +def test_six_frame_batch_closes_cross_frame_and_exact_inventory( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + outputs, audits = _fake_audits(tmp_path) + runtime = _Runtime() + iterator = iter(audits) + + def validate_locked_frame(*args: object, **kwargs: object): + del args, kwargs + for audit in audits: + lock_path = deep_acceptance._receipt_lock_path(audit.receipt_path) + descriptor = os.open(lock_path, os.O_RDWR) + try: + with pytest.raises(BlockingIOError): + fcntl.flock(descriptor, fcntl.LOCK_EX | fcntl.LOCK_NB) + finally: + os.close(descriptor) + return next(iterator) + + monkeypatch.setattr( + deep_acceptance, + "_validate_frame", + validate_locked_frame, + ) + audit_lock = tmp_path / "deep-audit.lock" + audit_lock.write_bytes(b"") + + result = deep_acceptance.validate_six_frame_batch( + outputs, + output_dir=tmp_path, + allowed_output_lock_name=audit_lock.name, + builder=object(), + evaluator=object(), + hybrid_runtime=runtime, + ) + + assert result["status"] == "passed" + assert result["slots"] == [1, 2, 3, 4, 5, 6] + assert result["approved_slots"] == [1, 6] + assert result["device_id"] == "usb:2:7" + assert result["device_model"] == "Nikon LS-5000 ED 1.03" + assert result["reviewed_fingerprint_sha256"] == audits[0].ownership.reviewed_fingerprint_sha256 + assert result["manual_approval_bindings"] == [ + { + "slot": slot, + "binding_sha256": deep_acceptance._validate_manual_approval( + audits[slot - 1], + expected=True, + )["binding_sha256"], + } + for slot in sorted(deep_acceptance.APPROVED_SLOTS) + ] + assert result["inventory"]["visible_file_count"] == 42 + assert result["inventory"]["exact"] is True + assert result["inventory"]["allowed_output_lock_path"] == str(audit_lock) + assert len(result["referenced_files"]) == 48 + assert str(audit_lock) not in result["referenced_files"] + assert all(Path(path).is_absolute() for path in result["referenced_files"]) + assert runtime.validated is True + json.dumps(result, allow_nan=False) + + +def test_completed_frame_reports_deterministic_absolute_references( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + outputs, audits = _fake_audits(tmp_path) + audit = audits[0] + monkeypatch.setattr( + deep_acceptance, + "_validate_frame", + lambda *args, **kwargs: audit, + ) + + result = deep_acceptance.validate_completed_frame( + outputs[0], + output_dir=tmp_path, + builder=object(), + evaluator=object(), + hybrid_runtime=_Runtime(), + ) + + assert result["referenced_files"] == sorted(result["referenced_files"]) + assert result["referenced_file_count"] == len(result["referenced_files"]) + assert all(Path(path).is_absolute() for path in result["referenced_files"]) + assert any("/.negpy-locks/" in path for path in result["referenced_files"]) + + +def test_manual_approval_rejects_string_review_reasons(tmp_path: Path) -> None: + _, audits = _fake_audits(tmp_path) + approval = audits[0].receipt["manual_approval"] + assert isinstance(approval, dict) + approval["review_reasons"] = "not-a-json-list" + + with pytest.raises( + deep_acceptance.DeepAcceptanceError, + match="approval reasons are malformed", + ): + deep_acceptance._validate_manual_approval(audits[0], expected=True) + + +def test_collect_completed_frame_files_returns_exact_cheap_inventory( + tmp_path: Path, +) -> None: + output, _ = _completed_receipt_fixture(tmp_path) + + files = deep_acceptance.collect_completed_frame_files( + output, + output_dir=tmp_path, + expected_slot=1, + ) + + assert files == sorted(files) + assert len(files) == 23 + assert len(set(files)) == 23 + assert all(Path(path).is_absolute() and Path(path).is_file() for path in files) + assert output.receipt_path in files + assert any("/.negpy-locks/" in path for path in files) + + +def test_collect_completed_frame_files_rejects_extra_receipt_path( + tmp_path: Path, +) -> None: + output, document = _completed_receipt_fixture(tmp_path) + extra = tmp_path / "extra.bin" + extra.write_bytes(b"extra") + outputs = document["outputs"] + assert isinstance(outputs, dict) + positive = outputs["positive"] + assert isinstance(positive, dict) + positive["unexpected"] = {"path": str(extra)} + Path(output.receipt_path).write_text( + json.dumps(document, indent=2, ensure_ascii=True, allow_nan=False), + encoding="utf-8", + ) + + with pytest.raises( + deep_acceptance.DeepAcceptanceError, + match="receipt path inventory changed", + ): + deep_acceptance.collect_completed_frame_files( + output, + output_dir=tmp_path, + expected_slot=1, + ) + + +def test_collect_completed_frame_files_rejects_noncanonical_or_mismatched_output( + tmp_path: Path, +) -> None: + output, document = _completed_receipt_fixture(tmp_path) + Path(output.receipt_path).write_text(json.dumps(document), encoding="utf-8") + with pytest.raises( + deep_acceptance.DeepAcceptanceError, + match="canonical frame-receipt encoding", + ): + deep_acceptance.collect_completed_frame_files( + output, + output_dir=tmp_path, + expected_slot=1, + ) + + Path(output.receipt_path).write_text( + json.dumps(document, indent=2, ensure_ascii=True, allow_nan=False), + encoding="utf-8", + ) + wrong_positive = tmp_path / "wrong-positive.tif" + wrong_positive.write_bytes(b"wrong") + mismatched = replace(output, positive_path=str(wrong_positive)) + with pytest.raises( + deep_acceptance.DeepAcceptanceError, + match="positive_path differs from its receipt", + ): + deep_acceptance.collect_completed_frame_files( + mismatched, + output_dir=tmp_path, + expected_slot=1, + ) + + +def test_collect_completed_frame_files_rejects_nonabsolute_required_path( + tmp_path: Path, +) -> None: + output, document = _completed_receipt_fixture(tmp_path) + outputs = document["outputs"] + assert isinstance(outputs, dict) + unrepaired = outputs["unrepaired"] + assert isinstance(unrepaired, dict) + unrepaired["rgb_path"] = "relative-frame.tif" + Path(output.receipt_path).write_text( + json.dumps(document, indent=2, ensure_ascii=True, allow_nan=False), + encoding="utf-8", + ) + + with pytest.raises( + deep_acceptance.DeepAcceptanceError, + match="absolute artifact path", + ): + deep_acceptance.collect_completed_frame_files( + output, + output_dir=tmp_path, + expected_slot=1, + ) + + +def test_collect_completed_frame_files_refuses_a_busy_production_lock( + tmp_path: Path, +) -> None: + output, _ = _completed_receipt_fixture(tmp_path) + receipt_path = Path(output.receipt_path) + lock_path = deep_acceptance._receipt_lock_path(receipt_path) + descriptor = os.open(lock_path, os.O_RDWR) + try: + fcntl.flock(descriptor, fcntl.LOCK_EX | fcntl.LOCK_NB) + with pytest.raises( + deep_acceptance.DeepAcceptanceError, + match="busy in another process", + ): + deep_acceptance.collect_completed_frame_files( + output, + output_dir=tmp_path, + expected_slot=1, + ) + finally: + fcntl.flock(descriptor, fcntl.LOCK_UN) + os.close(descriptor) + + +def test_completed_frame_runs_full_production_replay_end_to_end( + fake_repair_engine: object, + tmp_path: Path, +) -> None: + del fake_repair_engine + runtime = _Runtime() + frame, expected_acquisition = _production_frame() + + class Engine: + def repair( + self, + acquisition: roll_repair.RepairAcquisition, + mode: roll_repair.RepairMode, + **kwargs: object, + ) -> roll_repair.RepairResult: + del kwargs + assert mode is roll_repair.RepairMode.HYBRID + assert acquisition.acquisition_id == expected_acquisition.acquisition_id + return _runtime_bound_hybrid_result(acquisition, runtime) + + roll_repair.register_engine(Engine()) + output = RollScanningService(hybrid_runtime=runtime).write_frame( + frame, + str(tmp_path), + '20260723_{{ "%03d" % seq }}', + write_unrepaired=True, + write_repaired=True, + write_positive=True, + repair_mode="hybrid", + positive_mode="nikon-exact", + ) + + cheap_files = deep_acceptance.collect_completed_frame_files( + output, + output_dir=tmp_path, + expected_slot=5, + ) + result = deep_acceptance.validate_completed_frame( + output, + output_dir=tmp_path, + expected_slot=5, + hybrid_runtime=runtime, + ) + + assert result["status"] == "passed" + assert result["slot"] == 5 + assert result["referenced_file_count"] == 23 + assert result["referenced_files"] == cheap_files + assert result["acquisition_id"] == expected_acquisition.acquisition_id + assert runtime.validated is True + + +def test_six_frame_batch_rejects_one_unreferenced_file( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + outputs, audits = _fake_audits(tmp_path) + iterator = iter(audits) + monkeypatch.setattr( + deep_acceptance, + "_validate_frame", + lambda *args, **kwargs: next(iterator), + ) + (tmp_path / "unexpected.txt").write_text("not part of the batch") + + with pytest.raises( + deep_acceptance.DeepAcceptanceError, + match="inventory file set changed", + ): + deep_acceptance.validate_six_frame_batch( + outputs, + output_dir=tmp_path, + builder=object(), + evaluator=object(), + hybrid_runtime=_Runtime(), + ) + + +def test_six_frame_batch_ignores_regular_finder_metadata( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + outputs, audits = _fake_audits(tmp_path) + iterator = iter(audits) + monkeypatch.setattr( + deep_acceptance, + "_validate_frame", + lambda *args, **kwargs: next(iterator), + ) + (tmp_path / ".DS_Store").write_bytes(b"finder metadata") + + result = deep_acceptance.validate_six_frame_batch( + outputs, + output_dir=tmp_path, + builder=object(), + evaluator=object(), + hybrid_runtime=_Runtime(), + ) + + assert result["status"] == "passed" + assert result["inventory"]["visible_file_count"] == 42 + + +def test_six_frame_batch_rejects_finder_metadata_symlink( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + outputs, audits = _fake_audits(tmp_path) + iterator = iter(audits) + monkeypatch.setattr( + deep_acceptance, + "_validate_frame", + lambda *args, **kwargs: next(iterator), + ) + outside = tmp_path.parent / "outside-metadata" + outside.write_bytes(b"not metadata") + (tmp_path / ".DS_Store").symlink_to(outside) + + with pytest.raises( + deep_acceptance.DeepAcceptanceError, + match="file entry is unsafe", + ): + deep_acceptance.validate_six_frame_batch( + outputs, + output_dir=tmp_path, + builder=object(), + evaluator=object(), + hybrid_runtime=_Runtime(), + ) + + +def test_six_frame_batch_rejects_cross_frame_fingerprint_drift( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + outputs, audits = _fake_audits(tmp_path) + audits[3].ownership.fresh_fingerprint_sha256 = _digest("different-fresh") + iterator = iter(audits) + monkeypatch.setattr( + deep_acceptance, + "_validate_frame", + lambda *args, **kwargs: next(iterator), + ) + + with pytest.raises( + deep_acceptance.DeepAcceptanceError, + match="disagree on reservation, preview, transport, or fingerprint", + ): + deep_acceptance.validate_six_frame_batch( + outputs, + output_dir=tmp_path, + builder=object(), + evaluator=object(), + hybrid_runtime=_Runtime(), + ) + + +def test_six_frame_batch_binds_canonical_live_run_receipt( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + output_dir = tmp_path / "outputs" + output_dir.mkdir() + outputs, audits = _fake_audits(output_dir) + iterator = iter(audits) + monkeypatch.setattr( + deep_acceptance, + "_validate_frame", + lambda *args, **kwargs: next(iterator), + ) + contact_path = tmp_path / "reviewed-contact-sheet.png" + contact_path.write_bytes(b"pinned contact sheet") + contact_sha256 = hashlib.sha256(contact_path.read_bytes()).hexdigest() + approval_rows = [ + deep_acceptance._validate_manual_approval( + audits[slot - 1], + expected=True, + ) + for slot in sorted(deep_acceptance.APPROVED_SLOTS) + ] + review_document = { + "approvals": approval_rows, + "contact_sheet": { + "bytes": contact_path.stat().st_size, + "path": str(contact_path), + "sha256": contact_sha256, + }, + "preview_session": { + "bytes": 2, + "path": str(tmp_path / "preview-session.json"), + "sha256": hashlib.sha256(b"{}").hexdigest(), + }, + "review_basis": ("visual-inspection-of-six-frame-contact-sheet-and-canonical-restored-thumbnails"), + "reviewed_fingerprint_sha256": (audits[0].ownership.reviewed_fingerprint_sha256), + "schema": "negpy.ls5000-reviewed-approval.v1", + } + review_path = tmp_path / "reviewed-approval.json" + review_path.write_bytes( + deep_acceptance._canonical_json( + review_document, + newline=True, + ensure_ascii=True, + ) + ) + review_sha256 = hashlib.sha256(review_path.read_bytes()).hexdigest() + run_receipt = { + "approved_slots": [1, 6], + "close": { + "iterator": {"attempted": True, "succeeded": True}, + "roll": {"attempted": True, "succeeded": True}, + }, + "deep_acceptance": {"slots": [1, 2, 3, 4, 5, 6], "status": "passed"}, + "device_id": "usb:2:7", + "eject_requested": False, + "frames": [ + { + "artifacts": frame.output_artifacts, + "expected_slot": frame.summary["slot"], + "frame_receipt_sha256": frame.summary["frame_receipt"]["sha256"], + "slot": frame.summary["slot"], + } + for frame in audits + ], + "operation_state": { + "batch_exhausted": True, + "verified_slots": [1, 2, 3, 4, 5, 6], + }, + "output_dir": str(output_dir), + "output_lease": { + "acquired": True, + "release_attempted": True, + "released": True, + }, + "phase": "succeeded", + "reviewed_approval": { + "bytes": review_path.stat().st_size, + "contact_sheet": { + "path": str(contact_path), + "sha256": contact_sha256, + }, + "expected_sha256": review_sha256, + "path": str(review_path), + "reviewed_fingerprint_sha256": (audits[0].ownership.reviewed_fingerprint_sha256), + "verified_sha256": review_sha256, + }, + "retry_count": 0, + "schema": "negpy.ls5000-live-acceptance.v2", + "settings": { + "filename_pattern": 'acceptance_slot{{ "%02d" % seq }}', + "positive_mode": "nikon-exact", + "repair_mode": "hybrid", + "write_positive": True, + "write_repaired": True, + "write_unrepaired": True, + }, + "slots": [1, 2, 3, 4, 5, 6], + "status": "succeeded", + } + run_path = tmp_path / "run-receipt.json" + run_path.write_bytes( + deep_acceptance._canonical_json( + run_receipt, + newline=True, + ensure_ascii=True, + ) + ) + + result = deep_acceptance.validate_six_frame_batch( + outputs, + output_dir=output_dir, + run_receipt_path=run_path, + builder=object(), + evaluator=object(), + hybrid_runtime=_Runtime(), + ) + + assert result["run_receipt"]["path"] == str(run_path) + assert result["run_receipt"]["sha256"] == hashlib.sha256(run_path.read_bytes()).hexdigest() + + +def test_artifact_row_rejects_hash_drift(tmp_path: Path) -> None: + artifact = tmp_path / "artifact.bin" + artifact.write_bytes(b"bound bytes") + row = { + "path": str(artifact), + "bytes": artifact.stat().st_size, + "sha256": "0" * 64, + } + + with pytest.raises( + deep_acceptance.DeepAcceptanceError, + match="SHA-256 changed", + ): + deep_acceptance._artifact_row( + row, + root=tmp_path.resolve(), + label="test artifact", + ) + + +def test_regular_file_rejects_a_symlinked_parent(tmp_path: Path) -> None: + real = tmp_path / "real" + real.mkdir() + artifact = real / "artifact.bin" + artifact.write_bytes(b"payload") + alias = tmp_path / "alias" + os.symlink(real, alias) + + with pytest.raises( + deep_acceptance.DeepAcceptanceError, + match="symlink or non-directory parent", + ): + deep_acceptance._regular_file( + alias / artifact.name, + root=tmp_path.resolve(), + label="test artifact", + ) + + +def test_runtime_receipt_binding_rejects_retained_input_drift() -> None: + main = np.arange(24, dtype=np.uint16).reshape(2, 3, 4) + prepass = np.arange(16, dtype=np.uint16).reshape(2, 2, 4) + + def raw_digest(array: np.ndarray) -> str: + canonical = np.array(array, dtype=" None: + evidence = tmp_path / ".negpy-dice-acquisition" / "token" + evidence.mkdir(parents=True) + binding_path = evidence / "acquisition-binding.json" + rgb_path = tmp_path / "frame.tif" + ir_path = tmp_path / "frame_IR.tif" + rgb_path.write_bytes(b"rgb") + ir_path.write_bytes(b"ir") + + outer_artifacts: dict[str, dict[str, object]] = {} + bound_artifacts: dict[str, dict[str, object]] = {} + for key, filename, payload in ( + ("prepass_rgbi", "prepass.rgbi16.npy", b"prepass"), + ("ir_validity", "ir-validity.npy", b"validity"), + ): + path = evidence / filename + path.write_bytes(payload) + bound = { + "bytes": len(payload), + "dtype": " None: + """Only the external engine may be optional; broken NegPy code is fatal.""" + + target = "negpy.services.repair.fauxice_ir_repair" + real_import = builtins.__import__ + + def fail_internal_import(name, *args, **kwargs): + if name == target: + raise ImportError("sentinel internal packaging failure") + return real_import(name, *args, **kwargs) + + monkeypatch.setattr(builtins, "__import__", fail_internal_import) + path = Path(fauxice_bridge.__file__) + spec = importlib.util.spec_from_file_location( + "negpy.infrastructure.roll._fauxice_bridge_import_probe", + path, + ) + assert spec is not None and spec.loader is not None + probe = importlib.util.module_from_spec(spec) + + with pytest.raises(ImportError, match="sentinel internal packaging failure"): + spec.loader.exec_module(probe) + + +def _png(mask: np.ndarray) -> bytes: + stream = io.BytesIO() + Image.fromarray(mask.astype(np.uint8) * 255, mode="L").save(stream, format="PNG") + return stream.getvalue() + + +def _acquisition() -> roll_repair.RepairAcquisition: + main = np.arange(4 * 3 * 4, dtype=np.uint16).reshape(4, 3, 4) + prepass = np.arange(2 * 2 * 4, dtype=np.uint16).reshape(2, 2, 4) + validity = np.ones(main.shape[:2], dtype=np.bool_) + validity[0, -1] = False + return roll_repair.RepairAcquisition.from_arrays( + acquisition_id="dice-" + "1" * 64, + slot=9, + reservation_id="reservation-009", + capture_attempt_id="fine-slot-9-attempt-001", + storage_transform=roll_repair.DIGITAL_ICE_STORAGE_TRANSFORM, + evidence_sha256="2" * 64, + main_rgbi=main, + prepass_rgbi=prepass, + ir_validity=validity, + ) + + +def _runtime(tmp_path: Path) -> HybridRuntimeConfig: + return HybridRuntimeConfig( + hybrid_python=tmp_path / "hybrid" / "bin" / "python", + executable=tmp_path / "hybrid" / "bin" / "fauxce-hybrid", + core_source_manifest_sha256="1" * 64, + hybrid_source_manifest_sha256="2" * 64, + iopaint_python=tmp_path / "python", + iopaint_executable=tmp_path / "iopaint", + iopaint_source_manifest_sha256="3" * 64, + model_dir=tmp_path / "models", + model_weights=tmp_path / "models" / "big-lama.pt", + model_weights_sha256="4" * 64, + ) + + +def test_bridge_runs_hybrid_in_scanner_native_orientation_and_rotates_once(monkeypatch, tmp_path: Path) -> None: + acquisition = _acquisition() + runtime = _runtime(tmp_path) + native_repaired = np.ascontiguousarray( + np.where( + acquisition.ir_validity[..., None], + acquisition.main_rgbi[..., :3] + 19, + acquisition.main_rgbi[..., :3], + ) + ) + native_mask = np.zeros(acquisition.main_rgbi.shape[:2], dtype=np.bool_) + native_mask[1, 2] = True + native_mask[0, -1] = True # routed, but invalid IR means not applied + applied_mask = np.ascontiguousarray(native_mask & acquisition.ir_validity) + native_mask_png = _png(native_mask) + hybrid_receipt = b'{"schema":"fauxce-hybrid-receipt-v2"}' + calls = [] + + def fake_repair(rgb, ir, **kwargs): + calls.append((rgb, ir, kwargs)) + return FauxiceRepairResult( + status=RepairStatus.APPLIED, + reason="hybrid applied", + mode_requested=FauxiceMode.HYBRID, + mode_resolved=FauxiceMode.HYBRID, + repaired_rgb16=native_repaired, + engine_version="0.3.0", + backend_requested="auto", + backend_used="cpu-fast", + backend_selection_reason="parity self-test passed", + hybrid_mask_png=native_mask_png, + hybrid_mask_sha256=hashlib.sha256(native_mask_png).hexdigest(), + hybrid_mask=native_mask, + hybrid_synthesis_fraction=2 / native_mask.size, + hybrid_routing_counts={ + "final_regions": 1, + "synthesis_pixels": 2, + "frame_pixels": native_mask.size, + "at_floor_pixels": 2, + }, + hybrid_receipt=hybrid_receipt, + hybrid_receipt_sha256=hashlib.sha256(hybrid_receipt).hexdigest(), + hybrid_provenance_class="caller_asserted_bare_npy", + native_output_rgb_sha256=hashlib.sha256(native_repaired.astype(" None: + acquisition = _acquisition() + native_repaired = np.ascontiguousarray(acquisition.main_rgbi[..., :3] + 7) + + def fake_repair(rgb, ir, **kwargs): + assert kwargs["hybrid_runtime"] is None + return FauxiceRepairResult( + status=RepairStatus.APPLIED, + reason=("hybrid mode requested but no hybrid runtime is configured; degraded to exact repair"), + mode_requested=FauxiceMode.HYBRID, + mode_resolved=FauxiceMode.EXACT, + repaired_rgb16=native_repaired, + engine_version="0.3.0", + backend_requested="auto", + backend_used="cpu-fast", + backend_selection_reason="parity self-test passed", + native_output_rgb_sha256=hashlib.sha256(native_repaired.astype(" None: + self.validated = False + + def validate_files(self) -> None: + self.validated = True + + +@dataclasses.dataclass(frozen=True) +class _Approval: + payload: dict[str, object] + + def to_payload(self) -> dict[str, object]: + return dict(self.payload) + + +@dataclasses.dataclass(frozen=True) +class _AcceptanceFixture: + request: LiveAcceptanceRequest + review: ValidatedReviewedApproval + thumbnails: dict[int, np.ndarray] + + +def _json_sha256(document: object) -> str: + payload = json.dumps( + document, + sort_keys=True, + separators=(",", ":"), + ensure_ascii=False, + allow_nan=False, + ).encode("utf-8") + return hashlib.sha256(payload).hexdigest() + + +def _approval(slot: int, image: np.ndarray, fingerprint: str) -> _Approval: + payload: dict[str, object] = { + "boundary_offset_rows": 0, + "review_reasons": [f"slot-{slot}-edge-review"], + "reviewed_fingerprint_sha256": fingerprint, + "reviewed_lookup_row": (slot - 1) * 143, + "reviewed_native_origin": (slot - 1) * 5_959, + "schema_version": 1, + "slot": slot, + "thumbnail_sha256": thumbnail_sha256(image), + } + return _Approval({**payload, "binding_sha256": _json_sha256(payload)}) + + +def _fixture( + tmp_path: Path, + *, + confirm_live: bool = True, +) -> _AcceptanceFixture: + session_path = tmp_path / "preview-session.json" + session_path.write_text('{"version":1}', encoding="utf-8") + session_bytes = session_path.read_bytes() + + thumbnails = {slot: (np.arange(60, dtype=np.uint16).reshape(4, 5, 3) + np.uint16(slot * 1_000)) for slot in _SLOTS} + fingerprint = "c" * 64 + approvals = {slot: _approval(slot, thumbnails[slot], fingerprint) for slot in _APPROVED_SLOTS} + + contact_path = tmp_path / "reviewed-contact-sheet.png" + contact = np.concatenate( + tuple((thumbnails[slot] >> 8).astype(np.uint8) for slot in _SLOTS), + axis=1, + ) + Image.fromarray(contact).save(contact_path) + contact_bytes = contact_path.read_bytes() + + review_document = { + "approvals": [approvals[slot].to_payload() for slot in _APPROVED_SLOTS], + "contact_sheet": { + "bytes": len(contact_bytes), + "path": str(contact_path), + "sha256": hashlib.sha256(contact_bytes).hexdigest(), + }, + "preview_session": { + "bytes": len(session_bytes), + "path": str(session_path), + "sha256": hashlib.sha256(session_bytes).hexdigest(), + }, + "review_basis": REVIEW_BASIS, + "reviewed_fingerprint_sha256": fingerprint, + "schema": REVIEW_SCHEMA, + } + review_path = tmp_path / "reviewed-approval.json" + review_path.write_text( + json.dumps( + review_document, + sort_keys=True, + separators=(",", ":"), + ensure_ascii=False, + allow_nan=False, + ), + encoding="utf-8", + ) + review_bytes = review_path.read_bytes() + review = ValidatedReviewedApproval( + sha256=hashlib.sha256(review_bytes).hexdigest(), + byte_length=len(review_bytes), + reviewed_fingerprint_sha256=fingerprint, + contact_sheet_path=contact_path, + contact_sheet_sha256=hashlib.sha256(contact_bytes).hexdigest(), + approvals=approvals, + ) + + output_dir = tmp_path / "outputs" + output_dir.mkdir() + attempts_root = tmp_path / "scanner-attempts" + attempts_root.mkdir() + request = LiveAcceptanceRequest( + device_id="usb:2:7", + preview_session_path=session_path, + preview_session_sha256=hashlib.sha256(session_bytes).hexdigest(), + reviewed_approval_path=review_path, + reviewed_approval_sha256=review.sha256, + output_dir=output_dir, + run_receipt_path=tmp_path / "run-receipt.json", + attempts_root_path=attempts_root, + confirm_live=confirm_live, + ) + return _AcceptanceFixture( + request=request, + review=review, + thumbnails=thumbnails, + ) + + +def _write_sparse_zeros(path: Path, size: int) -> None: + with path.open("xb") as stream: + stream.truncate(size) + + +def _write_completed_attempt_evidence( + attempts_root: Path, + *, + reviewed_fingerprint: str, + approvals: dict[int, _Approval], +) -> Path: + """Mirror Coolscan's durable batch tree after fine-stream cleanup.""" + + batch = attempts_root / "batch-slot01-slot06-test" + batch.mkdir() + session_id = batch.name + fresh_fingerprint = "e" * 64 + from coolscanpy.protocol.ls5000_single_pass.bundle import ( + CAPTURE_BUNDLE_SHA256, + CAPTURE_WORKER_SHA256, + ) + + engine_sha256 = CAPTURE_WORKER_SHA256 + bundle_sha256 = CAPTURE_BUNDLE_SHA256 + plan_payload = b"pinned first-frame plan\n" + continuation_payload = b'{"kind":"pinned-continuation"}\n' + plan_sha256 = hashlib.sha256(plan_payload).hexdigest() + continuation_sha256 = hashlib.sha256(continuation_payload).hexdigest() + (batch / "replay-first-rgbi4-plan.jsonl").write_bytes(plan_payload) + (batch / "replay-next-rgbi4-plan.json").write_bytes(continuation_payload) + (batch / "replay-first-rgbi4-manifest.json").write_text( + json.dumps({"plan_sha256": plan_sha256}), + encoding="utf-8", + ) + (batch / "stdout.txt").write_bytes(b"") + (batch / "stderr.txt").write_bytes(b"") + + approval_payloads = {slot: approval.to_payload() for slot, approval in approvals.items()} + frames = [ + { + "ack": f"frame-{slot:03d}/parent-ack.json", + "boundary_offset_rows": 0, + "journal": f"frame-{slot:03d}/journal.json", + "manual_review_approval": approval_payloads.get(slot), + "output": f"frame-{slot:03d}/capture.bin", + "slot": slot, + } + for slot in _SLOTS + ] + job = { + "apply_all_boundary_offsets_before_first_frame": True, + "capture_plan_sha256": plan_sha256, + "continuation_plan_sha256": continuation_sha256, + "expected_usb_address": 7, + "expected_usb_bus": 3, + "frames": frames, + "parent_ack_required_after_every_frame": True, + "release_once_after_last_frame": True, + "reviewed_roll_fingerprint": { + "binding_sha256": reviewed_fingerprint, + }, + "schema_version": 3, + "session_id": session_id, + "session_contract": "one-process-one-reservation", + } + job_bytes = json.dumps( + job, + sort_keys=True, + separators=(",", ":"), + ).encode("utf-8") + (batch / "batch-job.json").write_bytes(job_bytes) + job_sha256 = hashlib.sha256(job_bytes).hexdigest() + density_calibration = {"session_id": session_id, "fixture": "validated"} + (batch / "session-journal.json").write_text( + json.dumps( + { + "status": "complete", + "session_id": session_id, + "density_calibration_session_id": session_id, + "nikon_density_calibration": density_calibration, + "selected_slots": list(_SLOTS), + "completed_slots": list(_SLOTS), + "active_frame_index": None, + "active_slot": None, + "batch_job_sha256": job_sha256, + "capture_engine_sha256": engine_sha256, + "capture_bundle_sha256": bundle_sha256, + "plan_sha256": plan_sha256, + "continuation_plan_sha256": continuation_sha256, + "manual_review_approval_sha256_by_slot": { + str(slot): (None if slot not in approval_payloads else approval_payloads[slot]["binding_sha256"]) for slot in _SLOTS + }, + "reviewed_roll_fingerprint_sha256": reviewed_fingerprint, + "expected_usb_bus": 3, + "expected_usb_address": 7, + "actual_usb_bus": 3, + "actual_usb_address": 7, + "reservation_acquired": True, + "unit_release_attempts": 1, + "unit_released": True, + "recovery_required": "none", + }, + sort_keys=True, + ), + encoding="utf-8", + ) + + for frame_index, slot in enumerate(_SLOTS, start=1): + frame = batch / f"frame-{slot:03d}" + frame.mkdir() + meter = frame / "capture-meter.bin" + _write_sparse_zeros(meter, _METER_BYTES) + nonce = f"{slot:032x}" + selection = { + "frame": slot, + "boundary_offset": { + "requested_rows": 0, + "applied_rows": 0, + }, + "roll_identity": { + "reviewed_fingerprint_sha256": reviewed_fingerprint, + "fresh_fingerprint_sha256": fresh_fingerprint, + "comparison": {"matches": True, "reason": "matched"}, + "selected_slot_comparison": { + "matches": True, + "reason": "matched", + "slot": slot, + }, + }, + } + journal: dict[str, object] = { + "status": "frame-complete", + "frame_complete": True, + "session_reservation_retained": True, + "unit_released": False, + "recovery_required": None, + "batch_session": { + "session_id": session_id, + "frame_index": frame_index, + "frame_total": len(_SLOTS), + "selected_slots": list(_SLOTS), + }, + "capture_mode": "full", + "requested_frame": slot, + "requested_boundary_offset_rows": 0, + "expected_reads": 2_980, + "completed_reads": 2_980, + "expected_bytes": 619_458_560, + "completed_bytes": 619_458_560, + "disk_bytes": 619_458_560, + "output": str(frame / "capture.bin"), + "output_sha256": hashlib.sha256(f"capture-{slot}".encode()).hexdigest(), + "plan_sha256": plan_sha256, + "continuation_plan_sha256": continuation_sha256, + "capture_engine_sha256": engine_sha256, + "capture_bundle_sha256": bundle_sha256, + "manual_review_approval": approval_payloads.get(slot), + "reviewed_roll_fingerprint_sha256": reviewed_fingerprint, + "expected_usb_bus": 3, + "expected_usb_address": 7, + "actual_usb_bus": 3, + "actual_usb_address": 7, + "ack_nonce": nonce, + "density_calibration_session_id": session_id, + "nikon_density_calibration": density_calibration, + "live_frame_selection": selection, + "meter_evidence": { + "path": str(meter), + "bytes": _METER_BYTES, + "sha256": _METER_SHA256, + "complete": True, + "durable_completed_passes": 3, + }, + "meter_evidence_persisted_before_fine_arm": True, + "meter_group_bytes": [1_088_000, 1_088_000, 1_088_000], + "meter_group_offsets": [0, 1_088_000, 2_176_000], + "meter_completed_reads": 15, + "meter_completed_bytes": _METER_BYTES, + "meter_layout": _METER_LAYOUT, + } + if slot == 1: + preview = frame / "capture-preview.bin" + _write_sparse_zeros(preview, _PREVIEW_BYTES) + table_payload = b"six-strip-transport-table" + table = frame / "capture-008e.bin" + table.write_bytes(table_payload) + mapping = frame / "capture-frame-map.json" + mapping.write_text(json.dumps(selection, sort_keys=True), encoding="utf-8") + journal["live_index_artifacts"] = { + "preview": str(preview), + "table": str(table), + "mapping": str(mapping), + } + journal["live_index_evidence"] = { + "status": "persisted-before-frame-detection", + "preview_bytes": _PREVIEW_BYTES, + "preview_sha256": _PREVIEW_SHA256, + "table_bytes": len(table_payload), + "table_sha256": hashlib.sha256(table_payload).hexdigest(), + } + (frame / "journal.json").write_text( + json.dumps(journal, sort_keys=True), + encoding="utf-8", + ) + (frame / "parent-ack.json").write_text( + json.dumps( + { + "ack_nonce": nonce, + "action": "continue", + "frame_index": frame_index, + "schema_version": 1, + "session_id": session_id, + "slot": slot, + }, + sort_keys=True, + ), + encoding="utf-8", + ) + return batch + + +class _Service: + def __init__( + self, + fixture: _AcceptanceFixture, + *, + restored_slots: tuple[int, ...] = _SLOTS, + approval_slots: tuple[int, ...] = _APPROVED_SLOTS, + wrong_approval_slot: int | None = None, + scan_error: Exception | None = None, + write_error_slot: int | None = None, + ) -> None: + self.output_dir = fixture.request.output_dir + self.thumbnails = fixture.thumbnails + self.approvals = fixture.review.approvals + self.reviewed_fingerprint = fixture.review.reviewed_fingerprint_sha256 + self.restored_slots = restored_slots + self.approval_slots = approval_slots + self.wrong_approval_slot = wrong_approval_slot + self.scan_error = scan_error + self.write_error_slot = write_error_slot + self.calls: list[object] = [] + self.approved: set[int] = set() + self.closed = False + self.safe_stop_calls = 0 + self.eject_calls = 0 + + def open_roll(self, device_id: str, *, attempts_root=None) -> None: + self.calls.append(("open_roll", device_id, attempts_root)) + _write_completed_attempt_evidence( + Path(attempts_root), + reviewed_fingerprint=self.reviewed_fingerprint, + approvals=self.approvals, + ) + + def restore_preview_session(self, payload: str, slots=None): + self.calls.append(("restore_preview_session", payload, slots)) + return [ + SimpleNamespace( + slot=slot, + image=np.array( + self.thumbnails.get( + slot, + np.full((4, 5, 3), slot, dtype=np.uint16), + ), + copy=True, + ), + ) + for slot in self.restored_slots + ] + + def needs_approval(self, slot: int) -> bool: + return slot in self.approval_slots + + def approve(self, slot: int) -> object: + self.calls.append(("approve", slot)) + self.approved.add(slot) + approval = self.approvals[slot] + if slot != self.wrong_approval_slot: + return approval + wrong = approval.to_payload() + wrong["binding_sha256"] = "0" * 64 + return _Approval(wrong) + + def prepare_batch(self) -> None: + self.calls.append("prepare_batch") + + def scan_many(self, slots, *, on_progress=None): + ordered = tuple(slots) + self.calls.append(("scan_many", ordered)) + if self.scan_error is not None: + raise self.scan_error + for index, slot in enumerate(ordered, start=1): + if on_progress is not None: + on_progress( + SimpleNamespace( + stage="fine-scan", + slot=slot, + index=index, + total=len(ordered), + fraction=index / len(ordered), + message=f"slot {slot} complete", + ) + ) + yield SimpleNamespace(slot=slot) + + def write_frame( + self, + frame, + output_folder: str, + filename_pattern: str, + **kwargs, + ) -> RollFrameOutput: + self.calls.append(("write_frame", frame.slot, output_folder, filename_pattern, kwargs)) + on_repair_progress = kwargs.get("on_repair_progress") + if callable(on_repair_progress): + on_repair_progress(0.5) + if frame.slot == self.write_error_slot: + raise RuntimeError(f"write failed for slot {frame.slot}") + base = self.output_dir / f"acceptance_slot{frame.slot:02d}" + paths = { + "rgb_path": base.with_suffix(".tif"), + "ir_path": base.with_name(base.name + "_IR.tif"), + "repaired_rgb_path": base.with_name(base.name + "_repaired.tif"), + "repaired_ir_path": base.with_name(base.name + "_repaired_IR.tif"), + "positive_path": base.with_name(base.name + "_positive.tif"), + "synthesis_mask_path": base.with_name(base.name + "_repaired_SYNTH.png"), + "native_synthesis_mask_path": base.with_name(base.name + "_native_SYNTH.png"), + "hybrid_receipt_path": base.with_name(base.name + "_hybrid_receipt.json"), + } + for path in paths.values(): + path.write_bytes(b"artifact") + receipt_path = base.with_name(base.name + "_receipt.json") + receipt_path.write_text( + json.dumps( + { + "depth": 16, + "dpi": 4_000, + "slot": frame.slot, + "outputs": { + "unrepaired": {"written": True}, + "repair_acquisition_evidence": { + "retained": True, + "replayable": True, + }, + "native_color_evidence": {"retained": True}, + "repaired": { + "written": True, + "mode_requested": "hybrid", + "mode_resolved": "hybrid", + "degraded": False, + }, + "positive": { + "written": True, + "color_mode": "nikon-exact", + "exact_nikon_color": True, + "native_per_acquisition_builder": True, + "builder_validated": True, + "cms_verified": True, + }, + }, + } + ), + encoding="utf-8", + ) + return RollFrameOutput( + slot=frame.slot, + receipt_path=str(receipt_path), + **{name: str(path) for name, path in paths.items()}, + ) + + def close(self) -> None: + self.calls.append("close") + self.closed = True + + def safe_stop(self) -> None: + self.safe_stop_calls += 1 + + def eject(self) -> None: + self.eject_calls += 1 + + +class _ExplodingProgress: + @property + def stage(self) -> str: + raise RuntimeError("progress extraction exploded") + + +class _UnrepresentableProgressValue: + def __repr__(self) -> str: + raise RuntimeError("progress repr exploded") + + +class _PoisonTelemetryService(_Service): + def scan_many(self, slots, *, on_progress=None): + ordered = tuple(slots) + self.calls.append(("scan_many", ordered)) + for slot in ordered: + if on_progress is not None: + on_progress(_ExplodingProgress()) + yield SimpleNamespace(slot=slot) + + def write_frame( + self, + frame, + output_folder: str, + filename_pattern: str, + **kwargs, + ) -> RollFrameOutput: + callback = kwargs.get("on_repair_progress") + if callable(callback): + callback(_UnrepresentableProgressValue()) + without_callback = dict(kwargs) + without_callback["on_repair_progress"] = None + return super().write_frame( + frame, + output_folder, + filename_pattern, + **without_callback, + ) + + +class _MutatingEvidenceLease: + def __init__(self, delegate: FixedOutputLease, target: Path) -> None: + self._delegate = delegate + self._target = target + + @property + def released(self) -> bool: + return self._delegate.released + + def assert_inventory(self, *args, **kwargs): + return self._delegate.assert_inventory(*args, **kwargs) + + def release(self) -> None: + self._delegate.release() + + def release_verified(self, owned_files, *, previous, finalize): + def finalize_then_mutate() -> None: + finalize() + self._target.write_bytes(b"mutated-after-receipt-publication") + + return self._delegate.release_verified( + owned_files, + previous=previous, + finalize=finalize_then_mutate, + ) + + +def _mutating_evidence_lease_factory(target: Path): + def acquire(root, lock_document, *, require_empty): + return _MutatingEvidenceLease( + FixedOutputLease.acquire( + root, + lock_document, + require_empty=require_empty, + ), + target, + ) + + return acquire + + +def _output_paths(output: RollFrameOutput) -> list[Path]: + paths = [Path(getattr(output, field)) for field in _OUTPUT_FIELDS] + assert all(path.is_absolute() and path.is_file() for path in paths) + return sorted(paths) + + +class _ValidationHarness: + def __init__(self, service: _Service) -> None: + self.service = service + self.inventory_closed_states: list[bool] = [] + self.batch_closed_states: list[bool] = [] + + def collect( + self, + output: RollFrameOutput, + *, + output_dir: Path, + expected_slot: int, + ) -> list[str]: + assert output.slot == expected_slot + assert Path(output_dir) == self.service.output_dir.resolve() + self.inventory_closed_states.append(self.service.closed) + return [str(path) for path in _output_paths(output)] + + def validate_batch( + self, + outputs, + **kwargs, + ) -> dict[str, Any]: + self.batch_closed_states.append(self.service.closed) + assert self.service.closed is True + output_dir = Path(kwargs["output_dir"]) + assert output_dir == self.service.output_dir.resolve() + assert kwargs["allowed_output_lock_name"] == OUTPUT_LOCK_NAME + ordered = tuple(outputs) + assert tuple(output.slot for output in ordered) == _SLOTS + frames = [] + all_paths: set[Path] = set() + for output in ordered: + paths = _output_paths(output) + all_paths.update(paths) + receipt_path = Path(output.receipt_path) + receipt_bytes = receipt_path.read_bytes() + frames.append( + { + "slot": output.slot, + "frame_receipt": { + "path": str(receipt_path), + "bytes": len(receipt_bytes), + "sha256": hashlib.sha256(receipt_bytes).hexdigest(), + }, + "referenced_file_count": len(paths), + "referenced_files": [str(path) for path in paths], + } + ) + referenced = [str(path) for path in sorted(all_paths)] + approval_payloads = {slot: self.service.approvals[slot].to_payload() for slot in _APPROVED_SLOTS} + return { + "schema": "negpy.ls5000-deep-acceptance.v1", + "status": "passed", + "scope": "six-frame-batch", + "output_dir": str(output_dir), + "slots": list(_SLOTS), + "approved_slots": list(_APPROVED_SLOTS), + "device_id": "usb:2:7", + "device_model": "Nikon LS-5000 ED 1.03", + "reviewed_fingerprint_sha256": approval_payloads[1]["reviewed_fingerprint_sha256"], + "manual_approval_bindings": [ + { + "slot": slot, + "binding_sha256": approval_payloads[slot]["binding_sha256"], + } + for slot in _APPROVED_SLOTS + ], + "frames": frames, + "referenced_files": referenced, + "inventory": { + "regular_file_count": len(referenced) + 1, + "visible_file_count": len(referenced), + "allowed_output_lock_path": str(output_dir / OUTPUT_LOCK_NAME), + "exact": True, + }, + } + + +def _review_loader(fixture: _AcceptanceFixture): + def load(request: LiveAcceptanceRequest, session) -> ValidatedReviewedApproval: + assert request.reviewed_approval_path == fixture.request.reviewed_approval_path + assert request.reviewed_approval_sha256 == fixture.review.sha256 + assert session.sha256 == fixture.request.preview_session_sha256 + return fixture.review + + return load + + +def _invoke( + fixture: _AcceptanceFixture, + service: _Service, + runtime: _Runtime, + **overrides, +) -> dict[str, Any]: + dependencies: dict[str, Any] = { + "service_factory": lambda *, hybrid_runtime: service, + "hybrid_runtime_loader": lambda _path: runtime, + "review_loader": _review_loader(fixture), + "emit": lambda _event: None, + } + dependencies.update(overrides) + return run_live_acceptance(fixture.request, **dependencies) + + +def _assert_no_recovery_or_eject( + service: _Service, + receipt: dict[str, Any], +) -> None: + assert service.safe_stop_calls == 0 + assert service.eject_calls == 0 + assert receipt["retry_count"] == 0 + assert receipt["eject_requested"] is False + + +def test_runs_one_six_slot_hybrid_nikon_exact_batch_and_records_truthful_state( + tmp_path: Path, +) -> None: + fixture = _fixture(tmp_path) + runtime = _Runtime() + service = _Service(fixture) + harness = _ValidationHarness(service) + events: list[dict[str, Any]] = [] + + receipt = _invoke( + fixture, + service, + runtime, + frame_inventory_collector=harness.collect, + batch_validator=harness.validate_batch, + emit=events.append, + ) + + assert receipt["status"] == "succeeded" + assert receipt["slots"] == list(_SLOTS) + assert receipt["approved_slots"] == list(_APPROVED_SLOTS) + assert [frame["slot"] for frame in receipt["frames"]] == list(_SLOTS) + assert all(frame["deep_acceptance"]["slot"] == frame["slot"] for frame in receipt["frames"]) + assert receipt["deep_acceptance"]["status"] == "passed" + assert runtime.validated is True + assert service.calls[:5] == [ + ( + "open_roll", + "usb:2:7", + fixture.request.attempts_root_path.resolve(), + ), + ("restore_preview_session", '{"version":1}', _SLOTS), + ("approve", 1), + ("approve", 6), + "prepare_batch", + ] + assert [call for call in service.calls if isinstance(call, tuple) and call[0] == "scan_many"] == [("scan_many", _SLOTS)] + writes = [call for call in service.calls if isinstance(call, tuple) and call[0] == "write_frame"] + assert [call[1] for call in writes] == list(_SLOTS) + assert all(call[4]["write_unrepaired"] is True for call in writes) + assert all(call[4]["write_repaired"] is True for call in writes) + assert all(call[4]["write_positive"] is True for call in writes) + assert all(call[4]["repair_mode"] == RepairMode.HYBRID.value for call in writes) + assert all(call[4]["positive_mode"] == PositiveColorMode.NIKON_EXACT.value for call in writes) + assert harness.inventory_closed_states == [False] * 6 + assert harness.batch_closed_states == [True] + assert service.closed is True + assert service.calls[-1] == "close" + state = receipt["operation_state"] + assert state == { + "phase": "succeeded", + "service_constructed": True, + "roll_opened": True, + "requested_slots": list(_SLOTS), + "restored_slots": list(_SLOTS), + "required_approval_slots": list(_APPROVED_SLOTS), + "approved_slots": list(_APPROVED_SLOTS), + "batch_prepared": True, + "yielded_slots": list(_SLOTS), + "committed_slots": list(_SLOTS), + "verified_slots": list(_SLOTS), + "processing_slot": None, + "batch_exhausted": True, + "last_progress": { + "event": "repair_progress", + "slot": 6, + "fraction": 0.5, + }, + "transport_may_have_advanced_beyond_yielded": False, + } + assert receipt["output_lease"]["released"] is True + assert receipt["capture_evidence_lease"]["released"] is True + evidence = receipt["capture_evidence"] + assert evidence["retained"] is True + assert evidence["file_count"] == 28 + assert evidence["snapshot_error"] is None + assert {Path(row["path"]).name for row in evidence["files"]} == { + "batch-job.json", + "capture-preview.bin", + "capture-008e.bin", + "capture-frame-map.json", + "capture-meter.bin", + "journal.json", + "parent-ack.json", + "replay-first-rgbi4-manifest.json", + "replay-first-rgbi4-plan.jsonl", + "replay-next-rgbi4-plan.json", + "session-journal.json", + "stderr.txt", + "stdout.txt", + } + assert not any(Path(row["path"]).name == "capture.bin" for row in evidence["files"]) + assert evidence["batch_binding"]["selected_slots"] == list(_SLOTS) + assert evidence["batch_binding"]["first_frame_directory"].endswith("/frame-001") + assert not (fixture.request.attempts_root_path / OUTPUT_LOCK_NAME).exists() + assert not (fixture.request.output_dir / OUTPUT_LOCK_NAME).exists() + assert json.loads(fixture.request.run_receipt_path.read_text(encoding="utf-8")) == receipt + assert any(event["event"] == "scan_progress" for event in events) + _assert_no_recovery_or_eject(service, receipt) + + +def test_telemetry_callback_exceptions_never_abort_the_live_run( + tmp_path: Path, +) -> None: + fixture = _fixture(tmp_path) + runtime = _Runtime() + service = _Service(fixture) + harness = _ValidationHarness(service) + + def broken_emit(_event: dict[str, Any]) -> None: + raise RuntimeError("telemetry sink is unavailable") + + receipt = _invoke( + fixture, + service, + runtime, + frame_inventory_collector=harness.collect, + batch_validator=harness.validate_batch, + emit=broken_emit, + ) + + assert receipt["status"] == "succeeded" + assert len(receipt["telemetry_errors"]) == 16 + assert all(error["type"] == "RuntimeError" for error in receipt["telemetry_errors"]) + assert all(error["message"] == "telemetry sink is unavailable" for error in receipt["telemetry_errors"]) + _assert_no_recovery_or_eject(service, receipt) + + +def test_poison_progress_extraction_and_repr_never_abort_the_live_run( + tmp_path: Path, +) -> None: + fixture = _fixture(tmp_path) + runtime = _Runtime() + service = _PoisonTelemetryService(fixture) + harness = _ValidationHarness(service) + + receipt = _invoke( + fixture, + service, + runtime, + frame_inventory_collector=harness.collect, + batch_validator=harness.validate_batch, + ) + + assert receipt["status"] == "succeeded" + assert receipt["telemetry_error_count"] == 6 + assert all( + error + == { + "event": "scan_progress", + "type": "RuntimeError", + "message": "progress extraction exploded", + } + for error in receipt["telemetry_errors"] + ) + assert receipt["operation_state"]["last_progress"] == { + "event": "repair_progress", + "slot": 6, + "fraction": "", + } + _assert_no_recovery_or_eject(service, receipt) + + +@pytest.mark.parametrize( + ("field", "wrong_value"), + [ + ("device_id", "usb:another-scanner"), + ("device_model", "Nikon LS-4000 ED 1.10"), + ("reviewed_fingerprint_sha256", "d" * 64), + ( + "manual_approval_bindings", + [ + {"slot": 1, "binding_sha256": "0" * 64}, + {"slot": 6, "binding_sha256": "1" * 64}, + ], + ), + ], +) +def test_deep_batch_identity_mismatch_can_never_publish_success( + tmp_path: Path, + field: str, + wrong_value: object, +) -> None: + fixture = _fixture(tmp_path) + runtime = _Runtime() + service = _Service(fixture) + harness = _ValidationHarness(service) + + def mismatched_batch(outputs, **kwargs): + result = harness.validate_batch(outputs, **kwargs) + result[field] = wrong_value + return result + + with pytest.raises(LiveAcceptanceError, match="invalid result") as raised: + _invoke( + fixture, + service, + runtime, + frame_inventory_collector=harness.collect, + batch_validator=mismatched_batch, + ) + + assert raised.value.receipt["status"] == "failed" + assert raised.value.receipt["operation_state"]["verified_slots"] == [] + assert json.loads(fixture.request.run_receipt_path.read_text(encoding="utf-8"))["status"] == "failed" + _assert_no_recovery_or_eject(service, raised.value.receipt) + + +@pytest.mark.parametrize( + "trigger_event", + ["six_frame_batch_verified", "run_finished"], +) +def test_output_path_swap_during_finalization_overwrites_any_success_with_failure( + tmp_path: Path, + trigger_event: str, +) -> None: + fixture = _fixture(tmp_path) + runtime = _Runtime() + service = _Service(fixture) + harness = _ValidationHarness(service) + moved = tmp_path / "moved-outputs" + swapped = False + + def swapping_emit(event: dict[str, Any]) -> None: + nonlocal swapped + if event.get("event") != trigger_event or swapped: + return + swapped = True + fixture.request.output_dir.rename(moved) + fixture.request.output_dir.mkdir() + + with pytest.raises( + LiveAcceptanceError, + match="pathname no longer identifies", + ) as raised: + _invoke( + fixture, + service, + runtime, + frame_inventory_collector=harness.collect, + batch_validator=harness.validate_batch, + emit=swapping_emit, + ) + + assert swapped is True + assert raised.value.receipt["status"] == "failed" + assert json.loads(fixture.request.run_receipt_path.read_text(encoding="utf-8"))["status"] == "failed" + assert not (moved / OUTPUT_LOCK_NAME).exists() + assert list(fixture.request.output_dir.iterdir()) == [] + assert len(list(moved.iterdir())) == len(_SLOTS) * len(_OUTPUT_FIELDS) + _assert_no_recovery_or_eject(service, raised.value.receipt) + + +def test_capture_evidence_mutation_after_success_publication_is_truthfully_failed( + tmp_path: Path, +) -> None: + fixture = _fixture(tmp_path) + runtime = _Runtime() + service = _Service(fixture) + harness = _ValidationHarness(service) + target = fixture.request.attempts_root_path / "batch-slot01-slot06-test" / "frame-001" / "capture-preview.bin" + + with pytest.raises(LiveAcceptanceError, match="previously owned output changed") as raised: + _invoke( + fixture, + service, + runtime, + frame_inventory_collector=harness.collect, + batch_validator=harness.validate_batch, + evidence_lease_factory=_mutating_evidence_lease_factory(target), + ) + + receipt = raised.value.receipt + assert receipt["status"] == "failed" + assert receipt["capture_evidence"]["retained"] is False + assert receipt["capture_evidence"]["snapshot_error"] is not None + assert json.loads(fixture.request.run_receipt_path.read_text(encoding="utf-8")) == receipt + _assert_no_recovery_or_eject(service, receipt) + + +def test_failed_scan_evidence_mutation_is_truthfully_failed( + tmp_path: Path, +) -> None: + fixture = _fixture(tmp_path) + runtime = _Runtime() + service = _Service(fixture, scan_error=RuntimeError("scan failed")) + target = fixture.request.attempts_root_path / "batch-slot01-slot06-test" / "frame-001" / "capture-preview.bin" + + with pytest.raises(LiveAcceptanceError, match="scan failed") as raised: + _invoke( + fixture, + service, + runtime, + evidence_lease_factory=_mutating_evidence_lease_factory(target), + ) + + receipt = raised.value.receipt + assert receipt["status"] == "failed" + assert receipt["capture_evidence"]["retained"] is False + assert receipt["capture_evidence"]["snapshot_error"] is not None + assert json.loads(fixture.request.run_receipt_path.read_text(encoding="utf-8")) == receipt + _assert_no_recovery_or_eject(service, receipt) + + +def test_required_capture_basenames_split_across_directories_never_pass( + tmp_path: Path, +) -> None: + fixture = _fixture(tmp_path) + runtime = _Runtime() + service = _Service(fixture) + harness = _ValidationHarness(service) + original_open = service.open_roll + + def open_with_split_evidence(device_id: str, *, attempts_root=None) -> None: + original_open(device_id, attempts_root=attempts_root) + frame = Path(attempts_root) / "batch-slot01-slot06-test" / "frame-001" + for index, name in enumerate(("capture-preview.bin", "capture-008e.bin", "capture-frame-map.json")): + decoy = Path(attempts_root) / f"decoy-{index}" + decoy.mkdir() + (frame / name).rename(decoy / name) + + service.open_roll = open_with_split_evidence # type: ignore[method-assign] + + with pytest.raises(LiveAcceptanceError, match="accepted batch capture evidence"): + _invoke( + fixture, + service, + runtime, + frame_inventory_collector=harness.collect, + batch_validator=harness.validate_batch, + ) + + receipt = json.loads(fixture.request.run_receipt_path.read_text(encoding="utf-8")) + assert receipt["status"] == "failed" + _assert_no_recovery_or_eject(service, receipt) + + +def test_missing_sixth_frame_journal_never_passes_completed_batch_evidence( + tmp_path: Path, +) -> None: + fixture = _fixture(tmp_path) + runtime = _Runtime() + service = _Service(fixture) + harness = _ValidationHarness(service) + original_open = service.open_roll + + def open_without_sixth_journal(device_id: str, *, attempts_root=None) -> None: + original_open(device_id, attempts_root=attempts_root) + missing = Path(attempts_root) / "batch-slot01-slot06-test" / "frame-006" / "journal.json" + missing.unlink() + + service.open_roll = open_without_sixth_journal # type: ignore[method-assign] + + with pytest.raises(LiveAcceptanceError, match="accepted batch capture evidence"): + _invoke( + fixture, + service, + runtime, + frame_inventory_collector=harness.collect, + batch_validator=harness.validate_batch, + ) + + +def test_session_journal_must_bind_hashed_batch_job(tmp_path: Path) -> None: + fixture = _fixture(tmp_path) + runtime = _Runtime() + service = _Service(fixture) + harness = _ValidationHarness(service) + original_open = service.open_roll + + def open_with_wrong_job_hash(device_id: str, *, attempts_root=None) -> None: + original_open(device_id, attempts_root=attempts_root) + session_path = Path(attempts_root) / "batch-slot01-slot06-test" / "session-journal.json" + session = json.loads(session_path.read_text(encoding="utf-8")) + session["batch_job_sha256"] = "0" * 64 + session_path.write_text(json.dumps(session), encoding="utf-8") + + service.open_roll = open_with_wrong_job_hash # type: ignore[method-assign] + + with pytest.raises(LiveAcceptanceError, match="completed six-frame batch"): + _invoke( + fixture, + service, + runtime, + frame_inventory_collector=harness.collect, + batch_validator=harness.validate_batch, + ) + + +def _never_constructed_factory(calls: list[str]): + def factory(*, hybrid_runtime): + calls.append("constructed") + raise AssertionError("service must not be constructed") + + return factory + + +def test_missing_confirmation_fails_before_constructing_service( + tmp_path: Path, +) -> None: + fixture = _fixture(tmp_path, confirm_live=False) + factory_calls: list[str] = [] + + with pytest.raises(LiveAcceptanceError, match="--confirm-live is required") as raised: + run_live_acceptance( + fixture.request, + service_factory=_never_constructed_factory(factory_calls), + emit=lambda _event: None, + ) + + assert factory_calls == [] + assert raised.value.receipt["status"] == "failed" + assert raised.value.receipt["operation_state"]["roll_opened"] is False + assert not fixture.request.run_receipt_path.exists() + + +def test_session_hash_mismatch_fails_before_constructing_service( + tmp_path: Path, +) -> None: + fixture = _fixture(tmp_path) + request = dataclasses.replace( + fixture.request, + preview_session_sha256="0" * 64, + ) + factory_calls: list[str] = [] + + with pytest.raises(LiveAcceptanceError, match="SHA-256 mismatch"): + run_live_acceptance( + request, + service_factory=_never_constructed_factory(factory_calls), + emit=lambda _event: None, + ) + + assert factory_calls == [] + receipt = json.loads(request.run_receipt_path.read_text(encoding="utf-8")) + assert receipt["status"] == "failed" + assert receipt["operation_state"]["roll_opened"] is False + + +def test_missing_session_fails_before_constructing_service(tmp_path: Path) -> None: + fixture = _fixture(tmp_path) + fixture.request.preview_session_path.unlink() + factory_calls: list[str] = [] + + with pytest.raises(LiveAcceptanceError, match="preview session is missing"): + run_live_acceptance( + fixture.request, + service_factory=_never_constructed_factory(factory_calls), + emit=lambda _event: None, + ) + + assert factory_calls == [] + receipt = json.loads(fixture.request.run_receipt_path.read_text(encoding="utf-8")) + assert receipt["status"] == "failed" + assert receipt["operation_state"]["roll_opened"] is False + + +def test_preexisting_output_collision_is_preserved_without_opening_roll( + tmp_path: Path, +) -> None: + fixture = _fixture(tmp_path) + existing = fixture.request.output_dir / "existing.tif" + existing.write_bytes(b"do not overwrite") + factory_calls: list[str] = [] + + with pytest.raises(LiveAcceptanceError, match="unexpected files"): + run_live_acceptance( + fixture.request, + service_factory=_never_constructed_factory(factory_calls), + emit=lambda _event: None, + ) + + assert factory_calls == [] + assert existing.read_bytes() == b"do not overwrite" + assert sorted(path.name for path in fixture.request.output_dir.iterdir()) == ["existing.tif"] + receipt = json.loads(fixture.request.run_receipt_path.read_text(encoding="utf-8")) + assert receipt["status"] == "failed" + assert receipt["operation_state"]["roll_opened"] is False + + +def test_preexisting_capture_evidence_is_preserved_without_opening_roll( + tmp_path: Path, +) -> None: + fixture = _fixture(tmp_path) + existing = fixture.request.attempts_root_path / "existing.bin" + existing.write_bytes(b"do not overwrite") + factory_calls: list[str] = [] + + with pytest.raises(LiveAcceptanceError, match="unexpected files"): + run_live_acceptance( + fixture.request, + service_factory=_never_constructed_factory(factory_calls), + emit=lambda _event: None, + ) + + assert factory_calls == [] + assert existing.read_bytes() == b"do not overwrite" + assert sorted(path.name for path in fixture.request.attempts_root_path.iterdir()) == ["existing.bin"] + receipt = json.loads(fixture.request.run_receipt_path.read_text(encoding="utf-8")) + assert receipt["status"] == "failed" + assert receipt["operation_state"]["roll_opened"] is False + + +def test_capture_evidence_root_must_not_overlap_output(tmp_path: Path) -> None: + fixture = _fixture(tmp_path) + request = dataclasses.replace( + fixture.request, + attempts_root_path=fixture.request.output_dir, + ) + factory_calls: list[str] = [] + + with pytest.raises(LiveAcceptanceError, match="disjoint"): + run_live_acceptance( + request, + service_factory=_never_constructed_factory(factory_calls), + emit=lambda _event: None, + ) + + assert factory_calls == [] + assert not request.run_receipt_path.exists() + + +def test_capture_evidence_root_must_not_be_a_symlink(tmp_path: Path) -> None: + fixture = _fixture(tmp_path) + linked = tmp_path / "linked-attempts" + linked.symlink_to(fixture.request.attempts_root_path, target_is_directory=True) + request = dataclasses.replace(fixture.request, attempts_root_path=linked) + factory_calls: list[str] = [] + + with pytest.raises(LiveAcceptanceError, match="non-symlink"): + run_live_acceptance( + request, + service_factory=_never_constructed_factory(factory_calls), + emit=lambda _event: None, + ) + + assert factory_calls == [] + assert not request.run_receipt_path.exists() + + +def test_preexisting_run_receipt_is_never_overwritten_or_live_opened( + tmp_path: Path, +) -> None: + fixture = _fixture(tmp_path) + fixture.request.run_receipt_path.write_bytes(b"existing receipt") + before = fixture.request.run_receipt_path.stat() + factory_calls: list[str] = [] + + with pytest.raises(LiveAcceptanceError, match="already exists") as raised: + run_live_acceptance( + fixture.request, + service_factory=_never_constructed_factory(factory_calls), + emit=lambda _event: None, + ) + + after = fixture.request.run_receipt_path.stat() + assert factory_calls == [] + assert fixture.request.run_receipt_path.read_bytes() == b"existing receipt" + assert (after.st_dev, after.st_ino) == (before.st_dev, before.st_ino) + assert raised.value.receipt["operation_state"]["roll_opened"] is False + + +def test_wrong_restored_slots_fails_without_scanning_and_closes( + tmp_path: Path, +) -> None: + fixture = _fixture(tmp_path) + runtime = _Runtime() + service = _Service(fixture, restored_slots=(1, 2, 3, 4, 5, 7)) + + with pytest.raises(LiveAcceptanceError, match="restored slots") as raised: + _invoke(fixture, service, runtime) + + assert not any(isinstance(call, tuple) and call[0] == "scan_many" for call in service.calls) + assert service.closed is True + _assert_no_recovery_or_eject(service, raised.value.receipt) + + +def test_missing_edge_approval_fails_without_scanning_and_closes( + tmp_path: Path, +) -> None: + fixture = _fixture(tmp_path) + runtime = _Runtime() + service = _Service(fixture, approval_slots=(1,)) + + with pytest.raises(LiveAcceptanceError, match="approval set") as raised: + _invoke(fixture, service, runtime) + + assert not any(isinstance(call, tuple) and call[0] == "scan_many" for call in service.calls) + assert service.closed is True + _assert_no_recovery_or_eject(service, raised.value.receipt) + + +def test_wrong_service_approval_return_fails_before_scan_with_truthful_partial_state( + tmp_path: Path, +) -> None: + fixture = _fixture(tmp_path) + runtime = _Runtime() + service = _Service(fixture, wrong_approval_slot=6) + + with pytest.raises(LiveAcceptanceError, match="differs from reviewed evidence") as raised: + _invoke(fixture, service, runtime) + + assert [call for call in service.calls if isinstance(call, tuple) and call[0] == "approve"] == [("approve", 1), ("approve", 6)] + assert "prepare_batch" not in service.calls + assert not any(isinstance(call, tuple) and call[0] == "scan_many" for call in service.calls) + state = raised.value.receipt["operation_state"] + assert state["required_approval_slots"] == list(_APPROVED_SLOTS) + assert state["approved_slots"] == [1] + assert state["yielded_slots"] == [] + assert service.closed is True + _assert_no_recovery_or_eject(service, raised.value.receipt) + + +def test_write_failure_records_yielded_committed_verified_and_transport_state( + tmp_path: Path, +) -> None: + fixture = _fixture(tmp_path) + runtime = _Runtime() + service = _Service(fixture, write_error_slot=3) + harness = _ValidationHarness(service) + + def batch_must_not_run(*_args, **_kwargs): + raise AssertionError("deep batch validation must not run after write failure") + + with pytest.raises(LiveAcceptanceError, match="write failed for slot 3") as raised: + _invoke( + fixture, + service, + runtime, + frame_inventory_collector=harness.collect, + batch_validator=batch_must_not_run, + ) + + receipt = raised.value.receipt + state = receipt["operation_state"] + assert state["yielded_slots"] == [1, 2, 3] + assert state["committed_slots"] == [1, 2] + assert state["verified_slots"] == [] + assert state["processing_slot"] == 3 + assert state["batch_exhausted"] is False + assert state["transport_may_have_advanced_beyond_yielded"] is True + assert [frame["slot"] for frame in receipt["frames"]] == [1, 2] + assert harness.inventory_closed_states == [False, False] + assert harness.batch_closed_states == [] + assert service.closed is True + assert json.loads(fixture.request.run_receipt_path.read_text(encoding="utf-8")) == receipt + _assert_no_recovery_or_eject(service, receipt) + + +def test_scan_exception_closes_without_retry_safe_stop_or_eject( + tmp_path: Path, +) -> None: + fixture = _fixture(tmp_path) + runtime = _Runtime() + service = _Service(fixture, scan_error=RuntimeError("scan failed")) + harness = _ValidationHarness(service) + + with pytest.raises(LiveAcceptanceError, match="scan failed") as raised: + _invoke( + fixture, + service, + runtime, + frame_inventory_collector=harness.collect, + batch_validator=harness.validate_batch, + ) + + state = raised.value.receipt["operation_state"] + assert state["yielded_slots"] == [] + assert state["committed_slots"] == [] + assert state["verified_slots"] == [] + assert state["batch_exhausted"] is False + assert state["transport_may_have_advanced_beyond_yielded"] is True + assert service.closed is True + assert raised.value.receipt["capture_evidence"]["retained"] is True + assert raised.value.receipt["capture_evidence"]["file_count"] == 28 + assert raised.value.receipt["capture_evidence_lease"]["released"] is True + _assert_no_recovery_or_eject(service, raised.value.receipt) + + +def test_default_deep_batch_validator_rejects_literal_fake_artifacts_after_close( + tmp_path: Path, +) -> None: + fixture = _fixture(tmp_path) + runtime = _Runtime() + service = _Service(fixture) + harness = _ValidationHarness(service) + + with pytest.raises(LiveAcceptanceError) as raised: + _invoke( + fixture, + service, + runtime, + frame_inventory_collector=harness.collect, + ) + + receipt = raised.value.receipt + state = receipt["operation_state"] + assert service.closed is True + assert state["yielded_slots"] == list(_SLOTS) + assert state["committed_slots"] == list(_SLOTS) + assert state["verified_slots"] == [] + assert state["batch_exhausted"] is True + assert state["transport_may_have_advanced_beyond_yielded"] is False + assert receipt["deep_acceptance"] is None + assert receipt["error"]["type"] == "DeepAcceptanceError" + _assert_no_recovery_or_eject(service, receipt) + + +def test_injected_deep_batch_failure_fails_only_after_scanner_close( + tmp_path: Path, +) -> None: + fixture = _fixture(tmp_path) + runtime = _Runtime() + service = _Service(fixture) + harness = _ValidationHarness(service) + close_states: list[bool] = [] + + def reject_batch(_outputs, **_kwargs): + close_states.append(service.closed) + raise RuntimeError("deep six-frame audit rejected") + + with pytest.raises( + LiveAcceptanceError, + match="deep six-frame audit rejected", + ) as raised: + _invoke( + fixture, + service, + runtime, + frame_inventory_collector=harness.collect, + batch_validator=reject_batch, + ) + + assert close_states == [True] + state = raised.value.receipt["operation_state"] + assert state["yielded_slots"] == list(_SLOTS) + assert state["committed_slots"] == list(_SLOTS) + assert state["verified_slots"] == [] + assert state["batch_exhausted"] is True + assert state["transport_may_have_advanced_beyond_yielded"] is False + assert service.closed is True + _assert_no_recovery_or_eject(service, raised.value.receipt) + + +def test_sane_style_device_id_fails_before_constructing_service( + tmp_path: Path, +) -> None: + fixture = _fixture(tmp_path) + request = dataclasses.replace( + fixture.request, + device_id="coolscan3:usb:libusb:002:007", + ) + factory_calls: list[str] = [] + + with pytest.raises(LiveAcceptanceError, match="usb:BUS:ADDRESS") as raised: + run_live_acceptance( + request, + service_factory=_never_constructed_factory(factory_calls), + emit=lambda _event: None, + ) + + assert factory_calls == [] + assert raised.value.receipt["operation_state"]["roll_opened"] is False + + +def test_receipt_inside_attempts_root_fails_before_constructing_service( + tmp_path: Path, +) -> None: + fixture = _fixture(tmp_path) + request = dataclasses.replace( + fixture.request, + run_receipt_path=fixture.request.attempts_root_path / "receipt.json", + ) + factory_calls: list[str] = [] + + with pytest.raises(LiveAcceptanceError, match="outside the attempts root"): + run_live_acceptance( + request, + service_factory=_never_constructed_factory(factory_calls), + emit=lambda _event: None, + ) + + assert factory_calls == [] + + +def test_parser_requires_attempts_root() -> None: + from negpy.services.roll import live_acceptance as module + + argv = [ + "--device-id", + "usb:2:7", + "--preview-session", + "session.json", + "--preview-session-sha256", + "0" * 64, + "--reviewed-approval", + "review.json", + "--reviewed-approval-sha256", + "0" * 64, + "--output-dir", + "outputs", + "--run-receipt", + "receipt.json", + "--confirm-live", + ] + with pytest.raises(SystemExit): + module._parser().parse_args(argv) + parsed = module._parser().parse_args([*argv, "--attempts-root", "attempts"]) + assert parsed.attempts_root == Path("attempts") + + +def test_interrupt_handlers_route_signals_to_truthful_failure( + monkeypatch: pytest.MonkeyPatch, +) -> None: + import signal as signal_module + + from negpy.services.roll import live_acceptance as module + + recorded: dict[int, object] = {} + monkeypatch.setattr( + module.signal, + "signal", + lambda number, handler: recorded.__setitem__(number, handler), + ) + + assert module.install_interrupt_handlers() is True + assert set(recorded) == {signal_module.SIGINT, signal_module.SIGTERM} + + handler = recorded[signal_module.SIGTERM] + assert callable(handler) + with pytest.raises(module.LiveAcceptanceInterrupted, match="signal"): + handler(signal_module.SIGTERM, None) + assert recorded[signal_module.SIGINT] is signal_module.SIG_IGN + assert recorded[signal_module.SIGTERM] is signal_module.SIG_IGN + + +def test_interrupt_mid_scan_closes_and_writes_truthful_failed_receipt( + tmp_path: Path, +) -> None: + from negpy.services.roll.live_acceptance import LiveAcceptanceInterrupted + + fixture = _fixture(tmp_path) + runtime = _Runtime() + service = _Service( + fixture, + scan_error=LiveAcceptanceInterrupted("received signal 15"), + ) + harness = _ValidationHarness(service) + + with pytest.raises(LiveAcceptanceError, match="received signal 15") as raised: + _invoke( + fixture, + service, + runtime, + frame_inventory_collector=harness.collect, + batch_validator=harness.validate_batch, + ) + + state = raised.value.receipt["operation_state"] + assert state["committed_slots"] == [] + assert service.closed is True + assert raised.value.receipt["status"] == "failed" + _assert_no_recovery_or_eject(service, raised.value.receipt) diff --git a/tests/roll/test_live_evidence.py b/tests/roll/test_live_evidence.py new file mode 100644 index 00000000..a24cee55 --- /dev/null +++ b/tests/roll/test_live_evidence.py @@ -0,0 +1,633 @@ +from __future__ import annotations + +import hashlib +import json +from pathlib import Path + +import pytest + +from negpy.services.roll import live_evidence +from negpy.services.roll.live_evidence import ( + bind_capture_evidence_inventory, + snapshot_capture_evidence, + validate_six_frame_batch_capture_evidence, +) +from negpy.services.roll.live_reservation import ( + FixedOutputLease, + InventoryConflict, +) + + +def _lease(root: Path) -> FixedOutputLease: + return FixedOutputLease.acquire( + root, + {"schema": "test.capture-evidence-owner.v1"}, + require_empty=True, + ) + + +_SLOTS = (1, 2, 3, 4, 5, 6) +_METER_BYTES = 3_264_000 +_METER_SHA256 = "2d417c2ed40641cd243f33989601b5a06c7a7c5b893c092c1868e0b9addd03e1" +_PREVIEW_BYTES = 6_250_496 +_PREVIEW_SHA256 = "690563a295100f3bb51b5cedbfc3e4a3df467d171d96483420810fd63e75a380" +_METER_LAYOUT = { + "passes": 3, + "rows_per_pass": 425, + "columns": 281, + "decoded_raster_channel_order": ["R", "G", "B", "IR"], + "wire_window_color_order": [9, 1, 2, 3], + "wire_color_to_controller_channel": { + "9": "IR", + "1": "R", + "2": "G", + "3": "B", + }, + "sample_byte_order": "big-endian-u16", + "row_core_bytes": 2_248, + "row_stride_bytes": 2_560, + "row_tail_bytes": 312, +} + + +def _write_sparse_zeros(path: Path, size: int) -> None: + with path.open("xb") as stream: + stream.truncate(size) + + +def _write_completed_batch(root: Path) -> dict[str, object]: + """Materialize Coolscan's durable tree after raw fine-stream cleanup.""" + + batch = root / "batch-slot01-slot06-fixture" + batch.mkdir() + session_id = batch.name + reviewed_sha256 = "a" * 64 + fresh_sha256 = "b" * 64 + from coolscanpy.protocol.ls5000_single_pass.bundle import ( + CAPTURE_BUNDLE_SHA256, + CAPTURE_WORKER_SHA256, + ) + + engine_sha256 = CAPTURE_WORKER_SHA256 + bundle_sha256 = CAPTURE_BUNDLE_SHA256 + plan_payload = b"pinned first-frame plan\n" + continuation_payload = b'{"kind":"pinned-continuation"}\n' + plan_sha256 = hashlib.sha256(plan_payload).hexdigest() + continuation_sha256 = hashlib.sha256(continuation_payload).hexdigest() + (batch / "replay-first-rgbi4-plan.jsonl").write_bytes(plan_payload) + (batch / "replay-next-rgbi4-plan.json").write_bytes(continuation_payload) + (batch / "replay-first-rgbi4-manifest.json").write_text( + json.dumps({"plan_sha256": plan_sha256}), + encoding="utf-8", + ) + (batch / "stdout.txt").write_bytes(b"") + (batch / "stderr.txt").write_bytes(b"") + + approvals = { + 1: {"binding_sha256": "1" * 64}, + 6: {"binding_sha256": "6" * 64}, + } + frames = [ + { + "ack": f"frame-{slot:03d}/parent-ack.json", + "boundary_offset_rows": 0, + "journal": f"frame-{slot:03d}/journal.json", + "manual_review_approval": approvals.get(slot), + "output": f"frame-{slot:03d}/capture.bin", + "slot": slot, + } + for slot in _SLOTS + ] + job = { + "apply_all_boundary_offsets_before_first_frame": True, + "capture_plan_sha256": plan_sha256, + "continuation_plan_sha256": continuation_sha256, + "expected_usb_address": 7, + "expected_usb_bus": 3, + "frames": frames, + "parent_ack_required_after_every_frame": True, + "release_once_after_last_frame": True, + "reviewed_roll_fingerprint": {"binding_sha256": reviewed_sha256}, + "schema_version": 3, + "session_id": session_id, + "session_contract": "one-process-one-reservation", + } + job_payload = json.dumps(job, sort_keys=True, separators=(",", ":")).encode() + (batch / "batch-job.json").write_bytes(job_payload) + job_sha256 = hashlib.sha256(job_payload).hexdigest() + + density_calibration = {"session_id": session_id, "fixture": "validated"} + (batch / "session-journal.json").write_text( + json.dumps( + { + "status": "complete", + "session_id": session_id, + "density_calibration_session_id": session_id, + "nikon_density_calibration": density_calibration, + "selected_slots": list(_SLOTS), + "completed_slots": list(_SLOTS), + "active_frame_index": None, + "active_slot": None, + "batch_job_sha256": job_sha256, + "capture_engine_sha256": engine_sha256, + "capture_bundle_sha256": bundle_sha256, + "plan_sha256": plan_sha256, + "continuation_plan_sha256": continuation_sha256, + "manual_review_approval_sha256_by_slot": { + str(slot): (None if slot not in approvals else approvals[slot]["binding_sha256"]) for slot in _SLOTS + }, + "reviewed_roll_fingerprint_sha256": reviewed_sha256, + "expected_usb_bus": 3, + "expected_usb_address": 7, + "actual_usb_bus": 3, + "actual_usb_address": 7, + "reservation_acquired": True, + "unit_release_attempts": 1, + "unit_released": True, + "recovery_required": "none", + }, + sort_keys=True, + ), + encoding="utf-8", + ) + + output_sha256_by_slot: dict[str, str] = {} + for frame_index, slot in enumerate(_SLOTS, start=1): + frame = batch / f"frame-{slot:03d}" + frame.mkdir() + meter = frame / "capture-meter.bin" + _write_sparse_zeros(meter, _METER_BYTES) + nonce = f"{slot:032x}" + selection = { + "frame": slot, + "boundary_offset": { + "requested_rows": 0, + "applied_rows": 0, + }, + "roll_identity": { + "reviewed_fingerprint_sha256": reviewed_sha256, + "fresh_fingerprint_sha256": fresh_sha256, + "comparison": {"matches": True, "reason": "matched"}, + "selected_slot_comparison": { + "matches": True, + "reason": "matched", + "slot": slot, + }, + }, + } + output_sha256 = hashlib.sha256(f"capture-{slot}".encode()).hexdigest() + output_sha256_by_slot[str(slot)] = output_sha256 + journal: dict[str, object] = { + "status": "frame-complete", + "frame_complete": True, + "session_reservation_retained": True, + "unit_released": False, + "recovery_required": None, + "batch_session": { + "session_id": session_id, + "frame_index": frame_index, + "frame_total": len(_SLOTS), + "selected_slots": list(_SLOTS), + }, + "capture_mode": "full", + "requested_frame": slot, + "requested_boundary_offset_rows": 0, + "expected_reads": 2_980, + "completed_reads": 2_980, + "expected_bytes": 619_458_560, + "completed_bytes": 619_458_560, + "disk_bytes": 619_458_560, + "output": str(frame / "capture.bin"), + "output_sha256": output_sha256, + "plan_sha256": plan_sha256, + "continuation_plan_sha256": continuation_sha256, + "capture_engine_sha256": engine_sha256, + "capture_bundle_sha256": bundle_sha256, + "manual_review_approval": approvals.get(slot), + "reviewed_roll_fingerprint_sha256": reviewed_sha256, + "expected_usb_bus": 3, + "expected_usb_address": 7, + "actual_usb_bus": 3, + "actual_usb_address": 7, + "ack_nonce": nonce, + "density_calibration_session_id": session_id, + "nikon_density_calibration": density_calibration, + "live_frame_selection": selection, + "meter_evidence": { + "path": str(meter), + "bytes": _METER_BYTES, + "sha256": _METER_SHA256, + "complete": True, + "durable_completed_passes": 3, + }, + "meter_evidence_persisted_before_fine_arm": True, + "meter_group_bytes": [1_088_000, 1_088_000, 1_088_000], + "meter_group_offsets": [0, 1_088_000, 2_176_000], + "meter_completed_reads": 15, + "meter_completed_bytes": _METER_BYTES, + "meter_layout": _METER_LAYOUT, + } + if slot == 1: + preview = frame / "capture-preview.bin" + _write_sparse_zeros(preview, _PREVIEW_BYTES) + table_payload = b"six-strip-transport-table" + table = frame / "capture-008e.bin" + table.write_bytes(table_payload) + mapping = frame / "capture-frame-map.json" + mapping.write_text(json.dumps(selection, sort_keys=True), encoding="utf-8") + journal["live_index_artifacts"] = { + "preview": str(preview), + "table": str(table), + "mapping": str(mapping), + } + journal["live_index_evidence"] = { + "status": "persisted-before-frame-detection", + "preview_bytes": _PREVIEW_BYTES, + "preview_sha256": _PREVIEW_SHA256, + "table_bytes": len(table_payload), + "table_sha256": hashlib.sha256(table_payload).hexdigest(), + } + (frame / "journal.json").write_text( + json.dumps(journal, sort_keys=True), + encoding="utf-8", + ) + (frame / "parent-ack.json").write_text( + json.dumps( + { + "ack_nonce": nonce, + "action": "continue", + "frame_index": frame_index, + "schema_version": 1, + "session_id": session_id, + "slot": slot, + }, + sort_keys=True, + ), + encoding="utf-8", + ) + return { + "batch": batch, + "output_sha256_by_slot": output_sha256_by_slot, + "reviewed_sha256": reviewed_sha256, + } + + +def test_completed_post_finalization_batch_binds_all_six_capture_records( + tmp_path: Path, +) -> None: + root = tmp_path / "attempts" + root.mkdir() + expected = _write_completed_batch(root) + + snapshot = snapshot_capture_evidence(root) + binding = validate_six_frame_batch_capture_evidence(snapshot) + + assert snapshot.manifest["file_count"] == 28 + assert binding["frame_output_sha256_by_slot"] == expected["output_sha256_by_slot"] + assert binding["reviewed_fingerprint_sha256"] == expected["reviewed_sha256"] + + +def test_finder_metadata_is_ignored_without_weakening_capture_evidence_inventory( + tmp_path: Path, +) -> None: + root = tmp_path / "attempts" + root.mkdir() + lease = _lease(root) + completed = _write_completed_batch(root) + batch = completed["batch"] + assert isinstance(batch, Path) + (root / ".DS_Store").write_bytes(b"finder root metadata") + (batch / ".DS_Store").write_bytes(b"finder batch metadata") + + try: + snapshot = snapshot_capture_evidence(root) + inventory = lease.assert_inventory(snapshot.files) + binding = validate_six_frame_batch_capture_evidence(snapshot) + receipt_evidence = bind_capture_evidence_inventory(snapshot, inventory) + finally: + lease.release() + + assert snapshot.manifest["file_count"] == 28 + assert all(not row["path"].endswith(".DS_Store") for row in snapshot.manifest["files"]) + assert binding["session_id"] == batch.name + assert receipt_evidence["retained"] is True + + +def test_finder_metadata_name_does_not_bypass_symlink_rejection(tmp_path: Path) -> None: + root = tmp_path / "attempts" + root.mkdir() + outside = tmp_path / "outside" + outside.write_bytes(b"must not be followed") + (root / ".DS_Store").symlink_to(outside) + + with pytest.raises(InventoryConflict, match="exclusively owned regular file"): + snapshot_capture_evidence(root) + + +def _rewrite_json(path: Path, mutate) -> None: + document = json.loads(path.read_text(encoding="utf-8")) + mutate(document) + path.write_text(json.dumps(document, sort_keys=True), encoding="utf-8") + + +def test_synthetic_journal_skeleton_without_meter_or_ack_artifacts_is_rejected( + tmp_path: Path, +) -> None: + root = tmp_path / "attempts" + root.mkdir() + completed = _write_completed_batch(root) + batch = completed["batch"] + assert isinstance(batch, Path) + for slot in _SLOTS: + frame = batch / f"frame-{slot:03d}" + (frame / "capture-meter.bin").unlink() + (frame / "parent-ack.json").unlink() + + snapshot = snapshot_capture_evidence(root) + + with pytest.raises(InventoryConflict, match="capture evidence is incomplete"): + validate_six_frame_batch_capture_evidence(snapshot) + + +def test_nonempty_second_attempt_tree_is_rejected(tmp_path: Path) -> None: + root = tmp_path / "attempts" + root.mkdir() + _write_completed_batch(root) + partial = root / "batch-slot01-slot06-aborted" / "frame-001" + partial.mkdir(parents=True) + (partial / "partial.bin").write_bytes(b"partial") + + snapshot = snapshot_capture_evidence(root) + + with pytest.raises(InventoryConflict, match="outside the completed six-frame batch"): + validate_six_frame_batch_capture_evidence(snapshot) + + +def test_additional_diagnostic_inside_completed_batch_is_allowed( + tmp_path: Path, +) -> None: + root = tmp_path / "attempts" + root.mkdir() + completed = _write_completed_batch(root) + batch = completed["batch"] + assert isinstance(batch, Path) + diagnostic = batch / "diagnostics" / "capture-timing.txt" + diagnostic.parent.mkdir() + diagnostic.write_text("retained diagnostic\n", encoding="utf-8") + + snapshot = snapshot_capture_evidence(root) + binding = validate_six_frame_batch_capture_evidence(snapshot) + + assert binding["session_id"] == batch.name + assert any(row["path"].endswith("capture-timing.txt") for row in snapshot.manifest["files"]) + + +@pytest.mark.parametrize( + ("target", "mutation", "message"), + [ + ( + "frame-003/journal.json", + lambda document: document.__setitem__("completed_bytes", 1), + "frame 3 journal", + ), + ( + "frame-004/parent-ack.json", + lambda document: document.__setitem__("action", "stop"), + "frame 4 parent ACK", + ), + ( + "session-journal.json", + lambda document: document.__setitem__("actual_usb_address", 8), + "completed six-frame batch", + ), + ( + "session-journal.json", + lambda document: document.pop("batch_job_sha256"), + "completed six-frame batch", + ), + ( + "session-journal.json", + lambda document: document.__setitem__("batch_job_sha256", "0" * 64), + "completed six-frame batch", + ), + ( + "frame-001/capture-frame-map.json", + lambda document: document.__setitem__("frame", 2), + "live preview, transport table, or frame map", + ), + ( + "frame-005/journal.json", + lambda document: document["live_frame_selection"]["boundary_offset"].__setitem__("applied_rows", 1), + "frame 5 journal", + ), + ], +) +def test_mutated_capture_contract_is_rejected( + tmp_path: Path, + target: str, + mutation, + message: str, +) -> None: + root = tmp_path / "attempts" + root.mkdir() + completed = _write_completed_batch(root) + batch = completed["batch"] + assert isinstance(batch, Path) + _rewrite_json(batch / target, mutation) + + snapshot = snapshot_capture_evidence(root) + + with pytest.raises(InventoryConflict, match=message): + validate_six_frame_batch_capture_evidence(snapshot) + + +def test_meter_payload_must_match_its_frame_journal_digest(tmp_path: Path) -> None: + root = tmp_path / "attempts" + root.mkdir() + completed = _write_completed_batch(root) + batch = completed["batch"] + assert isinstance(batch, Path) + meter = batch / "frame-005" / "capture-meter.bin" + with meter.open("r+b") as stream: + stream.write(b"changed") + + snapshot = snapshot_capture_evidence(root) + + with pytest.raises(InventoryConflict, match="frame 5 meter sidecar"): + validate_six_frame_batch_capture_evidence(snapshot) + + +def test_parent_ack_nonce_must_use_the_worker_safe_alphabet(tmp_path: Path) -> None: + root = tmp_path / "attempts" + root.mkdir() + completed = _write_completed_batch(root) + batch = completed["batch"] + assert isinstance(batch, Path) + _rewrite_json( + batch / "frame-002" / "journal.json", + lambda document: document.__setitem__("ack_nonce", "../unsafe"), + ) + _rewrite_json( + batch / "frame-002" / "parent-ack.json", + lambda document: document.__setitem__("ack_nonce", "../unsafe"), + ) + + snapshot = snapshot_capture_evidence(root) + + with pytest.raises(InventoryConflict, match="frame 2 parent ACK"): + validate_six_frame_batch_capture_evidence(snapshot) + + +@pytest.mark.parametrize("name", ["capture-preview.bin", "capture-008e.bin"]) +def test_live_index_payload_must_match_first_frame_journal( + tmp_path: Path, + name: str, +) -> None: + root = tmp_path / "attempts" + root.mkdir() + completed = _write_completed_batch(root) + batch = completed["batch"] + assert isinstance(batch, Path) + artifact = batch / "frame-001" / name + with artifact.open("r+b") as stream: + stream.write(b"changed") + + snapshot = snapshot_capture_evidence(root) + + with pytest.raises( + InventoryConflict, + match="live preview, transport table, or frame map", + ): + validate_six_frame_batch_capture_evidence(snapshot) + + +def test_retained_plan_bytes_must_match_job_and_journals(tmp_path: Path) -> None: + root = tmp_path / "attempts" + root.mkdir() + completed = _write_completed_batch(root) + batch = completed["batch"] + assert isinstance(batch, Path) + (batch / "replay-first-rgbi4-plan.jsonl").write_bytes(b"changed plan\n") + + snapshot = snapshot_capture_evidence(root) + + with pytest.raises(InventoryConflict, match="retained plans"): + validate_six_frame_batch_capture_evidence(snapshot) + + +def test_snapshot_hashes_and_binds_retained_attempt_tree(tmp_path: Path) -> None: + root = tmp_path / "attempts" + root.mkdir() + lease = _lease(root) + try: + frame = root / "batch-a" / "frame-001" + frame.mkdir(parents=True) + preview = frame / "capture-preview.bin" + table = frame / "capture-008e.bin" + preview.write_bytes(b"preview") + table.write_bytes(b"table") + + snapshot = snapshot_capture_evidence(root) + inventory = lease.assert_inventory(snapshot.files) + receipt = bind_capture_evidence_inventory(snapshot, inventory) + + assert receipt["file_count"] == 2 + assert receipt["directory_count"] == 2 + assert receipt["total_bytes"] == 12 + assert receipt["files"] == [ + { + "path": "batch-a/frame-001/capture-008e.bin", + "bytes": 5, + "sha256": hashlib.sha256(b"table").hexdigest(), + }, + { + "path": "batch-a/frame-001/capture-preview.bin", + "bytes": 7, + "sha256": hashlib.sha256(b"preview").hexdigest(), + }, + ] + finally: + lease.release() + + +def test_snapshot_rejects_symbolic_link(tmp_path: Path) -> None: + root = tmp_path / "attempts" + root.mkdir() + target = tmp_path / "outside.bin" + target.write_bytes(b"outside") + (root / "linked.bin").symlink_to(target) + + with pytest.raises(InventoryConflict, match="regular file"): + snapshot_capture_evidence(root) + + +def test_inventory_binding_rejects_post_hash_mutation(tmp_path: Path) -> None: + root = tmp_path / "attempts" + root.mkdir() + lease = _lease(root) + try: + artifact = root / "journal.json" + artifact.write_bytes(b"one") + snapshot = snapshot_capture_evidence(root) + artifact.write_bytes(b"two") + inventory = lease.assert_inventory(snapshot.files) + + with pytest.raises(InventoryConflict, match="changed between hashing"): + bind_capture_evidence_inventory(snapshot, inventory) + finally: + lease.release() + + +def test_snapshot_refuses_manifest_that_cannot_fit_in_run_receipt( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + root = tmp_path / "attempts" + root.mkdir() + (root / "capture-preview.bin").write_bytes(b"preview") + monkeypatch.setattr(live_evidence, "_MAX_MANIFEST_BYTES", 1) + + with pytest.raises(InventoryConflict, match="receipt-safe size limit"): + snapshot_capture_evidence(root) + + +def test_duplicate_json_keys_in_capture_evidence_are_rejected(tmp_path: Path) -> None: + root = tmp_path / "attempts" + root.mkdir() + completed = _write_completed_batch(root) + batch = completed["batch"] + assert isinstance(batch, Path) + session_path = batch / "session-journal.json" + document = json.loads(session_path.read_text(encoding="utf-8")) + body = json.dumps(document, sort_keys=True) + assert body.startswith("{") + duplicated = '{"batch_job_sha256": "' + "0" * 64 + '", ' + body[1:] + assert json.loads(duplicated)["batch_job_sha256"] == document["batch_job_sha256"] + session_path.write_text(duplicated, encoding="utf-8") + + snapshot = snapshot_capture_evidence(root) + + with pytest.raises(InventoryConflict, match="repeats key"): + validate_six_frame_batch_capture_evidence(snapshot) + + +def test_session_engine_identity_must_match_the_pinned_runtime(tmp_path: Path) -> None: + from coolscanpy.protocol.ls5000_single_pass.bundle import CAPTURE_BUNDLE_SHA256 + + root = tmp_path / "attempts" + root.mkdir() + completed = _write_completed_batch(root) + batch = completed["batch"] + assert isinstance(batch, Path) + foreign = "f" * 64 + assert foreign != CAPTURE_BUNDLE_SHA256 + for name in ("session-journal.json", *(f"frame-{slot:03d}/journal.json" for slot in _SLOTS)): + _rewrite_json( + batch / name, + lambda document: document.__setitem__("capture_bundle_sha256", foreign), + ) + + snapshot = snapshot_capture_evidence(root) + + with pytest.raises(InventoryConflict, match="completed six-frame batch"): + validate_six_frame_batch_capture_evidence(snapshot) diff --git a/tests/roll/test_live_reservation.py b/tests/roll/test_live_reservation.py new file mode 100644 index 00000000..d241abdf --- /dev/null +++ b/tests/roll/test_live_reservation.py @@ -0,0 +1,376 @@ +from __future__ import annotations + +import json +import os +import stat +from pathlib import Path + +import pytest + +from negpy.services.roll import live_reservation +from negpy.services.roll.live_reservation import ( + OUTPUT_LOCK_NAME, + ExclusiveReceiptReservation, + FixedOutputLease, + InventoryConflict, + LiveReservationError, + OwnershipLost, + ReservationConflict, +) + + +def _in_progress() -> dict[str, object]: + return { + "schema": "negpy.test-live-reservation.v1", + "status": "in_progress", + } + + +def _lock_document() -> dict[str, object]: + return { + "reservation_id": "0123456789abcdef", + "run_receipt": "/outside/output/run-receipt.json", + } + + +def test_receipt_is_exclusive_durable_canonical_and_finalized_in_place( + tmp_path: Path, +) -> None: + path = tmp_path / "run-receipt.json" + reservation = ExclusiveReceiptReservation.reserve(path, _in_progress()) + try: + initial_inode = reservation.inode + assert path.read_bytes() == (b'{"schema":"negpy.test-live-reservation.v1","status":"in_progress"}\n') + + final = { + "schema": "negpy.test-live-reservation.v1", + "status": "succeeded", + "frames": [1, 2, 3, 4, 5, 6], + } + reservation.publish(final) + + assert reservation.inode == initial_inode + assert (path.stat().st_dev, path.stat().st_ino) == initial_inode + assert json.loads(path.read_bytes()) == final + finally: + reservation.close() + + +def test_preexisting_receipt_is_preserved_and_never_opened_for_writing( + tmp_path: Path, +) -> None: + path = tmp_path / "run-receipt.json" + path.write_bytes(b"existing receipt") + before = path.stat() + + with pytest.raises(ReservationConflict, match="already exists"): + ExclusiveReceiptReservation.reserve(path, _in_progress()) + + after = path.stat() + assert path.read_bytes() == b"existing receipt" + assert (after.st_dev, after.st_ino) == (before.st_dev, before.st_ino) + + +def test_receipt_reservation_requires_in_progress_status(tmp_path: Path) -> None: + path = tmp_path / "run-receipt.json" + + with pytest.raises(LiveReservationError, match="exactly 'in_progress'"): + ExclusiveReceiptReservation.reserve(path, {"status": "succeeded"}) + + assert not path.exists() + + +def test_receipt_publication_refuses_replaced_path_and_preserves_collision( + tmp_path: Path, +) -> None: + path = tmp_path / "run-receipt.json" + reservation = ExclusiveReceiptReservation.reserve(path, _in_progress()) + path.unlink() + path.write_bytes(b"collision") + + try: + with pytest.raises(OwnershipLost, match="no longer identifies"): + reservation.publish({"status": "succeeded"}) + assert path.read_bytes() == b"collision" + finally: + reservation.close() + + +def test_receipt_publication_refuses_an_added_hard_link(tmp_path: Path) -> None: + path = tmp_path / "run-receipt.json" + linked = tmp_path / "unexpected-hard-link.json" + reservation = ExclusiveReceiptReservation.reserve(path, _in_progress()) + os.link(path, linked) + + try: + with pytest.raises(OwnershipLost, match="no longer identifies"): + reservation.publish({"status": "succeeded"}) + assert json.loads(path.read_bytes()) == _in_progress() + assert linked.read_bytes() == path.read_bytes() + finally: + reservation.close() + + +def test_oversized_final_receipt_is_rejected_before_mutating_reservation( + tmp_path: Path, +) -> None: + path = tmp_path / "run-receipt.json" + reservation = ExclusiveReceiptReservation.reserve( + path, + {"status": "in_progress"}, + maximum_bytes=64, + ) + before = path.read_bytes() + + try: + with pytest.raises(LiveReservationError, match="size limit"): + reservation.publish({"status": "failed", "error": "x" * 100}) + assert path.read_bytes() == before + finally: + reservation.close() + + +def test_fixed_output_lease_is_exclusive_visible_and_removed_on_release( + tmp_path: Path, +) -> None: + output = tmp_path / "outputs" + output.mkdir() + + lease = FixedOutputLease.acquire(output, _lock_document()) + assert lease.lock_path == output / OUTPUT_LOCK_NAME + assert json.loads(lease.lock_path.read_bytes()) == _lock_document() + assert sorted(path.name for path in output.iterdir()) == [OUTPUT_LOCK_NAME] + assert lease.assert_inventory(()).directories == () + + lease.release() + + assert lease.released is True + assert list(output.iterdir()) == [] + + +def test_preexisting_fixed_lock_is_preserved(tmp_path: Path) -> None: + output = tmp_path / "outputs" + output.mkdir() + lock = output / OUTPUT_LOCK_NAME + lock.write_bytes(b"existing lock") + before = lock.stat() + + with pytest.raises(ReservationConflict, match="already reserved"): + FixedOutputLease.acquire(output, _lock_document()) + + after = lock.stat() + assert lock.read_bytes() == b"existing lock" + assert (after.st_dev, after.st_ino) == (before.st_dev, before.st_ino) + + +def test_nonempty_output_fails_and_removes_only_its_new_lock(tmp_path: Path) -> None: + output = tmp_path / "outputs" + output.mkdir() + existing = output / "existing.tif" + existing.write_bytes(b"do not overwrite") + + with pytest.raises(InventoryConflict, match="unexpected files: existing.tif"): + FixedOutputLease.acquire(output, _lock_document()) + + assert existing.read_bytes() == b"do not overwrite" + assert not (output / OUTPUT_LOCK_NAME).exists() + + +def test_inventory_allows_only_declared_files_and_implied_directories( + tmp_path: Path, +) -> None: + output = tmp_path / "outputs" + output.mkdir() + lease = FixedOutputLease.acquire(output, _lock_document()) + evidence_dir = output / ".negpy-native-builder" / "receipt-a" + evidence_dir.mkdir(parents=True) + first = evidence_dir / "evidence.json" + first.write_bytes(b"one") + + try: + first_snapshot = lease.assert_inventory([first]) + assert first_snapshot.directories == ( + ".negpy-native-builder", + os.path.join(".negpy-native-builder", "receipt-a"), + ) + + second = output / "acceptance_slot01.tif" + second.write_bytes(b"two") + second_snapshot = lease.assert_inventory( + [first, second], + previous=first_snapshot, + ) + assert dict(second_snapshot.files)[os.fspath(second.relative_to(output))].size == 3 + + rogue = output / "acceptance_slot02.tif" + rogue.write_bytes(b"collision") + with pytest.raises(InventoryConflict, match="unexpected files: acceptance_slot02.tif"): + lease.assert_inventory([first, second], previous=second_snapshot) + rogue.unlink() + finally: + lease.release() + + +def test_inventory_ignores_only_regular_finder_metadata(tmp_path: Path) -> None: + output = tmp_path / "outputs" + output.mkdir() + lease = FixedOutputLease.acquire(output, _lock_document()) + artifact = output / "acceptance_slot01.tif" + artifact.write_bytes(b"captured") + (output / ".DS_Store").write_bytes(b"finder metadata") + + try: + snapshot = lease.assert_inventory([artifact]) + finally: + lease.release() + + assert [name for name, _identity in snapshot.files] == [ + OUTPUT_LOCK_NAME, + artifact.name, + ] + + +def test_inventory_detects_mutation_of_previously_owned_file(tmp_path: Path) -> None: + output = tmp_path / "outputs" + output.mkdir() + lease = FixedOutputLease.acquire(output, _lock_document()) + artifact = output / "acceptance_slot01.tif" + artifact.write_bytes(b"before") + snapshot = lease.assert_inventory([artifact]) + + try: + artifact.write_bytes(b"after!") + with pytest.raises(InventoryConflict, match="previously owned output changed"): + lease.assert_inventory([artifact], previous=snapshot) + finally: + lease.release() + + +def test_inventory_rejects_symlink_and_owned_path_escape(tmp_path: Path) -> None: + output = tmp_path / "outputs" + output.mkdir() + outside = tmp_path / "outside.tif" + outside.write_bytes(b"outside") + lease = FixedOutputLease.acquire(output, _lock_document()) + symlink = output / "linked.tif" + symlink.symlink_to(outside) + + try: + with pytest.raises(InventoryConflict, match="symbolic link"): + lease.assert_inventory([symlink]) + symlink.unlink() + with pytest.raises(InventoryConflict, match="escapes"): + lease.assert_inventory([outside]) + finally: + lease.release() + + +def test_output_path_replacement_is_detected_but_original_lock_is_released( + tmp_path: Path, +) -> None: + output = tmp_path / "outputs" + moved = tmp_path / "moved-outputs" + output.mkdir() + lease = FixedOutputLease.acquire(output, _lock_document()) + output.rename(moved) + output.mkdir() + + with pytest.raises(OwnershipLost, match="pathname no longer identifies"): + lease.assert_owned() + + lease.release() + + assert not (moved / OUTPUT_LOCK_NAME).exists() + assert list(output.iterdir()) == [] + + +def test_release_refuses_to_unlink_replacement_lock(tmp_path: Path) -> None: + output = tmp_path / "outputs" + output.mkdir() + lease = FixedOutputLease.acquire(output, _lock_document()) + lease.lock_path.unlink() + lease.lock_path.write_bytes(b"replacement lock") + + with pytest.raises(OwnershipLost, match="no longer exclusively owned"): + lease.release() + + assert lease.released is True + assert lease.lock_path.read_bytes() == b"replacement lock" + + +def test_verified_release_checks_inventory_around_final_publication( + tmp_path: Path, +) -> None: + output = tmp_path / "outputs" + output.mkdir() + artifact = output / "acceptance_slot01.tif" + lease = FixedOutputLease.acquire(output, _lock_document()) + artifact.write_bytes(b"stable") + snapshot = lease.assert_inventory([artifact]) + finalized: list[bool] = [] + + result = lease.release_verified( + [artifact], + previous=snapshot, + finalize=lambda: finalized.append(True), + ) + + assert finalized == [True] + assert lease.released is True + assert [name for name, _identity in result.files] == [artifact.name] + assert sorted(path.name for path in output.iterdir()) == [artifact.name] + + +def test_verified_release_rejects_output_path_swap_during_final_publication( + tmp_path: Path, +) -> None: + output = tmp_path / "outputs" + moved = tmp_path / "moved-outputs" + output.mkdir() + artifact = output / "acceptance_slot01.tif" + lease = FixedOutputLease.acquire(output, _lock_document()) + artifact.write_bytes(b"stable") + snapshot = lease.assert_inventory([artifact]) + + def replace_output_path() -> None: + output.rename(moved) + output.mkdir() + + with pytest.raises(OwnershipLost, match="pathname no longer identifies"): + lease.release_verified( + [artifact], + previous=snapshot, + finalize=replace_output_path, + ) + + assert lease.released is True + assert not (moved / OUTPUT_LOCK_NAME).exists() + assert (moved / artifact.name).read_bytes() == b"stable" + assert list(output.iterdir()) == [] + + +def test_receipt_lock_and_directories_are_fsynced( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + real_fsync = os.fsync + synced_modes: list[int] = [] + + def recording_fsync(descriptor: int) -> None: + synced_modes.append(stat.S_IFMT(os.fstat(descriptor).st_mode)) + real_fsync(descriptor) + + monkeypatch.setattr(live_reservation.os, "fsync", recording_fsync) + receipt = ExclusiveReceiptReservation.reserve( + tmp_path / "run-receipt.json", + _in_progress(), + ) + receipt.publish({"status": "succeeded"}) + receipt.close() + output = tmp_path / "outputs" + output.mkdir() + lease = FixedOutputLease.acquire(output, _lock_document()) + lease.release() + + assert synced_modes.count(stat.S_IFREG) >= 3 + assert synced_modes.count(stat.S_IFDIR) >= 4 diff --git a/tests/roll/test_live_review.py b/tests/roll/test_live_review.py new file mode 100644 index 00000000..2e16c1df --- /dev/null +++ b/tests/roll/test_live_review.py @@ -0,0 +1,218 @@ +from __future__ import annotations + +import hashlib +import json +from pathlib import Path +from types import SimpleNamespace + +import numpy as np +import pytest +from PIL import Image + +from negpy.services.roll.live_review import ( + REVIEW_BASIS, + SCHEMA, + load_reviewed_approval, + thumbnail_sha256, + validate_restored_thumbnails, +) + + +class _Approval: + def __init__(self, payload: dict) -> None: + self._payload = dict(payload) + + def to_payload(self) -> dict: + return dict(self._payload) + + +class _Session: + def __init__(self, fingerprint: str, approvals: dict[int, dict]) -> None: + self._fingerprint = fingerprint + self._approvals = approvals + + def reviewed_fingerprint(self): + return SimpleNamespace(binding_sha256=self._fingerprint) + + def approve_manual_origin(self, slot: int, offset: int): + assert self._approvals[slot]["boundary_offset_rows"] == offset + return _Approval(self._approvals[slot]) + + +def _approval(slot: int, thumbnail: str, fingerprint: str) -> dict: + payload = { + "boundary_offset_rows": 0, + "review_reasons": [f"slot-{slot}-edge-review"], + "reviewed_fingerprint_sha256": fingerprint, + "reviewed_lookup_row": slot, + "reviewed_native_origin": slot * 100, + "schema_version": 1, + "slot": slot, + "thumbnail_sha256": thumbnail, + } + encoded = json.dumps( + payload, + sort_keys=True, + separators=(",", ":"), + ensure_ascii=False, + allow_nan=False, + ).encode() + return {**payload, "binding_sha256": hashlib.sha256(encoded).hexdigest()} + + +def _fixture(tmp_path: Path): + fingerprint = "c" * 64 + images = {slot: np.full((2, 3, 3), slot, dtype=np.uint16) for slot in range(1, 7)} + approvals = {slot: _approval(slot, thumbnail_sha256(images[slot]), fingerprint) for slot in (1, 6)} + session_payload = '{"version":1}' + session_path = tmp_path / "session.json" + session_path.write_text(session_payload, encoding="utf-8") + session_sha = hashlib.sha256(session_payload.encode()).hexdigest() + contact_path = tmp_path / "contact.png" + Image.new("RGB", (6, 2), "white").save(contact_path) + contact_bytes = contact_path.read_bytes() + document = { + "approvals": [approvals[1], approvals[6]], + "contact_sheet": { + "bytes": len(contact_bytes), + "path": str(contact_path), + "sha256": hashlib.sha256(contact_bytes).hexdigest(), + }, + "preview_session": { + "bytes": len(session_payload.encode()), + "path": str(session_path), + "sha256": session_sha, + }, + "review_basis": REVIEW_BASIS, + "reviewed_fingerprint_sha256": fingerprint, + "schema": SCHEMA, + } + review_path = tmp_path / "review.json" + review_path.write_text(json.dumps(document), encoding="utf-8") + review_sha = hashlib.sha256(review_path.read_bytes()).hexdigest() + session = _Session(fingerprint, approvals) + return ( + review_path, + review_sha, + session_path, + session_payload, + session_sha, + session, + images, + document, + ) + + +def _load(values): + review_path, review_sha, session_path, payload, session_sha, session, *_ = values + return load_reviewed_approval( + review_path, + review_sha, + preview_session_path=session_path, + preview_session_payload=payload, + preview_session_sha256=session_sha, + session_loader=lambda _payload: session, + approval_parser=lambda row: _Approval(row), + ) + + +def test_loads_pinned_review_and_rederives_both_approvals(tmp_path: Path) -> None: + values = _fixture(tmp_path) + reviewed = _load(values) + + assert reviewed.reviewed_fingerprint_sha256 == "c" * 64 + assert tuple(reviewed.approvals) == (1, 6) + + images = values[6] + thumbnails = [SimpleNamespace(slot=slot, image=images[slot]) for slot in range(1, 7)] + validate_restored_thumbnails(thumbnails, reviewed) + + +def test_derives_required_approvals_from_the_restored_preview(tmp_path: Path) -> None: + values = list(_fixture(tmp_path)) + approvals = {1: values[7]["approvals"][0]} + document = dict(values[7]) + document["approvals"] = [approvals[1]] + values[0].write_text(json.dumps(document), encoding="utf-8") + values[1] = hashlib.sha256(values[0].read_bytes()).hexdigest() + + session = _Session("c" * 64, approvals) + session.slots = tuple( + SimpleNamespace( + slot_id=slot, + base_origin=SimpleNamespace(manual_review=(slot in (1, 7))), + ) + for slot in range(1, 8) + ) + session.selected_slots = (1, 2, 3, 4, 5, 6) + values[5] = session + + reviewed = _load(tuple(values)) + assert tuple(reviewed.approvals) == (1,) + + +def test_wrong_review_hash_fails_before_parsing(tmp_path: Path) -> None: + values = list(_fixture(tmp_path)) + values[1] = "0" * 64 + + with pytest.raises(ValueError, match="SHA-256 mismatch"): + _load(tuple(values)) + + +@pytest.mark.parametrize( + ("field", "value", "message"), + [ + ("schema", "wrong", "schema or review basis"), + ("reviewed_fingerprint_sha256", "d" * 64, "fingerprint"), + ], +) +def test_review_identity_changes_fail_closed( + tmp_path: Path, + field: str, + value: str, + message: str, +) -> None: + values = list(_fixture(tmp_path)) + document = dict(values[7]) + document[field] = value + values[0].write_text(json.dumps(document), encoding="utf-8") + values[1] = hashlib.sha256(values[0].read_bytes()).hexdigest() + + with pytest.raises(ValueError, match=message): + _load(tuple(values)) + + +def test_changed_approval_is_not_reaccepted_under_a_new_file_hash(tmp_path: Path) -> None: + values = list(_fixture(tmp_path)) + document = dict(values[7]) + rows = [dict(row) for row in document["approvals"]] + rows[0]["thumbnail_sha256"] = "e" * 64 + document["approvals"] = rows + values[0].write_text(json.dumps(document), encoding="utf-8") + values[1] = hashlib.sha256(values[0].read_bytes()).hexdigest() + + with pytest.raises(ValueError, match="does not match its thumbnail"): + _load(tuple(values)) + + +def test_restored_thumbnail_change_fails(tmp_path: Path) -> None: + values = _fixture(tmp_path) + reviewed = _load(values) + images = dict(values[6]) + images[1] = images[1].copy() + images[1][0, 0, 0] += 1 + thumbnails = [SimpleNamespace(slot=slot, image=images[slot]) for slot in range(1, 7)] + + with pytest.raises(ValueError, match="slot 1 changed"): + validate_restored_thumbnails(thumbnails, reviewed) + + +def test_review_mapping_is_immutable(tmp_path: Path) -> None: + reviewed = _load(_fixture(tmp_path)) + with pytest.raises(TypeError): + reviewed.approvals[1] = object() # type: ignore[index] + + +def test_thumbnail_validator_rejects_non_rgb16() -> None: + with pytest.raises(ValueError, match="HxWx3 uint16"): + thumbnail_sha256(np.zeros((2, 2), dtype=np.uint8)) diff --git a/tests/roll/test_native_builder.py b/tests/roll/test_native_builder.py new file mode 100644 index 00000000..54bfb087 --- /dev/null +++ b/tests/roll/test_native_builder.py @@ -0,0 +1,1064 @@ +"""Native Nikon Stage-1 builder tests; no scanner or VM required.""" + +from __future__ import annotations + +import json +import os +import struct +import types +from dataclasses import dataclass, replace +from hashlib import sha256 +from pathlib import Path + +import numpy as np +import pytest + +from negpy.infrastructure.roll import repair as roll_repair +from negpy.services.roll import exact_color +from negpy.services.roll import service as roll_service +from negpy.services.roll.native_builder import ( + NativeBuilderEvidence, + adapt_native_builder_evidence, + build_native_builder_receipt, +) +from negpy.services.roll.portable_builder import PortableStage1Builder +from negpy.services.roll.portable_cms import PortableCMSOnEvaluator +from negpy.services.roll.service import RollScanningService + + +CAPTURE_D = Path( + os.environ.get( + "CAPTURE_D", + "/Volumes/isos/NikonRE/session20260719/capture-d", + ) +) +CAPTURE_D_PREF_SHA256 = ( + "46a0d68ae20c72088e64a1144a0d38bf692f15f506539bbe94eb563fe437c976", + "23eda81294817e7a2a31f1488544a6f8d3e7ac817f22d43c8a39882565c34b95", + "3cfc61c06bac49c4c28e69afe99af01366fae6bf5ea88954f688592a8e2756bb", +) +DENSITY_ALGORITHM_ID = "ls5000-md3-10088810-layout1-u16-proven-inputs-macos-binary64-exact-v6" +FRAME_OWNERSHIP_STATUS = "proven-exact-reservation-preview-registration-and-transport" +RESERVATION_ID = "reservation-test-001" +PREVIEW_SHA256 = sha256(b"test-preview").hexdigest() +PREVIEW_IDENTITY_SHA256 = sha256(b"test-preview-identity").hexdigest() +TRANSPORT_TABLE_SHA256 = sha256(b"test-transport-table").hexdigest() +REVIEWED_FINGERPRINT_SHA256 = sha256(b"test-reviewed-registration").hexdigest() +FRESH_FINGERPRINT_SHA256 = sha256(b"test-fresh-registration").hexdigest() +DENSITY_WIRE_SHA256 = PREVIEW_SHA256 +DENSITY_CHILD_SHA256 = sha256(b"test-only-capture-d-density-child").hexdigest() + + +def _canonical_json(payload: dict[str, object]) -> bytes: + return json.dumps(payload, sort_keys=True, separators=(",", ":"), allow_nan=False).encode() + + +def _ownership_document() -> dict[str, object]: + material = { + "reservation_id": RESERVATION_ID, + "batch_session_id": RESERVATION_ID, + "preview_sha256": PREVIEW_SHA256, + "preview_identity_sha256": PREVIEW_IDENTITY_SHA256, + "transport_table_sha256": TRANSPORT_TABLE_SHA256, + "reviewed_fingerprint_sha256": REVIEWED_FINGERPRINT_SHA256, + "fresh_fingerprint_sha256": FRESH_FINGERPRINT_SHA256, + "selected_slots": [5], + } + return { + "schema_version": 1, + "scope": "reservation-preview-frame", + "binding_status": FRAME_OWNERSHIP_STATUS, + "session_reservation_retained": True, + **material, + "transport_identity_sha256": sha256(_canonical_json(material)).hexdigest(), + "frame_capture_attempt_id": "capture-d", + "frame_index": 1, + "frame_total": 1, + "selected_slot": 5, + } + + +def _density_document( + *, + numerators: tuple[int, int, int] = (57_114, 48_036, 32_683), + density_f03: tuple[int, int, int] = (70_307, 136_614, 125_470), + source_payload_bytes: int = 6_250_496, + densities: tuple[float, float, float] = ( + struct.unpack(">d", bytes.fromhex("3fd8b159777b9d5f"))[0], + struct.unpack(">d", bytes.fromhex("3fe9cc75f7f6705a"))[0], + struct.unpack(">d", bytes.fromhex("3ff0b0dae0533338"))[0], + ), +) -> dict[str, object]: + binding_identity = { + "session_id": RESERVATION_ID, + "capture_attempt_id": "capture-d", + "scan_identity": "capture-d-slot", + } + return { + "schema_version": 1, + "scope": "reservation-preview", + "per_frame_binding_status": "requires-explicit-frame-ownership-receipt", + "preview_identity_sha256": PREVIEW_IDENTITY_SHA256, + "source_payload_bytes": source_payload_bytes, + "calibration_binding": { + "calibration": { + "session_id": RESERVATION_ID, + "numerators_rgb": list(numerators), + }, + "capture_attempt_id": binding_identity["capture_attempt_id"], + "scan_identity": binding_identity["scan_identity"], + }, + "source_binding": { + **binding_identity, + "resolution_dpi": 97, + "wire_sha256": DENSITY_WIRE_SHA256, + "child_buffer_sha256": DENSITY_CHILD_SHA256, + }, + "exposure_binding": { + **binding_identity, + "density_f03_exposures_raw_10ns_rgb": list(density_f03), + }, + "result": { + **binding_identity, + "algorithm_id": DENSITY_ALGORITHM_ID, + "promotable": True, + "source_wire_sha256": DENSITY_WIRE_SHA256, + "source_child_buffer_sha256": DENSITY_CHILD_SHA256, + "numerators_rgb": list(numerators), + "density_f03_denominators_raw_10ns_rgb": list(density_f03), + "densities_rgb": list(densities), + "density_binary64_be_hex_rgb": [struct.pack(">d", value).hex() for value in densities], + }, + } + + +@dataclass(frozen=True) +class _PayloadReceipt: + payload: dict[str, object] + + def to_dict(self) -> dict[str, object]: + return self.payload + + +@dataclass(frozen=True) +class _RejectingOwnershipReceipt(_PayloadReceipt): + def validate_evidence(self, evidence: object) -> None: + del evidence + raise ValueError("preview identity changed") + + +@dataclass(frozen=True) +class _FrameReceipt: + version: int + slot: int + nikon_density_ownership: object | None + + +def _bound_repair_frame_fields(*, slot: int = 5) -> dict[str, object]: + """Create one valid, immutable scanner-native repair acquisition.""" + + storage_rgb = np.full((2, 3, 3), 12_000, dtype=np.uint16) + storage_ir = np.full((2, 3), 3_000, dtype=np.uint16) + storage_validity = np.ones(storage_rgb.shape[:2], dtype=np.bool_) + meter_rgbi = np.zeros((1, 1, 4), dtype=np.uint16) + native_rgbi = np.ascontiguousarray( + np.rot90( + np.dstack((storage_rgb, storage_ir)), + k=-1, + axes=(0, 1), + ) + ) + native_validity = np.ascontiguousarray(np.rot90(storage_validity, k=-1, axes=(0, 1))) + capture_attempt_id = f"capture-d-slot-{slot}" + acquisition_id, evidence_sha256 = roll_service._derive_digital_ice_producer_binding( + slot=slot, + reservation_id=RESERVATION_ID, + capture_attempt_id=capture_attempt_id, + main_rgbi=native_rgbi, + prepass_rgbi=meter_rgbi, + ir_validity=native_validity, + ) + acquisition = roll_repair.RepairAcquisition.from_arrays( + acquisition_id=acquisition_id, + slot=slot, + reservation_id=RESERVATION_ID, + capture_attempt_id=capture_attempt_id, + storage_transform=roll_repair.DIGITAL_ICE_STORAGE_TRANSFORM, + evidence_sha256=evidence_sha256, + main_rgbi=native_rgbi, + prepass_rgbi=meter_rgbi, + ir_validity=native_validity, + ) + return { + "rgb": storage_rgb, + "ir": storage_ir, + "ir_validity": storage_validity, + "meter_rgbi": meter_rgbi, + "prepare_digital_ice": lambda: acquisition, + } + + +def _capture_d_evidence() -> NativeBuilderEvidence: + if not CAPTURE_D.is_dir(): + pytest.skip("archived capture-d is unavailable") + descriptor = (CAPTURE_D / "analyzer-desc.bin").read_bytes() + width, height, r0, c0, r1, c1 = struct.unpack_from("<6I", descriptor) + numerators = struct.unpack_from("<3I", descriptor, 0x1C) + final_f02 = struct.unpack_from("<3I", descriptor, 0x28) + densities = struct.unpack_from("<3d", descriptor, 0x40) + analyzer = np.frombuffer((CAPTURE_D / "analyzer-pixels.bin").read_bytes(), dtype=" NativeBuilderEvidence: + """Small deterministic provenance fixture whose analyzer is self-contained.""" + + numerators = (57_114, 48_036, 32_683) + density_f03 = (70_307, 136_614, 125_470) + densities = ( + struct.unpack(">d", bytes.fromhex("3fd8b159777b9d5f"))[0], + struct.unpack(">d", bytes.fromhex("3fe9cc75f7f6705a"))[0], + struct.unpack(">d", bytes.fromhex("3ff0b0dae0533338"))[0], + ) + analyzer = np.full((425, 281, 3), 20_000, dtype=np.uint16) + ownership = _ownership_document() + ownership_payload = _canonical_json(ownership) + density_payload = _canonical_json( + _density_document( + numerators=numerators, + density_f03=density_f03, + source_payload_bytes=source_payload_bytes, + densities=densities, + ) + ) + return NativeBuilderEvidence( + session_id=RESERVATION_ID, + capture_attempt_id="capture-d", + scan_identity="capture-d-slot", + slot=5, + density_source_wire_sha256=DENSITY_WIRE_SHA256, + density_source_child_sha256=DENSITY_CHILD_SHA256, + calibration_numerators=numerators, + density_f03_denominators=density_f03, + densities=densities, + density_arithmetic=DENSITY_ALGORITHM_ID, + frame_ownership_status=FRAME_OWNERSHIP_STATUS, + frame_ownership_receipt=ownership_payload, + frame_ownership_receipt_sha256=sha256(ownership_payload).hexdigest(), + density_evidence_receipt=density_payload, + density_evidence_receipt_sha256=sha256(density_payload).hexdigest(), + reservation_id=RESERVATION_ID, + batch_session_id=RESERVATION_ID, + preview_sha256=PREVIEW_SHA256, + preview_identity_sha256=PREVIEW_IDENTITY_SHA256, + transport_table_sha256=TRANSPORT_TABLE_SHA256, + transport_identity_sha256=ownership["transport_identity_sha256"], + reviewed_fingerprint_sha256=REVIEWED_FINGERPRINT_SHA256, + fresh_fingerprint_sha256=FRESH_FINGERPRINT_SHA256, + frame_index=1, + frame_total=1, + selected_slots=(5,), + analyzer_rgb=analyzer, + analyzer_rgb_sha256=sha256(analyzer.astype(" object: + ownership = _PayloadReceipt(_ownership_document()) + return types.SimpleNamespace( + slot=5, + **_bound_repair_frame_fields(), + receipt=_FrameReceipt( + version=1, + slot=5, + nikon_density_ownership=ownership, + ), + nikon_density_evidence=_PayloadReceipt(_density_document()), + nikon_density_ownership=ownership, + nikon_exact_builder_evidence=_synthetic_native_evidence(), + ) + + +def _coolscanpy_capture_d_evidence() -> object: + try: + from coolscanpy.protocol.ls5000_single_pass.density import ( + NikonExactBuilderEvidence, + ) + except ImportError: + pytest.skip("installed Coolscanpy lacks the exact builder producer") + native = _capture_d_evidence() + return NikonExactBuilderEvidence(**{name: getattr(native, name) for name in NativeBuilderEvidence.__dataclass_fields__}) + + +def _coolscanpy_built_frame() -> object: + try: + from coolscanpy.protocol.ls5000_single_pass.density import ( + assemble_density_calibration, + build_nikon_density_evidence, + build_nikon_density_frame_ownership, + build_nikon_exact_builder_evidence, + decode_density_calibration_read, + ) + from coolscanpy.types import ( + Frame, + build_digital_ice_acquisition_evidence, + ) + except ImportError: + pytest.skip("installed Coolscanpy lacks the exact builder producer") + native = _capture_d_evidence() + calibration = assemble_density_calibration( + [ + decode_density_calibration_read( + bytes.fromhex(f"28008c000{color}0300000a80"), + bytes.fromhex(payload), + ) + for color, payload in enumerate( + ( + "8c20000000040000df1a", + "8c20000000040000bba4", + "8c200000000400007fab", + ), + start=1, + ) + ], + session_id=RESERVATION_ID, + ) + samples = np.concatenate( + ( + np.full(96, 45_000, dtype=np.uint16), + np.full(96, 40_000, dtype=np.uint16), + np.full(96, 32_000, dtype=np.uint16), + ) + ).astype(">u2") + density_source = bytes(100 * 1_024) + samples.tobytes() + bytes(448) + bytes((6_104 - 101) * 1_024) + density = build_nikon_density_evidence( + density_source, + calibration=calibration, + density_f03_exposures_raw_10ns=(70_307, 136_614, 125_470), + session_id=RESERVATION_ID, + capture_attempt_id="preview-attempt-1", + scan_identity="reservation-test-001-density-97dpi-preview", + ) + ownership = build_nikon_density_frame_ownership( + density, + reservation_id=RESERVATION_ID, + batch_session_id=RESERVATION_ID, + transport_table_sha256=TRANSPORT_TABLE_SHA256, + reviewed_fingerprint_sha256=REVIEWED_FINGERPRINT_SHA256, + fresh_fingerprint_sha256=FRESH_FINGERPRINT_SHA256, + frame_capture_attempt_id="fine-slot-5-attempt-1", + frame_index=1, + frame_total=1, + selected_slots=(5,), + selected_slot=5, + ) + exact_builder = build_nikon_exact_builder_evidence( + density, + ownership, + analyzer_rgb=native.analyzer_rgb, + final_f02_denominators=native.final_f02_denominators, + ) + repair_fields = _bound_repair_frame_fields() + digital_ice_evidence = build_digital_ice_acquisition_evidence( + slot=5, + reservation_id=RESERVATION_ID, + capture_attempt_id="fine-slot-5-attempt-1", + storage_rgb=repair_fields["rgb"], + storage_ir=repair_fields["ir"], + storage_ir_validity=repair_fields["ir_validity"], + meter_rgbi=repair_fields["meter_rgbi"], + ) + return Frame( + slot=5, + rgb=repair_fields["rgb"], + ir=repair_fields["ir"], + ir_validity=repair_fields["ir_validity"], + receipt=_FrameReceipt( + version=1, + slot=5, + nikon_density_ownership=ownership, + ), + meter_rgbi=repair_fields["meter_rgbi"], + nikon_density_evidence=density, + nikon_exact_builder_evidence=exact_builder, + digital_ice_evidence=digital_ice_evidence, + ) + + +def test_exact_coolscanpy_producer_is_narrowly_adapted_and_revalidated() -> None: + producer = _coolscanpy_capture_d_evidence() + adapted = adapt_native_builder_evidence(producer) + + assert type(adapted) is NativeBuilderEvidence + assert adapted.analyzer_rgb_sha256 == producer.analyzer_rgb_sha256 + receipt = build_native_builder_receipt(adapted) + assert tuple(sha256(blob).hexdigest() for blob in receipt.pre_f_luts) == (CAPTURE_D_PREF_SHA256) + + +def test_coolscanpy_built_frame_runs_through_negpy_native_service( + fake_repair_engine, + tmp_path: Path, +) -> None: + frame = _coolscanpy_built_frame() + + output = RollScanningService().write_frame( + frame, + str(tmp_path), + '{{ "%03d" % seq }}', + write_unrepaired=False, + write_positive=True, + positive_mode="nikon-exact", + ) + + assert output.positive_path is not None + positive = json.loads(Path(output.receipt_path).read_text())["outputs"]["positive"] + assert positive["native_per_acquisition_builder"] is True + + +def test_archived_capture_builder_math_reproduces_all_three_pref_luts_byte_exact() -> None: + receipt = build_native_builder_receipt(_capture_d_evidence()) + + assert tuple(sha256(blob).hexdigest() for blob in receipt.pre_f_luts) == CAPTURE_D_PREF_SHA256 + payload = exact_color.builder_receipt_payload(receipt) + assert payload["native_per_acquisition_builder"] is True + assert payload["density_source"]["resolution_dpi"] == 97 + assert payload["analyzer_source"]["resolution_dpi"] == 285 + assert payload["density_source"]["f03_denominators_raw_10ns_rgb"] != payload["analyzer_source"]["final_f02_denominators_raw_10ns_rgb"] + + +@pytest.mark.parametrize("source_payload_bytes", (6_250_496, 5_804_032)) +def test_native_builder_accepts_each_proven_roll_preview_geometry( + source_payload_bytes: int, +) -> None: + receipt = build_native_builder_receipt( + _synthetic_native_evidence(source_payload_bytes=source_payload_bytes) + ) + + assert exact_color.builder_receipt_payload(receipt)["native_per_acquisition_builder"] is True + + +@pytest.mark.parametrize("source_payload_bytes", (5_804_031, 5_804_033, 6_250_495, 6_250_497)) +def test_native_builder_rejects_near_miss_roll_preview_geometry( + source_payload_bytes: int, +) -> None: + with pytest.raises( + exact_color.ExactColorUnavailable, + match="density evidence belongs to a different preview or reservation", + ): + build_native_builder_receipt( + _synthetic_native_evidence(source_payload_bytes=source_payload_bytes) + ) + + +def test_native_receipt_runs_through_stage1_and_cms_with_native_application_binding() -> None: + receipt = build_native_builder_receipt(_capture_d_evidence()) + source = np.array([[[0, 1, 2], [65_535, 42_000, 17]]], dtype=np.uint16) + + result = exact_color.evaluate_exact_color( + source, + builder_receipt=receipt, + builder=PortableStage1Builder(chunk_pixels=1), + evaluator=PortableCMSOnEvaluator(chunk_pixels=1), + ) + + application = exact_color.receipt_payload(result.builder_application_receipt) + assert application["native_per_acquisition_builder"] is True + assert application["scope"] == exact_color.NATIVE_BUILDER_SCOPE + assert application["native_evidence_sha256"] == receipt.evidence_sha256 + + +@pytest.mark.parametrize("evidence_source", ["negpy", "coolscanpy"]) +def test_service_auto_builds_native_receipt_and_retains_analyzer_evidence( + fake_repair_engine, + tmp_path: Path, + evidence_source: str, +) -> None: + ownership = _PayloadReceipt(_ownership_document()) + density = _PayloadReceipt(_density_document()) + + @dataclass(frozen=True) + class Receipt: + version: int = 1 + slot: int = 5 + nikon_density_ownership: object = ownership + + frame = types.SimpleNamespace( + slot=5, + **_bound_repair_frame_fields(), + receipt=Receipt(), + nikon_density_evidence=density, + nikon_density_ownership=ownership, + nikon_exact_builder_evidence=(_capture_d_evidence() if evidence_source == "negpy" else _coolscanpy_capture_d_evidence()), + ) + + service = RollScanningService() + output = service.write_frame( + frame, + str(tmp_path), + '{{ "%03d" % seq }}', + write_unrepaired=False, + write_positive=True, + positive_mode="nikon-exact", + ) + + assert output.positive_path is not None + entry = json.loads(Path(output.receipt_path).read_text())["outputs"]["positive"] + assert entry["native_per_acquisition_builder"] is True + assert entry["inversion_path"] == "native-per-acquisition-builder-and-verified-portable-cms" + retained = entry["retained_builder_evidence"] + assert retained["scope"] == exact_color.NATIVE_BUILDER_SCOPE + analyzer = Path(retained["analyzer_rgb"]["path"]).read_bytes() + assert sha256(analyzer).hexdigest() == retained["analyzer_rgb"]["sha256"] + ownership_payload = Path(retained["frame_ownership_receipt"]["path"]).read_bytes() + assert sha256(ownership_payload).hexdigest() == retained["frame_ownership_receipt"]["sha256"] + density_payload = Path(retained["density_evidence_receipt"]["path"]).read_bytes() + assert sha256(density_payload).hexdigest() == retained["density_evidence_receipt"]["sha256"] + + +def test_service_retains_native_builder_evidence_when_positive_is_not_selected( + tmp_path: Path, +) -> None: + output = RollScanningService().write_frame( + _synthetic_native_frame(), + str(tmp_path), + '{{ "%03d" % seq }}', + write_unrepaired=True, + write_repaired=False, + write_positive=False, + ) + + receipt = json.loads(Path(output.receipt_path).read_text()) + assert receipt["outputs"]["positive"] == { + "written": False, + "status": "not selected", + } + entry = receipt["outputs"]["native_color_evidence"] + assert entry["retained"] is True + assert entry["builder_receipt_sha256"] == entry["retained_builder_evidence"]["builder_receipt"]["sha256"] + for artifact in ( + entry["retained_builder_evidence"]["builder_receipt"], + entry["retained_builder_evidence"]["evidence_receipt"], + entry["retained_builder_evidence"]["frame_ownership_receipt"], + entry["retained_builder_evidence"]["density_evidence_receipt"], + entry["retained_builder_evidence"]["analyzer_rgb"], + *entry["retained_builder_evidence"]["pre_f_luts"], + ): + payload = Path(artifact["path"]).read_bytes() + assert len(payload) == artifact["bytes"] + assert sha256(payload).hexdigest() == artifact["sha256"] + + reloaded = exact_color.load_native_builder_receipt(entry["retained_builder_evidence"]["builder_receipt"]["path"]) + assert reloaded.sha256 == entry["builder_receipt_sha256"] + replay = exact_color.evaluate_exact_color( + np.array([[[1, 2, 3], [65_535, 42_000, 17]]], dtype=np.uint16), + builder_receipt=reloaded, + builder=PortableStage1Builder(chunk_pixels=1), + evaluator=PortableCMSOnEvaluator(chunk_pixels=1), + ) + assert replay.builder_receipt.sha256 == entry["builder_receipt_sha256"] + + +def test_retained_native_builder_loader_rejects_a_symlinked_artifact( + tmp_path: Path, +) -> None: + output = RollScanningService().write_frame( + _synthetic_native_frame(), + str(tmp_path), + '{{ "%03d" % seq }}', + write_unrepaired=False, + write_repaired=False, + write_positive=False, + ) + retained = json.loads(Path(output.receipt_path).read_text())["outputs"]["native_color_evidence"]["retained_builder_evidence"] + target = Path(retained["pre_f_luts"][1]["path"]) + real = target.with_name("real-builder-preF-g.bin") + target.replace(real) + target.symlink_to(real) + + with pytest.raises(exact_color.ExactColorUnavailable, match="non-symlink"): + exact_color.load_native_builder_receipt(retained["builder_receipt"]["path"]) + + +def test_retained_native_builder_loader_does_not_block_on_a_fifo_swap( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + output = RollScanningService().write_frame( + _synthetic_native_frame(), + str(tmp_path), + '{{ "%03d" % seq }}', + write_unrepaired=False, + write_repaired=False, + write_positive=False, + ) + retained = json.loads(Path(output.receipt_path).read_text())["outputs"]["native_color_evidence"]["retained_builder_evidence"] + target = Path(retained["pre_f_luts"][1]["path"]) + saved = target.with_name("saved-builder-preF-g.bin") + real_open = os.open + swapped = False + + def swap_to_fifo_then_open(path, flags, *args, **kwargs): + nonlocal swapped + if not swapped and Path(path) == target: + swapped = True + target.replace(saved) + os.mkfifo(target) + return real_open(path, flags, *args, **kwargs) + + monkeypatch.setattr(exact_color.os, "open", swap_to_fifo_then_open) + + with pytest.raises( + exact_color.ExactColorIntegrityError, + match="changed while being read", + ): + exact_color.load_native_builder_receipt(retained["builder_receipt"]["path"]) + + +def test_retained_native_builder_loader_rederives_luts_instead_of_trusting_rehashed_files( + tmp_path: Path, +) -> None: + output = RollScanningService().write_frame( + _synthetic_native_frame(), + str(tmp_path), + '{{ "%03d" % seq }}', + write_unrepaired=False, + write_repaired=False, + write_positive=False, + ) + retained = json.loads(Path(output.receipt_path).read_text())["outputs"]["native_color_evidence"]["retained_builder_evidence"] + lut_path = Path(retained["pre_f_luts"][0]["path"]) + changed_lut = bytearray(lut_path.read_bytes()) + changed_lut[13_579] ^= 1 + lut_path.write_bytes(changed_lut) + receipt_path = Path(retained["builder_receipt"]["path"]) + envelope = json.loads(receipt_path.read_bytes()) + envelope["pre_f_luts"][0]["sha256"] = sha256(changed_lut).hexdigest() + receipt_path.write_bytes(_canonical_json(envelope)) + + with pytest.raises( + exact_color.ExactColorIntegrityError, + match="fresh native derivation", + ): + exact_color.load_native_builder_receipt(receipt_path) + + +def test_retained_native_builder_loader_requires_its_content_addressed_directory( + tmp_path: Path, +) -> None: + output = RollScanningService().write_frame( + _synthetic_native_frame(), + str(tmp_path), + '{{ "%03d" % seq }}', + write_unrepaired=False, + write_repaired=False, + write_positive=False, + ) + retained = json.loads(Path(output.receipt_path).read_text())["outputs"]["native_color_evidence"]["retained_builder_evidence"] + original_receipt = Path(retained["builder_receipt"]["path"]) + renamed_directory = original_receipt.parent.with_name("wrong-content-address") + original_receipt.parent.rename(renamed_directory) + + with pytest.raises( + exact_color.ExactColorIntegrityError, + match="content-addressed directory", + ): + exact_color.load_native_builder_receipt(renamed_directory / original_receipt.name) + + +def test_service_keeps_native_builder_evidence_when_exact_tiff_verification_fails( + fake_repair_engine, + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + def reject_tiff(*args: object, **kwargs: object) -> dict[str, object]: + del args, kwargs + raise exact_color.ExactColorIntegrityError("test TIFF rejection") + + monkeypatch.setattr(roll_service, "_verify_exact_positive_tiff", reject_tiff) + + output = RollScanningService().write_frame( + _synthetic_native_frame(), + str(tmp_path), + '{{ "%03d" % seq }}', + write_unrepaired=False, + write_repaired=False, + write_positive=True, + positive_mode="nikon-exact", + ) + + assert output.positive_path is None + receipt = json.loads(Path(output.receipt_path).read_text()) + assert "test TIFF rejection" in receipt["outputs"]["positive"]["status"] + retained = receipt["outputs"]["native_color_evidence"] + assert retained["retained"] is True + assert not (tmp_path / "005_positive.tif").exists() + for artifact in ( + retained["retained_builder_evidence"]["builder_receipt"], + retained["retained_builder_evidence"]["evidence_receipt"], + retained["retained_builder_evidence"]["analyzer_rgb"], + *retained["retained_builder_evidence"]["pre_f_luts"], + ): + assert Path(artifact["path"]).is_file() + + +def test_noncanonical_dice_binding_withholds_positive_but_keeps_native_color_evidence( + fake_repair_engine, + tmp_path: Path, +) -> None: + frame = _synthetic_native_frame() + forged_acquisition = replace( + frame.prepare_digital_ice(), + evidence_sha256="0" * 64, + ) + frame.prepare_digital_ice = lambda: forged_acquisition + + output = RollScanningService().write_frame( + frame, + str(tmp_path), + '{{ "%03d" % seq }}', + write_unrepaired=False, + write_repaired=False, + write_positive=True, + positive_mode="nikon-exact", + ) + + assert fake_repair_engine.calls == [] + assert output.positive_path is None + receipt = json.loads(Path(output.receipt_path).read_text())["outputs"] + assert "producer evidence SHA-256 changed" in receipt["repaired"]["status"] + assert "Tier 2" in receipt["positive"]["status"] + native = receipt["native_color_evidence"] + assert native["retained"] is True + assert Path(native["retained_builder_evidence"]["builder_receipt"]["path"]).is_file() + + +def test_service_refuses_explicit_builder_evidence_without_current_frame_ownership(fake_repair_engine, tmp_path: Path) -> None: + @dataclass(frozen=True) + class Receipt: + version: int = 1 + slot: int = 5 + + frame = types.SimpleNamespace( + slot=5, + **_bound_repair_frame_fields(), + receipt=Receipt(), + nikon_exact_builder_evidence=_capture_d_evidence(), + ) + + output = RollScanningService().write_frame( + frame, + str(tmp_path), + '{{ "%03d" % seq }}', + write_unrepaired=False, + write_positive=True, + positive_mode="nikon-exact", + ) + + assert output.positive_path is None + status = json.loads(Path(output.receipt_path).read_text())["outputs"]["positive"]["status"] + assert "frame ownership receipt is missing" in status + + +@pytest.mark.parametrize( + ("field", "replacement"), + [ + ("preview_sha256", sha256(b"new-preview").hexdigest()), + ("reservation_id", "new-reservation"), + ("batch_session_id", "new-reservation"), + ("transport_table_sha256", sha256(b"moved-transport-table").hexdigest()), + ("reviewed_fingerprint_sha256", sha256(b"new-registration").hexdigest()), + ("fresh_fingerprint_sha256", sha256(b"film-moved").hexdigest()), + ("frame_capture_attempt_id", "another-frame-attempt"), + ("selected_slot", 6), + ], + ids=[ + "new-preview", + "new-reservation", + "new-batch", + "transport-change", + "re-registration", + "film-move", + "new-frame-attempt", + "different-slot", + ], +) +def test_service_refuses_changed_frame_ownership_identity( + fake_repair_engine, + tmp_path: Path, + field: str, + replacement: object, +) -> None: + ownership_document = _ownership_document() + ownership_document[field] = replacement + ownership = _PayloadReceipt(ownership_document) + density = _PayloadReceipt(_density_document()) + frame = types.SimpleNamespace( + slot=5, + **_bound_repair_frame_fields(), + receipt=_FrameReceipt(version=1, slot=5, nikon_density_ownership=ownership), + nikon_density_evidence=density, + nikon_density_ownership=ownership, + nikon_exact_builder_evidence=_capture_d_evidence(), + ) + + output = RollScanningService().write_frame( + frame, + str(tmp_path), + '{{ "%03d" % seq }}', + write_unrepaired=False, + write_positive=True, + positive_mode="nikon-exact", + ) + + assert output.positive_path is None + status = json.loads(Path(output.receipt_path).read_text())["outputs"]["positive"]["status"] + assert "frame ownership does not match the native builder evidence" in status + + +def test_service_refuses_density_evidence_from_a_new_preview(fake_repair_engine, tmp_path: Path) -> None: + ownership = _PayloadReceipt(_ownership_document()) + density_document = _density_document() + density_document["preview_identity_sha256"] = sha256(b"new-preview-identity").hexdigest() + density = _PayloadReceipt(density_document) + frame = types.SimpleNamespace( + slot=5, + **_bound_repair_frame_fields(), + receipt=_FrameReceipt(version=1, slot=5, nikon_density_ownership=ownership), + nikon_density_evidence=density, + nikon_density_ownership=ownership, + nikon_exact_builder_evidence=_capture_d_evidence(), + ) + + output = RollScanningService().write_frame( + frame, + str(tmp_path), + '{{ "%03d" % seq }}', + write_unrepaired=False, + write_positive=True, + positive_mode="nikon-exact", + ) + + assert output.positive_path is None + status = json.loads(Path(output.receipt_path).read_text())["outputs"]["positive"]["status"] + assert "density evidence does not match the native builder evidence" in status + + +def test_service_refuses_frame_and_public_receipt_ownership_disagreement(fake_repair_engine, tmp_path: Path) -> None: + ownership = _PayloadReceipt(_ownership_document()) + changed = _ownership_document() + changed["fresh_fingerprint_sha256"] = sha256(b"film-moved").hexdigest() + public_ownership = _PayloadReceipt(changed) + frame = types.SimpleNamespace( + slot=5, + **_bound_repair_frame_fields(), + receipt=_FrameReceipt(version=1, slot=5, nikon_density_ownership=public_ownership), + nikon_density_evidence=_PayloadReceipt(_density_document()), + nikon_density_ownership=ownership, + nikon_exact_builder_evidence=_capture_d_evidence(), + ) + + output = RollScanningService().write_frame( + frame, + str(tmp_path), + '{{ "%03d" % seq }}', + write_unrepaired=False, + write_positive=True, + positive_mode="nikon-exact", + ) + + assert output.positive_path is None + status = json.loads(Path(output.receipt_path).read_text())["outputs"]["positive"]["status"] + assert "frame and public receipt disagree" in status + + +def test_service_honors_ownership_objects_runtime_validation(fake_repair_engine, tmp_path: Path) -> None: + ownership = _RejectingOwnershipReceipt(_ownership_document()) + frame = types.SimpleNamespace( + slot=5, + **_bound_repair_frame_fields(), + receipt=_FrameReceipt(version=1, slot=5, nikon_density_ownership=ownership), + nikon_density_evidence=_PayloadReceipt(_density_document()), + nikon_density_ownership=ownership, + nikon_exact_builder_evidence=_capture_d_evidence(), + ) + + output = RollScanningService().write_frame( + frame, + str(tmp_path), + '{{ "%03d" % seq }}', + write_unrepaired=False, + write_positive=True, + positive_mode="nikon-exact", + ) + + assert output.positive_path is None + status = json.loads(Path(output.receipt_path).read_text())["outputs"]["positive"]["status"] + assert "frame ownership is invalid: preview identity changed" in status + + +def test_service_refuses_generic_session_density_evidence_without_frame_ownership(fake_repair_engine, tmp_path: Path) -> None: + @dataclass(frozen=True) + class Receipt: + version: int = 1 + slot: int = 5 + + storage_rgb = np.full((2, 3, 3), 12_000, dtype=np.uint16) + storage_ir = np.full((2, 3), 3_000, dtype=np.uint16) + storage_rgbi = np.concatenate((storage_rgb, storage_ir[..., None]), axis=2) + native_rgbi = np.ascontiguousarray(np.rot90(storage_rgbi, k=-1, axes=(0, 1))) + prepass_rgbi = np.zeros((1, 1, 4), dtype=np.uint16) + native_validity = np.ones(native_rgbi.shape[:2], dtype=np.bool_) + acquisition_id, evidence_sha256 = roll_service._derive_digital_ice_producer_binding( + slot=5, + reservation_id="reservation-005", + capture_attempt_id="fine-slot-05-attempt-001", + main_rgbi=native_rgbi, + prepass_rgbi=prepass_rgbi, + ir_validity=native_validity, + ) + acquisition = roll_repair.RepairAcquisition.from_arrays( + acquisition_id=acquisition_id, + slot=5, + reservation_id="reservation-005", + capture_attempt_id="fine-slot-05-attempt-001", + storage_transform=roll_repair.DIGITAL_ICE_STORAGE_TRANSFORM, + evidence_sha256=evidence_sha256, + main_rgbi=native_rgbi, + prepass_rgbi=prepass_rgbi, + ir_validity=native_validity, + ) + frame = types.SimpleNamespace( + slot=5, + rgb=storage_rgb, + ir=storage_ir, + ir_validity=None, + receipt=Receipt(), + meter_rgbi=None, + nikon_density_session_evidence=object(), + prepare_digital_ice=lambda: acquisition, + ) + + output = RollScanningService().write_frame( + frame, + str(tmp_path), + '{{ "%03d" % seq }}', + write_unrepaired=False, + write_positive=True, + positive_mode="nikon-exact", + ) + + assert output.positive_path is None + entry = json.loads(Path(output.receipt_path).read_text())["outputs"]["positive"] + assert "no native builder evidence" in entry["status"] + + +def test_native_builder_rejects_conflated_density_and_final_exposure_triplets() -> None: + evidence = _capture_d_evidence() + density_payload = _canonical_json( + _density_document( + numerators=evidence.calibration_numerators, + density_f03=evidence.final_f02_denominators, + densities=evidence.densities, + ) + ) + candidate = replace( + evidence, + density_f03_denominators=evidence.final_f02_denominators, + density_evidence_receipt=density_payload, + density_evidence_receipt_sha256=sha256(density_payload).hexdigest(), + ) + + with pytest.raises(exact_color.ExactColorUnavailable, match="conflated"): + build_native_builder_receipt(candidate) + + +def test_native_builder_rejects_analyzer_mutation_after_identity_was_bound() -> None: + evidence = _capture_d_evidence() + analyzer = evidence.analyzer_rgb.copy() + candidate = replace(evidence, analyzer_rgb=analyzer) + analyzer[0, 0, 0] ^= 1 + + with pytest.raises(exact_color.ExactColorUnavailable, match="does not match its SHA-256"): + build_native_builder_receipt(candidate) + + +def test_service_rejects_native_evidence_for_a_different_frame_slot(fake_repair_engine, tmp_path: Path) -> None: + @dataclass(frozen=True) + class Receipt: + version: int = 1 + slot: int = 6 + + frame = types.SimpleNamespace( + slot=6, + **_bound_repair_frame_fields(slot=6), + receipt=Receipt(), + nikon_exact_builder_evidence=_capture_d_evidence(), + ) + + output = RollScanningService().write_frame( + frame, + str(tmp_path), + '{{ "%03d" % seq }}', + write_unrepaired=False, + write_positive=True, + positive_mode="nikon-exact", + ) + + assert output.positive_path is None + status = json.loads(Path(output.receipt_path).read_text())["outputs"]["positive"]["status"] + assert "different slot" in status + + +def test_native_receipt_rejects_evidence_mutation() -> None: + receipt = build_native_builder_receipt(_capture_d_evidence()) + tampered = replace(receipt, evidence_payload=receipt.evidence_payload + b" ") + + with pytest.raises(exact_color.ExactColorIntegrityError, match="evidence does not match"): + exact_color.builder_receipt_payload(tampered) diff --git a/tests/roll/test_nikon_icc.py b/tests/roll/test_nikon_icc.py new file mode 100644 index 00000000..06a8fdf3 --- /dev/null +++ b/tests/roll/test_nikon_icc.py @@ -0,0 +1,48 @@ +"""Exact Nikon output-profile identity and fail-closed tests.""" + +from __future__ import annotations + +import base64 +import hashlib +from io import BytesIO + +import pytest +from PIL import ImageCms + +from negpy.services.roll import nikon_icc + + +def test_embedded_profile_is_the_pinned_nikon_scan_rgb_profile() -> None: + profile = nikon_icc.nikon_adobe_rgb_profile() + + assert len(profile) == 492 + assert int.from_bytes(profile[:4], "big") == len(profile) + assert profile[12:16] == b"mntr" + assert profile[16:20] == b"RGB " + assert profile[20:24] == b"XYZ " + assert profile[36:40] == b"acsp" + assert hashlib.sha256(profile).hexdigest() == ("a8d0d753bd6129357cc2647435ce675e8637a679eb526fa180fba460874ce1d3") + parsed = ImageCms.ImageCmsProfile(BytesIO(profile)) + assert ImageCms.getProfileName(parsed).strip() == "Nikon Adobe RGB 4.0.0.3000" + assert ImageCms.getProfileCopyright(parsed).strip() == ("Nikon Inc. & Nikon Corporation 2001") + assert nikon_icc.profile_receipt_binding() == { + "bytes": 492, + "name": "Nikon Adobe RGB 4.0.0.3000", + "sha256": "a8d0d753bd6129357cc2647435ce675e8637a679eb526fa180fba460874ce1d3", + } + + +def test_embedded_profile_one_byte_change_fails_closed( + monkeypatch: pytest.MonkeyPatch, +) -> None: + profile = bytearray(nikon_icc.nikon_adobe_rgb_profile()) + profile[100] ^= 1 + monkeypatch.setattr( + nikon_icc, + "_NIKON_ADOBE_RGB_PROFILE_BASE64", + base64.b64encode(profile).decode("ascii"), + ) + nikon_icc.nikon_adobe_rgb_profile.cache_clear() + + with pytest.raises(nikon_icc.NikonICCProfileError, match="pinned SHA-256"): + nikon_icc.nikon_adobe_rgb_profile() diff --git a/tests/roll/test_portable_builder.py b/tests/roll/test_portable_builder.py new file mode 100644 index 00000000..cd84f88a --- /dev/null +++ b/tests/roll/test_portable_builder.py @@ -0,0 +1,248 @@ +"""Portable Stage-1 builder tests; all hardware and VM free.""" + +from __future__ import annotations + +import dataclasses +import json +import shutil +from hashlib import sha256 +from pathlib import Path + +import numpy as np +import pytest + +from negpy.services.roll import exact_color +from negpy.services.roll.portable_builder import ( + FIXED_LUT_FILENAME, + MAX_CHUNK_PIXELS, + PortableStage1Builder, +) +from negpy.services.roll.portable_cms import PortableCMSOnEvaluator +from tests.roll._exact_fixtures import make_stage3_replay_receipt + + +CAPTURED_PREF = Path("/Volumes/isos/NikonRE/session20260719/capture-d") +CAPTURED_PREF_SHA256 = ( + "46a0d68ae20c72088e64a1144a0d38bf692f15f506539bbe94eb563fe437c976", + "23eda81294817e7a2a31f1488544a6f8d3e7ac817f22d43c8a39882565c34b95", + "3cfc61c06bac49c4c28e69afe99af01366fae6bf5ea88954f688592a8e2756bb", +) +# These are post-F only. The different `final_luts` hashes in the older +# CMS-off per-pixel report include an additional Adobe gamma encoding stage. +CAPTURED_POSTF_SHA256 = { + "r": "4fcc07a38a64fa004e252705905775952e73a2c5f0ab1f06c56de6f51073a907", + "g": "e4965e69289dacb21a3241db1fbbee2aa7dba4a1a15f458ed3ca55638b684a30", + "b": "95ec797e1afed10990ba3609f728af85b7547a33a99a2eee2d67dcae1832bf70", +} + + +def _receipt(pre_f_arrays: tuple[np.ndarray, np.ndarray, np.ndarray]) -> exact_color.ValidatedBuilderReceipt: + return make_stage3_replay_receipt(pre_f_arrays) + + +def _synthetic_luts() -> tuple[np.ndarray, np.ndarray, np.ndarray]: + values = np.arange(65_536, dtype=np.uint32) + return ( + ((values + 17) & 0xFFFF).astype(np.uint16), + (65_535 - values).astype(np.uint16), + (values // 3).astype(np.uint16), + ) + + +def test_builder_composes_fixed_after_each_pref_plane() -> None: + source = np.array([[[0, 1, 2], [65_535, 40_000, 17]]], dtype=np.uint16) + pre_f = _synthetic_luts() + fixed_path = Path(__file__).resolve().parents[2] / "negpy/assets/portable_builder" / FIXED_LUT_FILENAME + fixed = np.frombuffer(fixed_path.read_bytes(), dtype=" None: + source = np.random.default_rng(44).integers(0, 65_536, size=(2, 5, 3), dtype=np.uint16) + receipt = _receipt(_synthetic_luts()) + outputs = [PortableStage1Builder(chunk_pixels=size).apply(source, builder_receipt=receipt).rgb for size in (1, 3, 10, 11)] + + for output in outputs[1:]: + np.testing.assert_array_equal(output, outputs[0]) + + +def test_exact_boundary_runs_builder_then_cms_and_keeps_receipts_separate() -> None: + source = np.random.default_rng(46).integers(0, 65_536, size=(2, 4, 3), dtype=np.uint16) + receipt = _receipt(_synthetic_luts()) + + result = exact_color.evaluate_exact_color( + source, + builder_receipt=receipt, + builder=PortableStage1Builder(chunk_pixels=3), + evaluator=PortableCMSOnEvaluator(chunk_pixels=3), + ) + + assert result.source_rgb_sha256 == exact_color.rgb16_content_sha256(source) + assert result.builder_application_receipt is not None + builder_application = exact_color.receipt_payload(result.builder_application_receipt) + cms = exact_color.receipt_payload(result.cms_receipt) + assert result.input_rgb_sha256 == builder_application["stage1_input_rgb_sha256"] + assert cms["input_rgb_sha256"] == result.input_rgb_sha256 + assert cms["output_rgb_sha256"] == result.output_rgb_sha256 + assert builder_application["kind"] == "negpy.verified-stage1-builder-application" + assert cms["kind"] == "negpy.portable-cms-on-receipt" + + +@pytest.mark.parametrize( + "field", + ["pre_f_blob", "stage3_report", "stage3_artifact_binding", "fixed_identity"], +) +def test_builder_receipt_tampering_fails_closed(field: str) -> None: + receipt = _receipt(_synthetic_luts()) + if field == "pre_f_blob": + tampered = bytearray(receipt.pre_f_luts[0]) + tampered[99] ^= 1 + candidate = dataclasses.replace( + receipt, + pre_f_luts=(bytes(tampered), *receipt.pre_f_luts[1:]), + ) + elif field == "stage3_report": + tampered = bytearray(receipt.stage3_receipt) + tampered[-2] ^= 1 + candidate = dataclasses.replace(receipt, stage3_receipt=bytes(tampered)) + elif field == "stage3_artifact_binding": + report = json.loads(receipt.stage3_receipt) + next(row for row in report["artifacts"] if row["name"] == "builder-preF-r.bin")["sha256"] = "0" * 64 + stage3 = json.dumps(report, sort_keys=True, separators=(",", ":")).encode() + stage3_hash = sha256(stage3).hexdigest() + envelope = json.loads(receipt.payload) + envelope["stage3_receipt_sha256"] = stage3_hash + payload = json.dumps(envelope, sort_keys=True, separators=(",", ":")).encode() + candidate = dataclasses.replace( + receipt, + payload=payload, + sha256=sha256(payload).hexdigest(), + stage3_receipt=stage3, + stage3_receipt_sha256=stage3_hash, + ) + else: + candidate = dataclasses.replace(receipt, fixed_composition_sha256="0" * 64) + + with pytest.raises(exact_color.ExactColorIntegrityError): + PortableStage1Builder().apply( + np.zeros((1, 1, 3), dtype=np.uint16), + builder_receipt=candidate, + ) + + +def test_stage3_report_with_errors_is_rejected_even_when_summary_claims_pass() -> None: + receipt = _receipt(_synthetic_luts()) + report = json.loads(receipt.stage3_receipt) + report["errors"] = [{"code": "suppressed", "message": "must remain fatal"}] + stage3 = json.dumps(report, sort_keys=True, separators=(",", ":")).encode() + stage3_hash = sha256(stage3).hexdigest() + envelope = json.loads(receipt.payload) + envelope["stage3_receipt_sha256"] = stage3_hash + payload = json.dumps(envelope, sort_keys=True, separators=(",", ":")).encode() + candidate = dataclasses.replace( + receipt, + payload=payload, + sha256=sha256(payload).hexdigest(), + stage3_receipt=stage3, + stage3_receipt_sha256=stage3_hash, + ) + + with pytest.raises(exact_color.ExactColorIntegrityError, match="errors"): + exact_color.builder_receipt_payload(candidate) + + +def test_stage3_replay_receipts_require_the_trusted_file_loader() -> None: + assert callable(exact_color.load_stage3_replay_builder_receipt) + + +@pytest.mark.parametrize("failure", ["missing", "tampered"]) +def test_fixed_lut_asset_verification_fails_closed(tmp_path: Path, failure: str) -> None: + packaged = Path(__file__).resolve().parents[2] / "negpy/assets/portable_builder" + copied = tmp_path / "assets" + shutil.copytree(packaged, copied) + target = copied / FIXED_LUT_FILENAME + if failure == "missing": + target.unlink() + expected = exact_color.ExactColorUnavailable + else: + payload = bytearray(target.read_bytes()) + payload[1000] ^= 1 + target.write_bytes(payload) + expected = exact_color.ExactColorIntegrityError + + with pytest.raises(expected): + PortableStage1Builder(assets_dir=copied) + + +def test_builder_preserves_input_and_freezes_stage1_output() -> None: + source = np.random.default_rng(45).integers(0, 65_536, size=(2, 3, 3), dtype=np.uint16) + original = source.copy() + + result = PortableStage1Builder(chunk_pixels=2).apply( + source, + builder_receipt=_receipt(_synthetic_luts()), + ) + + np.testing.assert_array_equal(source, original) + assert not result.rgb.flags.writeable + with pytest.raises(ValueError): + result.rgb[0, 0, 0] ^= 1 + + +@pytest.mark.parametrize( + "source", + [ + np.zeros((2, 2, 3), dtype=np.uint8), + np.zeros((2, 2), dtype=np.uint16), + np.zeros((2, 2, 4), dtype=np.uint16), + np.zeros((0, 2, 3), dtype=np.uint16), + ], +) +def test_builder_rejects_wrong_input_dtype_or_shape(source: np.ndarray) -> None: + with pytest.raises(exact_color.ExactColorIntegrityError): + PortableStage1Builder().apply( + source, + builder_receipt=_receipt(_synthetic_luts()), + ) + + +@pytest.mark.parametrize("chunk_pixels", [0, MAX_CHUNK_PIXELS + 1, True, 1.5]) +def test_builder_chunk_size_is_bounded(chunk_pixels: object) -> None: + with pytest.raises(exact_color.ExactColorUnavailable): + PortableStage1Builder(chunk_pixels=chunk_pixels) # ty: ignore[invalid-argument-type] + + +def test_accessible_captured_pref_luts_reproduce_pinned_postf_hashes() -> None: + paths = [CAPTURED_PREF / f"builder-preF-{channel}.bin" for channel in ("r", "g", "b")] + if not all(path.is_file() for path in paths): + pytest.skip("stored capture-d builder LUTs are unavailable") + blobs = tuple(path.read_bytes() for path in paths) + assert tuple(sha256(blob).hexdigest() for blob in blobs) == CAPTURED_PREF_SHA256 + luts = ( + np.frombuffer(blobs[0], dtype=" np.ndarray: + path = ORACLE_LAB / f"event{event}-{kind}.bin" + if not path.is_file(): + pytest.skip(f"stored CML fixture is unavailable: {path}") + width, height = GEOMETRY[event] + raw = np.frombuffer(path.read_bytes(), dtype=" exact_color.ValidatedBuilderReceipt: + identity = np.frombuffer(IDENTITY_LUT, dtype=" None: + source = _event(event, "source") + expected = _event(event, "expected") + transforms = PortableCMSOnEvaluator()._transforms # noqa: SLF001 - oracle-stage verification + flat_source = source.reshape(-1, 3) + + if event in STAGE1_EVENTS: + actual = transforms.stage1(flat_source) + else: + actual = transforms.stage2(flat_source) + + np.testing.assert_array_equal(actual.reshape(source.shape), expected) + + +@pytest.mark.parametrize( + ("stage1_event", "stage2_event"), + [(30, 31), (34, 35), (38, 39), (42, 43), (47, 48), (52, 53)], +) +def test_adapter_replays_all_stage1_to_stage2_fixture_pairs( + stage1_event: int, + stage2_event: int, +) -> None: + source = _event(stage1_event, "source") + expected = _event(stage2_event, "expected") + builder_receipt = _builder_receipt() + + result = PortableCMSOnEvaluator(chunk_pixels=997).evaluate( + source, + builder_receipt=builder_receipt, + ) + + np.testing.assert_array_equal(result.rgb, expected) + cms = exact_color.receipt_payload(result.cms_receipt) + assert cms["scope"] == "captured-cml4-stage1-stage2-only" + assert cms["builder_receipt_sha256"] == builder_receipt.sha256 + assert cms["input_rgb_sha256"] == exact_color.rgb16_content_sha256(source) + assert cms["output_rgb_sha256"] == exact_color.rgb16_content_sha256(expected) + assert len(cms["assets"]) == 9 + assert cms["validation"] == { + "events": 12, + "full_payload_mismatched_bytes": 0, + "full_payload_total_bytes": 698_880, + "mismatched_u16": 0, + "receipt_sha256": VALIDATION_RECEIPT_SHA256, + "total_u16": 265_440, + } + + +def test_chunk_boundaries_do_not_change_integer_results() -> None: + source = np.random.default_rng(22).integers(0, 65_536, size=(2, 5, 3), dtype=np.uint16) + builder_receipt = _builder_receipt() + outputs = [] + + for chunk_pixels in (1, 3, 10, 11): + result = PortableCMSOnEvaluator(chunk_pixels=chunk_pixels).evaluate( + source, + builder_receipt=builder_receipt, + ) + outputs.append(result.rgb) + + for output in outputs[1:]: + np.testing.assert_array_equal(output, outputs[0]) + + +@pytest.mark.parametrize("failure", ["missing", "tampered"]) +def test_asset_verification_fails_closed(tmp_path: Path, failure: str) -> None: + packaged = Path(__file__).resolve().parents[2] / "negpy/assets/portable_cms" + copied = tmp_path / "assets" + shutil.copytree(packaged, copied) + target = copied / next(iter(ASSET_SHA256)) + if failure == "missing": + target.unlink() + expected = exact_color.ExactColorUnavailable + else: + payload = bytearray(target.read_bytes()) + payload[len(payload) // 2] ^= 1 + target.write_bytes(payload) + expected = exact_color.ExactColorIntegrityError + + with pytest.raises(expected): + PortableCMSOnEvaluator(assets_dir=copied) + + +def test_packaged_validation_receipt_is_exact_and_closes_the_claimed_totals() -> None: + packaged = Path(__file__).resolve().parents[2] / "negpy/assets/portable_cms" + payload = (packaged / VALIDATION_RECEIPT_FILENAME).read_bytes() + + assert len(payload) == VALIDATION_RECEIPT_BYTES + assert hashlib.sha256(payload).hexdigest() == VALIDATION_RECEIPT_SHA256 + summary = portable_cms._validate_validation_receipt(payload) # noqa: SLF001 - release provenance contract + assert summary == { + "events": 12, + "mismatched_u16": 0, + "total_u16": 265_440, + "full_payload_mismatched_bytes": 0, + "full_payload_total_bytes": 698_880, + } + + +@pytest.mark.parametrize("failure", ["missing", "one_byte"]) +def test_validation_receipt_file_failure_is_fatal(tmp_path: Path, failure: str) -> None: + packaged = Path(__file__).resolve().parents[2] / "negpy/assets/portable_cms" + copied = tmp_path / "assets" + shutil.copytree(packaged, copied) + target = copied / VALIDATION_RECEIPT_FILENAME + if failure == "missing": + target.unlink() + expected = exact_color.ExactColorUnavailable + else: + payload = bytearray(target.read_bytes()) + payload[len(payload) // 2] ^= 1 + target.write_bytes(payload) + expected = exact_color.ExactColorIntegrityError + + with pytest.raises(expected): + PortableCMSOnEvaluator(assets_dir=copied) + + +def test_correctly_hashed_but_incomplete_validation_schema_is_fatal( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + packaged = Path(__file__).resolve().parents[2] / "negpy/assets/portable_cms" + copied = tmp_path / "assets" + shutil.copytree(packaged, copied) + target = copied / VALIDATION_RECEIPT_FILENAME + document = json.loads(target.read_bytes()) + del document["stage1"]["per_event"]["30"] + mutated = json.dumps(document, sort_keys=True, separators=(",", ":")).encode() + target.write_bytes(mutated) + monkeypatch.setattr(portable_cms, "VALIDATION_RECEIPT_SHA256", hashlib.sha256(mutated).hexdigest()) + monkeypatch.setattr(portable_cms, "VALIDATION_RECEIPT_BYTES", len(mutated)) + + with pytest.raises(exact_color.ExactColorIntegrityError, match="validation receipt"): + PortableCMSOnEvaluator(assets_dir=copied) + + +@pytest.mark.parametrize( + "source", + [ + np.zeros((2, 2, 3), dtype=np.uint8), + np.zeros((2, 2), dtype=np.uint16), + np.zeros((2, 2, 4), dtype=np.uint16), + ], +) +def test_adapter_rejects_wrong_input_dtype_or_shape(source: np.ndarray) -> None: + with pytest.raises(exact_color.ExactColorIntegrityError): + PortableCMSOnEvaluator().evaluate(source, builder_receipt=_builder_receipt()) + + +def test_adapter_rejects_empty_input_geometry() -> None: + source = np.zeros((0, 2, 3), dtype=np.uint16) + + with pytest.raises(exact_color.ExactColorIntegrityError): + PortableCMSOnEvaluator().evaluate(source, builder_receipt=_builder_receipt()) + + +def test_adapter_does_not_mutate_input_and_returns_immutable_output() -> None: + source = np.random.default_rng(23).integers(0, 65_536, size=(2, 3, 3), dtype=np.uint16) + original = source.copy() + + result = PortableCMSOnEvaluator(chunk_pixels=2).evaluate( + source, + builder_receipt=_builder_receipt(), + ) + + np.testing.assert_array_equal(source, original) + assert not result.rgb.flags.writeable + with pytest.raises(ValueError): + result.rgb[0, 0, 0] ^= 1 + + +@pytest.mark.parametrize("chunk_pixels", [0, MAX_CHUNK_PIXELS + 1, True, 1.5]) +def test_chunk_size_is_bounded(chunk_pixels: object) -> None: + with pytest.raises(exact_color.ExactColorUnavailable): + PortableCMSOnEvaluator(chunk_pixels=chunk_pixels) # ty: ignore[invalid-argument-type] + + +def test_preloaded_oracle_module_substitution_is_rejected(monkeypatch: pytest.MonkeyPatch) -> None: + canonical_name = "negpy.services.roll.portable_oracle_evaluator" + substitute = types.ModuleType(canonical_name) + substitute.lookup_three = lambda *_args: pytest.fail("substituted oracle executed") + monkeypatch.setitem(sys.modules, canonical_name, substitute) + + with pytest.raises(exact_color.ExactColorIntegrityError, match="preloaded"): + PortableCMSOnEvaluator() + + +def test_verified_oracle_executes_isolated_without_registering_the_canonical_module() -> None: + canonical_name = "negpy.services.roll.portable_oracle_evaluator" + assert canonical_name not in sys.modules + + PortableCMSOnEvaluator() + + assert canonical_name not in sys.modules + + +def test_oracle_source_hash_mutation_is_rejected(tmp_path: Path) -> None: + source = Path(portable_cms.__file__).with_name("portable_oracle_evaluator.py") + copied = tmp_path / source.name + copied.write_bytes(source.read_bytes() + b"\n# mutation\n") + + with pytest.raises(exact_color.ExactColorIntegrityError, match="source hash mismatch"): + portable_cms._load_verified_oracle(copied) # noqa: SLF001 - adversarial loader contract + + +def test_oracle_source_swap_between_lstat_and_open_is_rejected( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + packaged = Path(portable_cms.__file__).with_name("portable_oracle_evaluator.py") + copied = tmp_path / packaged.name + copied.write_bytes(packaged.read_bytes()) + replacement = tmp_path / "replacement.py" + replacement.write_bytes(packaged.read_bytes() + b"\n# swapped\n") + real_open = os.open + swapped = False + + def swap_then_open(path, flags, *args, **kwargs): + nonlocal swapped + if not swapped and Path(path) == copied: + swapped = True + os.replace(replacement, copied) + return real_open(path, flags, *args, **kwargs) + + monkeypatch.setattr(portable_cms.os, "open", swap_then_open) + + with pytest.raises(exact_color.ExactColorIntegrityError, match="changed while being read"): + portable_cms._load_verified_oracle(copied) # noqa: SLF001 - adversarial loader contract diff --git a/tests/roll/test_positive.py b/tests/roll/test_positive.py new file mode 100644 index 00000000..8fbd365a --- /dev/null +++ b/tests/roll/test_positive.py @@ -0,0 +1,82 @@ +"""Tests for Tier-3 rendering (negpy.services.roll.positive): a thin wrapper +around NegPy's own ImageProcessor.run_pipeline, exercised for real -- no +scanner or GPU involved, `prefer_gpu=False` forces the CPU engine, matching +how the rest of this test suite already drives ImageProcessor directly +(see tests/test_image_processor.py). +""" + +from __future__ import annotations + +import numpy as np + +from negpy.services.rendering.image_processor import ImageProcessor +from negpy.services.roll import positive + + +def _negative_like(seed: int, shape=(40, 60)) -> np.ndarray: + """A plausible scanner-linear C41 negative: red-biased (orange mask), + with real per-pixel variance so the pipeline's percentile-based bounds + analysis has something non-degenerate to measure.""" + rng = np.random.default_rng(seed) + rgb = np.zeros((*shape, 3), dtype=np.uint16) + rgb[..., 0] = rng.integers(40000, 60000, size=shape) + rgb[..., 1] = rng.integers(25000, 45000, size=shape) + rgb[..., 2] = rng.integers(15000, 35000, size=shape) + return rgb + + +class TestAvailable: + def test_always_true(self) -> None: + """Unlike coolscanpy or a Tier-2 repair engine, NegPy's own rendering + pipeline ships with this application -- there is nothing to install.""" + assert positive.available() is True + + +class TestRenderPositive: + def test_returns_uint16_rgb_of_the_same_shape(self) -> None: + rgb = _negative_like(seed=1) + result = positive.render_positive(rgb, processor=ImageProcessor()) + + assert result.rgb.shape == rgb.shape + assert result.rgb.dtype == np.uint16 + + def test_uses_the_default_c41_print_conversion(self) -> None: + """Stock WorkspaceConfig(): the same conversion a freshly opened + negative gets before any slider is touched.""" + rgb = _negative_like(seed=2) + result = positive.render_positive(rgb, processor=ImageProcessor()) + + assert result.process_mode == "C41" + assert result.render_intent == "print" + assert result.auto_exposure is True + + def test_records_the_negpy_version(self) -> None: + import negpy + + rgb = _negative_like(seed=3) + result = positive.render_positive(rgb, processor=ImageProcessor()) + + assert result.negpy_version == negpy.__version__ + + def test_actually_inverts_not_a_passthrough(self) -> None: + """A scanner-linear negative and its rendered positive must not be the + same array -- catches a wrapper that accidentally short-circuits the + engine (e.g. a stubbed process()) and returns the source untouched.""" + rgb = _negative_like(seed=4) + result = positive.render_positive(rgb, processor=ImageProcessor()) + + assert not np.array_equal(result.rgb, rgb) + + def test_two_different_frames_on_one_processor_render_differently(self) -> None: + """Regression guard: render_positive must pass a fresh source_hash per + call. DarkroomEngine's stage cache reuses a prior render whenever + source_hash *and* the settings hash both match, and every Tier-3 call + shares the same stock WorkspaceConfig -- a reused/stable hash across + two different frames would silently hand back the first frame's + pixels for the second. Two distinct frames rendered on the *same* + ImageProcessor (so its cache is actually in play) must differ.""" + processor = ImageProcessor() + first = positive.render_positive(_negative_like(seed=5), processor=processor) + second = positive.render_positive(_negative_like(seed=6), processor=processor) + + assert not np.array_equal(first.rgb, second.rgb) diff --git a/tests/roll/test_repair.py b/tests/roll/test_repair.py new file mode 100644 index 00000000..238ae166 --- /dev/null +++ b/tests/roll/test_repair.py @@ -0,0 +1,91 @@ +"""Tests for the Tier-2 repair engine seam (negpy.infrastructure.roll.repair). + +The optional digital-fauxice bridge auto-registers only when its engine is +installed. These tests force the seam's independently-unavailable state, +then pin that degrade contract down because `RollScanningService.write_frame`'s +Tier 2/3 path depends on it. `fake_repair_engine` (tests/roll/conftest.py) +scripts a registered engine for tests that need one. +""" + +from __future__ import annotations + +import numpy as np +import pytest + +from negpy.infrastructure.roll import repair as roll_repair + + +def _acquisition() -> roll_repair.RepairAcquisition: + main = np.full((3, 2, 4), 100, dtype=np.uint16) + return roll_repair.RepairAcquisition.from_arrays( + acquisition_id="dice-" + "a" * 64, + slot=7, + reservation_id="reservation-007", + capture_attempt_id="fine-slot-7-attempt-001", + storage_transform=roll_repair.DIGITAL_ICE_STORAGE_TRANSFORM, + evidence_sha256="b" * 64, + main_rgbi=main, + prepass_rgbi=np.full((2, 2, 4), 50, dtype=np.uint16), + ir_validity=np.ones((3, 2), dtype=np.bool_), + ) + + +class TestUnavailableByDefault: + def test_available_is_false_with_nothing_registered(self, no_repair_engine) -> None: + assert roll_repair.available() is False + + def test_repair_raises_when_unavailable(self, no_repair_engine) -> None: + with pytest.raises(RuntimeError, match="no dust-repair engine is registered"): + roll_repair.repair(_acquisition(), roll_repair.RepairMode.EXACT) + + +class TestRegisteredEngine: + def test_available_becomes_true(self, fake_repair_engine) -> None: + assert roll_repair.available() is True + + def test_repair_delegates_to_the_registered_engine(self, fake_repair_engine) -> None: + main = np.full((3, 2, 4), 100, dtype=np.uint16) + prepass = np.full((2, 2, 4), 50, dtype=np.uint16) + validity = np.ones((3, 2), dtype=np.bool_) + acquisition = roll_repair.RepairAcquisition.from_arrays( + acquisition_id="dice-" + "a" * 64, + slot=7, + reservation_id="reservation-007", + capture_attempt_id="fine-slot-7-attempt-001", + storage_transform="rot90-k1-scanner-native-to-upright-v1", + evidence_sha256="b" * 64, + main_rgbi=main, + prepass_rgbi=prepass, + ir_validity=validity, + ) + + result = roll_repair.repair(acquisition, roll_repair.RepairMode.HYBRID) + + assert fake_repair_engine.calls == [(acquisition, roll_repair.RepairMode.HYBRID, None)] + assert result.engine == "test-repair-engine" + assert result.engine_version == "0.0.1-test" + assert result.mode_requested == roll_repair.RepairMode.HYBRID + assert result.mode_resolved == roll_repair.RepairMode.EXACT + assert result.degraded is True + np.testing.assert_array_equal( + result.rgb, + np.rot90(main[..., :3], k=1, axes=(0, 1)), + ) + + def test_unregister_reverts_to_unavailable(self, fake_repair_engine) -> None: + roll_repair.unregister_engine() + assert roll_repair.available() is False + with pytest.raises(RuntimeError): + roll_repair.repair(_acquisition(), roll_repair.RepairMode.EXACT) + + +class TestRepairMode: + def test_values_are_exact_and_hybrid(self) -> None: + assert roll_repair.RepairMode.EXACT.value == "exact" + assert roll_repair.RepairMode.HYBRID.value == "hybrid" + + def test_constructible_from_its_own_string_value(self) -> None: + """RollScanningService.write_frame coerces a persisted settings string + back to a RepairMode this way; pin the round-trip.""" + assert roll_repair.RepairMode("exact") is roll_repair.RepairMode.EXACT + assert roll_repair.RepairMode("hybrid") is roll_repair.RepairMode.HYBRID diff --git a/tests/roll/test_service.py b/tests/roll/test_service.py new file mode 100644 index 00000000..d450f83a --- /dev/null +++ b/tests/roll/test_service.py @@ -0,0 +1,2626 @@ +"""Tests for RollScanningService: lifecycle orchestration and output writing. + +Lifecycle tests use `fake_coolscanpy` (see tests/roll/conftest.py) the +same way test_coolscanpy_roll.py does. write_frame() tests construct a fake +Frame/Receipt directly -- writing to disk never touches coolscanpy itself, +so no module injection is needed for those. +""" + +from __future__ import annotations + +import dataclasses +import io +import json +import os +import time +from hashlib import sha256 +from pathlib import Path +from unittest.mock import MagicMock + +import numpy as np +import pytest +import tifffile +from PIL import Image + +from negpy.infrastructure.roll import repair as roll_repair +from negpy.services.roll import service as roll_service +from negpy.services.roll import exact_color +from negpy.services.roll.portable_cms import PortableCMSOnEvaluator +from negpy.services.roll.service import RollScanningError, RollScanningService +from tests.roll._exact_fixtures import make_stage3_replay_receipt, production_cms_payload + + +class TestAvailable: + def test_reexports_backend_availability(self, fake_coolscanpy) -> None: + assert roll_service.available() is True + + +class TestRollLifecycle: + def _open_service(self, fake_coolscanpy, roll=None): + roll = roll if roll is not None else fake_coolscanpy.Roll() + device = fake_coolscanpy.Device(roll) + fake_coolscanpy.state["open_device"] = device + service = RollScanningService() + service.open_roll("ls5000-usb-001") + return service, roll, device + + def test_open_then_close(self, fake_coolscanpy) -> None: + service, roll, device = self._open_service(fake_coolscanpy) + service.close() + assert roll.closed is True + assert device.closed is True + + def test_open_forwards_caller_owned_attempts_root( + self, + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, + ) -> None: + sentinel = object() + calls: list[tuple[object, object, object]] = [] + + def open_roll(device_id, *, material=None, attempts_root=None): + calls.append((device_id, material, attempts_root)) + return sentinel + + monkeypatch.setattr(roll_service.coolscanpy_roll, "open_roll", open_roll) + service = RollScanningService() + evidence = tmp_path / "scanner-attempts" + service.open_roll("ls5000-usb-001", attempts_root=evidence) + + assert calls == [("ls5000-usb-001", None, evidence)] + + def test_failed_close_retains_the_open_handle_for_a_safe_retry(self, fake_coolscanpy) -> None: + ownership_error = RuntimeError("USB ownership is retained") + + class UncertainRoll(fake_coolscanpy.Roll): + def close(self) -> None: + raise ownership_error + + service, _roll, device = self._open_service( + fake_coolscanpy, + UncertainRoll(), + ) + + with pytest.raises(RuntimeError) as raised: + service.close() + + assert raised.value is ownership_error + assert service._roll is not None + assert device.closed is False + + def test_double_open_raises(self, fake_coolscanpy) -> None: + service, _roll, _device = self._open_service(fake_coolscanpy) + with pytest.raises(RollScanningError, match="already open"): + service.open_roll() + + def test_methods_before_open_raise(self) -> None: + service = RollScanningService() + with pytest.raises(RollScanningError, match="no roll is open"): + service.preview() + + def test_close_without_open_is_a_no_op(self) -> None: + RollScanningService().close() # must not raise + + def test_preview_and_approve_delegate_to_handle(self, fake_coolscanpy) -> None: + approval = object() + + class ReturningApprovalRoll(fake_coolscanpy.Roll): + def approve(self, slot): + super().approve(slot) + return approval + + thumb = fake_coolscanpy.Thumbnail( + slot=1, + image=np.zeros((2, 2, 3)), + boundary_rows=(0, 2), + spacing_offset=0, + needs_approval=True, + ) + service, roll, _device = self._open_service( + fake_coolscanpy, + ReturningApprovalRoll(thumbnails=[thumb]), + ) + + assert service.preview() == [thumb] + assert service.approve(1) is approval + assert roll.approved == [1] + + def test_restore_preview_session_delegates_to_open_handle(self, fake_coolscanpy) -> None: + thumbnails = [ + fake_coolscanpy.Thumbnail( + slot=slot, + image=np.zeros((2, 2, 3)), + boundary_rows=(0, 2), + spacing_offset=0, + needs_approval=slot == 1, + ) + for slot in (1, 2) + ] + service, roll, _device = self._open_service( + fake_coolscanpy, + fake_coolscanpy.Roll(thumbnails=thumbnails), + ) + + result = service.restore_preview_session("saved-session", [2]) + + assert [thumbnail.slot for thumbnail in result] == [2] + assert roll.restore_preview_session_calls == [("saved-session", (2,))] + + def test_restore_preview_session_requires_an_open_roll(self) -> None: + service = RollScanningService() + + with pytest.raises(RollScanningError, match="no roll is open"): + service.restore_preview_session("saved-session") + + def test_scan_many_delegates_to_handle(self, fake_coolscanpy) -> None: + frame = fake_coolscanpy.Frame(slot=1, rgb=np.zeros((2, 2, 3), dtype=np.uint16), ir=None, ir_validity=None, receipt=None) + service, _roll, _device = self._open_service(fake_coolscanpy, fake_coolscanpy.Roll(frames=[frame])) + + assert list(service.scan_many([1])) == [frame] + + def test_safe_stop_before_open_is_a_no_op(self) -> None: + RollScanningService().safe_stop() # must not raise + + def test_context_manager_closes(self, fake_coolscanpy) -> None: + roll = fake_coolscanpy.Roll() + device = fake_coolscanpy.Device(roll) + fake_coolscanpy.state["open_device"] = device + + with RollScanningService() as service: + service.open_roll() + assert roll.closed is True + + +class TestWriteFrame: + def _frame(self, fake_coolscanpy, *, slot=7, ir=None, meter_rgbi=None): + rgb = np.random.randint(0, 65535, (40, 60, 3), dtype=np.uint16) + receipt = fake_coolscanpy.Receipt(version=1, slot=slot, dpi=4000, depth=16, device_id="usb:1:2", transport_smear_verdict="clean") + return fake_coolscanpy.Frame(slot=slot, rgb=rgb, ir=ir, ir_validity=None, receipt=receipt, meter_rgbi=meter_rgbi) + + def test_writes_rgb_tiff(self, fake_coolscanpy, tmp_path) -> None: + frame = self._frame(fake_coolscanpy) + service = RollScanningService() + + output = service.write_frame(frame, str(tmp_path), '{{ date }}_slot{{ "%02d" % seq }}') + + assert os.path.exists(output.rgb_path) + assert output.rgb_path.endswith(".tif") + readback = tifffile.imread(output.rgb_path) + assert readback.shape == (40, 60, 3) + assert readback.dtype == np.uint16 + np.testing.assert_array_equal(readback, frame.rgb) + + def test_seq_seeded_from_slot_number(self, fake_coolscanpy, tmp_path) -> None: + frame = self._frame(fake_coolscanpy, slot=23) + service = RollScanningService() + + output = service.write_frame(frame, str(tmp_path), '{{ "%03d" % seq }}') + + assert "023" in os.path.basename(output.rgb_path) + + def test_writes_ir_sidecar_when_present(self, fake_coolscanpy, tmp_path) -> None: + ir = np.random.randint(0, 65535, (40, 60), dtype=np.uint16) + frame = self._frame(fake_coolscanpy, ir=ir) + service = RollScanningService() + + output = service.write_frame(frame, str(tmp_path), '{{ "%03d" % seq }}') + + assert output.ir_path is not None + assert output.ir_path.endswith("_IR.tif") + assert os.path.exists(output.ir_path) + readback = tifffile.imread(output.ir_path) + np.testing.assert_array_equal(readback, ir) + + def test_no_ir_sidecar_when_absent(self, fake_coolscanpy, tmp_path) -> None: + frame = self._frame(fake_coolscanpy, ir=None) + service = RollScanningService() + + output = service.write_frame(frame, str(tmp_path), '{{ "%03d" % seq }}') + + assert output.ir_path is None + assert not any(name.endswith("_IR.tif") for name in os.listdir(tmp_path)) + + def test_writes_receipt_json_sidecar(self, fake_coolscanpy, tmp_path) -> None: + frame = self._frame(fake_coolscanpy, slot=9) + service = RollScanningService() + + output = service.write_frame(frame, str(tmp_path), '{{ "%03d" % seq }}') + + assert output.receipt_path.endswith("_receipt.json") + with open(output.receipt_path) as fh: + payload = json.load(fh) + assert payload["slot"] == 9 + assert payload["dpi"] == 4000 + assert payload["transport_smear_verdict"] == "clean" + + def test_writes_current_coolscan_receipt_with_immutable_artifacts(self, tmp_path) -> None: + from coolscanpy.types import ( + ArtifactEvidence, + ClippingTelemetry, + ExposureVector, + FocusDetailTelemetry, + Frame, + Receipt, + TransportSmearAssessment, + ) + + rgb = np.zeros((2, 3, 3), dtype=np.uint16) + artifact = ArtifactEvidence( + sha256=sha256(memoryview(rgb).cast("B")).hexdigest(), + byte_length=rgb.nbytes, + shape=rgb.shape, + dtype=rgb.dtype.str, + ) + receipt = Receipt( + version=1, + slot=1, + spacing_offset=0, + dpi=4000, + depth=16, + device_id="usb:2:7", + device_model="LS-5000 ED", + reviewed_fingerprint_sha256="a" * 64, + fresh_fingerprint_sha256="a" * 64, + manual_approval=None, + exposure=ExposureVector( + focus_position=1, + exposure_multiplier=1.0, + red_exposure_us=1.0, + green_exposure_us=1.0, + blue_exposure_us=1.0, + ), + split_alignment=None, + clipping=ClippingTelemetry( + fractions=(0.0, 0.0, 0.0), + clip_level=65535.0, + warning_fraction=0.01, + warning=False, + ), + focus_detail=FocusDetailTelemetry( + method="laplacian", + verdict="measured", + score=1.0, + texture_span=1.0, + ), + transport_smear=TransportSmearAssessment( + verdict="clean", + start_row=None, + suffix_rows=0, + minimum_matches=0, + tail_median_rms=None, + tail_min_corr=None, + pre_tail_median_rms=None, + texture_span=None, + reason="no repeated tail", + ), + artifacts={"rgb": artifact}, + ) + assert type(receipt.artifacts).__name__ == "_ImmutableArtifacts" + frame = Frame( + slot=1, + rgb=rgb, + ir=None, + ir_validity=None, + receipt=receipt, + ) + + output = RollScanningService().write_frame( + frame, + str(tmp_path), + '{{ "%03d" % seq }}', + ) + + with open(output.receipt_path, encoding="utf-8") as stream: + payload = json.load(stream) + assert payload["artifacts"] == { + "rgb": { + "byte_length": rgb.nbytes, + "dtype": rgb.dtype.str, + "sha256": artifact.sha256, + "shape": list(rgb.shape), + } + } + + def test_rescanning_same_slot_overwrites(self, fake_coolscanpy, tmp_path) -> None: + service = RollScanningService() + pattern = '{{ "%03d" % seq }}' + + first = service.write_frame(self._frame(fake_coolscanpy, slot=4), str(tmp_path), pattern) + second_frame = self._frame(fake_coolscanpy, slot=4) + second = service.write_frame(second_frame, str(tmp_path), pattern) + + assert first.rgb_path == second.rgb_path + readback = tifffile.imread(second.rgb_path) + np.testing.assert_array_equal(readback, second_frame.rgb) + + def test_creates_output_folder(self, fake_coolscanpy, tmp_path) -> None: + nested = tmp_path / "does" / "not" / "exist" / "yet" + frame = self._frame(fake_coolscanpy) + service = RollScanningService() + + output = service.write_frame(frame, str(nested), '{{ "%03d" % seq }}') + + assert os.path.exists(output.rgb_path) + + +def _tier_frame( + fake_coolscanpy, + *, + slot=11, + ir=True, + seed=0, + meter_rgbi=None, + attempt=1, +): + """A frame with real per-pixel variance (not a flat fill) so a rendered + Tier-3 positive has something non-degenerate to measure -- red-biased + like a plausible C41 negative, matching tests/roll/test_positive.py's + own synthetic data.""" + rng = np.random.default_rng(seed) + shape = (24, 32) + rgb = np.zeros((*shape, 3), dtype=np.uint16) + rgb[..., 0] = rng.integers(40000, 60000, size=shape) + rgb[..., 1] = rng.integers(25000, 45000, size=shape) + rgb[..., 2] = rng.integers(15000, 35000, size=shape) + ir_plane = rng.integers(0, 65535, size=shape, dtype=np.uint16) if ir else None + acquisition = None + validity = None + if ir_plane is not None and 1 <= slot <= 40: + if meter_rgbi is None: + meter_rgbi = rng.integers( + 0, + 65535, + size=(4, 5, 4), + dtype=np.uint16, + ) + storage_rgbi = np.dstack((rgb, ir_plane)) + native_rgbi = np.ascontiguousarray(np.rot90(storage_rgbi, k=-1, axes=(0, 1))) + validity = np.ones(shape, dtype=np.bool_) + native_validity = np.ascontiguousarray(np.rot90(validity, k=-1, axes=(0, 1))) + reservation_id = f"reservation-{slot:03d}" + capture_attempt_id = f"fine-slot-{slot}-attempt-{attempt:03d}" + acquisition_id, evidence_sha256 = roll_service._derive_digital_ice_producer_binding( + slot=slot, + reservation_id=reservation_id, + capture_attempt_id=capture_attempt_id, + main_rgbi=native_rgbi, + prepass_rgbi=meter_rgbi, + ir_validity=native_validity, + ) + acquisition = roll_repair.RepairAcquisition.from_arrays( + acquisition_id=acquisition_id, + slot=slot, + reservation_id=reservation_id, + capture_attempt_id=capture_attempt_id, + storage_transform=roll_repair.DIGITAL_ICE_STORAGE_TRANSFORM, + evidence_sha256=evidence_sha256, + main_rgbi=native_rgbi, + prepass_rgbi=meter_rgbi, + ir_validity=native_validity, + ) + receipt = fake_coolscanpy.Receipt(version=1, slot=slot, dpi=4000, depth=16, device_id="usb:1:2", transport_smear_verdict="clean") + return fake_coolscanpy.Frame( + slot=slot, + rgb=rgb, + ir=ir_plane, + ir_validity=validity, + receipt=receipt, + meter_rgbi=meter_rgbi, + digital_ice_acquisition=acquisition, + ) + + +def _valid_hybrid_result( + acquisition: roll_repair.RepairAcquisition, + *, + routed_pixel_count: int = 1, + at_floor_pixel_count: int = 1, +) -> roll_repair.RepairResult: + native_rgb = np.ascontiguousarray(acquisition.main_rgbi[..., :3] + 3) + routed_mask = np.zeros(acquisition.main_rgbi.shape[:2], dtype=np.bool_) + routed_mask.reshape(-1)[:routed_pixel_count] = True + applied_mask = np.ascontiguousarray(routed_mask & acquisition.ir_validity) + + def png(mask: np.ndarray) -> bytes: + stream = io.BytesIO() + Image.fromarray(mask.astype(np.uint8) * 255, mode="L").save( + stream, + format="PNG", + ) + return stream.getvalue() + + routed_png = png(routed_mask) + applied_png = png(applied_mask) + storage_mask = acquisition.storage_mask(applied_mask) + storage_png = png(storage_mask) + native_hash = sha256(native_rgb.astype(" exact_color.VerifiedCMSReceipt: + encoded = payload if type(payload) is bytes else json.dumps(payload, sort_keys=True, separators=(",", ":")).encode("utf-8") + return dataclasses.replace(receipt, payload=encoded, sha256=sha256(encoded).hexdigest()) + + +def _self_attested_cms_receipt(payload: dict) -> exact_color.VerifiedCMSReceipt: + encoded = json.dumps(payload, sort_keys=True, separators=(",", ":")).encode("utf-8") + return exact_color.VerifiedCMSReceipt( + payload=encoded, + sha256=sha256(encoded).hexdigest(), + _factory_token=object(), + ) + + +def _builder_receipt() -> exact_color.ValidatedBuilderReceipt: + identity = np.arange(65_536, dtype=np.uint16) + return make_stage3_replay_receipt((identity, identity, identity)) + + +class _ExactStage1Builder: + def __init__(self) -> None: + self.calls = [] + + def apply(self, rgb, *, builder_receipt): + self.calls.append((rgb, builder_receipt)) + output = rgb.copy() + source_hash = exact_color.rgb16_content_sha256(rgb) + stage1_hash = exact_color.rgb16_content_sha256(output) + return exact_color.Stage1BuilderResult( + rgb=output, + source_rgb_sha256=source_hash, + stage1_input_rgb_sha256=stage1_hash, + builder_receipt=builder_receipt, + application_receipt=_receipt_blob( + exact_color.VerifiedBuilderApplicationReceipt, + { + "builder_receipt_sha256": builder_receipt.sha256, + "fixed_composition": { + "lut_sha256": exact_color.FIXED_COMPOSITION_SHA256, + "order": "F[B_c(i)]", + }, + "kind": "negpy.verified-stage1-builder-application", + "native_per_acquisition_builder": False, + "pre_f_lut_sha256": dict(zip(("r", "g", "b"), builder_receipt.pre_f_lut_sha256, strict=True)), + "source_rgb_sha256": source_hash, + "scope": exact_color.STAGE3_REPLAY_SCOPE, + "stage1_input_rgb_sha256": stage1_hash, + "stage3_receipt_sha256": builder_receipt.stage3_receipt_sha256, + "version": 1, + }, + attested=True, + ), + ) + + +class _ExactColorEvaluator: + def __init__(self) -> None: + self.calls = [] + self.results = [] + self._evaluator = PortableCMSOnEvaluator(chunk_pixels=17) + + def evaluate(self, rgb, *, builder_receipt): + self.calls.append((rgb, builder_receipt)) + result = self._evaluator.evaluate(rgb, builder_receipt=builder_receipt) + self.results.append(result) + return result + + +class TestThreeTierWriting: + """`write_frame`'s three independently-selectable output tiers: unrepaired + (Tier 1), repaired (Tier 2, needs a registered repair engine), and + positive (Tier 3, always derived from Tier 2's in-memory result). Naming, + receipt provenance, and every degrade path get covered here; the plain + default-settings behavior (Tier 1 only) is already covered above by + TestWriteFrame, unchanged.""" + + def _receipt(self, path: str) -> dict: + with open(path) as fh: + return json.load(fh) + + # -- selecting nothing -------------------------------------------------- + + def test_no_tier_selected_writes_only_the_receipt(self, fake_coolscanpy, tmp_path) -> None: + frame = _tier_frame(fake_coolscanpy) + service = RollScanningService() + + output = service.write_frame( + frame, str(tmp_path), '{{ "%03d" % seq }}', write_unrepaired=False, write_repaired=False, write_positive=False + ) + + assert (output.rgb_path, output.ir_path, output.repaired_rgb_path, output.repaired_ir_path, output.positive_path) == ( + None, + None, + None, + None, + None, + ) + assert os.path.exists(output.receipt_path) + payload = self._receipt(output.receipt_path) + assert payload["outputs"]["unrepaired"] == {"written": False, "status": "not selected"} + assert payload["outputs"]["repaired"] == {"written": False, "status": "not selected"} + assert payload["outputs"]["positive"] == {"written": False, "status": "not selected"} + + # -- Tier 2 without a registered engine ---------------------------------- + + def test_repaired_without_an_engine_degrades_but_unrepaired_still_writes(self, fake_coolscanpy, no_repair_engine, tmp_path) -> None: + frame = _tier_frame(fake_coolscanpy) + service = RollScanningService() + + output = service.write_frame(frame, str(tmp_path), '{{ "%03d" % seq }}', write_unrepaired=True, write_repaired=True) + + assert output.rgb_path is not None and os.path.exists(output.rgb_path) + assert output.repaired_rgb_path is None + assert output.repaired_ir_path is None + payload = self._receipt(output.receipt_path) + assert payload["outputs"]["unrepaired"]["written"] is True + assert payload["outputs"]["repaired"] == {"written": False, "status": "unavailable: no dust-repair engine registered"} + + def test_tier1_retains_replayable_dice_acquisition_when_repair_is_unavailable( + self, + fake_coolscanpy, + no_repair_engine, + tmp_path, + ) -> None: + frame = _tier_frame(fake_coolscanpy, slot=12) + original = frame.prepare_digital_ice() + archive = tmp_path / "archive" + + output = RollScanningService().write_frame( + frame, + str(archive), + 'nested/{{ "%03d" % seq }}', + write_unrepaired=True, + write_repaired=True, + ) + + receipt = self._receipt(output.receipt_path) + evidence = receipt["outputs"]["repair_acquisition_evidence"] + assert evidence["retained"] is True + assert receipt["outputs"]["repaired"]["written"] is False + replay = roll_service.load_repair_acquisition_evidence(evidence["binding"]["path"]) + assert replay.acquisition_id == original.acquisition_id + assert replay.capture_attempt_id == original.capture_attempt_id + np.testing.assert_array_equal(replay.main_rgbi, original.main_rgbi) + np.testing.assert_array_equal(replay.prepass_rgbi, original.prepass_rgbi) + np.testing.assert_array_equal(replay.ir_validity, original.ir_validity) + + # Artifact references are relative to the binding as well as recorded + # absolutely in the receipt, so moving the complete archive preserves + # its ability to replay after the scanner media is gone. + relative_binding = Path(evidence["binding"]["path"]).relative_to(archive) + moved_archive = tmp_path / "moved-archive" + archive.rename(moved_archive) + moved_replay = roll_service.load_repair_acquisition_evidence(moved_archive / relative_binding) + np.testing.assert_array_equal(moved_replay.main_rgbi, original.main_rgbi) + + def test_tier1_survives_disclosed_dice_evidence_retention_failure( + self, + fake_coolscanpy, + tmp_path, + monkeypatch, + ) -> None: + real_atomic_write_bytes = roll_service._atomic_write_bytes + + def fail_prepass(path, payload): + if path.endswith("prepass.rgbi16.npy"): + raise OSError("synthetic evidence volume failure") + return real_atomic_write_bytes(path, payload) + + monkeypatch.setattr(roll_service, "_atomic_write_bytes", fail_prepass) + output = RollScanningService().write_frame( + _tier_frame(fake_coolscanpy, slot=13), + str(tmp_path), + '{{ "%03d" % seq }}', + write_unrepaired=True, + write_repaired=False, + ) + + assert output.rgb_path is not None and Path(output.rgb_path).is_file() + assert output.ir_path is not None and Path(output.ir_path).is_file() + evidence = self._receipt(output.receipt_path)["outputs"]["repair_acquisition_evidence"] + assert evidence["retained"] is False + assert "synthetic evidence volume failure" in evidence["status"] + assert not (tmp_path / ".negpy-dice-acquisition").exists() + + @pytest.mark.parametrize( + ("tamper", "match"), + [ + ("producer-hash", "producer evidence SHA-256 changed"), + ("missing-prepass", "No such file"), + ("symlink-prepass", "regular non-symlink"), + ("storage-rgb", "storage RGB SHA-256 changed"), + ], + ) + def test_dice_acquisition_replay_rejects_tampered_or_missing_archive_parts( + self, + fake_coolscanpy, + tmp_path, + tamper, + match, + ) -> None: + output = RollScanningService().write_frame( + _tier_frame(fake_coolscanpy, slot=14), + str(tmp_path), + '{{ "%03d" % seq }}', + write_unrepaired=True, + write_repaired=False, + ) + evidence = self._receipt(output.receipt_path)["outputs"]["repair_acquisition_evidence"] + binding_path = Path(evidence["binding"]["path"]) + document = json.loads(binding_path.read_bytes()) + assert str(tmp_path) not in binding_path.read_text(encoding="utf-8") + + if tamper == "producer-hash": + document["acquisition"]["evidence_sha256"] = "0" * 64 + binding_path.write_bytes( + json.dumps( + document, + sort_keys=True, + separators=(",", ":"), + ).encode() + + b"\n" + ) + elif tamper in {"missing-prepass", "symlink-prepass"}: + relative = document["artifacts"]["prepass_rgbi"]["relative_path"] + prepass_path = (binding_path.parent / relative).resolve() + if tamper == "missing-prepass": + prepass_path.unlink() + else: + real_prepass = prepass_path.with_suffix(".real") + prepass_path.rename(real_prepass) + prepass_path.symlink_to(real_prepass.name) + else: + assert output.rgb_path is not None + tifffile.imwrite( + output.rgb_path, + np.zeros_like(tifffile.imread(output.rgb_path)), + photometric="rgb", + ) + + with pytest.raises((OSError, ValueError), match=match): + roll_service.load_repair_acquisition_evidence(binding_path) + + @pytest.mark.skipif(not hasattr(os, "mkfifo"), reason="POSIX FIFO test") + def test_dice_replay_tiff_regular_to_fifo_swap_never_blocks( + self, + fake_coolscanpy, + tmp_path, + monkeypatch, + ) -> None: + output = RollScanningService().write_frame( + _tier_frame(fake_coolscanpy, slot=14), + str(tmp_path), + '{{ "%03d" % seq }}', + write_unrepaired=True, + ) + evidence = self._receipt(output.receipt_path)["outputs"]["repair_acquisition_evidence"] + binding_path = Path(evidence["binding"]["path"]) + rgb_path = Path(output.rgb_path) + real_open = roll_service.os.open + swapped = False + + def swap_before_open(path, flags, *args): + nonlocal swapped + if not swapped and Path(path) == rgb_path: + swapped = True + rgb_path.unlink() + os.mkfifo(rgb_path) + return real_open(path, flags, *args) + + monkeypatch.setattr(roll_service.os, "open", swap_before_open) + started = time.monotonic() + with pytest.raises(ValueError, match="regular non-symlink|changed"): + roll_service.load_repair_acquisition_evidence(binding_path) + assert time.monotonic() - started < 1.0 + + def test_dice_replay_rejects_npy_header_before_array_allocation( + self, + fake_coolscanpy, + tmp_path, + monkeypatch, + ) -> None: + output = RollScanningService().write_frame( + _tier_frame(fake_coolscanpy, slot=14), + str(tmp_path), + '{{ "%03d" % seq }}', + write_unrepaired=True, + ) + evidence = self._receipt(output.receipt_path)["outputs"]["repair_acquisition_evidence"] + binding_path = Path(evidence["binding"]["path"]) + document = json.loads(binding_path.read_bytes()) + prepass_row = document["artifacts"]["prepass_rgbi"] + prepass_path = binding_path.parent / prepass_row["relative_path"] + malicious = io.BytesIO() + np.lib.format.write_array_header_1_0( + malicious, + { + "descr": " None: + service = RollScanningService() + first = service.write_frame( + _tier_frame(fake_coolscanpy, slot=15, attempt=1), + str(tmp_path), + '{{ "%03d" % seq }}', + write_unrepaired=True, + ) + first_evidence = self._receipt(first.receipt_path)["outputs"]["repair_acquisition_evidence"] + old_directory = Path(first_evidence["binding"]["path"]).parent + assert old_directory.is_dir() + + second = service.write_frame( + _tier_frame(fake_coolscanpy, slot=15, seed=2, attempt=2), + str(tmp_path), + '{{ "%03d" % seq }}', + write_unrepaired=True, + ) + second_evidence = self._receipt(second.receipt_path)["outputs"]["repair_acquisition_evidence"] + + assert Path(second_evidence["binding"]["path"]).parent != old_directory + assert not old_directory.exists() + assert Path(second_evidence["binding"]["path"]).is_file() + + def test_rescan_preserves_dice_evidence_owned_by_a_sibling_receipt( + self, + fake_coolscanpy, + tmp_path, + ) -> None: + service = RollScanningService() + first = service.write_frame( + _tier_frame(fake_coolscanpy, slot=16, attempt=1), + str(tmp_path), + '{{ "%03d" % seq }}', + write_unrepaired=True, + ) + first_evidence = self._receipt(first.receipt_path)["outputs"]["repair_acquisition_evidence"] + old_binding = Path(first_evidence["binding"]["path"]) + sibling_receipt = tmp_path / "shared_receipt.json" + sibling_receipt.write_text( + json.dumps({"outputs": {"shared_dice": first_evidence}}), + encoding="utf-8", + ) + + with pytest.raises(OSError, match="owned by another receipt"): + service.write_frame( + _tier_frame(fake_coolscanpy, slot=16, seed=3, attempt=2), + str(tmp_path), + '{{ "%03d" % seq }}', + write_unrepaired=True, + ) + + assert old_binding.is_file() + replay = roll_service.load_repair_acquisition_evidence(old_binding) + assert replay.capture_attempt_id.endswith("001") + + def test_positive_without_an_engine_degrades_too_since_it_needs_tier_2(self, fake_coolscanpy, no_repair_engine, tmp_path) -> None: + frame = _tier_frame(fake_coolscanpy) + service = RollScanningService() + + output = service.write_frame(frame, str(tmp_path), '{{ "%03d" % seq }}', write_unrepaired=True, write_positive=True) + + assert output.rgb_path is not None and os.path.exists(output.rgb_path) + assert output.positive_path is None + payload = self._receipt(output.receipt_path) + assert payload["outputs"]["unrepaired"]["written"] is True + assert payload["outputs"]["positive"]["written"] is False + assert "Tier 2" in payload["outputs"]["positive"]["status"] + assert "no dust-repair engine registered" in payload["outputs"]["positive"]["status"] + + # -- Tier 2 with a registered engine ------------------------------------- + + def test_repair_runs_scanner_native_then_rotates_once_to_storage(self, fake_coolscanpy, fake_repair_engine, tmp_path) -> None: + native_main = np.arange(3 * 2 * 4, dtype=np.uint16).reshape(3, 2, 4) + storage_rgbi = np.ascontiguousarray(np.rot90(native_main, k=1, axes=(0, 1))) + meter = np.arange(2 * 2 * 4, dtype=np.uint16).reshape(2, 2, 4) + validity = np.ones(native_main.shape[:2], dtype=np.bool_) + validity[-1, 0] = False + acquisition_id, evidence_sha256 = roll_service._derive_digital_ice_producer_binding( + slot=11, + reservation_id="reservation-011", + capture_attempt_id="fine-slot-11-attempt-001", + main_rgbi=native_main, + prepass_rgbi=meter, + ir_validity=validity, + ) + acquisition = roll_repair.RepairAcquisition.from_arrays( + acquisition_id=acquisition_id, + slot=11, + reservation_id="reservation-011", + capture_attempt_id="fine-slot-11-attempt-001", + storage_transform=roll_repair.DIGITAL_ICE_STORAGE_TRANSFORM, + evidence_sha256=evidence_sha256, + main_rgbi=native_main, + prepass_rgbi=meter, + ir_validity=validity, + ) + receipt = fake_coolscanpy.Receipt( + version=1, + slot=11, + dpi=4000, + depth=16, + device_id="usb:1:2", + transport_smear_verdict="clean", + ) + + class _Frame: + slot = 11 + rgb = storage_rgbi[..., :3] + ir = storage_rgbi[..., 3] + ir_validity = np.rot90(validity, k=1, axes=(0, 1)) + meter_rgbi = meter + + def __init__(self) -> None: + self.receipt = receipt + + def prepare_digital_ice(self): + return acquisition + + frame = _Frame() + fake_repair_engine.transform = lambda rgb: np.ascontiguousarray(rgb + 17) + + output = RollScanningService().write_frame( + frame, + str(tmp_path), + '{{ "%03d" % seq }}', + write_unrepaired=False, + write_repaired=True, + ) + + assert output.repaired_rgb_path is not None + seen = fake_repair_engine.calls[0][0] + np.testing.assert_array_equal(seen.main_rgbi, native_main) + np.testing.assert_array_equal(seen.prepass_rgbi, meter) + np.testing.assert_array_equal(seen.ir_validity, validity) + expected = np.ascontiguousarray(np.rot90(native_main[..., :3] + 17, k=1, axes=(0, 1))) + np.testing.assert_array_equal( + tifffile.imread(output.repaired_rgb_path), + expected, + ) + + def test_hybrid_persists_output_aligned_mask_and_native_verification_evidence( + self, fake_coolscanpy, fake_repair_engine, tmp_path, monkeypatch + ) -> None: + frame = _tier_frame(fake_coolscanpy) + acquisition = frame.prepare_digital_ice() + native_rgb = np.ascontiguousarray(acquisition.main_rgbi[..., :3] + 3) + native_mask = np.zeros(acquisition.main_rgbi.shape[:2], dtype=np.bool_) + native_mask[2, 4] = True + native_stream = io.BytesIO() + Image.fromarray(native_mask.astype(np.uint8) * 255, mode="L").save( + native_stream, + format="PNG", + ) + native_mask_png = native_stream.getvalue() + storage_mask = acquisition.storage_mask(native_mask) + storage_stream = io.BytesIO() + Image.fromarray(storage_mask.astype(np.uint8) * 255, mode="L").save( + storage_stream, + format="PNG", + ) + storage_mask_png = storage_stream.getvalue() + hybrid_output_hash = sha256(native_rgb.astype(" None: + acquisition = _tier_frame(fake_coolscanpy).prepare_digital_ice() + result = _valid_hybrid_result( + acquisition, + routed_pixel_count=2, + at_floor_pixel_count=1, + ) + + roll_service._validate_repair_result_binding( + acquisition, + result, + requested_mode=roll_repair.RepairMode.HYBRID, + ) + + @pytest.mark.parametrize( + ("tamper", "match"), + [ + ("mode", "resolved mode is invalid"), + ("native-output", "scanner-native RGB SHA-256 changed"), + ("receipt-output", "receipt output binding changed"), + ("receipt-composite", "receipt output binding changed"), + ("routing-regions", "routing counts disagree"), + ("noncanonical-receipt", "not canonical"), + ("permuted-mask", "routed mask binding changed"), + ], + ) + def test_hybrid_result_binding_rejects_tampered_result_surface( + self, + fake_coolscanpy, + tamper, + match, + ) -> None: + acquisition = _tier_frame(fake_coolscanpy).prepare_digital_ice() + result = _valid_hybrid_result(acquisition) + + if tamper == "mode": + result = dataclasses.replace(result, mode_resolved=None) + elif tamper == "native-output": + result = dataclasses.replace( + result, + native_output_rgb_sha256="0" * 64, + ) + elif tamper in { + "receipt-output", + "receipt-composite", + "routing-regions", + }: + document = json.loads(result.hybrid_receipt) + routing_counts = result.routing_counts + if tamper == "receipt-output": + document["artifacts"][0]["raw_sha256"] = "0" * 64 + elif tamper == "receipt-composite": + document["composite"]["hybrid_rgb16_raw_sha256"] = "0" * 64 + else: + document["routing"]["counts"]["final_regions"] = 2 + routing_counts = {**routing_counts, "final_regions": 2} + receipt = ( + json.dumps( + document, + sort_keys=True, + separators=(",", ":"), + ).encode() + + b"\n" + ) + result = dataclasses.replace( + result, + hybrid_receipt=receipt, + hybrid_receipt_sha256=sha256(receipt).hexdigest(), + routing_counts=routing_counts, + ) + elif tamper == "noncanonical-receipt": + receipt = json.dumps( + json.loads(result.hybrid_receipt), + indent=2, + ).encode() + result = dataclasses.replace( + result, + hybrid_receipt=receipt, + hybrid_receipt_sha256=sha256(receipt).hexdigest(), + ) + else: + routed = np.zeros(acquisition.main_rgbi.shape[:2], dtype=np.bool_) + routed.reshape(-1)[1] = True + applied = np.ascontiguousarray(routed & acquisition.ir_validity) + + def png(mask): + stream = io.BytesIO() + Image.fromarray(mask.astype(np.uint8) * 255, mode="L").save( + stream, + format="PNG", + ) + return stream.getvalue() + + routed_png = png(routed) + applied_png = png(applied) + storage_png = png(acquisition.storage_mask(applied)) + result = dataclasses.replace( + result, + routed_native_synthesis_mask_png=routed_png, + routed_native_synthesis_mask_sha256=sha256(routed_png).hexdigest(), + native_synthesis_mask_png=applied_png, + native_synthesis_mask_sha256=sha256(applied_png).hexdigest(), + storage_synthesis_mask_png=storage_png, + storage_synthesis_mask_sha256=sha256(storage_png).hexdigest(), + ) + + with pytest.raises(ValueError, match=match): + roll_service._validate_repair_result_binding( + acquisition, + result, + requested_mode=roll_repair.RepairMode.HYBRID, + ) + + def test_service_snapshots_mutable_hybrid_result_before_validation_and_write( + self, + fake_coolscanpy, + fake_repair_engine, + tmp_path, + monkeypatch, + ) -> None: + frame = _tier_frame(fake_coolscanpy, slot=18) + acquisition = frame.prepare_digital_ice() + valid = _valid_hybrid_result(acquisition) + mutable_rgb = np.array(valid.rgb, copy=True) + mutable_counts = dict(valid.routing_counts) + returned = dataclasses.replace( + valid, + rgb=mutable_rgb, + routing_counts=mutable_counts, + ) + expected = mutable_rgb.copy() + fake_repair_engine.repair = lambda *_args, **_kwargs: returned + real_validate = roll_service._validate_repair_result_binding + + def mutate_producer_after_snapshot(acquisition, result, *, requested_mode): + mutable_rgb.fill(0) + mutable_counts["synthesis_pixels"] = 999 + return real_validate( + acquisition, + result, + requested_mode=requested_mode, + ) + + monkeypatch.setattr( + roll_service, + "_validate_repair_result_binding", + mutate_producer_after_snapshot, + ) + output = RollScanningService().write_frame( + frame, + str(tmp_path), + '{{ "%03d" % seq }}', + write_unrepaired=False, + write_repaired=True, + repair_mode="hybrid", + ) + + assert output.repaired_rgb_path is not None + np.testing.assert_array_equal( + tifffile.imread(output.repaired_rgb_path), + expected, + ) + + def test_malformed_repair_result_degrades_without_losing_tier1( + self, + fake_coolscanpy, + fake_repair_engine, + tmp_path, + ) -> None: + frame = _tier_frame(fake_coolscanpy, slot=18) + malformed = dataclasses.replace( + _valid_hybrid_result(frame.prepare_digital_ice()), + rgb=object(), + ) + fake_repair_engine.repair = lambda *_args, **_kwargs: malformed + + output = RollScanningService().write_frame( + frame, + str(tmp_path), + '{{ "%03d" % seq }}', + write_unrepaired=True, + write_repaired=True, + repair_mode="hybrid", + ) + + assert output.rgb_path is not None and Path(output.rgb_path).is_file() + assert output.repaired_rgb_path is None + repaired = self._receipt(output.receipt_path)["outputs"]["repaired"] + assert repaired["written"] is False + assert "repair evidence failed" in repaired["status"] + + def test_tier2_only_rejects_noncanonical_producer_evidence_before_engine( + self, + fake_coolscanpy, + fake_repair_engine, + tmp_path, + ) -> None: + frame = _tier_frame(fake_coolscanpy, slot=18) + forged = dataclasses.replace( + frame.prepare_digital_ice(), + evidence_sha256="0" * 64, + ) + frame = dataclasses.replace(frame, digital_ice_acquisition=forged) + + output = RollScanningService().write_frame( + frame, + str(tmp_path), + '{{ "%03d" % seq }}', + write_unrepaired=False, + write_repaired=True, + ) + + assert output.repaired_rgb_path is None + assert fake_repair_engine.calls == [] + repaired = self._receipt(output.receipt_path)["outputs"]["repaired"] + assert repaired["written"] is False + assert "producer evidence SHA-256 changed" in repaired["status"] + + def test_repaired_writes_rgb_and_retains_original_ir(self, fake_coolscanpy, fake_repair_engine, tmp_path) -> None: + fake_repair_engine.transform = lambda rgb: np.clip(rgb.astype(np.int32) + 1000, 0, 65535).astype(np.uint16) + frame = _tier_frame(fake_coolscanpy) + service = RollScanningService() + + output = service.write_frame(frame, str(tmp_path), '{{ "%03d" % seq }}', write_unrepaired=True, write_repaired=True) + + assert output.repaired_rgb_path is not None + assert output.repaired_rgb_path.endswith("_repaired.tif") + readback = tifffile.imread(output.repaired_rgb_path) + np.testing.assert_array_equal(readback, fake_repair_engine.transform(frame.rgb)) + assert not np.array_equal(readback, frame.rgb) # actually repaired, not a copy + + assert output.repaired_ir_path is not None + assert output.repaired_ir_path.endswith("_repaired_IR.tif") + ir_readback = tifffile.imread(output.repaired_ir_path) + np.testing.assert_array_equal(ir_readback, frame.ir) # Tier 1's own IR, unchanged + + def test_transaction_failure_keeps_prior_artifacts_and_receipt( + self, + fake_coolscanpy, + fake_repair_engine, + tmp_path, + monkeypatch, + ) -> None: + first = _tier_frame(fake_coolscanpy, slot=13, seed=1) + service = RollScanningService() + published = service.write_frame( + first, + str(tmp_path), + '{{ "%03d" % seq }}', + write_unrepaired=True, + write_repaired=True, + ) + prior_receipt = Path(published.receipt_path).read_bytes() + prior_repaired = Path(published.repaired_rgb_path).read_bytes() + second = _tier_frame(fake_coolscanpy, slot=13, seed=2) + + def fail_receipt(_path, _payload): + raise OSError("synthetic receipt staging failure") + + monkeypatch.setattr(roll_service, "_atomic_write_json", fail_receipt) + with pytest.raises(OSError, match="receipt staging failure"): + service.write_frame( + second, + str(tmp_path), + '{{ "%03d" % seq }}', + write_unrepaired=True, + write_repaired=True, + ) + + assert Path(published.receipt_path).read_bytes() == prior_receipt + assert Path(published.repaired_rgb_path).read_bytes() == prior_repaired + assert not any(path.name.startswith(".negpy-frame-stage-") for path in tmp_path.iterdir()) + + def test_frame_lock_contention_fails_closed_without_blocking( + self, + tmp_path, + ) -> None: + receipt_path = str(tmp_path / "013_receipt.json") + first = roll_service._OutputTransaction(str(tmp_path)) + second = roll_service._OutputTransaction(str(tmp_path)) + first._acquire_frame_lock(receipt_path) + started = time.monotonic() + try: + with pytest.raises(OSError, match="busy in another NegPy process"): + second._acquire_frame_lock(receipt_path) + assert time.monotonic() - started < 0.5 + finally: + second.abort() + first.abort() + + # The failed contender closed its descriptor and the owner released + # normally, so a later transaction can acquire the same frame lock. + retry = roll_service._OutputTransaction(str(tmp_path)) + try: + retry._acquire_frame_lock(receipt_path) + finally: + retry.abort() + assert not any(path.name.startswith(".negpy-frame-stage-") for path in tmp_path.iterdir()) + + def test_frame_publication_fails_closed_when_platform_has_no_fcntl( + self, + tmp_path, + monkeypatch, + ) -> None: + transaction = roll_service._OutputTransaction(str(tmp_path)) + monkeypatch.setattr(roll_service, "fcntl", None) + try: + with pytest.raises(OSError, match="locking is unavailable"): + transaction._acquire_frame_lock(str(tmp_path / "013_receipt.json")) + finally: + transaction.abort() + assert not any(path.name.startswith(".negpy-frame-stage-") for path in tmp_path.iterdir()) + + def test_receipt_commit_failure_rolls_back_new_derived_artifacts( + self, + fake_coolscanpy, + fake_repair_engine, + tmp_path, + monkeypatch, + ) -> None: + service = RollScanningService() + first = service.write_frame( + _tier_frame(fake_coolscanpy, slot=14, seed=1), + str(tmp_path), + '{{ "%03d" % seq }}', + write_unrepaired=True, + write_repaired=True, + ) + prior_receipt = Path(first.receipt_path).read_bytes() + prior_repaired = Path(first.repaired_rgb_path).read_bytes() + real_replace = roll_service.os.replace + failed = False + + def replace_with_failed_receipt_commit(source, destination): + nonlocal failed + if ( + not failed + and os.path.abspath(destination) == os.path.abspath(first.receipt_path) + and ".negpy-frame-stage-" in os.path.abspath(source) + and "backups" not in os.path.abspath(source) + ): + failed = True + raise OSError("synthetic receipt commit failure") + return real_replace(source, destination) + + monkeypatch.setattr(roll_service.os, "replace", replace_with_failed_receipt_commit) + with pytest.raises(OSError, match="receipt commit failure"): + service.write_frame( + _tier_frame(fake_coolscanpy, slot=14, seed=2), + str(tmp_path), + '{{ "%03d" % seq }}', + write_unrepaired=True, + write_repaired=True, + ) + + assert failed is True + assert Path(first.receipt_path).read_bytes() == prior_receipt + assert Path(first.repaired_rgb_path).read_bytes() == prior_repaired + + def test_incomplete_rollback_retains_prior_file_in_recovery_directory( + self, + fake_coolscanpy, + fake_repair_engine, + tmp_path, + monkeypatch, + ) -> None: + service = RollScanningService() + first = service.write_frame( + _tier_frame(fake_coolscanpy, slot=15, seed=1), + str(tmp_path), + '{{ "%03d" % seq }}', + write_unrepaired=True, + write_repaired=True, + ) + prior_repaired = Path(first.repaired_rgb_path).read_bytes() + real_replace = roll_service.os.replace + receipt_commit_failed = False + repair_restore_failed = False + + def fail_commit_and_one_restore(source, destination): + nonlocal receipt_commit_failed, repair_restore_failed + source_path = os.path.abspath(source) + destination_path = os.path.abspath(destination) + if ( + not receipt_commit_failed + and destination_path == os.path.abspath(first.receipt_path) + and ".negpy-frame-stage-" in source_path + and "backups" not in source_path + ): + receipt_commit_failed = True + raise OSError("synthetic receipt commit failure") + if ( + receipt_commit_failed + and not repair_restore_failed + and destination_path == os.path.abspath(first.repaired_rgb_path) + and "backups" in source_path + ): + repair_restore_failed = True + raise OSError("synthetic repaired restore failure") + return real_replace(source, destination) + + monkeypatch.setattr( + roll_service.os, + "replace", + fail_commit_and_one_restore, + ) + with pytest.raises(roll_service.OutputRollbackError) as raised: + service.write_frame( + _tier_frame(fake_coolscanpy, slot=15, seed=2), + str(tmp_path), + '{{ "%03d" % seq }}', + write_unrepaired=True, + write_repaired=True, + ) + + assert receipt_commit_failed is True + assert repair_restore_failed is True + recovery = Path(raised.value.recovery_path) + assert recovery.is_dir() + assert recovery.name.startswith(".negpy-recovery-") + manifest = json.loads((recovery / "RECOVERY.json").read_text()) + backup_relative = manifest["unrestored_backups"][os.path.abspath(first.repaired_rgb_path)] + assert (recovery / backup_relative).read_bytes() == prior_repaired + + def test_output_parent_symlink_cannot_escape_selected_folder( + self, + fake_coolscanpy, + fake_repair_engine, + tmp_path, + ) -> None: + outside = tmp_path.parent / f"{tmp_path.name}-outside" + outside.mkdir() + (tmp_path / "sub").symlink_to(outside, target_is_directory=True) + + with pytest.raises(OSError, match="symbolic link"): + RollScanningService().write_frame( + _tier_frame(fake_coolscanpy, slot=16), + str(tmp_path), + 'sub/{{ "%03d" % seq }}', + write_unrepaired=True, + ) + + assert list(outside.iterdir()) == [] + assert not any(path.name.startswith(".negpy-frame-stage-") for path in tmp_path.iterdir()) + + def test_unreferenced_staged_output_aborts_publication( + self, + fake_coolscanpy, + tmp_path, + monkeypatch, + ) -> None: + real_atomic_write_json = roll_service._atomic_write_json + + def stage_orphan_before_receipt(path, payload): + roll_service._atomic_write_bytes( + str(tmp_path / "unreferenced.bin"), + b"orphan", + ) + return real_atomic_write_json(path, payload) + + monkeypatch.setattr( + roll_service, + "_atomic_write_json", + stage_orphan_before_receipt, + ) + with pytest.raises(OSError, match="absent from its receipt"): + RollScanningService().write_frame( + _tier_frame(fake_coolscanpy, slot=17), + str(tmp_path), + '{{ "%03d" % seq }}', + write_unrepaired=True, + ) + + assert not (tmp_path / "unreferenced.bin").exists() + assert not any(path.name.endswith("_receipt.json") for path in tmp_path.iterdir()) + assert not any(path.name.startswith(".negpy-frame-stage-") for path in tmp_path.iterdir()) + + @pytest.mark.skipif(not hasattr(os, "mkfifo"), reason="POSIX FIFO test") + def test_existing_fifo_receipt_fails_without_blocking_or_publishing( + self, + fake_coolscanpy, + tmp_path, + ) -> None: + receipt_path = tmp_path / "019_receipt.json" + os.mkfifo(receipt_path) + started = time.monotonic() + + with pytest.raises(OSError, match="regular non-symlink"): + RollScanningService().write_frame( + _tier_frame(fake_coolscanpy, slot=19), + str(tmp_path), + '{{ "%03d" % seq }}', + write_unrepaired=True, + ) + + assert time.monotonic() - started < 1.0 + assert not (tmp_path / "019.tif").exists() + assert not (tmp_path / "019_IR.tif").exists() + + @pytest.mark.parametrize( + "hostile_receipt", + [ + b'{"outputs":{"first":{"path":"owned"}},"outputs":{}}', + b'{"outputs":{"value":NaN}}', + ], + ) + def test_unsafe_sibling_receipt_conservatively_blocks_overwrite( + self, + fake_coolscanpy, + tmp_path, + hostile_receipt, + ) -> None: + (tmp_path / "hostile_receipt.json").write_bytes(hostile_receipt) + + with pytest.raises(OSError, match="owned by another receipt"): + RollScanningService().write_frame( + _tier_frame(fake_coolscanpy, slot=20), + str(tmp_path), + '{{ "%03d" % seq }}', + write_unrepaired=True, + ) + + assert not (tmp_path / "020.tif").exists() + assert not any(path.name.startswith(".negpy-frame-stage-") for path in tmp_path.iterdir()) + + def test_receipt_rejects_non_json_values_instead_of_stringifying_them( + self, + fake_coolscanpy, + tmp_path, + ) -> None: + @dataclasses.dataclass(frozen=True) + class InvalidReceipt: + version: int + slot: int + opaque: object + + frame = dataclasses.replace( + _tier_frame(fake_coolscanpy, slot=18), + receipt=InvalidReceipt(version=1, slot=18, opaque=object()), + ) + + with pytest.raises(TypeError, match="not JSON serializable"): + RollScanningService().write_frame( + frame, + str(tmp_path), + '{{ "%03d" % seq }}', + write_unrepaired=True, + ) + + assert not any(path.name.endswith("_receipt.json") for path in tmp_path.iterdir()) + assert not any(path.name.endswith(".tif") for path in tmp_path.iterdir()) + + def test_repaired_forwards_the_coolscanpy_meter_prepass_unchanged(self, fake_coolscanpy, fake_repair_engine, tmp_path) -> None: + prepass = np.zeros((4, 5, 4), dtype=np.uint16) + frame = _tier_frame( + fake_coolscanpy, + ir=True, + meter_rgbi=prepass, + ) + + RollScanningService().write_frame(frame, str(tmp_path), '{{ "%03d" % seq }}', write_unrepaired=False, write_repaired=True) + + assert len(fake_repair_engine.prepasses) == 1 + np.testing.assert_array_equal(fake_repair_engine.prepasses[0], prepass) + + def test_exact_nikon_positive_uses_injected_evaluator_and_binds_both_receipts( + self, fake_coolscanpy, fake_repair_engine, tmp_path + ) -> None: + fake_repair_engine.transform = lambda rgb: np.clip(rgb.astype(np.int32) + 731, 0, 65535).astype(np.uint16) + builder = _ExactStage1Builder() + evaluator = _ExactColorEvaluator() + builder_receipt = _builder_receipt() + frame = _tier_frame(fake_coolscanpy) + + output = RollScanningService( + exact_color_builder=builder, + exact_color_evaluator=evaluator, + ).write_frame( + frame, + str(tmp_path), + '{{ "%03d" % seq }}', + write_unrepaired=False, + write_positive=True, + positive_mode="nikon-exact", + builder_receipt=builder_receipt, + ) + + assert output.positive_path is not None + repaired = fake_repair_engine.transform(frame.rgb) + readback = tifffile.imread(output.positive_path) + assert readback.dtype == np.uint16 + np.testing.assert_array_equal(readback, evaluator.results[0].rgb) + with tifffile.TiffFile(output.positive_path) as exact_tiff: + embedded_profile = exact_tiff.pages[0].tags[34675].value + assert len(embedded_profile) == 492 + assert sha256(embedded_profile).hexdigest() == "a8d0d753bd6129357cc2647435ce675e8637a679eb526fa180fba460874ce1d3" + positive_receipt = self._receipt(output.receipt_path)["outputs"]["positive"] + tiff_artifact = positive_receipt["tiff_artifact"] + assert tiff_artifact["file_sha256"] == sha256(Path(output.positive_path).read_bytes()).hexdigest() + assert tiff_artifact["pixel_sha256"] == exact_color.rgb16_content_sha256(readback) + assert tiff_artifact["icc_sha256"] == sha256(embedded_profile).hexdigest() + assert tiff_artifact["page_count"] == 1 + assert tiff_artifact["bits_per_sample"] == [16, 16, 16] + assert tiff_artifact["orientation"] == "top-left" + np.testing.assert_array_equal(builder.calls[0][0], repaired) + np.testing.assert_array_equal(evaluator.calls[0][0], repaired) + entry = self._receipt(output.receipt_path)["outputs"]["positive"] + assert entry["color_mode"] == "nikon-exact" + assert entry["input_rgb_sha256"] == exact_color.rgb16_content_sha256(repaired) + assert entry["output_rgb_sha256"] == exact_color.rgb16_content_sha256(readback) + assert entry["builder_receipt_sha256"] == builder_receipt.sha256 + assert entry["builder_receipt"]["schema"] == exact_color.BUILDER_RECEIPT_SCHEMA + assert entry["builder_application_receipt"]["stage1_input_rgb_sha256"] == entry["input_rgb_sha256"] + assert entry["cms_receipt"]["algorithm"] == exact_color.CMS_ALGORITHM_ID + assert entry["cms_receipt"]["validation"]["mismatched_u16"] == 0 + assert entry["cms_receipt"]["input_rgb_sha256"] == entry["input_rgb_sha256"] + assert entry["icc_profile"] == { + "name": "Nikon Adobe RGB 4.0.0.3000", + "bytes": 492, + "sha256": "a8d0d753bd6129357cc2647435ce675e8637a679eb526fa180fba460874ce1d3", + } + evidence = entry["retained_builder_evidence"] + assert evidence["scope"] == exact_color.STAGE3_REPLAY_SCOPE + assert evidence["native_per_acquisition_builder"] is False + rows = [evidence["stage3_report"], *evidence["pre_f_luts"]] + for row in rows: + payload = Path(row["path"]).read_bytes() + assert len(payload) == row["bytes"] + assert sha256(payload).hexdigest() == row["sha256"] + reproduced = exact_color.load_stage3_replay_builder_receipt(evidence["stage3_report"]["path"]) + assert reproduced.sha256 == builder_receipt.sha256 + assert reproduced.pre_f_lut_sha256 == builder_receipt.pre_f_lut_sha256 + + @pytest.mark.parametrize("corruption", ["pixels", "icc", "orientation"]) + def test_exact_positive_rejects_tiff_write_corruption_before_publication( + self, + corruption, + fake_coolscanpy, + fake_repair_engine, + tmp_path, + monkeypatch, + ) -> None: + builder = _ExactStage1Builder() + evaluator = _ExactColorEvaluator() + real_imwrite = roll_service.tifffile.imwrite + + def corrupting_imwrite(path, array, **kwargs): + if kwargs.get("iccprofile") is not None: + if corruption == "pixels": + array = np.array(array, copy=True) + array[0, 0, 0] ^= np.uint16(1) + elif corruption == "icc": + kwargs.pop("iccprofile") + else: + kwargs["extratags"] = ((274, "H", 1, 6, False),) + return real_imwrite(path, array, **kwargs) + + monkeypatch.setattr(roll_service.tifffile, "imwrite", corrupting_imwrite) + output = RollScanningService( + exact_color_builder=builder, + exact_color_evaluator=evaluator, + ).write_frame( + _tier_frame(fake_coolscanpy), + str(tmp_path), + '{{ "%03d" % seq }}', + write_unrepaired=True, + write_repaired=True, + write_positive=True, + positive_mode="nikon-exact", + builder_receipt=_builder_receipt(), + ) + + assert output.rgb_path is not None and Path(output.rgb_path).is_file() + assert output.repaired_rgb_path is not None and Path(output.repaired_rgb_path).is_file() + assert output.positive_path is None + receipt = self._receipt(output.receipt_path) + assert receipt["outputs"]["unrepaired"]["written"] is True + assert receipt["outputs"]["repaired"]["written"] is True + positive = receipt["outputs"]["positive"] + assert positive["written"] is False + assert "unavailable: exact Nikon color" in positive["status"] + assert "exact_nikon_color" not in positive + assert "retained_builder_evidence" not in positive + assert not any(path.name.endswith("_positive.tif") for path in tmp_path.iterdir()) + assert not (tmp_path / ".negpy-stage3-replay").exists() + assert not (tmp_path / ".negpy-native-builder").exists() + assert not any(path.name.startswith(".negpy-frame-stage-") for path in tmp_path.iterdir()) + + def test_exact_positive_detects_path_swap_after_stable_read( + self, + fake_coolscanpy, + fake_repair_engine, + tmp_path, + monkeypatch, + ) -> None: + real_lstat = roll_service.os.lstat + positive_lstats = 0 + + def swap_before_final_identity_check(path, *args, **kwargs): + nonlocal positive_lstats + path_string = os.fspath(path) + if path_string.endswith("_positive.tif"): + positive_lstats += 1 + if positive_lstats == 2: + replacement = path_string + ".swapped" + Path(replacement).write_bytes(b"not the verified TIFF") + os.replace(replacement, path_string) + return real_lstat(path, *args, **kwargs) + + monkeypatch.setattr( + roll_service.os, + "lstat", + swap_before_final_identity_check, + ) + output = RollScanningService( + exact_color_builder=_ExactStage1Builder(), + exact_color_evaluator=_ExactColorEvaluator(), + ).write_frame( + _tier_frame(fake_coolscanpy), + str(tmp_path), + '{{ "%03d" % seq }}', + write_unrepaired=True, + write_repaired=True, + write_positive=True, + positive_mode="nikon-exact", + builder_receipt=_builder_receipt(), + ) + + assert positive_lstats >= 2 + assert output.rgb_path is not None and Path(output.rgb_path).is_file() + assert output.repaired_rgb_path is not None + assert output.positive_path is None + positive = self._receipt(output.receipt_path)["outputs"]["positive"] + assert "changed while it was verified" in positive["status"] + assert not any(path.name.endswith("_positive.tif") for path in tmp_path.iterdir()) + + @pytest.mark.skipif(not hasattr(os, "mkfifo"), reason="POSIX FIFO test") + def test_exact_positive_regular_to_fifo_swap_never_blocks( + self, + fake_coolscanpy, + fake_repair_engine, + tmp_path, + monkeypatch, + ) -> None: + real_open = roll_service.os.open + builder_receipt = _builder_receipt() + swapped = False + + def swap_before_open(path, flags, *args, **kwargs): + nonlocal swapped + path_string = os.fspath(path) + if not swapped and path_string.endswith("_positive.tif"): + swapped = True + os.unlink(path_string) + os.mkfifo(path_string) + return real_open(path, flags, *args, **kwargs) + + monkeypatch.setattr(roll_service.os, "open", swap_before_open) + started = time.monotonic() + output = RollScanningService( + exact_color_builder=_ExactStage1Builder(), + exact_color_evaluator=_ExactColorEvaluator(), + ).write_frame( + _tier_frame(fake_coolscanpy), + str(tmp_path), + '{{ "%03d" % seq }}', + write_unrepaired=True, + write_repaired=True, + write_positive=True, + positive_mode="nikon-exact", + builder_receipt=builder_receipt, + ) + + assert swapped is True + assert time.monotonic() - started < 1.0 + assert output.positive_path is None + assert not any(path.name.endswith("_positive.tif") for path in tmp_path.iterdir()) + + def test_omitted_c41_positive_mode_defaults_to_fail_closed_nikon_exact( + self, fake_coolscanpy, fake_repair_engine, tmp_path, monkeypatch + ) -> None: + monkeypatch.setattr( + roll_service.roll_positive, + "render_positive", + lambda *_args, **_kwargs: pytest.fail("approximate color must require explicit selection"), + ) + + output = RollScanningService().write_frame( + _tier_frame(fake_coolscanpy), + str(tmp_path), + '{{ "%03d" % seq }}', + write_unrepaired=False, + write_positive=True, + ) + + assert output.positive_path is None + entry = self._receipt(output.receipt_path)["outputs"]["positive"] + assert entry["written"] is False + assert entry["color_mode"] == "nikon-exact" + assert "frame has no native builder evidence" in entry["status"] + + def test_exact_nikon_positive_fails_closed_when_embedded_icc_identity_is_invalid( + self, fake_coolscanpy, fake_repair_engine, tmp_path, monkeypatch + ) -> None: + monkeypatch.setattr( + roll_service.roll_nikon_icc, + "nikon_adobe_rgb_profile", + MagicMock(side_effect=roll_service.roll_nikon_icc.NikonICCProfileError("profile hash mismatch")), + ) + + output = RollScanningService( + exact_color_builder=_ExactStage1Builder(), + exact_color_evaluator=_ExactColorEvaluator(), + ).write_frame( + _tier_frame(fake_coolscanpy), + str(tmp_path), + '{{ "%03d" % seq }}', + write_unrepaired=False, + write_positive=True, + builder_receipt=_builder_receipt(), + ) + + assert output.positive_path is None + entry = self._receipt(output.receipt_path)["outputs"]["positive"] + assert entry["written"] is False + assert entry["color_mode"] == "nikon-exact" + assert "profile hash mismatch" in entry["status"] + + def test_exact_nikon_positive_is_unavailable_without_portable_evaluator( + self, fake_coolscanpy, fake_repair_engine, tmp_path, monkeypatch + ) -> None: + builder_receipt = _builder_receipt() + monkeypatch.setattr( + roll_service.roll_positive, + "render_positive", + lambda *_args, **_kwargs: pytest.fail("approximate renderer must not satisfy exact mode"), + ) + + output = RollScanningService(exact_color_builder=_ExactStage1Builder()).write_frame( + _tier_frame(fake_coolscanpy), + str(tmp_path), + '{{ "%03d" % seq }}', + write_unrepaired=False, + write_positive=True, + positive_mode="nikon-exact", + builder_receipt=builder_receipt, + ) + + assert output.positive_path is None + entry = self._receipt(output.receipt_path)["outputs"]["positive"] + assert entry["written"] is False + assert entry["color_mode"] == "nikon-exact" + assert "verified portable CMS evaluator is not supplied" in entry["status"] + + def test_exact_nikon_positive_fails_closed_when_replay_evidence_cannot_be_retained( + self, fake_coolscanpy, fake_repair_engine, tmp_path, monkeypatch + ) -> None: + monkeypatch.setattr( + roll_service, + "_atomic_write_bytes", + MagicMock(side_effect=OSError("read-only evidence volume")), + ) + + output = RollScanningService( + exact_color_builder=_ExactStage1Builder(), + exact_color_evaluator=_ExactColorEvaluator(), + ).write_frame( + _tier_frame(fake_coolscanpy), + str(tmp_path), + '{{ "%03d" % seq }}', + write_unrepaired=False, + write_positive=True, + positive_mode="nikon-exact", + builder_receipt=_builder_receipt(), + ) + + assert output.positive_path is None + assert not (tmp_path / "011_positive.tif").exists() + entry = self._receipt(output.receipt_path)["outputs"]["positive"] + assert "cannot retain Stage-3 replay evidence" in entry["status"] + + def test_exact_nikon_positive_is_unavailable_without_stage1_builder(self, fake_coolscanpy, fake_repair_engine, tmp_path) -> None: + evaluator = _ExactColorEvaluator() + + output = RollScanningService(exact_color_evaluator=evaluator).write_frame( + _tier_frame(fake_coolscanpy), + str(tmp_path), + '{{ "%03d" % seq }}', + write_unrepaired=False, + write_positive=True, + positive_mode="nikon-exact", + builder_receipt=_builder_receipt(), + ) + + assert output.positive_path is None + assert evaluator.calls == [] + entry = self._receipt(output.receipt_path)["outputs"]["positive"] + assert "verified Stage-1 builder applicator is not supplied" in entry["status"] + + def test_exact_nikon_positive_is_unavailable_without_validated_builder_receipt( + self, fake_coolscanpy, fake_repair_engine, tmp_path + ) -> None: + evaluator = _ExactColorEvaluator() + + output = RollScanningService( + exact_color_builder=_ExactStage1Builder(), + exact_color_evaluator=evaluator, + ).write_frame( + _tier_frame(fake_coolscanpy), + str(tmp_path), + '{{ "%03d" % seq }}', + write_unrepaired=False, + write_positive=True, + positive_mode="nikon-exact", + ) + + assert output.positive_path is None + assert evaluator.calls == [] + entry = self._receipt(output.receipt_path)["outputs"]["positive"] + assert "validated Stage-3 builder receipt is not supplied" in entry["status"] + + def test_exact_nikon_positive_rejects_tampered_builder_receipt(self, fake_coolscanpy, fake_repair_engine, tmp_path) -> None: + evaluator = _ExactColorEvaluator() + receipt = _builder_receipt() + tampered = dataclasses.replace(receipt, payload=receipt.payload + b" ") + + output = RollScanningService( + exact_color_builder=_ExactStage1Builder(), + exact_color_evaluator=evaluator, + ).write_frame( + _tier_frame(fake_coolscanpy), + str(tmp_path), + '{{ "%03d" % seq }}', + write_unrepaired=False, + write_positive=True, + positive_mode="nikon-exact", + builder_receipt=tampered, + ) + + assert output.positive_path is None + assert evaluator.calls == [] + entry = self._receipt(output.receipt_path)["outputs"]["positive"] + assert "builder receipt payload does not match its SHA-256" in entry["status"] + + def test_exact_nikon_positive_rejects_malformed_cms_receipt(self, fake_coolscanpy, fake_repair_engine, tmp_path) -> None: + builder_receipt = _builder_receipt() + frame = _tier_frame(fake_coolscanpy) + valid_result = _ExactColorEvaluator().evaluate(frame.rgb, builder_receipt=builder_receipt) + malformed_payload = b"this is not a JSON receipt" + evaluator = MagicMock() + evaluator.evaluate.return_value = dataclasses.replace( + valid_result, + cms_receipt=_replace_cms_payload(valid_result.cms_receipt, malformed_payload), + ) + + output = RollScanningService( + exact_color_builder=_ExactStage1Builder(), + exact_color_evaluator=evaluator, + ).write_frame( + frame, + str(tmp_path), + '{{ "%03d" % seq }}', + write_unrepaired=False, + write_positive=True, + positive_mode="nikon-exact", + builder_receipt=builder_receipt, + ) + + assert output.positive_path is None + entry = self._receipt(output.receipt_path)["outputs"]["positive"] + assert "CMS receipt is not valid JSON" in entry["status"] + + def test_exact_nikon_positive_rejects_cms_receipt_input_mismatch(self, fake_coolscanpy, fake_repair_engine, tmp_path) -> None: + builder_receipt = _builder_receipt() + frame = _tier_frame(fake_coolscanpy) + valid_result = _ExactColorEvaluator().evaluate(frame.rgb, builder_receipt=builder_receipt) + cms = exact_color.receipt_payload(valid_result.cms_receipt) + cms["input_rgb_sha256"] = "0" * 64 + evaluator = MagicMock() + evaluator.evaluate.return_value = dataclasses.replace( + valid_result, + cms_receipt=_replace_cms_payload(valid_result.cms_receipt, cms), + ) + + output = RollScanningService( + exact_color_builder=_ExactStage1Builder(), + exact_color_evaluator=evaluator, + ).write_frame( + frame, + str(tmp_path), + '{{ "%03d" % seq }}', + write_unrepaired=False, + write_positive=True, + positive_mode="nikon-exact", + builder_receipt=builder_receipt, + ) + + assert output.positive_path is None + entry = self._receipt(output.receipt_path)["outputs"]["positive"] + assert "CMS receipt does not bind its builder, input, and output" in entry["status"] + + def test_exact_nikon_positive_rejects_self_attested_xor_evaluator_even_with_full_cms_contract( + self, fake_coolscanpy, fake_repair_engine, tmp_path + ) -> None: + builder_receipt = _builder_receipt() + frame = _tier_frame(fake_coolscanpy) + valid_result = _ExactColorEvaluator().evaluate(frame.rgb, builder_receipt=builder_receipt) + xor_output = np.bitwise_xor(frame.rgb, np.uint16(0xFFFF)) + xor_output_hash = exact_color.rgb16_content_sha256(xor_output) + forged_payload = production_cms_payload( + builder_receipt_sha256=builder_receipt.sha256, + input_rgb_sha256=valid_result.input_rgb_sha256, + output_rgb_sha256=xor_output_hash, + ) + evaluator = MagicMock() + evaluator.evaluate.return_value = dataclasses.replace( + valid_result, + rgb=xor_output, + output_rgb_sha256=xor_output_hash, + cms_receipt=_self_attested_cms_receipt(forged_payload), + ) + + output = RollScanningService( + exact_color_builder=_ExactStage1Builder(), + exact_color_evaluator=evaluator, + ).write_frame( + frame, + str(tmp_path), + '{{ "%03d" % seq }}', + write_unrepaired=False, + write_positive=True, + positive_mode="nikon-exact", + builder_receipt=builder_receipt, + ) + + assert output.positive_path is None + entry = self._receipt(output.receipt_path)["outputs"]["positive"] + assert "trusted portable CMS adapter" in entry["status"] + + @pytest.mark.parametrize( + "mutate", + [ + pytest.param(lambda cms: cms.update(kind="wrong"), id="kind"), + pytest.param(lambda cms: cms.update(version=True), id="version-type"), + pytest.param(lambda cms: cms.update(algorithm="unverified"), id="algorithm"), + pytest.param(lambda cms: cms["assets"].pop(next(iter(cms["assets"])), None), id="nine-assets"), + pytest.param(lambda cms: cms["oracle_source"].update(sha256="0" * 64), id="oracle-source"), + pytest.param(lambda cms: cms["validation"].update(mismatched_u16=1), id="validation"), + pytest.param(lambda cms: cms.update(scope="expanded"), id="scope"), + pytest.param(lambda cms: cms.update(stage_order=["stage2", "stage1"]), id="stage-order"), + pytest.param(lambda cms: cms.update(dll_free=False), id="dll-free"), + pytest.param(lambda cms: cms.update(upstream_builder_included=True), id="builder-scope"), + pytest.param(lambda cms: cms.update(chunk_pixels=0), id="chunk-size"), + ], + ) + def test_exact_nikon_positive_requires_every_cms_contract_field(self, fake_coolscanpy, fake_repair_engine, tmp_path, mutate) -> None: + builder_receipt = _builder_receipt() + frame = _tier_frame(fake_coolscanpy) + valid_result = _ExactColorEvaluator().evaluate(frame.rgb, builder_receipt=builder_receipt) + cms = exact_color.receipt_payload(valid_result.cms_receipt) + mutate(cms) + evaluator = MagicMock() + evaluator.evaluate.return_value = dataclasses.replace( + valid_result, + cms_receipt=_replace_cms_payload(valid_result.cms_receipt, cms), + ) + + output = RollScanningService( + exact_color_builder=_ExactStage1Builder(), + exact_color_evaluator=evaluator, + ).write_frame( + frame, + str(tmp_path), + '{{ "%03d" % seq }}', + write_unrepaired=False, + write_positive=True, + positive_mode="nikon-exact", + builder_receipt=builder_receipt, + ) + + assert output.positive_path is None + assert self._receipt(output.receipt_path)["outputs"]["positive"]["written"] is False + + def test_exact_nikon_positive_rejects_input_identity_mismatch(self, fake_coolscanpy, fake_repair_engine, tmp_path) -> None: + builder_receipt = _builder_receipt() + frame = _tier_frame(fake_coolscanpy) + valid_result = _ExactColorEvaluator().evaluate(frame.rgb, builder_receipt=builder_receipt) + evaluator = MagicMock() + evaluator.evaluate.return_value = dataclasses.replace( + valid_result, + input_rgb_sha256="0" * 64, + ) + + output = RollScanningService( + exact_color_builder=_ExactStage1Builder(), + exact_color_evaluator=evaluator, + ).write_frame( + frame, + str(tmp_path), + '{{ "%03d" % seq }}', + write_unrepaired=False, + write_positive=True, + positive_mode="nikon-exact", + builder_receipt=builder_receipt, + ) + + assert output.positive_path is None + entry = self._receipt(output.receipt_path)["outputs"]["positive"] + assert "input hash does not match the Stage-1 input content" in entry["status"] + + def test_exact_nikon_positive_rejects_unbound_builder_output(self, fake_coolscanpy, fake_repair_engine, tmp_path) -> None: + builder_receipt = _builder_receipt() + honest_builder = _ExactStage1Builder() + builder = MagicMock() + + def unbound(rgb, *, builder_receipt): + valid = honest_builder.apply(rgb, builder_receipt=builder_receipt) + return dataclasses.replace(valid, stage1_input_rgb_sha256="0" * 64) + + builder.apply.side_effect = unbound + evaluator = _ExactColorEvaluator() + + output = RollScanningService( + exact_color_builder=builder, + exact_color_evaluator=evaluator, + ).write_frame( + _tier_frame(fake_coolscanpy), + str(tmp_path), + '{{ "%03d" % seq }}', + write_unrepaired=False, + write_positive=True, + positive_mode="nikon-exact", + builder_receipt=builder_receipt, + ) + + assert output.positive_path is None + assert evaluator.calls == [] + entry = self._receipt(output.receipt_path)["outputs"]["positive"] + assert "builder output hash does not match the Stage-1 input content" in entry["status"] + + def test_exact_nikon_positive_rejects_unbound_output_hash(self, fake_coolscanpy, fake_repair_engine, tmp_path) -> None: + builder_receipt = _builder_receipt() + frame = _tier_frame(fake_coolscanpy) + valid_result = _ExactColorEvaluator().evaluate(frame.rgb, builder_receipt=builder_receipt) + evaluator = MagicMock() + evaluator.evaluate.return_value = dataclasses.replace( + valid_result, + output_rgb_sha256="f" * 64, + ) + + output = RollScanningService( + exact_color_builder=_ExactStage1Builder(), + exact_color_evaluator=evaluator, + ).write_frame( + frame, + str(tmp_path), + '{{ "%03d" % seq }}', + write_unrepaired=False, + write_positive=True, + positive_mode="nikon-exact", + builder_receipt=builder_receipt, + ) + + assert output.positive_path is None + assert not (tmp_path / "011_positive.tif").exists() + entry = self._receipt(output.receipt_path)["outputs"]["positive"] + assert "output hash does not match the returned RGB content" in entry["status"] + + def test_repaired_naming_does_not_collide_with_unrepaired(self, fake_coolscanpy, fake_repair_engine, tmp_path) -> None: + frame = _tier_frame(fake_coolscanpy) + service = RollScanningService() + + output = service.write_frame(frame, str(tmp_path), '{{ "%03d" % seq }}', write_unrepaired=True, write_repaired=True) + + paths = {output.rgb_path, output.ir_path, output.repaired_rgb_path, output.repaired_ir_path} + assert len(paths) == 4 # four distinct files + assert all(os.path.exists(p) for p in paths) + + def test_repaired_receipt_records_engine_provenance(self, fake_coolscanpy, fake_repair_engine, tmp_path) -> None: + frame = _tier_frame(fake_coolscanpy) + service = RollScanningService() + + output = service.write_frame( + frame, str(tmp_path), '{{ "%03d" % seq }}', write_unrepaired=False, write_repaired=True, repair_mode="hybrid" + ) + + payload = self._receipt(output.receipt_path) + entry = payload["outputs"]["repaired"] + assert entry["written"] is True + assert entry["engine"] == "test-repair-engine" + assert entry["engine_version"] == "0.0.1-test" + assert entry["mode_requested"] == "hybrid" + assert entry["mode_resolved"] == "exact" + assert entry["degraded"] is True + assert "no hybrid runtime" in entry["reason"] + assert entry["rgb_path"] == output.repaired_rgb_path + + def test_repair_engine_failure_degrades_without_losing_unrepaired(self, fake_coolscanpy, fake_repair_engine, tmp_path) -> None: + fake_repair_engine.raise_error = RuntimeError("inpainting model unavailable") + frame = _tier_frame(fake_coolscanpy) + service = RollScanningService() + + output = service.write_frame( + frame, str(tmp_path), '{{ "%03d" % seq }}', write_unrepaired=True, write_repaired=True, write_positive=True + ) + + assert output.rgb_path is not None and os.path.exists(output.rgb_path) + assert output.repaired_rgb_path is None + assert output.positive_path is None + payload = self._receipt(output.receipt_path) + assert payload["outputs"]["unrepaired"]["written"] is True + assert "repair failed" in payload["outputs"]["repaired"]["status"] + assert "inpainting model unavailable" in payload["outputs"]["repaired"]["status"] + assert "Tier 2" in payload["outputs"]["positive"]["status"] + + def test_frame_without_infrared_degrades_repaired_and_positive(self, fake_coolscanpy, fake_repair_engine, tmp_path) -> None: + frame = _tier_frame(fake_coolscanpy, ir=False) + service = RollScanningService() + + output = service.write_frame( + frame, str(tmp_path), '{{ "%03d" % seq }}', write_unrepaired=True, write_repaired=True, write_positive=True + ) + + assert fake_repair_engine.calls == [] # never even attempted + assert output.repaired_rgb_path is None + assert output.positive_path is None + payload = self._receipt(output.receipt_path) + assert "no infrared plane" in payload["outputs"]["repaired"]["status"] + + def test_invalid_repair_mode_fails_before_writing_or_repair(self, fake_coolscanpy, fake_repair_engine, tmp_path) -> None: + frame = _tier_frame(fake_coolscanpy) + service = RollScanningService() + + with pytest.raises(ValueError, match="unknown repair mode"): + service.write_frame( + frame, + str(tmp_path), + '{{ "%03d" % seq }}', + write_repaired=True, + repair_mode="bogus-mode", + ) + + assert fake_repair_engine.calls == [] + assert list(tmp_path.iterdir()) == [] + + def test_positive_requested_without_write_repaired_still_repairs_in_memory(self, fake_coolscanpy, fake_repair_engine, tmp_path) -> None: + frame = _tier_frame(fake_coolscanpy) + service = RollScanningService() + + output = service.write_frame( + frame, + str(tmp_path), + '{{ "%03d" % seq }}', + write_unrepaired=False, + write_positive=True, + positive_mode="negpy-approximate", + ) + + assert len(fake_repair_engine.calls) == 1 # repair ran to feed Tier 3... + assert output.repaired_rgb_path is None # ...but Tier 2 itself was never written + assert output.positive_path is not None + payload = self._receipt(output.receipt_path) + entry = payload["outputs"]["repaired"] + assert entry["written"] is False + assert entry["status"] == "not selected (computed in memory for the positive)" + assert entry["engine"] == "test-repair-engine" # still recorded even though unwritten + + # -- Tier 3 ----------------------------------------------------------------- + + def test_positive_is_written_and_derived_from_repaired_not_raw_rgb( + self, fake_coolscanpy, fake_repair_engine, tmp_path, monkeypatch + ) -> None: + fake_repair_engine.transform = lambda rgb: np.clip(rgb.astype(np.int32) + 5000, 0, 65535).astype(np.uint16) + frame = _tier_frame(fake_coolscanpy) + service = RollScanningService() + + from negpy.services.roll import positive as roll_positive_module + + captured = {} + real_render = roll_positive_module.render_positive + + def _spy(rgb_u16, *, processor): + captured["rgb_u16"] = rgb_u16 + return real_render(rgb_u16, processor=processor) + + monkeypatch.setattr(roll_positive_module, "render_positive", _spy) + + output = service.write_frame( + frame, + str(tmp_path), + '{{ "%03d" % seq }}', + write_unrepaired=False, + write_positive=True, + positive_mode="negpy-approximate", + ) + + np.testing.assert_array_equal(captured["rgb_u16"], fake_repair_engine.transform(frame.rgb)) + assert not np.array_equal(captured["rgb_u16"], frame.rgb) + assert output.positive_path is not None + readback = tifffile.imread(output.positive_path) + assert readback.shape == frame.rgb.shape + assert readback.dtype == np.uint16 + + def test_positive_receipt_records_inversion_and_repair_provenance(self, fake_coolscanpy, fake_repair_engine, tmp_path) -> None: + frame = _tier_frame(fake_coolscanpy) + service = RollScanningService() + + output = service.write_frame( + frame, + str(tmp_path), + '{{ "%03d" % seq }}', + write_unrepaired=False, + write_positive=True, + positive_mode="negpy-approximate", + ) + + payload = self._receipt(output.receipt_path) + entry = payload["outputs"]["positive"] + assert entry["written"] is True + assert entry["rgb_path"] == output.positive_path + assert entry["color_mode"] == "negpy-approximate" + assert entry["exact_nikon_color"] is False + assert entry["inversion_path"] == "negpy.services.rendering.image_processor.ImageProcessor.run_pipeline" + assert entry["render_intent"] == "print" + assert entry["process_mode"] == "C41" + assert entry["auto_exposure"] is True + assert entry["negpy_version"] + assert entry["repair_engine"] == "test-repair-engine" + assert entry["repair_engine_version"] == "0.0.1-test" + assert entry["repair_mode"] == "exact" + assert "icc_profile" not in entry + with tifffile.TiffFile(output.positive_path) as approximate_tiff: + assert 34675 not in approximate_tiff.pages[0].tags + + def test_positive_filename_has_no_infrared_companion(self, fake_coolscanpy, fake_repair_engine, tmp_path) -> None: + frame = _tier_frame(fake_coolscanpy) + service = RollScanningService() + + output = service.write_frame( + frame, + str(tmp_path), + '{{ "%03d" % seq }}', + write_unrepaired=False, + write_positive=True, + positive_mode="negpy-approximate", + ) + + assert output.positive_path is not None + assert output.positive_path.endswith("_positive.tif") + assert os.path.exists(output.positive_path) + assert not any(name.endswith("_positive_IR.tif") for name in os.listdir(str(tmp_path))) + + def test_inversion_unavailable_degrades_positive_but_keeps_unrepaired_and_repaired( + self, fake_coolscanpy, fake_repair_engine, tmp_path, monkeypatch + ) -> None: + from negpy.services.roll import positive as roll_positive_module + + monkeypatch.setattr(roll_positive_module, "available", lambda: False) + frame = _tier_frame(fake_coolscanpy) + service = RollScanningService() + + output = service.write_frame( + frame, + str(tmp_path), + '{{ "%03d" % seq }}', + write_unrepaired=True, + write_repaired=True, + write_positive=True, + positive_mode="negpy-approximate", + ) + + assert output.rgb_path is not None and os.path.exists(output.rgb_path) + assert output.repaired_rgb_path is not None and os.path.exists(output.repaired_rgb_path) + assert output.positive_path is None + payload = self._receipt(output.receipt_path) + assert payload["outputs"]["unrepaired"]["written"] is True + assert payload["outputs"]["repaired"]["written"] is True + assert payload["outputs"]["positive"] == { + "written": False, + "status": "unavailable: inversion path not available", + "color_mode": "negpy-approximate", + } + + def test_inversion_failure_degrades_positive_only(self, fake_coolscanpy, fake_repair_engine, tmp_path, monkeypatch) -> None: + from negpy.services.roll import positive as roll_positive_module + + def _boom(rgb_u16, *, processor): + raise RuntimeError("no CPU render backend available") + + monkeypatch.setattr(roll_positive_module, "render_positive", _boom) + frame = _tier_frame(fake_coolscanpy) + service = RollScanningService() + + output = service.write_frame( + frame, + str(tmp_path), + '{{ "%03d" % seq }}', + write_unrepaired=True, + write_repaired=True, + write_positive=True, + positive_mode="negpy-approximate", + ) + + assert output.rgb_path is not None and os.path.exists(output.rgb_path) + assert output.repaired_rgb_path is not None and os.path.exists(output.repaired_rgb_path) + assert output.positive_path is None + payload = self._receipt(output.receipt_path) + assert payload["outputs"]["repaired"]["written"] is True + assert "inversion failed" in payload["outputs"]["positive"]["status"] + assert "no CPU render backend available" in payload["outputs"]["positive"]["status"] + + # -- all three together, and receipt backward-compatibility ----------------- + + def test_all_three_tiers_together(self, fake_coolscanpy, fake_repair_engine, tmp_path) -> None: + frame = _tier_frame(fake_coolscanpy) + service = RollScanningService() + + output = service.write_frame( + frame, + str(tmp_path), + '{{ "%03d" % seq }}', + write_unrepaired=True, + write_repaired=True, + write_positive=True, + positive_mode="negpy-approximate", + ) + + for path in (output.rgb_path, output.ir_path, output.repaired_rgb_path, output.repaired_ir_path, output.positive_path): + assert path is not None and os.path.exists(path) + assert len({output.rgb_path, output.ir_path, output.repaired_rgb_path, output.repaired_ir_path, output.positive_path}) == 5 + for untagged_path in (output.rgb_path, output.repaired_rgb_path, output.positive_path): + with tifffile.TiffFile(untagged_path) as untagged_tiff: + assert 34675 not in untagged_tiff.pages[0].tags + payload = self._receipt(output.receipt_path) + assert all(payload["outputs"][tier]["written"] for tier in ("unrepaired", "repaired", "positive")) + + def test_receipt_keeps_the_original_scan_receipt_fields_alongside_outputs(self, fake_coolscanpy, tmp_path) -> None: + """`outputs` must be additive -- write_frame's pre-tiering callers (and + TestWriteFrame's tests above) read coolscanpy's own receipt fields + (slot, dpi, transport_smear_verdict, ...) at the JSON root.""" + frame = _tier_frame(fake_coolscanpy, slot=42) + service = RollScanningService() + + output = service.write_frame(frame, str(tmp_path), '{{ "%03d" % seq }}') + + payload = self._receipt(output.receipt_path) + assert payload["slot"] == 42 + assert payload["dpi"] == 4000 + assert payload["transport_smear_verdict"] == "clean" + assert "outputs" in payload diff --git a/tests/roll/test_sidebar.py b/tests/roll/test_sidebar.py new file mode 100644 index 00000000..0cf03015 --- /dev/null +++ b/tests/roll/test_sidebar.py @@ -0,0 +1,869 @@ +"""Smoke and behavior tests for CoolscanRollSidebar. Mirrors +test_scanlight_sidebar.py: build the widget against a mock controller, then +drive it either through real widgets (clicks, combo selection) or by +calling its private `_on_*` handlers directly to simulate what the (mocked, +never-really-running) controller/worker would deliver. +""" + +from __future__ import annotations + +import sys +from types import SimpleNamespace + +import numpy as np +from PyQt6.QtWidgets import QApplication +from unittest.mock import MagicMock + +from negpy.desktop.view.sidebar.coolscan_roll import ( + CoolscanRollSidebar, + QMessageBox, + _thumbnail_rgb8, +) +from negpy.infrastructure.roll import coolscanpy_roll + +if not QApplication.instance(): + _app = QApplication(sys.argv) + + +def _sidebar(): + ctrl = MagicMock() + ctrl.session.repo.get_global_setting.return_value = {} + return CoolscanRollSidebar(ctrl) + + +def _thumb(fake_coolscanpy, slot, *, needs_approval=False, spacing_offset=0): + return fake_coolscanpy.Thumbnail( + slot=slot, + image=np.zeros((4, 4, 3), dtype=np.uint8), + boundary_rows=(0, 4), + spacing_offset=spacing_offset, + needs_approval=needs_approval, + ) + + +def _pick_device(w, device_id="ls5000-usb-001", label="Nikon LS-5000"): + w.device_combo.clear() + w.device_combo.addItem(label, device_id) + w.device_combo.setCurrentIndex(0) + + +class TestBuildsAndGating: + def test_sidebar_builds_with_all_controls(self) -> None: + w = _sidebar() + for attr in ( + "preview_btn", + "scan_btn", + "safe_stop_btn", + "eject_btn", + "select_all_btn", + "clear_selection_btn", + "device_combo", + "refresh_btn", + "folder_edit", + "pattern_edit", + "contact_sheet", + "offset_spin", + "offset_apply_btn", + "approve_btn", + "gate_hint", + "status_label", + "hybrid_synthesis_limit_spin", + "hybrid_guidance", + "preview_workspace", + "preview_display_combo", + "open_preview_workspace_btn", + ): + assert hasattr(w, attr), attr + assert w.contact_sheet is w.preview_workspace.contact_sheet + assert w.contact_sheet.minimumHeight() >= 360 + assert w.contact_sheet.iconSize().width() >= 200 + + def test_missing_coolscanpy_disables_preview_and_shows_hint(self, monkeypatch) -> None: + # The optional group may be installed by this test run, so exercise + # the sidebar's unavailable branch at its adapter seam instead of + # assuming an interpreter-level package state. + monkeypatch.setattr(coolscanpy_roll, "available", lambda: False) + w = _sidebar() + assert w.preview_btn.isEnabled() is False + assert "install coolscanpy" in w.gate_hint.text() + assert not w._setup_hint.isHidden() + + def test_device_selected_still_blocks_scan_until_folder_and_slots(self, fake_coolscanpy) -> None: + w = _sidebar() + _pick_device(w) + assert w.preview_btn.isEnabled() is True # coolscanpy available (faked) + device picked + assert w.scan_btn.isEnabled() is False + missing = w._missing_for_scan() + assert "select at least one slot" in missing + assert "choose an output folder" in missing + + def test_scan_enabled_once_folder_and_slot_selected(self, fake_coolscanpy, tmp_path) -> None: + w = _sidebar() + _pick_device(w) + w.folder_edit.setText(str(tmp_path)) + w._on_preview_ready([_thumb(fake_coolscanpy, 1)]) + w.contact_sheet.item(0).setSelected(True) + + assert w._missing_for_scan() == [] + assert w.scan_btn.isEnabled() is True + + def test_approve_button_hidden_without_a_selection(self) -> None: + w = _sidebar() + assert w.approve_btn.isVisible() is False + + +class TestPreview: + def test_positive_preview_auto_tones_scanner_linear_negative(self) -> None: + levels = np.linspace(0.0, 1.0, 64, dtype=np.float64).reshape(8, 8, 1) + black = np.array([500.0, 350.0, 250.0]).reshape(1, 1, 3) + film_base = np.array([35_000.0, 27_000.0, 22_000.0]).reshape(1, 1, 3) + scanner_linear = np.rint(black + levels * (film_base - black)).astype( + np.uint16 + ) + original = scanner_linear.copy() + + positive = _thumbnail_rgb8(scanner_linear, positive=True) + + assert positive.dtype == np.uint8 + assert positive.shape == scanner_linear.shape + assert int(np.percentile(positive, 5)) < 80 + assert int(np.percentile(positive, 95)) > 200 + np.testing.assert_array_equal(scanner_linear, original) + + def test_raw_preview_ignores_one_hot_pixel(self) -> None: + scanner_linear = np.full((20, 20, 3), 16_000, dtype=np.uint16) + scanner_linear[0, 0] = 65_535 + + raw = _thumbnail_rgb8(scanner_linear, positive=False) + + assert int(np.median(raw)) > 240 + assert raw[0, 0].tolist() == [255, 255, 255] + + def test_preview_click_starts_request_for_selected_device(self, fake_coolscanpy) -> None: + w = _sidebar() + _pick_device(w, device_id="dev-42") + + w._on_preview_clicked() + + from negpy.desktop.workers.roll_worker import RollPreviewRequest + + req = w.controller.start_coolscan_roll_preview.call_args[0][0] + assert isinstance(req, RollPreviewRequest) + assert req.device_id == "dev-42" + assert w._preview_pending is True + assert w.eject_btn.isEnabled() is False + + def test_preview_click_without_a_device_is_a_noop(self) -> None: + w = _sidebar() + w._on_preview_clicked() + w.controller.start_coolscan_roll_preview.assert_not_called() + + def test_workspace_buttons_switch_between_roll_preview_and_editor(self) -> None: + w = _sidebar() + opened: list[bool] = [] + closed: list[bool] = [] + w.workspace_requested.connect(lambda: opened.append(True)) + w.preview_workspace.back_requested.connect(lambda: closed.append(True)) + + w.open_preview_workspace_btn.click() + w.preview_workspace.back_btn.click() + + assert opened == [True] + assert closed == [True] + + def test_preview_ready_populates_the_contact_sheet(self, fake_coolscanpy) -> None: + w = _sidebar() + thumbs = [_thumb(fake_coolscanpy, s) for s in (2, 1, 3)] + + w._on_preview_ready(thumbs) + + assert w.contact_sheet.count() == 3 + assert sorted(w._thumbnails.keys()) == [1, 2, 3] + # the widget lists slots in ascending order regardless of preview()'s return order + from negpy.desktop.view.sidebar.coolscan_roll import _SLOT_ROLE + + listed = [w.contact_sheet.item(i).data(_SLOT_ROLE) for i in range(w.contact_sheet.count())] + assert listed == [1, 2, 3] + + def test_display_toggle_updates_icons_without_mutating_capture_data(self, fake_coolscanpy) -> None: + w = _sidebar() + image = np.zeros((8, 12, 3), dtype=np.uint8) + image[:, :6, :] = (10, 20, 30) + image[:, 6:, :] = (180, 190, 200) + original = image.copy() + + w._on_preview_ready( + [ + fake_coolscanpy.Thumbnail( + slot=1, + image=image, + boundary_rows=(0, 8), + spacing_offset=0, + needs_approval=False, + ) + ] + ) + + assert w.preview_display_combo.currentData() is True + positive_key = w.contact_sheet.item(0).icon().cacheKey() + w.preview_display_combo.setCurrentIndex(1) + raw_key = w.contact_sheet.item(0).icon().cacheKey() + w.preview_display_combo.setCurrentIndex(0) + positive_again_key = w.contact_sheet.item(0).icon().cacheKey() + + assert positive_key != raw_key + assert raw_key != positive_again_key + np.testing.assert_array_equal(w._thumbnails[1].image, original) + w.controller.start_coolscan_roll_preview.assert_not_called() + w.controller.start_roll_scan.assert_not_called() + + def test_needs_approval_slot_is_marked_and_shows_approve_button(self, fake_coolscanpy) -> None: + w = _sidebar() + w._on_preview_ready([_thumb(fake_coolscanpy, 5, needs_approval=True)]) + + item = w.contact_sheet.item(0) + assert "⚠" in item.text() + + w.contact_sheet.setCurrentItem(item) + assert not w.approve_btn.isHidden() + + def test_selecting_a_slot_seeds_the_offset_spinner(self, fake_coolscanpy) -> None: + w = _sidebar() + w._on_preview_ready([_thumb(fake_coolscanpy, 7, spacing_offset=-15)]) + + w.contact_sheet.setCurrentItem(w.contact_sheet.item(0)) + + assert w.slot_label.text() == "7" + assert w.offset_spin.value() == -15 + + def test_new_preview_clears_the_old_contact_sheet(self, fake_coolscanpy) -> None: + w = _sidebar() + w._on_preview_ready([_thumb(fake_coolscanpy, 1)]) + assert w.contact_sheet.count() == 1 + + w._clear_contact_sheet() # what _on_preview_clicked does before requesting a fresh preview + w._on_preview_ready([_thumb(fake_coolscanpy, 9)]) + + assert w.contact_sheet.count() == 1 + assert list(w._thumbnails.keys()) == [9] + + def test_select_all_frames_prepares_a_full_roll_selection( + self, + fake_coolscanpy, + tmp_path, + ) -> None: + w = _sidebar() + _pick_device(w) + w.folder_edit.setText(str(tmp_path)) + w._on_preview_ready([_thumb(fake_coolscanpy, slot) for slot in range(1, 37)]) + + w.select_all_btn.click() + + assert w._selected_slots() == list(range(1, 37)) + assert w.scan_btn.isEnabled() is True + assert w.clear_selection_btn.isEnabled() is True + + +class TestEject: + def test_preview_in_flight_blocks_eject(self, fake_coolscanpy, monkeypatch) -> None: + w = _sidebar() + _pick_device(w, device_id="usb:2:7") + w._on_preview_clicked() + monkeypatch.setattr( + QMessageBox, + "question", + lambda *args, **kwargs: QMessageBox.StandardButton.Yes, + ) + + w._on_eject_clicked() + + w.controller.eject_roll.assert_not_called() + assert w._preview_pending is True + assert w.eject_btn.isEnabled() is False + + def test_confirmation_ejects_selected_direct_usb_device( + self, + fake_coolscanpy, + monkeypatch, + ) -> None: + w = _sidebar() + _pick_device(w, device_id="usb:2:7") + w._on_preview_ready([_thumb(fake_coolscanpy, 1)]) + monkeypatch.setattr( + QMessageBox, + "question", + lambda *args, **kwargs: QMessageBox.StandardButton.Yes, + ) + + w._on_eject_clicked() + + w.controller.eject_roll.assert_called_once_with("usb:2:7") + assert w.contact_sheet.count() == 0 + assert w.eject_btn.isEnabled() is False + assert w.preview_btn.isEnabled() is False + assert "Ejecting" in w.status_label.text() + assert "do not press eject again" in w.status_label.text().lower() + + def test_confirmation_cancel_does_not_eject(self, monkeypatch) -> None: + w = _sidebar() + _pick_device(w, device_id="usb:2:7") + monkeypatch.setattr( + QMessageBox, + "question", + lambda *args, **kwargs: QMessageBox.StandardButton.No, + ) + + w._on_eject_clicked() + + w.controller.eject_roll.assert_not_called() + assert w.eject_btn.isEnabled() is True + + def test_success_clears_registration_and_latches_against_second_eject( + self, + fake_coolscanpy, + ) -> None: + w = _sidebar() + _pick_device(w, device_id="usb:2:7") + w._on_preview_ready([_thumb(fake_coolscanpy, 1)]) + w._eject_pending = True + + w._on_ejected(True) + + assert w.contact_sheet.count() == 0 + assert w.eject_btn.isEnabled() is False + assert "Eject started" in w.status_label.text() + + def test_late_preview_after_eject_cannot_restore_registration( + self, + fake_coolscanpy, + tmp_path, + ) -> None: + w = _sidebar() + _pick_device(w, device_id="usb:2:7") + w.folder_edit.setText(str(tmp_path)) + w._eject_latched = True + + w._on_preview_ready([_thumb(fake_coolscanpy, 1)]) + w.contact_sheet.selectAll() + w._on_scan_clicked() + + assert w.contact_sheet.count() == 0 + assert w._thumbnails == {} + assert w.select_all_btn.isEnabled() is False + assert w.scan_btn.isEnabled() is False + w.controller.start_roll_scan.assert_not_called() + + def test_fresh_preview_after_refeed_reestablishes_registration( + self, + fake_coolscanpy, + ) -> None: + w = _sidebar() + _pick_device(w, device_id="usb:2:7") + w._eject_latched = True + w._apply_gating() + + w._on_preview_clicked() + w._on_preview_ready([_thumb(fake_coolscanpy, 1)]) + + assert w._preview_pending is False + assert w._eject_latched is False + assert w.contact_sheet.count() == 1 + assert w.eject_btn.isEnabled() is True + + def test_uncertain_failure_blocks_preview_and_retry(self) -> None: + w = _sidebar() + _pick_device(w, device_id="usb:2:7") + w._eject_pending = True + + w._on_eject_error("transport status unknown") + + assert w.eject_btn.isEnabled() is False + assert w.preview_btn.isEnabled() is False + assert "do not retry" in w.status_label.text() + assert "physically checking" in w._missing_for_preview()[-1] + + +class TestSpacingAndApproval: + def test_apply_offset_forwards_the_spinner_value(self, fake_coolscanpy) -> None: + w = _sidebar() + w._on_preview_ready([_thumb(fake_coolscanpy, 4)]) + w.contact_sheet.setCurrentItem(w.contact_sheet.item(0)) + w.offset_spin.setValue(-30) + + w._on_apply_offset() + + w.controller.set_roll_spacing_offset.assert_called_once_with(4, -30) + + def test_spacing_offset_set_updates_local_thumbnail_state(self, fake_coolscanpy) -> None: + w = _sidebar() + w._on_preview_ready([_thumb(fake_coolscanpy, 4, spacing_offset=0)]) + + w._on_spacing_offset_set(4, -8) + + assert w._thumbnails[4].spacing_offset == -8 + + def test_approve_click_forwards_the_selected_slot(self, fake_coolscanpy) -> None: + w = _sidebar() + w._on_preview_ready([_thumb(fake_coolscanpy, 6, needs_approval=True)]) + w.contact_sheet.setCurrentItem(w.contact_sheet.item(0)) + + w._on_approve() + + w.controller.approve_roll_slot.assert_called_once_with(6) + + def test_approved_clears_the_warning_marker(self, fake_coolscanpy) -> None: + w = _sidebar() + w._on_preview_ready([_thumb(fake_coolscanpy, 6, needs_approval=True)]) + w.contact_sheet.setCurrentItem(w.contact_sheet.item(0)) + assert not w.approve_btn.isHidden() + + w._on_approved(6) + + assert w._thumbnails[6].needs_approval is False + assert "⚠" not in w.contact_sheet.item(0).text() + assert w.approve_btn.isHidden() + + +class TestScanning: + def test_scan_click_builds_request_from_selection_and_settings(self, fake_coolscanpy, tmp_path) -> None: + w = _sidebar() + _pick_device(w, device_id="dev-1") + w.folder_edit.setText(str(tmp_path)) + w.pattern_edit.setText('{{ "%03d" % seq }}') + w._on_preview_ready([_thumb(fake_coolscanpy, 1), _thumb(fake_coolscanpy, 2)]) + w.contact_sheet.item(0).setSelected(True) + w.contact_sheet.item(1).setSelected(True) + + w._on_scan_clicked() + + from negpy.desktop.workers.roll_worker import RollBatchScanRequest + + req = w.controller.start_roll_scan.call_args[0][0] + assert isinstance(req, RollBatchScanRequest) + assert req.device_id == "dev-1" + assert req.slots == (1, 2) + assert req.output_folder == str(tmp_path) + assert req.hybrid_synthesis_limit_percent == 10.0 + assert w._scanning is True + + def test_scan_click_without_a_selected_slot_is_a_noop(self, fake_coolscanpy, tmp_path) -> None: + w = _sidebar() + _pick_device(w) + w.folder_edit.setText(str(tmp_path)) + w._on_scan_clicked() + w.controller.start_roll_scan.assert_not_called() + + def test_finished_clears_scanning_state(self, fake_coolscanpy) -> None: + w = _sidebar() + w.set_scanning(True) + w._on_finished([]) + assert w._scanning is False + assert "Scanned 0" in w.status_label.text() + + def test_finished_reports_hybrid_to_exact_degradation_as_an_issue( + self, + fake_coolscanpy, + tmp_path, + ) -> None: + from negpy.desktop.workers.roll_worker import RollBatchScanRequest + + w = _sidebar() + w._active_scan_request = RollBatchScanRequest( + device_id="ls5000-usb-001", + slots=(1,), + output_folder=str(tmp_path), + filename_pattern='{{ "%03d" % seq }}', + write_unrepaired=True, + write_repaired=True, + write_positive=False, + repair_mode="hybrid", + ) + w.set_scanning(True) + + w._on_finished( + [ + SimpleNamespace( + slot=1, + rgb_path="001.tif", + repaired_rgb_path="001_repaired.tif", + positive_path=None, + native_synthesis_mask_path=None, + hybrid_receipt_path=None, + ) + ] + ) + + assert w._scanning is False + assert "Completed with issues" in w.status_label.text() + assert "Hybrid repair degraded or unavailable" in w.status_label.text() + assert "Scanned 1 frame" not in w.status_label.text() + + def test_finished_reports_unavailable_requested_exact_positive_as_an_issue( + self, + fake_coolscanpy, + tmp_path, + ) -> None: + from negpy.desktop.workers.roll_worker import RollBatchScanRequest + + w = _sidebar() + w._active_scan_request = RollBatchScanRequest( + device_id="ls5000-usb-001", + slots=(1,), + output_folder=str(tmp_path), + filename_pattern='{{ "%03d" % seq }}', + write_unrepaired=True, + write_repaired=False, + write_positive=True, + repair_mode="exact", + positive_mode="nikon-exact", + ) + w.set_scanning(True) + + w._on_finished( + [ + SimpleNamespace( + slot=1, + rgb_path="001.tif", + repaired_rgb_path=None, + positive_path=None, + native_synthesis_mask_path=None, + hybrid_receipt_path=None, + ) + ] + ) + + assert "Completed with issues" in w.status_label.text() + assert "Nikon exact positive unavailable" in w.status_label.text() + assert "Scanned 1 frame" not in w.status_label.text() + + def test_finished_reports_missing_requested_slot_as_an_issue( + self, + fake_coolscanpy, + tmp_path, + ) -> None: + from negpy.desktop.workers.roll_worker import RollBatchScanRequest + + w = _sidebar() + w._active_scan_request = RollBatchScanRequest( + device_id="ls5000-usb-001", + slots=(1, 2), + output_folder=str(tmp_path), + filename_pattern='{{ "%03d" % seq }}', + write_unrepaired=True, + write_repaired=False, + write_positive=False, + repair_mode="exact", + ) + + w._on_finished( + [ + SimpleNamespace( + slot=1, + rgb_path="001.tif", + repaired_rgb_path=None, + positive_path=None, + native_synthesis_mask_path=None, + hybrid_receipt_path=None, + ) + ] + ) + + assert "Completed with issues" in w.status_label.text() + assert "slot(s) 2 did not complete" in w.status_label.text() + assert "Scanned 1 frame" not in w.status_label.text() + + def test_finished_reports_each_missing_requested_file_tier_as_an_issue( + self, + fake_coolscanpy, + tmp_path, + ) -> None: + from negpy.desktop.workers.roll_worker import RollBatchScanRequest + + w = _sidebar() + w._active_scan_request = RollBatchScanRequest( + device_id="ls5000-usb-001", + slots=(1,), + output_folder=str(tmp_path), + filename_pattern='{{ "%03d" % seq }}', + write_unrepaired=True, + write_repaired=True, + write_positive=False, + repair_mode="exact", + ) + + w._on_finished( + [ + SimpleNamespace( + slot=1, + rgb_path=None, + repaired_rgb_path=None, + positive_path=None, + native_synthesis_mask_path=None, + hybrid_receipt_path=None, + ) + ] + ) + + status = w.status_label.text() + assert "Completed with issues" in status + assert "requested unrepaired output unavailable" in status + assert "requested repaired output unavailable" in status + assert "Scanned 1 frame" not in status + + def test_finished_reports_full_success_only_when_hybrid_and_exact_exist( + self, + fake_coolscanpy, + tmp_path, + ) -> None: + from negpy.desktop.workers.roll_worker import RollBatchScanRequest + + w = _sidebar() + w._active_scan_request = RollBatchScanRequest( + device_id="ls5000-usb-001", + slots=(1,), + output_folder=str(tmp_path), + filename_pattern='{{ "%03d" % seq }}', + write_unrepaired=True, + write_repaired=True, + write_positive=True, + repair_mode="hybrid", + positive_mode="nikon-exact", + ) + w.set_scanning(True) + + w._on_finished( + [ + SimpleNamespace( + slot=1, + rgb_path="001.tif", + repaired_rgb_path="001_repaired.tif", + positive_path="001_positive.tif", + native_synthesis_mask_path="native-mask.png", + hybrid_receipt_path="hybrid-receipt.json", + ) + ] + ) + + assert w.status_label.text() == "Scanned 1 frame(s)." + + def test_safe_stop_disables_itself_and_forwards_to_controller(self, fake_coolscanpy) -> None: + w = _sidebar() + w.set_scanning(True) + + w._on_safe_stop_clicked() + + w.controller.roll_safe_stop.assert_called_once() + assert w.safe_stop_btn.isEnabled() is False + assert "Stopping" in w.status_label.text() + + def test_safe_stop_suppresses_further_progress_chatter(self, fake_coolscanpy) -> None: + w = _sidebar() + w.set_scanning(True) + w._on_safe_stop_clicked() + + w._on_progress(0.5, "slot 3 complete") + + assert "Stopping" in w.status_label.text() # not overwritten by the progress message + + def test_cancelled_clears_scanning_state(self, fake_coolscanpy) -> None: + w = _sidebar() + w.set_scanning(True) + w._on_cancelled() + assert w._scanning is False + assert w.safe_stop_btn.isEnabled() is False + + def test_error_clears_scanning_state_and_shows_message(self, fake_coolscanpy) -> None: + w = _sidebar() + w.set_scanning(True) + w._on_error("transport jam") + assert w._scanning is False + assert "transport jam" in w.status_label.text() + + def test_safe_stop_button_only_enabled_while_scanning(self, fake_coolscanpy) -> None: + w = _sidebar() + assert w.safe_stop_btn.isEnabled() is False + w.set_scanning(True) + assert w.safe_stop_btn.isEnabled() is True + w.set_scanning(False) + assert w.safe_stop_btn.isEnabled() is False + + +class TestSettingsPersistence: + def test_settings_round_trip_through_the_repo(self, fake_coolscanpy, tmp_path) -> None: + w = _sidebar() + _pick_device(w, device_id="dev-7") + w.folder_edit.setText(str(tmp_path)) + w.pattern_edit.setText('{{ "%03d" % seq }}') + + w._update_settings_from_ui() + + saved = w.controller.session.repo.save_global_setting.call_args[0] + assert saved[0] == "roll_scan_settings" + assert saved[1]["last_device_id"] == "dev-7" + assert saved[1]["output_folder"] == str(tmp_path) + + def test_load_settings_restores_last_device_on_device_list(self, fake_coolscanpy) -> None: + ctrl = MagicMock() + ctrl.session.repo.get_global_setting.return_value = {"last_device_id": "dev-9", "output_folder": "", "filename_pattern": ""} + w = CoolscanRollSidebar(ctrl) + + class _Dev: + id = "dev-9" + vendor = "Nikon" + model = "LS-5000" + + w._on_devices_ready([_Dev()]) + + assert w.device_combo.currentData() == "dev-9" + + +class TestOutputTiers: + """The three independent output-tier checkboxes and the repair-mode + combo. See tests/roll/test_service.py and test_worker.py for what each + tier actually does; these tests only cover the sidebar's own wiring.""" + + def test_sidebar_builds_with_tier_controls(self) -> None: + w = _sidebar() + for attr in ("write_unrepaired_check", "write_repaired_check", "write_positive_check", "repair_mode_combo", "tier_hint"): + assert hasattr(w, attr), attr + + def test_defaults_match_settings_defaults(self) -> None: + w = _sidebar() + assert w.write_unrepaired_check.isChecked() is True + assert w.write_repaired_check.isChecked() is True + assert w.write_positive_check.isChecked() is True + assert w.repair_mode_combo.currentData() == "hybrid" + assert "generative" in w.repair_mode_combo.currentText().lower() + assert "not bit-deterministic" in w.repair_mode_combo.toolTip() + assert w.positive_mode_combo.currentData() == "nikon-exact" + assert w.positive_mode_combo.currentText() == "Nikon C-41 exact (parity)" + assert w.tier_hint.isHidden() is True # Tier 1 is on by default -- nothing to warn about + + def test_tier_hint_shown_when_unrepaired_is_turned_off(self) -> None: + w = _sidebar() + + w.write_unrepaired_check.setChecked(False) + + assert not w.tier_hint.isHidden() + assert "Unrepaired" in w.tier_hint.text() + + def test_tier_hint_hidden_again_once_unrepaired_is_back_on(self) -> None: + w = _sidebar() + w.write_unrepaired_check.setChecked(False) + assert not w.tier_hint.isHidden() + + w.write_unrepaired_check.setChecked(True) + + assert w.tier_hint.isHidden() is True + + def test_scan_blocked_when_no_tier_is_selected(self, fake_coolscanpy, tmp_path) -> None: + w = _sidebar() + _pick_device(w) + w.folder_edit.setText(str(tmp_path)) + w._on_preview_ready([_thumb(fake_coolscanpy, 1)]) + w.contact_sheet.item(0).setSelected(True) + w.write_unrepaired_check.setChecked(False) + w.write_repaired_check.setChecked(False) + w.write_positive_check.setChecked(False) + + assert "select at least one output tier" in w._missing_for_scan() + assert w.scan_btn.isEnabled() is False + + def test_scan_click_forwards_tier_flags_and_repair_mode(self, fake_coolscanpy, tmp_path) -> None: + w = _sidebar() + _pick_device(w, device_id="dev-1") + w.folder_edit.setText(str(tmp_path)) + w._on_preview_ready([_thumb(fake_coolscanpy, 1)]) + w.contact_sheet.item(0).setSelected(True) + w.write_unrepaired_check.setChecked(False) + w.write_repaired_check.setChecked(True) + w.write_positive_check.setChecked(True) + w.repair_mode_combo.setCurrentIndex(w.repair_mode_combo.findData("hybrid")) + w.positive_mode_combo.setCurrentIndex(w.positive_mode_combo.findData("negpy-approximate")) + + w._on_scan_clicked() + + req = w.controller.start_roll_scan.call_args[0][0] + assert req.write_unrepaired is False + assert req.write_repaired is True + assert req.write_positive is True + assert req.repair_mode == "hybrid" + assert req.positive_mode == "negpy-approximate" + + def test_scan_click_without_any_tier_selected_is_a_noop(self, fake_coolscanpy, tmp_path) -> None: + w = _sidebar() + _pick_device(w) + w.folder_edit.setText(str(tmp_path)) + w._on_preview_ready([_thumb(fake_coolscanpy, 1)]) + w.contact_sheet.item(0).setSelected(True) + w.write_unrepaired_check.setChecked(False) + w.write_repaired_check.setChecked(False) + w.write_positive_check.setChecked(False) + + w._on_scan_clicked() + + w.controller.start_roll_scan.assert_not_called() + + def test_tier_settings_round_trip_through_the_repo(self) -> None: + w = _sidebar() + w.write_unrepaired_check.setChecked(False) + w.write_repaired_check.setChecked(True) + w.repair_mode_combo.setCurrentIndex(w.repair_mode_combo.findData("hybrid")) + w.positive_mode_combo.setCurrentIndex(w.positive_mode_combo.findData("negpy-approximate")) + + w._update_settings_from_ui() + + saved = w.controller.session.repo.save_global_setting.call_args[0] + assert saved[0] == "roll_scan_settings" + assert saved[1]["write_unrepaired"] is False + assert saved[1]["write_repaired"] is True + assert saved[1]["repair_mode"] == "hybrid" + assert saved[1]["positive_mode"] == "negpy-approximate" + + def test_load_settings_restores_tier_choices(self) -> None: + ctrl = MagicMock() + ctrl.session.repo.get_global_setting.return_value = { + "write_unrepaired": False, + "write_repaired": True, + "write_positive": True, + "repair_mode": "hybrid", + "positive_mode": "negpy-approximate", + } + w = CoolscanRollSidebar(ctrl) + + assert w.write_unrepaired_check.isChecked() is False + assert w.write_repaired_check.isChecked() is True + assert w.write_positive_check.isChecked() is True + assert w.repair_mode_combo.currentData() == "hybrid" + assert w.positive_mode_combo.currentData() == "negpy-approximate" + + def test_existing_saved_exact_and_disabled_choices_override_new_defaults(self) -> None: + ctrl = MagicMock() + ctrl.session.repo.get_global_setting.return_value = { + "write_unrepaired": True, + "write_repaired": False, + "write_positive": False, + "repair_mode": "exact", + "positive_mode": "nikon-exact", + } + + w = CoolscanRollSidebar(ctrl) + + assert w.write_unrepaired_check.isChecked() is True + assert w.write_repaired_check.isChecked() is False + assert w.write_positive_check.isChecked() is False + assert w.repair_mode_combo.currentData() == "exact" + + +class TestActivation: + def test_on_activated_requests_devices_once_devices_have_arrived(self, fake_coolscanpy) -> None: + w = _sidebar() + w.on_activated() + assert w.controller.request_roll_devices.call_count == 1 + w._on_devices_ready([]) # simulate the controller's reply landing + w.on_activated() + assert w.controller.request_roll_devices.call_count == 1 # already loaded, no repeat + + def test_on_activated_refreshes_the_setup_hint(self, fake_coolscanpy) -> None: + w = _sidebar() + assert w._setup_hint.isVisible() is False # built while coolscanpy (faked) was available diff --git a/tests/roll/test_stage3_replay_loader.py b/tests/roll/test_stage3_replay_loader.py new file mode 100644 index 00000000..c797760f --- /dev/null +++ b/tests/roll/test_stage3_replay_loader.py @@ -0,0 +1,115 @@ +"""Adversarial contracts for the file-backed Stage-3 replay bridge.""" + +from __future__ import annotations + +import dataclasses +import json +import os +from pathlib import Path + +import numpy as np +import pytest + +from negpy.services.roll import exact_color +from tests.roll._exact_fixtures import write_stage3_replay_fixture + + +def _arrays() -> tuple[np.ndarray, np.ndarray, np.ndarray]: + identity = np.arange(65_536, dtype=np.uint16) + return identity, 65_535 - identity, identity // 3 + + +def _rewrite_report(path: Path, mutate) -> None: + report = json.loads(path.read_bytes()) + mutate(report) + path.write_bytes(json.dumps(report, sort_keys=True, separators=(",", ":")).encode()) + + +def test_trusted_loader_builds_a_bound_replay_receipt(tmp_path: Path) -> None: + report_path = write_stage3_replay_fixture(tmp_path / "capture", _arrays()) + + receipt = exact_color.load_stage3_replay_builder_receipt(report_path) + + payload = exact_color.builder_receipt_payload(receipt) + assert receipt.attested is True + assert payload["scope"] == exact_color.STAGE3_REPLAY_SCOPE + assert payload["native_per_acquisition_builder"] is False + assert receipt.stage3_receipt == report_path.read_bytes() + + +@pytest.mark.parametrize( + ("mutate", "message"), + [ + (lambda report: report.update(errors=[{"code": "hidden"}]), "errors"), + (lambda report: report["provenance"].pop("observer_source"), "provenance"), + (lambda report: report["provenance"]["observer_executable"].pop("sha256"), "observer executable"), + (lambda report: report["provenance"]["module"].update(sha256="0" * 64), "module provenance"), + (lambda report: report["provenance"]["resource"].update(bytes=1_023), "resource provenance"), + (lambda report: report["artifacts"].pop(), "exact required file set"), + (lambda report: report["artifacts"].append(dict(report["artifacts"][0])), "repeats"), + ], +) +def test_report_provenance_and_inventory_mutations_fail_closed( + tmp_path: Path, + mutate, + message: str, +) -> None: + report_path = write_stage3_replay_fixture(tmp_path / "capture", _arrays()) + _rewrite_report(report_path, mutate) + + with pytest.raises(exact_color.ExactColorIntegrityError, match=message): + exact_color.load_stage3_replay_builder_receipt(report_path) + + +def test_pre_f_file_must_match_the_report_inventory(tmp_path: Path) -> None: + report_path = write_stage3_replay_fixture(tmp_path / "capture", _arrays()) + path = report_path.parent / "builder-preF-g.bin" + payload = bytearray(path.read_bytes()) + payload[4_096] ^= 1 + path.write_bytes(payload) + + with pytest.raises(exact_color.ExactColorIntegrityError, match="does not bind"): + exact_color.load_stage3_replay_builder_receipt(report_path) + + +def test_symlinked_evidence_is_rejected(tmp_path: Path) -> None: + report_path = write_stage3_replay_fixture(tmp_path / "capture", _arrays()) + target = report_path.parent / "builder-preF-b.bin" + real = report_path.parent / "real-b.bin" + target.replace(real) + target.symlink_to(real) + + with pytest.raises(exact_color.ExactColorUnavailable, match="non-symlink"): + exact_color.load_stage3_replay_builder_receipt(report_path) + + +def test_report_swap_between_lstat_and_open_is_detected( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + report_path = write_stage3_replay_fixture(tmp_path / "capture", _arrays()) + replacement = tmp_path / "replacement.json" + replacement.write_bytes(report_path.read_bytes() + b" ") + real_open = os.open + swapped = False + + def swap_then_open(path, flags, *args, **kwargs): + nonlocal swapped + if not swapped and Path(path) == report_path: + swapped = True + os.replace(replacement, report_path) + return real_open(path, flags, *args, **kwargs) + + monkeypatch.setattr(exact_color.os, "open", swap_then_open) + + with pytest.raises(exact_color.ExactColorIntegrityError, match="changed while being read"): + exact_color.load_stage3_replay_builder_receipt(report_path) + + +def test_callers_cannot_self_attest_a_builder_receipt(tmp_path: Path) -> None: + report_path = write_stage3_replay_fixture(tmp_path / "capture", _arrays()) + receipt = exact_color.load_stage3_replay_builder_receipt(report_path) + forged = dataclasses.replace(receipt, _factory_token=object()) + + with pytest.raises(exact_color.ExactColorIntegrityError, match="trusted Stage-3 replay file loader"): + exact_color.builder_receipt_payload(forged) diff --git a/tests/roll/test_worker.py b/tests/roll/test_worker.py new file mode 100644 index 00000000..d7ae8174 --- /dev/null +++ b/tests/roll/test_worker.py @@ -0,0 +1,683 @@ +"""Tests for RollWorker: signal emissions, device-session reuse, and the +safe-stop / cancellation distinction. Mirrors test_capture_worker.py's +shape: fakes wired directly into the worker, signals captured with plain +list-appending slots, no QThread or QApplication involved. +""" + +from __future__ import annotations + +import os +import threading +from pathlib import Path +from unittest.mock import MagicMock + +import numpy as np +import pytest + +from negpy.desktop.workers.roll_worker import RollBatchScanRequest, RollPreviewRequest, RollWorker +from negpy.infrastructure.roll import repair as roll_repair +from negpy.services.repair.fauxice_hybrid_runner import HybridRuntimeConfig +from negpy.services.roll import service as roll_service + + +def _frame(fake_coolscanpy, *, slot, ir=None): + rgb = np.random.randint(0, 65535, (4, 6, 3), dtype=np.uint16) + receipt = fake_coolscanpy.Receipt(version=1, slot=slot, dpi=4000, depth=16, device_id="usb:1:2", transport_smear_verdict="clean") + acquisition = None + validity = None + meter = None + if ir is not None: + storage_rgbi = np.dstack((rgb, ir)) + native_rgbi = np.ascontiguousarray(np.rot90(storage_rgbi, k=-1, axes=(0, 1))) + validity = np.ones(ir.shape, dtype=np.bool_) + native_validity = np.ascontiguousarray(np.rot90(validity, k=-1, axes=(0, 1))) + meter = np.full((2, 2, 4), 500, dtype=np.uint16) + reservation_id = f"reservation-{slot:03d}" + capture_attempt_id = f"fine-slot-{slot}-attempt-001" + acquisition_id, evidence_sha256 = roll_service._derive_digital_ice_producer_binding( + slot=slot, + reservation_id=reservation_id, + capture_attempt_id=capture_attempt_id, + main_rgbi=native_rgbi, + prepass_rgbi=meter, + ir_validity=native_validity, + ) + acquisition = roll_repair.RepairAcquisition.from_arrays( + acquisition_id=acquisition_id, + slot=slot, + reservation_id=reservation_id, + capture_attempt_id=capture_attempt_id, + storage_transform=roll_repair.DIGITAL_ICE_STORAGE_TRANSFORM, + evidence_sha256=evidence_sha256, + main_rgbi=native_rgbi, + prepass_rgbi=meter, + ir_validity=native_validity, + ) + return fake_coolscanpy.Frame( + slot=slot, + rgb=rgb, + ir=ir, + ir_validity=validity, + receipt=receipt, + meter_rgbi=meter, + digital_ice_acquisition=acquisition, + ) + + +def _open(fake_coolscanpy, roll=None): + roll = roll if roll is not None else fake_coolscanpy.Roll() + device = fake_coolscanpy.Device(roll) + fake_coolscanpy.state["open_device"] = device + return roll, device + + +class TestListDevices: + def test_explicit_hybrid_runtime_is_injected_into_scanning_service(self, tmp_path: Path) -> None: + runtime = HybridRuntimeConfig( + hybrid_python=tmp_path / "hybrid" / "bin" / "python", + executable=tmp_path / "hybrid" / "bin" / "fauxce-hybrid", + core_source_manifest_sha256="c" * 64, + hybrid_source_manifest_sha256="d" * 64, + iopaint_python=tmp_path / "iopaint" / "bin" / "python", + iopaint_executable=tmp_path / "iopaint" / "bin" / "iopaint", + iopaint_source_manifest_sha256="a" * 64, + model_dir=tmp_path / "models", + model_weights=tmp_path / "models" / "big-lama.pt", + model_weights_sha256="b" * 64, + ) + + worker = RollWorker(hybrid_runtime=runtime) + + assert worker._service._hybrid_runtime is runtime + + def test_success_emits_devices(self, fake_coolscanpy) -> None: + fake_coolscanpy.state["devices"] = ["device-a", "device-b"] + worker = RollWorker() + seen = [] + worker.devices_ready.connect(seen.append) + + worker.list_devices() + + assert seen == [["device-a", "device-b"]] + + def test_failure_emits_error_and_empty_list(self, fake_coolscanpy, monkeypatch) -> None: + worker = RollWorker() + + def boom(local_only: bool = False): + raise RuntimeError("usb enumeration failed") + + monkeypatch.setattr(fake_coolscanpy.module, "get_devices", boom) + errors, devices = [], [] + worker.error.connect(errors.append) + worker.devices_ready.connect(devices.append) + + worker.list_devices() + + assert errors and "usb enumeration failed" in errors[-1] + assert devices == [[]] + + +class TestOpenAndPreview: + def test_preview_opens_then_returns_thumbnails(self, fake_coolscanpy) -> None: + thumb = fake_coolscanpy.Thumbnail(slot=1, image=np.zeros((2, 2, 3)), boundary_rows=(0, 2), spacing_offset=0, needs_approval=False) + roll, device = _open(fake_coolscanpy, fake_coolscanpy.Roll(thumbnails=[thumb])) + worker = RollWorker() + opened, ready, progress = [], [], [] + worker.opened.connect(opened.append) + worker.preview_ready.connect(ready.append) + worker.progress.connect(lambda frac, msg: progress.append((frac, msg))) + + worker.run_preview(RollPreviewRequest(device_id="ls5000-usb-001")) + + assert opened == ["ls5000-usb-001"] + assert ready == [[thumb]] + assert device.roll_called_with == fake_coolscanpy.module.Material.COLOR_NEGATIVE + # the fake preview() reports 0.0 then 1.0, forwarded as (fraction, message) pairs + assert progress[0][0] == 0.0 + assert progress[-1] == (1.0, "preview complete") + + def test_preview_filters_to_requested_slots(self, fake_coolscanpy) -> None: + thumbs = [ + fake_coolscanpy.Thumbnail(slot=i, image=np.zeros((2, 2, 3)), boundary_rows=(0, 2), spacing_offset=0, needs_approval=False) + for i in (1, 2, 3) + ] + _open(fake_coolscanpy, fake_coolscanpy.Roll(thumbnails=thumbs)) + worker = RollWorker() + ready = [] + worker.preview_ready.connect(ready.append) + + worker.run_preview(RollPreviewRequest(device_id="ls5000-usb-001", slots=(2,))) + + assert [t.slot for t in ready[0]] == [2] + + def test_second_preview_of_the_same_device_does_not_reopen(self, fake_coolscanpy) -> None: + _open(fake_coolscanpy) + open_calls = [] + real_open = fake_coolscanpy.module.open + fake_coolscanpy.module.open = lambda devname: (open_calls.append(devname), real_open(devname))[1] + worker = RollWorker() + + worker.run_preview(RollPreviewRequest(device_id="ls5000-usb-001")) + worker.run_preview(RollPreviewRequest(device_id="ls5000-usb-001")) + + assert open_calls == ["ls5000-usb-001"] # opened once, reused the second time + + def test_switching_device_closes_the_previous_roll(self, fake_coolscanpy) -> None: + roll_a, device_a = _open(fake_coolscanpy) + worker = RollWorker() + worker.run_preview(RollPreviewRequest(device_id="device-a")) + assert roll_a.closed is False + + roll_b, device_b = _open(fake_coolscanpy) + worker.run_preview(RollPreviewRequest(device_id="device-b")) + + assert roll_a.closed is True + assert device_a.closed is True + assert roll_b.closed is False + + def test_preview_failure_emits_error(self, fake_coolscanpy) -> None: + _open(fake_coolscanpy, fake_coolscanpy.Roll(raise_on={"preview": fake_coolscanpy.module.PyCoolscanError("fingerprint mismatch")})) + worker = RollWorker() + errors = [] + worker.error.connect(errors.append) + + worker.run_preview(RollPreviewRequest(device_id="ls5000-usb-001")) + + assert errors and "fingerprint mismatch" in errors[-1] + + def test_close_roll_forgets_the_open_device(self, fake_coolscanpy) -> None: + roll, device = _open(fake_coolscanpy) + worker = RollWorker() + worker.run_preview(RollPreviewRequest(device_id="ls5000-usb-001")) + closed = [] + worker.closed.connect(lambda: closed.append(True)) + + worker.close_roll() + + assert closed == [True] + assert roll.closed is True + assert worker._open_device_id is None + + def test_close_roll_failure_retains_identity_and_does_not_emit_closed( + self, + fake_coolscanpy, + ) -> None: + _open(fake_coolscanpy) + worker = RollWorker() + worker.run_preview(RollPreviewRequest(device_id="ls5000-usb-001")) + closed = [] + errors = [] + worker.closed.connect(lambda: closed.append(True)) + worker.error.connect(errors.append) + failure = RuntimeError("ownership remains uncertain") + worker._service.close = MagicMock(side_effect=failure) + + assert worker.close_roll() is False + + assert worker._open_device_id == "ls5000-usb-001" + assert closed == [] + assert errors and "ownership remains uncertain" in errors[-1] + + +class TestEject: + def test_eject_without_an_open_roll_emits_success(self) -> None: + worker = RollWorker() + worker._service.eject = MagicMock(return_value=True) + results: list[bool] = [] + worker.ejected.connect(results.append) + + worker.eject("usb:2:7") + + worker._service.eject.assert_called_once_with("usb:2:7") + assert results == [True] + + def test_eject_closes_the_matching_roll_before_motion( + self, + fake_coolscanpy, + ) -> None: + _open(fake_coolscanpy) + worker = RollWorker() + worker.run_preview(RollPreviewRequest(device_id="usb:2:7")) + events: list[str] = [] + real_close = worker._service.close + + def close() -> None: + events.append("close") + real_close() + + def eject(device_id: str) -> bool: + events.append(f"eject:{device_id}") + return True + + worker._service.close = close + worker._service.eject = eject + + worker.eject("usb:2:7") + + assert events == ["close", "eject:usb:2:7"] + assert worker._open_device_id is None + + def test_close_failure_preserves_identity_and_never_ejects( + self, + fake_coolscanpy, + ) -> None: + _open(fake_coolscanpy) + worker = RollWorker() + worker.run_preview(RollPreviewRequest(device_id="usb:2:7")) + worker._service.close = MagicMock(side_effect=RuntimeError("reservation uncertain")) + worker._service.eject = MagicMock(return_value=True) + errors: list[str] = [] + worker.eject_error.connect(errors.append) + + worker.eject("usb:2:7") + + assert worker._open_device_id == "usb:2:7" + worker._service.eject.assert_not_called() + assert errors == ["reservation uncertain"] + + def test_refuses_to_eject_a_device_other_than_the_open_roll( + self, + fake_coolscanpy, + ) -> None: + _open(fake_coolscanpy) + worker = RollWorker() + worker.run_preview(RollPreviewRequest(device_id="usb:2:7")) + worker._service.eject = MagicMock(return_value=True) + errors: list[str] = [] + worker.eject_error.connect(errors.append) + + worker.eject("usb:2:8") + + worker._service.eject.assert_not_called() + assert worker._open_device_id == "usb:2:7" + assert errors and "not requested device" in errors[0] + + +class TestSpacingAndApproval: + def test_set_spacing_offset_echoes_the_applied_value(self, fake_coolscanpy) -> None: + roll, _device = _open(fake_coolscanpy) + worker = RollWorker() + worker.run_preview(RollPreviewRequest(device_id="ls5000-usb-001")) + seen = [] + worker.spacing_offset_set.connect(lambda slot, offset: seen.append((slot, offset))) + + worker.set_spacing_offset(3, -12) + + assert seen == [(3, -12)] + assert roll.spacing_offsets[3] == -12 + + def test_set_spacing_offset_before_open_emits_error(self, fake_coolscanpy) -> None: + worker = RollWorker() + errors = [] + worker.error.connect(errors.append) + + worker.set_spacing_offset(1, 5) + + assert errors and "no roll is open" in errors[-1] + + def test_approve_records_slot(self, fake_coolscanpy) -> None: + roll, _device = _open(fake_coolscanpy) + worker = RollWorker() + worker.run_preview(RollPreviewRequest(device_id="ls5000-usb-001")) + approved = [] + worker.approved.connect(approved.append) + + worker.approve(4) + + assert approved == [4] + assert roll.approved == [4] + + def test_approve_failure_emits_error(self, fake_coolscanpy) -> None: + _open(fake_coolscanpy, fake_coolscanpy.Roll(raise_on={"approve": fake_coolscanpy.module.PyCoolscanError("slot 3 needs review")})) + worker = RollWorker() + worker.run_preview(RollPreviewRequest(device_id="ls5000-usb-001")) + errors = [] + worker.error.connect(errors.append) + + worker.approve(3) + + assert errors and "slot 3 needs review" in errors[-1] + + +class TestBatchScan: + def test_writes_every_frame_and_reports_progress(self, fake_coolscanpy, tmp_path) -> None: + frames = [_frame(fake_coolscanpy, slot=s) for s in (1, 2, 3)] + _open(fake_coolscanpy, fake_coolscanpy.Roll(frames=frames)) + worker = RollWorker() + written, finished, progress = [], [], [] + worker.frame_written.connect(written.append) + worker.finished.connect(finished.append) + worker.progress.connect(lambda frac, msg: progress.append((frac, msg))) + + worker.run_batch_scan( + RollBatchScanRequest( + device_id="ls5000-usb-001", slots=(1, 2, 3), output_folder=str(tmp_path), filename_pattern='{{ "%03d" % seq }}' + ) + ) + + assert [w.slot for w in written] == [1, 2, 3] + assert all(os.path.exists(w.rgb_path) for w in written) + assert len(finished) == 1 and finished[0] == written + assert [msg for _frac, msg in progress] == ["slot 1 complete", "slot 2 complete", "slot 3 complete"] + + def test_batch_scan_opens_the_requested_device_first(self, fake_coolscanpy, tmp_path) -> None: + _open(fake_coolscanpy, fake_coolscanpy.Roll(frames=[_frame(fake_coolscanpy, slot=1)])) + worker = RollWorker() + opened = [] + worker.opened.connect(opened.append) + + worker.run_batch_scan( + RollBatchScanRequest(device_id="ls5000-usb-001", slots=(1,), output_folder=str(tmp_path), filename_pattern='{{ "%03d" % seq }}') + ) + + assert opened == ["ls5000-usb-001"] + + def test_safe_stop_mid_batch_is_reported_as_cancelled_not_error(self, fake_coolscanpy, tmp_path) -> None: + stop_error = fake_coolscanpy.module.SafeStopRequested("safe stop requested; 1 of 2 requested frames completed") + frames = [_frame(fake_coolscanpy, slot=1)] + roll = fake_coolscanpy.Roll(frames=frames, raise_on={"scan_many_slots": {2: stop_error}}) + _open(fake_coolscanpy, roll) + worker = RollWorker() + written, finished, cancelled, errors = [], [], [], [] + worker.frame_written.connect(written.append) + worker.finished.connect(finished.append) + worker.cancelled.connect(lambda: cancelled.append(True)) + worker.error.connect(errors.append) + + worker.run_batch_scan( + RollBatchScanRequest( + device_id="ls5000-usb-001", slots=(1, 2), output_folder=str(tmp_path), filename_pattern='{{ "%03d" % seq }}' + ) + ) + + assert len(written) == 1 # slot 1 finished and was written before the stop landed + assert cancelled == [True] + assert finished == [] # a stop is not a completion + assert errors == [] + + def test_genuine_failure_mid_batch_is_reported_as_error_not_cancelled(self, fake_coolscanpy, tmp_path) -> None: + boom = fake_coolscanpy.module.PyCoolscanError("transport jam") + frames = [_frame(fake_coolscanpy, slot=1)] + roll = fake_coolscanpy.Roll(frames=frames, raise_on={"scan_many_slots": {2: boom}}) + _open(fake_coolscanpy, roll) + worker = RollWorker() + written, cancelled, errors = [], [], [] + worker.frame_written.connect(written.append) + worker.cancelled.connect(lambda: cancelled.append(True)) + worker.error.connect(errors.append) + + worker.run_batch_scan( + RollBatchScanRequest( + device_id="ls5000-usb-001", slots=(1, 2), output_folder=str(tmp_path), filename_pattern='{{ "%03d" % seq }}' + ) + ) + + assert len(written) == 1 + assert cancelled == [] + assert errors and "transport jam" in errors[-1] + + def test_repair_cancellation_aborts_frame_transaction_and_batch( + self, + fake_coolscanpy, + fake_repair_engine, + tmp_path, + ) -> None: + frame = _frame( + fake_coolscanpy, + slot=1, + ir=np.full((4, 6), 2_000, dtype=np.uint16), + ) + roll = fake_coolscanpy.Roll(frames=[frame]) + _open(fake_coolscanpy, roll) + started = threading.Event() + + def blocking_repair( + acquisition, + mode, + *, + hybrid_runtime=None, + progress=None, + cancel=None, + ): + started.set() + assert cancel is not None + assert cancel.wait(timeout=5) + raise roll_repair.RepairCancelled("repair cancelled by test") + + fake_repair_engine.repair = blocking_repair + worker = RollWorker() + written, finished, cancelled, errors = [], [], [], [] + worker.frame_written.connect(written.append) + worker.finished.connect(finished.append) + worker.cancelled.connect(lambda: cancelled.append(True)) + worker.error.connect(errors.append) + request = RollBatchScanRequest( + device_id="ls5000-usb-001", + slots=(1,), + output_folder=str(tmp_path), + filename_pattern='{{ "%03d" % seq }}', + write_unrepaired=True, + write_repaired=True, + ) + worker.prepare_batch() + stop_thread = threading.Thread( + target=lambda: ( + started.wait(timeout=5), + worker.safe_stop(), + ) + ) + stop_thread.start() + worker.run_batch_scan(request) + stop_thread.join(timeout=5) + + assert not stop_thread.is_alive() + assert written == [] + assert finished == [] + assert cancelled == [True] + assert errors == [] + assert not any(path.name.endswith(".tif") for path in tmp_path.iterdir()) + assert not any(path.name.endswith("_receipt.json") for path in tmp_path.iterdir()) + assert not any(path.name.startswith(".negpy-frame-stage-") for path in tmp_path.iterdir()) + + def test_stop_after_batch_is_queued_is_not_cleared_at_first_scan( + self, + fake_coolscanpy, + tmp_path, + ) -> None: + frame = _frame(fake_coolscanpy, slot=1) + roll = fake_coolscanpy.Roll(frames=[frame]) + _open(fake_coolscanpy, roll) + worker = RollWorker() + cancelled, finished = [], [] + worker.cancelled.connect(lambda: cancelled.append(True)) + worker.finished.connect(finished.append) + request = RollBatchScanRequest( + device_id="ls5000-usb-001", + slots=(1,), + output_folder=str(tmp_path), + filename_pattern='{{ "%03d" % seq }}', + ) + + worker.prepare_batch() + worker.safe_stop() + worker.run_batch_scan(request) + + assert cancelled == [True] + assert finished == [] + assert not any(tmp_path.iterdir()) + + def test_writes_ir_sidecar_when_frame_carries_one(self, fake_coolscanpy, tmp_path) -> None: + ir = np.random.randint(0, 65535, (4, 6), dtype=np.uint16) + _open(fake_coolscanpy, fake_coolscanpy.Roll(frames=[_frame(fake_coolscanpy, slot=1, ir=ir)])) + worker = RollWorker() + written = [] + worker.frame_written.connect(written.append) + + worker.run_batch_scan( + RollBatchScanRequest(device_id="ls5000-usb-001", slots=(1,), output_folder=str(tmp_path), filename_pattern='{{ "%03d" % seq }}') + ) + + assert written[0].ir_path is not None + assert written[0].ir_path.endswith("_IR.tif") + assert os.path.exists(written[0].ir_path) + + +class TestSafeStopAndShutdown: + def test_safe_stop_before_any_roll_is_open_is_a_no_op(self, fake_coolscanpy) -> None: + RollWorker().safe_stop() # must not raise + + def test_safe_stop_forwards_to_the_open_roll(self, fake_coolscanpy) -> None: + roll, _device = _open(fake_coolscanpy) + worker = RollWorker() + worker.run_preview(RollPreviewRequest(device_id="ls5000-usb-001")) + + worker.safe_stop() + + assert roll.safe_stop_called is True + + def test_shutdown_stops_and_closes(self, fake_coolscanpy) -> None: + roll, _device = _open(fake_coolscanpy) + worker = RollWorker() + worker.run_preview(RollPreviewRequest(device_id="ls5000-usb-001")) + + worker.shutdown() + + assert roll.safe_stop_called is True + assert roll.closed is True + + def test_shutdown_close_failure_is_raised_and_retains_open_identity( + self, + fake_coolscanpy, + ) -> None: + _open(fake_coolscanpy) + worker = RollWorker() + worker.run_preview(RollPreviewRequest(device_id="ls5000-usb-001")) + failure = RuntimeError("ownership remains uncertain") + worker._service.close = MagicMock(side_effect=failure) + + with pytest.raises(RuntimeError) as raised: + worker.shutdown(timeout_seconds=0) + + assert raised.value is failure + assert worker._open_device_id == "ls5000-usb-001" + + def test_queued_preview_cannot_reopen_after_shutdown_was_proven( + self, + fake_coolscanpy, + ) -> None: + _open(fake_coolscanpy) + worker = RollWorker() + errors = [] + worker.error.connect(errors.append) + + worker.shutdown(timeout_seconds=0) + worker.run_preview(RollPreviewRequest(device_id="ls5000-usb-001")) + + assert worker._open_device_id is None + assert errors and "no new operation may begin" in errors[-1] + + def test_shutdown_without_any_roll_open_does_not_raise(self) -> None: + RollWorker().shutdown() + + +class TestBatchScanOutputTiers: + """RollBatchScanRequest's tier fields, forwarded to RollScanningService + .write_frame -- see tests/roll/test_service.py for the full tier-behavior + matrix (naming, receipt provenance, every degrade path); these tests only + pin the plumbing between the request and the service call.""" + + def test_request_defaults_match_the_complete_parity_workflow(self) -> None: + """An unnamed request follows the new-user Color + DICE target.""" + req = RollBatchScanRequest(device_id="d", slots=(1,), output_folder="/tmp/x", filename_pattern="p") + + assert req.write_unrepaired is True + assert req.write_repaired is True + assert req.write_positive is True + assert req.repair_mode == "hybrid" + assert req.positive_mode == "nikon-exact" + assert req.hybrid_synthesis_limit_percent == 10.0 + + @pytest.mark.parametrize( + "value", + (float("nan"), float("inf"), -0.1, 100.1, True), + ) + def test_request_rejects_invalid_hybrid_synthesis_limit( + self, + value: float | bool, + ) -> None: + with pytest.raises(ValueError, match="hybrid_synthesis_limit_percent"): + RollBatchScanRequest( + device_id="d", + slots=(1,), + output_folder="/tmp/x", + filename_pattern="p", + hybrid_synthesis_limit_percent=value, + ) + + def test_tier_flags_are_forwarded_to_write_frame( + self, + fake_coolscanpy, + no_repair_engine, + tmp_path, + monkeypatch, + ) -> None: + _open(fake_coolscanpy, fake_coolscanpy.Roll(frames=[_frame(fake_coolscanpy, slot=1)])) + worker = RollWorker() + captured = {} + real_write_frame = worker._service.write_frame + + def _spy(frame, output_folder, filename_pattern, **kwargs): + captured.update(kwargs) + return real_write_frame(frame, output_folder, filename_pattern, **kwargs) + + monkeypatch.setattr(worker._service, "write_frame", _spy) + + worker.run_batch_scan( + RollBatchScanRequest( + device_id="ls5000-usb-001", + slots=(1,), + output_folder=str(tmp_path), + filename_pattern='{{ "%03d" % seq }}', + write_unrepaired=False, + write_repaired=True, + write_positive=True, + repair_mode="hybrid", + positive_mode="negpy-approximate", + ) + ) + + # write_repaired/write_positive=True with no repair engine registered degrades + # gracefully (see test_service.py) rather than raising, so this exercises the + # real write_frame end to end without needing a fake engine here. + assert callable(captured.pop("on_repair_progress")) + assert captured == { + "write_unrepaired": False, + "write_repaired": True, + "write_positive": True, + "repair_mode": "hybrid", + "positive_mode": "negpy-approximate", + "hybrid_runtime": None, + } + + def test_repaired_tier_writes_through_the_worker_with_a_registered_engine(self, fake_coolscanpy, fake_repair_engine, tmp_path) -> None: + ir = np.zeros((4, 6), dtype=np.uint16) + _open(fake_coolscanpy, fake_coolscanpy.Roll(frames=[_frame(fake_coolscanpy, slot=1, ir=ir)])) + worker = RollWorker() + written = [] + worker.frame_written.connect(written.append) + + worker.run_batch_scan( + RollBatchScanRequest( + device_id="ls5000-usb-001", + slots=(1,), + output_folder=str(tmp_path), + filename_pattern='{{ "%03d" % seq }}', + write_unrepaired=True, + write_repaired=True, + ) + ) + + assert len(written) == 1 + assert written[0].repaired_rgb_path is not None + assert os.path.exists(written[0].repaired_rgb_path) + assert fake_repair_engine.calls # the worker's request actually reached the engine diff --git a/tests/scanners/test_sane_session.py b/tests/scanners/test_sane_session.py index ba4e5a2d..bdf6c740 100644 --- a/tests/scanners/test_sane_session.py +++ b/tests/scanners/test_sane_session.py @@ -252,6 +252,7 @@ def _held(self, monkeypatch: pytest.MonkeyPatch, *, eject: bool): session = backend.open_session(_DEV_ID) observed: dict[str, int] = {} run = FakeRun(on_call=lambda: observed.__setitem__("close_calls_at_press", module.dev.close_calls)) + monkeypatch.setattr(sane_backend, "_scanimage_executable", lambda: "scanimage") monkeypatch.setattr(sane_backend.subprocess, "run", run) return backend, module, session, run, observed diff --git a/tests/scanners/test_scanner_eject.py b/tests/scanners/test_scanner_eject.py index 104616a5..798796e0 100644 --- a/tests/scanners/test_scanner_eject.py +++ b/tests/scanners/test_scanner_eject.py @@ -138,6 +138,7 @@ def __call__(self, args: list[str], **kwargs: Any) -> _FakeCompleted: def _patch_run(monkeypatch: pytest.MonkeyPatch, **kwargs: Any) -> FakeRun: fake = FakeRun(**kwargs) + monkeypatch.setattr(sane_backend, "_scanimage_executable", lambda: "scanimage") monkeypatch.setattr(sane_backend.subprocess, "run", fake) return fake @@ -175,6 +176,36 @@ def test_matches_hyphenated_spelling(self) -> None: assert _find_eject_option({"eject": FakeOption()}) == "eject" +class TestDirectUsbEject: + def test_maps_the_exact_direct_usb_topology_to_coolscan3( + self, + monkeypatch: pytest.MonkeyPatch, + ) -> None: + calls: list[str] = [] + monkeypatch.setattr(sane_backend, "_scanimage_eject", calls.append) + + assert sane_backend.scanimage_eject_direct_usb("usb:2:7") is True + assert calls == ["coolscan3:usb:libusb:002:007"] + + @pytest.mark.parametrize( + "device_id", + ( + "ls5000", + "usb:2", + "usb:0:7", + "usb:2:0", + "usb:2:7:extra", + "usb:1000:7", + ), + ) + def test_rejects_ambiguous_or_out_of_range_topologies( + self, + device_id: str, + ) -> None: + with pytest.raises(ValueError, match="direct Coolscan eject"): + sane_backend.scanimage_eject_direct_usb(device_id) + + class TestSaneBackendEject: def test_presses_eject_via_scanimage_and_returns_true(self, monkeypatch: pytest.MonkeyPatch) -> None: dev = FakeSaneDev(dict(COOLSCAN3_OPT_WITH_EJECT)) @@ -204,13 +235,36 @@ def test_detection_handle_is_closed_before_scanimage_opens_it(self, monkeypatch: def test_spurious_out_of_documents_exit_is_treated_as_success(self, monkeypatch: pytest.MonkeyPatch) -> None: dev = FakeSaneDev(dict(COOLSCAN3_OPT_WITH_EJECT)) backend = _make_backend(FakeSaneModule(dev)) - _patch_run( + run = _patch_run( monkeypatch, returncode=7, stderr="scanimage: sane_start: Document feeder out of documents\n", ) assert backend.eject(_DEV) is True + assert run.calls[0][1]["timeout"] == sane_backend._EJECT_TIMEOUT_S + + def test_timeout_reports_that_eject_may_have_dispatched_and_forbids_retry( + self, + monkeypatch: pytest.MonkeyPatch, + ) -> None: + dev = FakeSaneDev(dict(COOLSCAN3_OPT_WITH_EJECT)) + backend = _make_backend(FakeSaneModule(dev)) + _patch_run( + monkeypatch, + raises=sane_backend.subprocess.TimeoutExpired( + ["scanimage", "-d", _DEV, "--eject"], + timeout=sane_backend._EJECT_TIMEOUT_S, + ), + ) + + with pytest.raises(RuntimeError) as exc_info: + backend.eject(_DEV) + + message = str(exc_info.value) + assert "may already have been dispatched" in message + assert "Check the film physically" in message + assert "do not retry" in message def test_capability_gated_skip_when_device_has_no_eject_option(self, monkeypatch: pytest.MonkeyPatch) -> None: dev = FakeSaneDev(dict(COOLSCAN3_OPT_NO_EJECT)) diff --git a/tests/test_controller.py b/tests/test_controller.py index 4ca740bb..f9fbfc87 100644 --- a/tests/test_controller.py +++ b/tests/test_controller.py @@ -180,6 +180,23 @@ def test_start_roll_preview_prepares_worker_and_emits_preview_only(self): # No "started": a preview must not flip the main scan UI into scanning state. self.assertEqual(events, ["prepare", ("preview", request)]) + def test_coolscan_roll_preview_uses_its_separate_worker_lane(self): + from negpy.desktop.workers.roll_worker import RollPreviewRequest + + events: list[object] = [] + request = RollPreviewRequest(device_id="usb:1:2", slots=(1, 2)) + controller = SimpleNamespace( + _ensure_roll_thread=lambda: events.append("ensure-coolscan-thread"), + roll_preview_requested=SimpleNamespace(emit=lambda value: events.append(("coolscan-preview", value))), + ) + + AppController.start_coolscan_roll_preview(controller, request) + + self.assertEqual( + events, + ["ensure-coolscan-thread", ("coolscan-preview", request)], + ) + def test_thumbnail_refreshes_on_config_changed_settle(self): """Filmstrip thumbnail is re-captured on every settled render whose config differs from the last capture (covers in-place edits and reset), but not on a diff --git a/tests/test_crosstalk_profiles.py b/tests/test_crosstalk_profiles.py index 182205d5..e892aa6b 100644 --- a/tests/test_crosstalk_profiles.py +++ b/tests/test_crosstalk_profiles.py @@ -1,4 +1,6 @@ import os +import subprocess +import sys import pytest @@ -51,6 +53,48 @@ def test_malformed_skipped(tmp_path, monkeypatch): assert CrosstalkProfiles.list_profiles() == ["Default"] +@pytest.mark.skipif(not hasattr(os, "mkfifo"), reason="named pipes are not available on this platform") +def test_list_profiles_ignores_special_files_and_symlinks_without_blocking(tmp_path): + user_dir = tmp_path / "user" + crosstalk_dir = user_dir / "crosstalk" + crosstalk_dir.mkdir(parents=True) + linked_target = tmp_path / "outside.toml" + _write( + linked_target, + 'name = "Unsafe Symlink"\nmatrix = [[1.0, 0.0, 0.0], [0.0, 1.0, 0.0], [0.0, 0.0, 1.0]]\n', + ) + os.symlink(linked_target, crosstalk_dir / "linked.toml") + os.mkfifo(crosstalk_dir / "stale.toml") + + env = os.environ.copy() + env["NEGPY_USER_DIR"] = str(user_dir) + completed = subprocess.run( + [ + sys.executable, + "-c", + ("from negpy.services.assets.crosstalk import CrosstalkProfiles; print(CrosstalkProfiles.list_profiles())"), + ], + cwd=os.getcwd(), + env=env, + capture_output=True, + text=True, + timeout=3, + check=True, + ) + + assert "Unsafe Symlink" not in completed.stdout + + +def test_list_profiles_ignores_unreasonably_large_profile(tmp_path, monkeypatch): + monkeypatch.setattr(APP_CONFIG, "crosstalk_dir", str(tmp_path)) + _write( + tmp_path / "oversized.toml", + "#" + ("x" * (64 * 1024)) + '\nname = "Oversized"\nmatrix = [[1, 0, 0], [0, 1, 0], [0, 0, 1]]\n', + ) + + assert "Oversized" not in CrosstalkProfiles.list_profiles() + + def test_ensure_user_dir_creates_directory(tmp_path, monkeypatch): target = tmp_path / "nested" / "crosstalk" monkeypatch.setattr(APP_CONFIG, "crosstalk_dir", str(target)) diff --git a/tests/test_ls5000_frozen_packaging.py b/tests/test_ls5000_frozen_packaging.py new file mode 100644 index 00000000..d87a1a48 --- /dev/null +++ b/tests/test_ls5000_frozen_packaging.py @@ -0,0 +1,902 @@ +"""Offline contracts for the frozen LS-5000 capture entry point.""" + +import ast +import hashlib +import os +import platform +import plistlib +import runpy +import subprocess +import sys +import types +from pathlib import Path +from types import SimpleNamespace + +import pytest +from PyInstaller.utils.hooks import collect_all + +from coolscanpy.protocol.ls5000_single_pass.capture_process import ( + CAPTURE_HELPER_FLAG, +) + + +REPO = Path(__file__).resolve().parents[1] + + +def test_desktop_script_dispatches_frozen_auxiliary_before_gui_import( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """The real entry point must not import Qt/desktop code for helper mode.""" + + dispatched: list[list[str]] = [] + frozen_entry = types.ModuleType("negpy.desktop.frozen_entry") + frozen_entry.dispatch_frozen_auxiliary = lambda argv: dispatched.append(argv) or True + monkeypatch.setitem(sys.modules, "negpy.desktop.frozen_entry", frozen_entry) + monkeypatch.delitem(sys.modules, "negpy.desktop.main", raising=False) + real_import = __import__ + + def reject_gui_import(name, *args, **kwargs): + if name == "negpy.desktop.main": + raise AssertionError("GUI desktop imported during frozen auxiliary dispatch") + return real_import(name, *args, **kwargs) + + monkeypatch.setattr("builtins.__import__", reject_gui_import) + monkeypatch.setattr(sys, "argv", ["NegPy", "--negpy-packaging-smoke"]) + + runpy.run_path(str(REPO / "desktop.py"), run_name="__main__") + + assert dispatched == [["NegPy", "--negpy-packaging-smoke"]] + + +def test_live_acceptance_dispatch_forwards_exact_arguments_without_qt( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """The private live runner must enter before the frozen Qt desktop.""" + + from negpy.desktop import frozen_entry + + forwarded: list[list[str]] = [] + live_acceptance = types.ModuleType("negpy.services.roll.live_acceptance") + live_acceptance.main = lambda argv: forwarded.append(argv) or 0 + monkeypatch.setitem( + sys.modules, + "negpy.services.roll.live_acceptance", + live_acceptance, + ) + real_import = __import__ + + def reject_gui_import(name, *args, **kwargs): + if name == "negpy.desktop.main" or name.startswith("PyQt6"): + raise AssertionError("Qt desktop imported during live acceptance dispatch") + return real_import(name, *args, **kwargs) + + monkeypatch.setattr("builtins.__import__", reject_gui_import) + arguments = [ + "--device-id", + "usb:2:7", + "--confirm-live", + ] + + handled = frozen_entry.dispatch_frozen_auxiliary(["NegPy", frozen_entry.LIVE_ACCEPTANCE_FLAG, *arguments]) + + assert handled is True + assert forwarded == [arguments] + assert "LIVE_ACCEPTANCE_FLAG" in frozen_entry.__all__ + assert "dispatch_live_acceptance" in frozen_entry.__all__ + + +def test_live_acceptance_dispatch_propagates_failure_exit_code( + monkeypatch: pytest.MonkeyPatch, +) -> None: + from negpy.desktop import frozen_entry + + live_acceptance = types.ModuleType("negpy.services.roll.live_acceptance") + live_acceptance.main = lambda _argv: 23 + monkeypatch.setitem( + sys.modules, + "negpy.services.roll.live_acceptance", + live_acceptance, + ) + + with pytest.raises(SystemExit) as stopped: + frozen_entry.dispatch_frozen_auxiliary(["NegPy", frozen_entry.LIVE_ACCEPTANCE_FLAG]) + + assert stopped.value.code == 23 + + +def test_capture_helper_dispatch_runs_the_real_worker_without_usb(capsys) -> None: + from negpy.desktop.main import _dispatch_capture_helper + + handled = _dispatch_capture_helper( + [ + "NegPy", + CAPTURE_HELPER_FLAG, + "--frame", + "18", + "--boundary-offset-rows", + "0", + "--confirm-full-capture", + ] + ) + + assert handled is True + output = capsys.readouterr().out + assert "619458560 bytes" in output + assert "dry run only; scanner was not accessed" in output + + +def test_desktop_main_stops_before_bootstrap_for_capture_helper( + monkeypatch: pytest.MonkeyPatch, +) -> None: + import negpy.desktop.main as desktop_main + + monkeypatch.setattr(desktop_main, "_dispatch_capture_helper", lambda _argv: True) + + def fail_if_bootstrapped(*_args, **_kwargs): + raise AssertionError("desktop bootstrap ran in capture-helper mode") + + monkeypatch.setattr(desktop_main, "load_override", fail_if_bootstrapped) + monkeypatch.setattr( + desktop_main, + "_macos_frozen_documents_handoff", + lambda: (_ for _ in ()).throw(AssertionError("handoff ran in capture-helper mode")), + ) + + desktop_main.main() + + +class _FakeMacApplication: + events: list[str] = [] + + @staticmethod + def instance(): + return None + + def __init__(self, _argv) -> None: + self.events.append("application") + + def processEvents(self) -> None: + self.events.append("events") + + +class _FakeMacProgress: + window_flags: list[tuple[object, bool]] = [] + + def __init__(self, *_args) -> None: + _FakeMacApplication.events.append("progress") + + def setWindowTitle(self, _title: str) -> None: + pass + + def setCancelButton(self, _button) -> None: + pass + + def setAutoClose(self, _enabled: bool) -> None: + pass + + def setAutoReset(self, _enabled: bool) -> None: + pass + + def setWindowFlag(self, _flag, _enabled: bool) -> None: + self.window_flags.append((_flag, _enabled)) + + def setWindowModality(self, _modality) -> None: + pass + + def show(self) -> None: + _FakeMacApplication.events.append("show") + + def close(self) -> None: + _FakeMacApplication.events.append("close") + + +def _configure_frozen_macos_handoff(monkeypatch: pytest.MonkeyPatch, desktop_main) -> None: + _FakeMacApplication.events = [] + _FakeMacProgress.window_flags = [] + monkeypatch.setattr(sys, "platform", "darwin") + monkeypatch.setattr(sys, "frozen", True, raising=False) + monkeypatch.delenv(desktop_main._MACOS_DOCUMENTS_READY_ENV, raising=False) # noqa: SLF001 - startup seam + monkeypatch.setattr(desktop_main, "QApplication", _FakeMacApplication) + monkeypatch.setattr(desktop_main, "_MacOSDocumentsHandoff", _FakeMacProgress) + + +def test_macos_frozen_handoff_shows_ui_before_probe_and_reexecs( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + import negpy.desktop.main as desktop_main + + _configure_frozen_macos_handoff(monkeypatch, desktop_main) + monkeypatch.setattr(desktop_main, "BASE_USER_DIR", str(tmp_path)) + + def probe(_path: str) -> None: + _FakeMacApplication.events.append("probe") + + class ReexecCalled(Exception): + pass + + def reexec(executable: str, argv: list[str], env: dict[str, str]) -> None: + _FakeMacApplication.events.append("reexec") + assert executable == str(Path(sys.executable).absolute()) + assert argv[0] == executable + assert env[desktop_main._MACOS_DOCUMENTS_READY_ENV] == "1" # noqa: SLF001 - startup seam + raise ReexecCalled + + monkeypatch.setattr(desktop_main, "_probe_user_data_access", probe) + monkeypatch.setattr(desktop_main.os, "execve", reexec) + + with pytest.raises(ReexecCalled): + desktop_main._macos_frozen_documents_handoff() # noqa: SLF001 - startup seam + + assert _FakeMacApplication.events.index("application") < _FakeMacApplication.events.index("probe") + assert _FakeMacApplication.events.index("show") < _FakeMacApplication.events.index("probe") + assert _FakeMacApplication.events[-1] == "reexec" + assert (desktop_main.Qt.WindowType.WindowCloseButtonHint, False) in _FakeMacProgress.window_flags + + +def test_macos_frozen_handoff_consumes_its_one_process_sentinel( + monkeypatch: pytest.MonkeyPatch, +) -> None: + import negpy.desktop.main as desktop_main + + _configure_frozen_macos_handoff(monkeypatch, desktop_main) + monkeypatch.setenv(desktop_main._MACOS_DOCUMENTS_READY_ENV, "1") # noqa: SLF001 - startup seam + monkeypatch.setattr( + desktop_main, + "_probe_user_data_access", + lambda _path: (_ for _ in ()).throw(AssertionError("child probed Documents again")), + ) + + assert desktop_main._macos_frozen_documents_handoff() is False # noqa: SLF001 - startup seam + assert desktop_main._MACOS_DOCUMENTS_READY_ENV not in os.environ # noqa: SLF001 - startup seam + assert _FakeMacApplication.events == [] + + +def test_macos_frozen_handoff_denial_stops_startup_without_bootstrap( + monkeypatch: pytest.MonkeyPatch, +) -> None: + import negpy.desktop.main as desktop_main + + _configure_frozen_macos_handoff(monkeypatch, desktop_main) + errors: list[tuple] = [] + monkeypatch.setattr(desktop_main, "_dispatch_packaging_smoke", lambda _argv: False) + monkeypatch.setattr(desktop_main, "_dispatch_capture_helper", lambda _argv: False) + monkeypatch.setattr(desktop_main, "_probe_user_data_access", lambda _path: (_ for _ in ()).throw(PermissionError("denied"))) + monkeypatch.setattr( + desktop_main, + "QMessageBox", + SimpleNamespace(critical=lambda *args: errors.append(args)), + ) + monkeypatch.setattr( + desktop_main, + "load_override", + lambda _path: (_ for _ in ()).throw(AssertionError("override loaded after denial")), + ) + monkeypatch.setattr( + desktop_main, + "_bootstrap_environment", + lambda: (_ for _ in ()).throw(AssertionError("bootstrap ran after denial")), + ) + + desktop_main.main() + + assert errors + assert "Documents access" in errors[0][1] + assert "progress" in _FakeMacApplication.events + + +def test_macos_handoff_ignores_close_events(qapp) -> None: + from PyQt6.QtGui import QCloseEvent + + from negpy.desktop.main import _MacOSDocumentsHandoff + + handoff = _MacOSDocumentsHandoff("Opening workspace", None, 0, 0) + event = QCloseEvent() + handoff.closeEvent(event) + + assert not event.isAccepted() + + +def test_macos_access_probe_removes_its_empty_file(tmp_path: Path) -> None: + from negpy.desktop.main import _MACOS_ACCESS_PROBE_PREFIX, _probe_user_data_access + + _probe_user_data_access(str(tmp_path)) + + assert not list(tmp_path.glob(f"{_MACOS_ACCESS_PROBE_PREFIX}*")) + + +def test_desktop_packaging_smoke_dispatches_before_bootstrap( + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: + from negpy.desktop import frozen_entry + + checked: list[bool] = [] + monkeypatch.setattr( + frozen_entry, + "run_packaging_smoke_checks", + lambda: checked.append(True), + ) + + handled = frozen_entry.dispatch_packaging_smoke(["NegPy", frozen_entry.PACKAGING_SMOKE_FLAG]) + + assert handled is True + assert checked == [True] + assert frozen_entry.PACKAGING_SMOKE_OK in capsys.readouterr().out + + +def test_pyinstaller_collects_the_frozen_scanner_and_dice_runtimes() -> None: + build_source = (REPO / "build.py").read_text(encoding="utf-8") + frozen_entry_source = (REPO / "negpy/desktop/frozen_entry.py").read_text(encoding="utf-8") + + required_options = { + "--collect-all=coolscanpy", + "--hidden-import=coolscanpy.protocol.ls5000_single_pass.worker", + "--hidden-import=negpy.services.roll.live_acceptance", + "--collect-all=portable_digital_ice", + } + assert all(option in build_source for option in required_options) + assert 'LIVE_ACCEPTANCE_FLAG = "--ls5000-live-acceptance"' in build_source + assert "--add-data=negpy/assets/portable_cms:negpy/assets/portable_cms" in build_source + # This constructor performs frozen hash/schema validation of the packaged + # 12-event receipt as well as the nine transform binaries. + smoke_function = next( + node + for node in ast.parse(frozen_entry_source).body + if isinstance(node, ast.FunctionDef) and node.name == "run_packaging_smoke_checks" + ) + smoke_calls = {node.func.id for node in ast.walk(smoke_function) if isinstance(node, ast.Call) and isinstance(node.func, ast.Name)} + assert "check_jit_execution" in smoke_calls + assert "PortableCMSOnEvaluator" in smoke_calls + assert "_require_registered_repair_engine" in smoke_calls + assert "_require_pinned_hybrid_runtime" in smoke_calls + assert "--codesign-identity=" in build_source + + +@pytest.mark.skipif(sys.platform != "darwin", reason="macOS bundle identity contract") +def test_macos_build_uses_a_stable_reverse_dns_bundle_identifier() -> None: + import build + + assert build.MACOS_BUNDLE_IDENTIFIER == "org.negpy.NegPy" + assert f"--osx-bundle-identifier={build.MACOS_BUNDLE_IDENTIFIER}" in build.params + + +@pytest.mark.skipif(sys.platform != "darwin", reason="macOS hardened-runtime contract") +def test_macos_build_uses_only_the_llvmlite_executable_memory_entitlement() -> None: + import build + + entitlements_path = Path(build.MACOS_ENTITLEMENTS_FILE) + entitlements = plistlib.loads(entitlements_path.read_bytes()) + + assert entitlements == { + "com.apple.security.cs.allow-unsigned-executable-memory": True, + } + assert f"--osx-entitlements-file={entitlements_path}" in build.params + + +@pytest.mark.skipif(sys.platform != "darwin", reason="macOS hardened-runtime contract") +def test_macos_embedded_entitlements_are_read_back_as_xml( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + import build + + app_path = tmp_path / "NegPy.app" + embedded = {"com.apple.security.cs.allow-unsigned-executable-memory": True} + calls: list[list[str]] = [] + + def emit_entitlements(argv, **_kwargs): + calls.append(list(argv)) + return subprocess.CompletedProcess(argv, 0, stdout=plistlib.dumps(embedded), stderr=b"") + + monkeypatch.setattr(build.subprocess, "run", emit_entitlements) + + assert build.verify_macos_runtime_entitlements(str(app_path)) == embedded + assert calls == [ + [ + "/usr/bin/codesign", + "--display", + "--entitlements", + "-", + "--xml", + str(app_path), + ] + ] + + +@pytest.mark.skipif(sys.platform != "darwin", reason="macOS hardened-runtime contract") +def test_macos_developer_id_signature_exposes_the_hardened_runtime_flag( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + import build + + app_path = tmp_path / "NegPy.app" + calls: list[list[str]] = [] + + def emit_signature_details(argv, **_kwargs): + calls.append(list(argv)) + return subprocess.CompletedProcess(argv, 0, stdout="", stderr="CodeDirectory v=20500 flags=0x10000(runtime)\n") + + monkeypatch.setattr(build.subprocess, "run", emit_signature_details) + + assert build.verify_macos_hardened_runtime(str(app_path)) == 0x10000 + assert calls == [["/usr/bin/codesign", "--display", "--verbose=4", str(app_path)]] + + +@pytest.mark.skipif(sys.platform != "darwin", reason="macOS hardened-runtime contract") +def test_macos_developer_id_signature_rejects_a_non_runtime_seal( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + import build + + monkeypatch.setattr( + build.subprocess, + "run", + lambda argv, **_kwargs: subprocess.CompletedProcess( + argv, + 0, + stdout="", + stderr="CodeDirectory v=20400 flags=0x2(adhoc)\n", + ), + ) + + with pytest.raises(RuntimeError, match="missing the hardened-runtime"): + build.verify_macos_hardened_runtime(str(tmp_path / "NegPy.app")) + + +@pytest.mark.skipif(sys.platform != "darwin", reason="macOS hardened-runtime contract") +@pytest.mark.parametrize( + "embedded", + [ + {}, + { + "com.apple.security.cs.allow-unsigned-executable-memory": True, + "com.apple.security.cs.allow-jit": True, + }, + { + "com.apple.security.cs.allow-unsigned-executable-memory": True, + "com.apple.security.cs.disable-library-validation": True, + }, + ], +) +def test_macos_embedded_entitlement_verification_fails_closed( + embedded: dict[str, bool], + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + import build + + def emit_entitlements(argv, **_kwargs): + return subprocess.CompletedProcess(argv, 0, stdout=plistlib.dumps(embedded), stderr=b"") + + monkeypatch.setattr(build.subprocess, "run", emit_entitlements) + + with pytest.raises(RuntimeError, match="unexpected hardened-runtime entitlements"): + build.verify_macos_runtime_entitlements(str(tmp_path / "NegPy.app")) + + +def test_frozen_smoke_rejects_an_unregistered_dice_bridge( + monkeypatch: pytest.MonkeyPatch, +) -> None: + from negpy.desktop import frozen_entry + from negpy.infrastructure.roll import repair as roll_repair + + monkeypatch.setattr(roll_repair, "available", lambda: False) + + with pytest.raises(RuntimeError, match="Digital ICE repair bridge"): + frozen_entry._require_registered_repair_engine() # noqa: SLF001 - frozen acceptance seam + + +def test_frozen_smoke_rehashes_the_pinned_hybrid_model(tmp_path: Path) -> None: + from negpy.desktop import frozen_entry + + model = tmp_path / "big-lama.pt" + model.write_bytes(b"offline pinned model fixture") + expected = hashlib.sha256(model.read_bytes()).hexdigest() + validated: list[bool] = [] + runtime = SimpleNamespace( + model_weights=model, + model_weights_sha256=expected, + validate_files=lambda: validated.append(True), + ) + + actual = frozen_entry._require_pinned_hybrid_runtime( # noqa: SLF001 - frozen acceptance seam + loader=lambda: runtime, + ) + + assert actual == expected + assert validated == [True] + + +@pytest.mark.parametrize("failure", ["missing-runtime", "wrong-model", "symlink-model"]) +def test_frozen_smoke_rejects_an_unpinned_hybrid_runtime( + failure: str, + tmp_path: Path, +) -> None: + from negpy.desktop import frozen_entry + + model = tmp_path / "big-lama.pt" + model.write_bytes(b"offline pinned model fixture") + expected = hashlib.sha256(model.read_bytes()).hexdigest() + runtime = SimpleNamespace( + model_weights=model, + model_weights_sha256=expected, + validate_files=lambda: None, + ) + if failure == "missing-runtime": + runtime_to_load = None + else: + runtime_to_load = runtime + if failure == "wrong-model": + runtime.model_weights_sha256 = "0" * 64 + elif failure == "symlink-model": + alias = tmp_path / "model-link.pt" + alias.symlink_to(model) + runtime.model_weights = alias + + def loader(): + return runtime_to_load + + with pytest.raises(RuntimeError, match="hybrid runtime|model weights"): + frozen_entry._require_pinned_hybrid_runtime( # noqa: SLF001 - frozen acceptance seam + loader=loader, + ) + + +def test_normal_desktop_startup_rejects_model_corruption_after_manifest_load( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + from negpy.desktop import main as desktop_main + from negpy.services.repair.fauxice_hybrid_runner import HybridRuntimeConfig + + executables = [tmp_path / name for name in ("python", "fauxce-hybrid", "iopaint-python", "iopaint")] + for executable in executables: + executable.write_bytes(b"offline executable fixture") + executable.chmod(0o755) + model_dir = tmp_path / "models" + model_dir.mkdir() + model = model_dir / "big-lama.pt" + model.write_bytes(b"corrupted after the manifest was pinned") + runtime = HybridRuntimeConfig( + hybrid_python=executables[0], + executable=executables[1], + core_source_manifest_sha256="1" * 64, + hybrid_source_manifest_sha256="2" * 64, + iopaint_python=executables[2], + iopaint_executable=executables[3], + iopaint_source_manifest_sha256="3" * 64, + model_dir=model_dir, + model_weights=model, + model_weights_sha256="0" * 64, + ) + monkeypatch.setattr(desktop_main, "load_default_hybrid_runtime_manifest", lambda: runtime) + + assert desktop_main._load_desktop_hybrid_runtime() is None # noqa: SLF001 - startup gate + + +def test_pyinstaller_hooks_find_capture_assets_and_dice_backends() -> None: + coolscan_data, _coolscan_binaries, coolscan_modules = collect_all("coolscanpy") + _dice_data, _dice_binaries, dice_modules = collect_all("portable_digital_ice") + + captured_names = {Path(source).name for source, _destination in coolscan_data} + assert { + "replay-first-rgbi4-manifest.json", + "replay-first-rgbi4-plan.jsonl", + "replay-next-rgbi4-plan.json", + } <= captured_names + assert "coolscanpy.protocol.ls5000_single_pass.worker" in coolscan_modules + assert { + "portable_digital_ice.backend", + "portable_digital_ice.fast_cpu.engine", + "portable_digital_ice.metal_backend.engine", + } <= set(dice_modules) + + +def test_build_preflight_rejects_a_stale_coolscan_api( + monkeypatch: pytest.MonkeyPatch, +) -> None: + import build + + real_import = build.importlib.import_module + + def import_without_exact_builder(module_name: str): + if module_name == "coolscanpy.protocol.ls5000_single_pass.density": + return SimpleNamespace( + build_nikon_density_evidence=lambda: None, + build_nikon_exact_builder_evidence=lambda: None, + ) + return real_import(module_name) + + monkeypatch.setattr(build.importlib, "import_module", import_without_exact_builder) + + with pytest.raises(RuntimeError, match="NikonExactBuilderEvidence"): + build.preflight_coolscan_runtime() + + +def test_build_preflight_rejects_coolscan_without_durable_attempts_api( + monkeypatch: pytest.MonkeyPatch, +) -> None: + import build + + real_import = build.importlib.import_module + + class StaleDevice: + def roll(self, *, material=None): + del material + + def import_with_stale_device(module_name: str): + module = real_import(module_name) + if module_name != "coolscanpy": + return module + return SimpleNamespace( + Device=StaleDevice, + DigitalIceAcquisition=module.DigitalIceAcquisition, + DigitalIceAcquisitionEvidence=module.DigitalIceAcquisitionEvidence, + build_digital_ice_acquisition_evidence=(module.build_digital_ice_acquisition_evidence), + ) + + monkeypatch.setattr(build.importlib, "import_module", import_with_stale_device) + + with pytest.raises(RuntimeError, match="attempts_root"): + build.preflight_coolscan_runtime() + + +def test_build_preflight_accepts_the_current_sealed_capture_bundle() -> None: + import build + from coolscanpy.protocol.ls5000_single_pass.bundle import CAPTURE_BUNDLE_SHA256 + + assert build.preflight_coolscan_runtime() == CAPTURE_BUNDLE_SHA256 + + +@pytest.mark.skipif(sys.platform != "darwin", reason="macOS dylib packaging contract") +def test_macos_build_declares_a_present_arch_compatible_libusb() -> None: + import build + + source = Path(build.MACOS_LIBUSB_SOURCE) + + assert source.is_absolute() + assert source.is_file() + assert source.name == build.COOLSCAN_LIBUSB_FILENAME + assert f"--add-binary={source}:coolscanpy/_native" in build.params + architecture = subprocess.run( + ["/usr/bin/lipo", str(source), "-verify_arch", platform.machine()], + check=False, + capture_output=True, + text=True, + ) + assert architecture.returncode == 0, architecture.stderr + + +@pytest.mark.skipif(sys.platform != "darwin", reason="macOS signing contract") +def test_macos_build_signs_libusb_before_resealing_the_app( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + import build + + app_path = tmp_path / "NegPy.app" + bundled_libusb = app_path / "Contents" / "Frameworks" / build.COOLSCAN_LIBUSB_DESTINATION / build.COOLSCAN_LIBUSB_FILENAME + bundled_libusb.parent.mkdir(parents=True) + bundled_libusb.write_bytes(b"offline signing fixture") + calls: list[list[str]] = [] + entitlement_checks: list[str] = [] + + def record_run(argv, **_kwargs): + calls.append(list(argv)) + return subprocess.CompletedProcess(argv, 0) + + monkeypatch.setattr(build.subprocess, "run", record_run) + monkeypatch.setattr( + build, + "verify_macos_runtime_entitlements", + lambda path: entitlement_checks.append(path), + ) + monkeypatch.setenv("NEGPY_CODESIGN_IDENTITY", "-") + + signed_path = build.sign_macos_scanner_runtime(str(app_path)) + + assert signed_path == str(bundled_libusb) + signing_targets = [call[-1] for call in calls if "--sign" in call] + assert signing_targets == [str(bundled_libusb), str(app_path)] + verification_targets = [call[-1] for call in calls if "--verify" in call] + assert verification_targets == [str(bundled_libusb), str(app_path)] + assert entitlement_checks == [str(app_path)] + + +@pytest.mark.skipif(sys.platform != "darwin", reason="macOS signing contract") +def test_macos_reseal_reapplies_the_exact_pyinstaller_entitlements( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + import build + + app_path = tmp_path / "NegPy.app" + bundled_libusb = app_path / "Contents" / "Frameworks" / build.COOLSCAN_LIBUSB_DESTINATION / build.COOLSCAN_LIBUSB_FILENAME + bundled_libusb.parent.mkdir(parents=True) + bundled_libusb.write_bytes(b"offline signing fixture") + calls: list[list[str]] = [] + + def record_run(argv, **_kwargs): + calls.append(list(argv)) + return subprocess.CompletedProcess(argv, 0) + + monkeypatch.setattr(build.subprocess, "run", record_run) + monkeypatch.setattr(build, "verify_macos_runtime_entitlements", lambda _path: None) + monkeypatch.setenv("NEGPY_CODESIGN_IDENTITY", "-") + + build.sign_macos_scanner_runtime(str(app_path)) + + app_sign = next(call for call in calls if "--sign" in call and call[-1] == str(app_path)) + entitlement_index = app_sign.index("--entitlements") + assert app_sign[entitlement_index + 1] == build.MACOS_ENTITLEMENTS_FILE + preserve_option = next(option for option in app_sign if option.startswith("--preserve-metadata=")) + assert "entitlements" not in preserve_option.split("=", 1)[1].split(",") + + +@pytest.mark.skipif(sys.platform != "darwin", reason="macOS signing contract") +def test_macos_developer_id_reseal_verifies_hardened_runtime( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + import build + + app_path = tmp_path / "NegPy.app" + bundled_libusb = app_path / "Contents" / "Frameworks" / build.COOLSCAN_LIBUSB_DESTINATION / build.COOLSCAN_LIBUSB_FILENAME + bundled_libusb.parent.mkdir(parents=True) + bundled_libusb.write_bytes(b"offline signing fixture") + runtime_checks: list[str] = [] + + monkeypatch.setattr( + build.subprocess, + "run", + lambda argv, **_kwargs: subprocess.CompletedProcess(argv, 0), + ) + monkeypatch.setattr(build, "verify_macos_runtime_entitlements", lambda _path: None) + monkeypatch.setattr( + build, + "verify_macos_hardened_runtime", + lambda path: runtime_checks.append(path), + ) + monkeypatch.setenv("NEGPY_CODESIGN_IDENTITY", "Developer ID Application: NegPy Test") + + build.sign_macos_scanner_runtime(str(app_path)) + + assert runtime_checks == [str(app_path)] + + +@pytest.mark.skipif(sys.platform != "darwin", reason="macOS app smoke contract") +def test_macos_postbuild_smoke_uses_only_offline_frozen_entry_points( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + import build + + app_path = tmp_path / "NegPy.app" + executable = app_path / "Contents" / "MacOS" / "NegPy" + executable.parent.mkdir(parents=True) + executable.write_bytes(b"offline executable fixture") + executable.chmod(0o755) + bundled_libusb = app_path / "Contents" / "Frameworks" / build.COOLSCAN_LIBUSB_DESTINATION / build.COOLSCAN_LIBUSB_FILENAME + bundled_libusb.parent.mkdir(parents=True) + bundled_libusb.write_bytes(b"offline dylib fixture") + + class LoadedLibusb: + libusb_init = object() + libusb_exit = object() + + monkeypatch.setattr(build.ctypes, "CDLL", lambda _path: LoadedLibusb()) + commands: list[list[str]] = [] + environments: list[dict[str, str]] = [] + ambient_loader_variables = { + "DYLD_FALLBACK_FRAMEWORK_PATH", + "DYLD_FALLBACK_LIBRARY_PATH", + "DYLD_FRAMEWORK_PATH", + "DYLD_INSERT_LIBRARIES", + "DYLD_LIBRARY_PATH", + } + for variable in ambient_loader_variables: + monkeypatch.setenv(variable, "/untrusted/host/path") + + def complete(argv, **kwargs): + command = list(argv) + commands.append(command) + environments.append(kwargs["env"]) + if build.PACKAGING_SMOKE_FLAG in command: + stdout = build.PACKAGING_SMOKE_OK + "\n" + else: + stdout = "validated RGBI4x plan: 619458560 bytes\ndry run only; scanner was not accessed\n" + return subprocess.CompletedProcess(command, 0, stdout, "") + + monkeypatch.setattr(build.subprocess, "run", complete) + + result = build.smoke_macos_scanner_runtime(str(app_path)) + + assert result["libusb"] == str(bundled_libusb) + assert [command[1] for command in commands] == [ + build.PACKAGING_SMOKE_FLAG, + build.CAPTURE_HELPER_FLAG, + ] + assert all("--live" not in command for command in commands) + assert all(build.LIVE_ACCEPTANCE_FLAG not in command for command in commands) + assert all(ambient_loader_variables.isdisjoint(environment) for environment in environments) + + +@pytest.mark.skipif(sys.platform != "darwin", reason="macOS app smoke contract") +def test_macos_postbuild_smoke_refuses_the_live_acceptance_flag( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + import build + + app_path = tmp_path / "NegPy.app" + executable = app_path / "Contents" / "MacOS" / "NegPy" + executable.parent.mkdir(parents=True) + executable.write_bytes(b"offline executable fixture") + executable.chmod(0o755) + bundled_libusb = app_path / "Contents" / "Frameworks" / build.COOLSCAN_LIBUSB_DESTINATION / build.COOLSCAN_LIBUSB_FILENAME + bundled_libusb.parent.mkdir(parents=True) + bundled_libusb.write_bytes(b"offline dylib fixture") + + class LoadedLibusb: + libusb_init = object() + libusb_exit = object() + + monkeypatch.setattr(build.ctypes, "CDLL", lambda _path: LoadedLibusb()) + monkeypatch.setattr(build, "PACKAGING_SMOKE_FLAG", build.LIVE_ACCEPTANCE_FLAG) + monkeypatch.setattr( + build.subprocess, + "run", + lambda *_args, **_kwargs: (_ for _ in ()).throw(AssertionError("live acceptance reached subprocess")), + ) + + with pytest.raises(RuntimeError, match="attempted to enable live acceptance"): + build.smoke_macos_scanner_runtime(str(app_path)) + + +@pytest.mark.skipif(sys.platform != "darwin", reason="macOS app smoke contract") +def test_macos_postbuild_smoke_resolves_a_relative_bundle_before_chdir( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """A build invoked from its project root must not resolve dist twice.""" + import build + + app_path = tmp_path / "dist" / "NegPy.app" + executable = app_path / "Contents" / "MacOS" / "NegPy" + executable.parent.mkdir(parents=True) + executable.write_bytes(b"offline executable fixture") + executable.chmod(0o755) + bundled_libusb = app_path / "Contents" / "Frameworks" / build.COOLSCAN_LIBUSB_DESTINATION / build.COOLSCAN_LIBUSB_FILENAME + bundled_libusb.parent.mkdir(parents=True) + bundled_libusb.write_bytes(b"offline dylib fixture") + + class LoadedLibusb: + libusb_init = object() + libusb_exit = object() + + monkeypatch.chdir(tmp_path) + monkeypatch.setattr(build.ctypes, "CDLL", lambda _path: LoadedLibusb()) + commands: list[list[str]] = [] + working_directories: list[str] = [] + + def complete(argv, **kwargs): + command = list(argv) + commands.append(command) + working_directories.append(kwargs["cwd"]) + stdout = ( + build.PACKAGING_SMOKE_OK + "\n" + if build.PACKAGING_SMOKE_FLAG in command + else "validated RGBI4x plan: 619458560 bytes\ndry run only; scanner was not accessed\n" + ) + return subprocess.CompletedProcess(command, 0, stdout, "") + + monkeypatch.setattr(build.subprocess, "run", complete) + + result = build.smoke_macos_scanner_runtime("dist/NegPy.app") + + assert result["executable"] == str(executable.resolve()) + assert all(command[0] == str(executable.resolve()) for command in commands) + assert working_directories == [str(app_path.parent.resolve())] * 2 diff --git a/tests/test_main_window.py b/tests/test_main_window.py index 3e75f23a..cd2dd711 100644 --- a/tests/test_main_window.py +++ b/tests/test_main_window.py @@ -2,6 +2,7 @@ from unittest.mock import patch import numpy as np +from PyQt6.QtCore import QEvent from negpy.desktop.view.main_window import _display_buffer_for_canvas @@ -55,5 +56,48 @@ def test_drop_auto_opens_like_add_dialogs(self): stub.controller.request_asset_discovery.assert_called_once_with(["/tmp/scan.dng"], auto_open=True) +class TestCloseEvent(unittest.TestCase): + def test_unresolved_scanner_close_ignores_window_close(self): + from types import SimpleNamespace + from unittest.mock import MagicMock + + from negpy.desktop.view.main_window import MainWindow + + controller = MagicMock() + controller.request_shutdown.return_value = False + controller.shutdown_block_reason = "ownership remains uncertain" + status_bar = MagicMock() + stub = SimpleNamespace( + controller=controller, + statusBar=lambda: status_bar, + show=MagicMock(), + raise_=MagicMock(), + activateWindow=MagicMock(), + ) + stub.request_shutdown_for_exit = lambda: MainWindow.request_shutdown_for_exit(stub) + event = MagicMock() + + with patch("negpy.desktop.view.main_window.QMessageBox.critical") as critical: + MainWindow.closeEvent(stub, event) + + event.ignore.assert_called_once_with() + controller.session.repo.save_global_setting.assert_not_called() + status_bar.showMessage.assert_called_once() + critical.assert_called_once() + + def test_application_quit_event_is_consumed_when_shutdown_is_blocked(self): + from unittest.mock import MagicMock + + from negpy.desktop.main import _ApplicationShutdownGate + + allow_shutdown = MagicMock(return_value=False) + gate = _ApplicationShutdownGate(allow_shutdown) + event = MagicMock() + event.type.return_value = QEvent.Type.Quit + + assert gate.eventFilter(None, event) is True + allow_shutdown.assert_called_once_with() + + if __name__ == "__main__": unittest.main() diff --git a/tests/test_roll_controller.py b/tests/test_roll_controller.py new file mode 100644 index 00000000..71c31f13 --- /dev/null +++ b/tests/test_roll_controller.py @@ -0,0 +1,125 @@ +"""Regression test for the roll-scan→import seam (AppController._on_roll_scan_finished). + +Mirrors test_capture_controller.py's shape: call the seam directly against a mock +controller, no real worker/thread/session involved. RollFrameOutput's shape (slot, +rgb_path, ir_path, receipt_path) is stood in with SimpleNamespace -- this seam only +reads .rgb_path, so nothing coolscanpy-shaped needs to be constructed. +""" + +from types import SimpleNamespace +from unittest.mock import MagicMock + +import pytest + +from negpy.desktop.controller import AppController +from negpy.services.roll.service import RollScanningError + + +def _output(slot: int, rgb_path: str): + return SimpleNamespace(slot=slot, rgb_path=rgb_path, ir_path=None, receipt_path=f"{rgb_path}_receipt.json") + + +def test_writes_are_discovered_in_batch_order(): + controller = MagicMock() + outputs = [_output(1, "a.tif"), _output(2, "b.tif"), _output(3, "c.tif")] + + AppController._on_roll_scan_finished(controller, outputs) + + controller.roll_finished.emit.assert_called_once_with(outputs) + controller.request_asset_discovery.assert_called_once_with(["a.tif", "b.tif", "c.tif"]) + + +def test_empty_batch_still_emits_but_does_not_request_discovery(): + controller = MagicMock() + + AppController._on_roll_scan_finished(controller, []) + + controller.roll_finished.emit.assert_called_once_with([]) + controller.request_asset_discovery.assert_not_called() + + +def test_ir_and_receipt_paths_are_not_sent_to_discovery(): + """Only the RGB TIFF is a NegPy-openable asset; the _IR sidecar and the JSON + receipt are not scannable frames and must not be handed to discovery.""" + controller = MagicMock() + outputs = [SimpleNamespace(slot=1, rgb_path="frame.tif", ir_path="frame_IR.tif", receipt_path="frame_receipt.json")] + + AppController._on_roll_scan_finished(controller, outputs) + + (discovered,), _kwargs = controller.request_asset_discovery.call_args + assert discovered == ["frame.tif"] + + +def test_outputs_with_no_rgb_path_are_skipped_for_discovery(): + """The roll sidebar's output-tier setting can leave Tier 1 (rgb_path) + unwritten -- e.g. a positive-only scan -- in which case rgb_path is None. + Those frames have nothing NegPy can open and must not reach discovery, + but a frame that did write Tier 1 in the same batch still should.""" + controller = MagicMock() + outputs = [ + SimpleNamespace(slot=1, rgb_path=None, ir_path=None, receipt_path="a_receipt.json"), + SimpleNamespace(slot=2, rgb_path="b.tif", ir_path=None, receipt_path="b_receipt.json"), + ] + + AppController._on_roll_scan_finished(controller, outputs) + + controller.roll_finished.emit.assert_called_once_with(outputs) + controller.request_asset_discovery.assert_called_once_with(["b.tif"]) + + +def test_batch_with_no_rgb_paths_at_all_does_not_request_discovery(): + controller = MagicMock() + outputs = [SimpleNamespace(slot=1, rgb_path=None, ir_path=None, receipt_path="a_receipt.json")] + + AppController._on_roll_scan_finished(controller, outputs) + + controller.request_asset_discovery.assert_not_called() + + +def test_request_shutdown_blocks_when_scanner_close_is_unresolved(): + worker = MagicMock() + worker.shutdown.side_effect = RuntimeError("ownership remains uncertain") + controller = SimpleNamespace( + _roll_shutdown_complete=False, + _shutdown_block_reason=None, + roll_worker=worker, + roll_status=MagicMock(), + ) + + assert AppController.request_shutdown(controller) is False + + assert controller._roll_shutdown_complete is False + assert controller._shutdown_block_reason == "ownership remains uncertain" + controller.roll_status.emit.assert_called_once() + + +def test_request_shutdown_is_idempotent_after_verified_close(): + worker = MagicMock() + controller = SimpleNamespace( + _roll_shutdown_complete=False, + _shutdown_block_reason="old failure", + roll_worker=worker, + roll_status=MagicMock(), + ) + + assert AppController.request_shutdown(controller) is True + assert AppController.request_shutdown(controller) is True + + worker.shutdown.assert_called_once_with() + assert controller._shutdown_block_reason is None + + +def test_cleanup_refuses_to_quit_worker_threads_before_scanner_close(): + roll_thread = MagicMock() + controller = SimpleNamespace( + _cleaned_up=False, + _shutdown_block_reason="ownership remains uncertain", + request_shutdown=lambda: False, + roll_thread=roll_thread, + ) + + with pytest.raises(RollScanningError, match="ownership remains uncertain"): + AppController.cleanup(controller) + + assert controller._cleaned_up is False + roll_thread.quit.assert_not_called() diff --git a/tests/test_user_dir_resolution.py b/tests/test_user_dir_resolution.py index 202dfede..7eeed2f7 100644 --- a/tests/test_user_dir_resolution.py +++ b/tests/test_user_dir_resolution.py @@ -42,6 +42,20 @@ def test_env_override_wins(monkeypatch, tmp_path): assert get_default_user_dir() == os.path.abspath(str(tmp_path / "custom")) +def test_macos_resolution_is_path_only_before_the_visible_startup_handoff(monkeypatch, tmp_path): + """Resolving the frozen app's default cannot itself touch protected Documents.""" + home = tmp_path / "home" + + monkeypatch.delenv("NEGPY_USER_DIR", raising=False) + monkeypatch.setattr("sys.platform", "darwin") + monkeypatch.setattr(os.path, "expanduser", lambda p: str(home) if p == "~" else p) + monkeypatch.setattr(os.path, "isdir", lambda _path: (_ for _ in ()).throw(AssertionError("resolution touched Documents"))) + monkeypatch.setattr(os, "makedirs", lambda *_args, **_kwargs: (_ for _ in ()).throw(AssertionError("resolution created Documents"))) + + assert get_default_user_dir() == str((home / "Documents" / "NegPy").absolute()) + assert not home.exists() + + def test_broken_documents_falls_back_to_home(monkeypatch, tmp_path): home = tmp_path / "home" home.mkdir() diff --git a/tests/test_wheel_packaging.py b/tests/test_wheel_packaging.py new file mode 100644 index 00000000..90e66715 --- /dev/null +++ b/tests/test_wheel_packaging.py @@ -0,0 +1,170 @@ +"""Regression coverage for the installable NegPy wheel boundary.""" + +from __future__ import annotations + +import hashlib +import os +import site as site_module +import subprocess +import sys +import zipfile +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[1] +ROLL_MODULES = { + f"negpy/services/roll/{name}" + for name in ( + "__init__.py", + "exact_color.py", + "nikon_icc.py", + "portable_builder.py", + "portable_cms.py", + "portable_oracle_evaluator.py", + "positive.py", + "service.py", + ) +} + + +def _run( + command: list[str], + *, + cwd: Path, + env: dict[str, str] | None = None, +) -> subprocess.CompletedProcess[str]: + completed = subprocess.run( + command, + cwd=cwd, + env=env, + check=False, + capture_output=True, + text=True, + ) + assert completed.returncode == 0, ( + f"command failed ({completed.returncode}): {' '.join(command)}\nstdout:\n{completed.stdout}\nstderr:\n{completed.stderr}" + ) + return completed + + +def test_wheel_contains_and_loads_all_negpy_packages_and_portable_assets(tmp_path: Path) -> None: + dist = tmp_path / "dist" + _run(["uv", "build", "--wheel", "--out-dir", str(dist)], cwd=ROOT) + wheels = list(dist.glob("*.whl")) + assert len(wheels) == 1 + wheel = wheels[0] + + expected_python = {path.relative_to(ROOT).as_posix() for path in (ROOT / "negpy").rglob("*.py")} + expected_assets = { + path.relative_to(ROOT).as_posix(): hashlib.sha256(path.read_bytes()).hexdigest() + for directory in ( + ROOT / "negpy/assets/portable_builder", + ROOT / "negpy/assets/portable_cms", + ) + for path in directory.iterdir() + if path.suffix in {".bin", ".json"} + } + assert len(expected_assets) == 11 + assert sum(name.endswith(".bin") for name in expected_assets) == 10 + assert sum(name.endswith(".json") for name in expected_assets) == 1 + + with zipfile.ZipFile(wheel) as archive: + packaged = set(archive.namelist()) + packaged_python = {name for name in packaged if name.startswith("negpy/") and name.endswith(".py")} + packaged_assets = { + name for name in packaged if name.startswith("negpy/assets/portable_") and Path(name).suffix in {".bin", ".json"} + } + assert packaged_python == expected_python + assert ROLL_MODULES <= packaged + assert packaged_assets == set(expected_assets) + assert {name: hashlib.sha256(archive.read(name)).hexdigest() for name in expected_assets} == expected_assets + + venv = tmp_path / "venv" + _run(["uv", "venv", "--python", sys.executable, str(venv)], cwd=tmp_path) + venv_python = venv / ("Scripts/python.exe" if os.name == "nt" else "bin/python") + _run( + ["uv", "pip", "install", "--python", str(venv_python), "--no-deps", str(wheel)], + cwd=tmp_path, + ) + venv_site = Path( + _run( + [str(venv_python), "-c", "import site; print(site.getsitepackages()[0])"], + cwd=tmp_path, + ).stdout.strip() + ) + dependency_site = next(Path(path) for path in site_module.getsitepackages() if (Path(path) / "numpy").is_dir()) + (venv_site / "negpy-test-dependencies.pth").write_text( + str(dependency_site) + "\n", + encoding="utf-8", + ) + smoke = """ +import importlib +from pathlib import Path +import negpy.services.roll.exact_color as exact_color +import negpy.services.roll.nikon_icc as nikon_icc +import negpy.services.roll.portable_builder as portable_builder +import negpy.services.roll.portable_cms as portable_cms +import negpy.services.roll.service as service + +site = Path(__import__('sys').argv[1]).resolve() +roll_modules = [ + importlib.import_module(name) + for name in ( + 'negpy.services.roll', + 'negpy.services.roll.exact_color', + 'negpy.services.roll.nikon_icc', + 'negpy.services.roll.portable_builder', + 'negpy.services.roll.portable_cms', + 'negpy.services.roll.positive', + 'negpy.services.roll.service', + ) +] +assert len(roll_modules) == 7 +for module in roll_modules: + assert Path(module.__file__).resolve().is_relative_to(site), module.__file__ +builder = portable_builder.PortableStage1Builder() +cms = portable_cms.PortableCMSOnEvaluator() +assert 'negpy.services.roll.portable_oracle_evaluator' not in __import__('sys').modules +assert len(list(builder.assets_dir.glob('*.bin'))) == 1 +assert len(list(cms.assets_dir.glob('*.bin'))) == 9 +validation = cms.assets_dir / portable_cms.VALIDATION_RECEIPT_FILENAME +assert validation.is_file() +assert validation.stat().st_size == portable_cms.VALIDATION_RECEIPT_BYTES +assert __import__('hashlib').sha256(validation.read_bytes()).hexdigest() == portable_cms.VALIDATION_RECEIPT_SHA256 +profile = nikon_icc.nikon_adobe_rgb_profile() +assert len(profile) == 492 +assert __import__('hashlib').sha256(profile).hexdigest() == nikon_icc.NIKON_ADOBE_RGB_PROFILE_SHA256 +""" + env = os.environ.copy() + env.pop("PYTHONPATH", None) + _run([str(venv_python), "-c", smoke, str(venv_site)], cwd=tmp_path, env=env) + + validation_path = venv_site / "negpy/assets/portable_cms/portable-oracle-receipt.json" + validation_payload = validation_path.read_bytes() + for failure, expected in ( + ("missing", "portable CMS validation receipt is unavailable"), + ("one-byte", "portable CMS validation receipt hash mismatch"), + ): + if failure == "missing": + validation_path.unlink() + else: + tampered = bytearray(validation_payload) + tampered[len(tampered) // 2] ^= 1 + validation_path.write_bytes(tampered) + try: + completed = subprocess.run( + [ + str(venv_python), + "-c", + "from negpy.services.roll.portable_cms import PortableCMSOnEvaluator; PortableCMSOnEvaluator()", + ], + cwd=tmp_path, + env=env, + check=False, + capture_output=True, + text=True, + ) + assert completed.returncode != 0 + assert expected in completed.stderr + finally: + validation_path.write_bytes(validation_payload) diff --git a/uv.lock b/uv.lock index cfbb8304..9cfb2626 100644 --- a/uv.lock +++ b/uv.lock @@ -11,6 +11,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/a9/ba/000a1996d4308bc65120167c21241a3b205464a2e0b58deda26ae8ac21d1/altgraph-0.17.5-py2.py3-none-any.whl", hash = "sha256:f3a22400bce1b0c701683820ac4f3b159cd301acab067c51c653e06961600597", size = 21228, upload-time = "2025-11-21T20:35:49.444Z" }, ] +[[package]] +name = "attrs" +version = "26.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/9a/8e/82a0fe20a541c03148528be8cac2408564a6c9a0cc7e9171802bc1d26985/attrs-26.1.0.tar.gz", hash = "sha256:d03ceb89cb322a8fd706d4fb91940737b6642aa36998fe130a9bc96c985eff32", size = 952055, upload-time = "2026-03-19T14:22:25.026Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/64/b4/17d4b0b2a2dc85a6df63d1157e028ed19f90d4cd97c36717afef2bc2f395/attrs-26.1.0-py3-none-any.whl", hash = "sha256:c647aa4a12dfbad9333ca4e71fe62ddc36f4e63b2d260a37a8b83d2f043ac309", size = 67548, upload-time = "2026-03-19T14:22:23.645Z" }, +] + [[package]] name = "cffi" version = "2.0.0" @@ -65,6 +74,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", size = 25335, upload-time = "2022-10-25T02:36:20.889Z" }, ] +[[package]] +name = "coolscanpy" +version = "0.1.3" +source = { git = "https://github.com/rohanpandula/coolscanpy.git?rev=970d18e#970d18e4b092737a633bd20141d3eaa62b25dcc6" } +dependencies = [ + { name = "imagecodecs" }, + { name = "jinja2" }, + { name = "numpy" }, + { name = "opencv-python-headless" }, + { name = "pyusb" }, + { name = "tifffile" }, +] + [[package]] name = "coverage" version = "7.14.0" @@ -134,6 +156,33 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/61/e8/cb8e80d6f9f55b99588625062822bf946cf03ed06315df4bd8397f5632a1/coverage-7.14.0-py3-none-any.whl", hash = "sha256:8de5b61163aee3d05c8a2beab6f47913df7981dad1baf82c414d99158c286ab1", size = 211764, upload-time = "2026-05-10T18:02:29.538Z" }, ] +[[package]] +name = "fauxce-hybrid" +version = "0.3.1" +source = { url = "https://github.com/rohanpandula/digital-fauxice/releases/download/v0.3.1/fauxce_hybrid-0.3.1-py3-none-any.whl" } +dependencies = [ + { name = "imageio" }, + { name = "jsonschema" }, + { name = "numpy" }, + { name = "portable-digital-ice", extra = ["fast"] }, + { name = "scipy" }, +] +wheels = [ + { url = "https://github.com/rohanpandula/digital-fauxice/releases/download/v0.3.1/fauxce_hybrid-0.3.1-py3-none-any.whl", hash = "sha256:f36c66d8e4e2d2a2b32caf79bc35241aaf92b0453781f713ffd5e5345ae3032f" }, +] + +[package.metadata] +requires-dist = [ + { name = "imageio", specifier = ">=2.37,<3" }, + { name = "jsonschema", specifier = ">=4.25,<5" }, + { name = "numpy", specifier = ">=2.4,<3" }, + { name = "portable-digital-ice", extras = ["fast"], specifier = "==0.3.1" }, + { name = "pytest", marker = "extra == 'test'", specifier = ">=9,<10" }, + { name = "ruff", marker = "extra == 'test'", specifier = ">=0.11" }, + { name = "scipy", specifier = ">=1.16,<2" }, +] +provides-extras = ["test"] + [[package]] name = "gphoto2" version = "2.6.4" @@ -206,6 +255,33 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/62/a1/3d680cbfd5f4b8f15abc1d571870c5fc3e594bb582bc3b64ea099db13e56/jinja2-3.1.6-py3-none-any.whl", hash = "sha256:85ece4451f492d0c13c5dd7c13a64681a86afae63a5f347908daf103ce6d2f67", size = 134899, upload-time = "2025-03-05T20:05:00.369Z" }, ] +[[package]] +name = "jsonschema" +version = "4.26.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "attrs" }, + { name = "jsonschema-specifications" }, + { name = "referencing" }, + { name = "rpds-py" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b3/fc/e067678238fa451312d4c62bf6e6cf5ec56375422aee02f9cb5f909b3047/jsonschema-4.26.0.tar.gz", hash = "sha256:0c26707e2efad8aa1bfc5b7ce170f3fccc2e4918ff85989ba9ffa9facb2be326", size = 366583, upload-time = "2026-01-07T13:41:07.246Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/69/90/f63fb5873511e014207a475e2bb4e8b2e570d655b00ac19a9a0ca0a385ee/jsonschema-4.26.0-py3-none-any.whl", hash = "sha256:d489f15263b8d200f8387e64b4c3a75f06629559fb73deb8fdfb525f2dab50ce", size = 90630, upload-time = "2026-01-07T13:41:05.306Z" }, +] + +[[package]] +name = "jsonschema-specifications" +version = "2025.9.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "referencing" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/19/74/a633ee74eb36c44aa6d1095e7cc5569bebf04342ee146178e2d36600708b/jsonschema_specifications-2025.9.1.tar.gz", hash = "sha256:b540987f239e745613c7a9176f3edb72b832a4ac465cf02712288397832b5e8d", size = 32855, upload-time = "2025-09-08T01:34:59.186Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/41/45/1a4ed80516f02155c51f51e8cedb3c1902296743db0bbc66608a0db2814f/jsonschema_specifications-2025.9.1-py3-none-any.whl", hash = "sha256:98802fee3a11ee76ecaca44429fda8a41bff98b00a0f2838151b113f210cc6fe", size = 18437, upload-time = "2025-09-08T01:34:57.871Z" }, +] + [[package]] name = "llvmlite" version = "0.47.0" @@ -315,6 +391,9 @@ dependencies = [ camera = [ { name = "gphoto2", marker = "sys_platform != 'win32'" }, ] +coolscan-roll = [ + { name = "coolscanpy" }, +] dev = [ { name = "pyinstaller" }, { name = "pytest" }, @@ -322,6 +401,12 @@ dev = [ { name = "ruff" }, { name = "ty" }, ] +fauxice = [ + { name = "portable-digital-ice" }, +] +fauxice-hybrid = [ + { name = "fauxce-hybrid" }, +] scanner = [ { name = "python-sane" }, ] @@ -347,6 +432,7 @@ requires-dist = [ [package.metadata.requires-dev] camera = [{ name = "gphoto2", marker = "sys_platform != 'win32'", specifier = ">=2.5" }] +coolscan-roll = [{ name = "coolscanpy", git = "https://github.com/rohanpandula/coolscanpy.git?rev=970d18e" }] dev = [ { name = "pyinstaller", specifier = ">=6.18.0" }, { name = "pytest", specifier = "==9.0.2" }, @@ -354,6 +440,8 @@ dev = [ { name = "ruff", specifier = "==0.14.10" }, { name = "ty", specifier = ">=0.0.26" }, ] +fauxice = [{ name = "portable-digital-ice", url = "https://github.com/rohanpandula/digital-fauxice/releases/download/v0.3.1/portable_digital_ice-0.3.1-py3-none-any.whl" }] +fauxice-hybrid = [{ name = "fauxce-hybrid", url = "https://github.com/rohanpandula/digital-fauxice/releases/download/v0.3.1/fauxce_hybrid-0.3.1-py3-none-any.whl" }] scanner = [{ name = "python-sane", specifier = ">=2.9" }] [[package]] @@ -542,6 +630,36 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/54/20/4d324d65cc6d9205fabedc306948156824eb9f0ee1633355a8f7ec5c66bf/pluggy-1.6.0-py3-none-any.whl", hash = "sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746", size = 20538, upload-time = "2025-05-15T12:30:06.134Z" }, ] +[[package]] +name = "portable-digital-ice" +version = "0.3.1" +source = { url = "https://github.com/rohanpandula/digital-fauxice/releases/download/v0.3.1/portable_digital_ice-0.3.1-py3-none-any.whl" } +dependencies = [ + { name = "numpy" }, +] +wheels = [ + { url = "https://github.com/rohanpandula/digital-fauxice/releases/download/v0.3.1/portable_digital_ice-0.3.1-py3-none-any.whl", hash = "sha256:54fc0cfd42456abdc91a33a5530b5a615a6ec05a98577d195f2b9d8d080bcfa4" }, +] + +[package.optional-dependencies] +fast = [ + { name = "numba" }, +] + +[package.metadata] +requires-dist = [ + { name = "cupy-cuda12x", marker = "extra == 'cuda'", specifier = ">=13" }, + { name = "numba", marker = "extra == 'cuda'", specifier = ">=0.65,<0.68" }, + { name = "numba", marker = "extra == 'fast'", specifier = ">=0.65,<0.68" }, + { name = "numba", marker = "extra == 'metal'", specifier = ">=0.65,<0.68" }, + { name = "numpy", specifier = ">=2.4,<3" }, + { name = "pyobjc-framework-metal", marker = "extra == 'metal'", specifier = ">=10.3" }, + { name = "pytest", marker = "extra == 'dev'", specifier = ">=9,<10" }, + { name = "pytest", marker = "extra == 'test'", specifier = ">=9,<10" }, + { name = "ruff", marker = "extra == 'dev'", specifier = ">=0.11" }, +] +provides-extras = ["cuda", "fast", "metal", "test", "dev"] + [[package]] name = "pycparser" version = "3.0" @@ -725,6 +843,15 @@ version = "2.9.2" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/45/e9/e8baff69fc2347606c547201204d4b4843c7ad8ecb9164eceee42016eff6/python_sane-2.9.2.tar.gz", hash = "sha256:50ab8e0b033cececad26c7231a7254f80ad8fe9ec6b5c25add2493d7e2a07bbe", size = 22513, upload-time = "2025-07-21T21:20:21.735Z" } +[[package]] +name = "pyusb" +version = "1.3.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/00/6b/ce3727395e52b7b76dfcf0c665e37d223b680b9becc60710d4bc08b7b7cb/pyusb-1.3.1.tar.gz", hash = "sha256:3af070b607467c1c164f49d5b0caabe8ac78dbed9298d703a8dbf9df4052d17e", size = 77281, upload-time = "2025-01-08T23:45:01.866Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/28/b8/27e6312e86408a44fe16bd28ee12dd98608b39f7e7e57884a24e8f29b573/pyusb-1.3.1-py3-none-any.whl", hash = "sha256:bf9b754557af4717fe80c2b07cc2b923a9151f5c08d17bdb5345dac09d6a0430", size = 58465, upload-time = "2025-01-08T23:45:00.029Z" }, +] + [[package]] name = "pywin32-ctypes" version = "0.2.3" @@ -777,6 +904,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/4d/01/1854b2f789d7a6d94152ba451bdaadae0010d4247099d9c2719b381d4b6b/rawpy-0.27.0-cp314-cp314-win_amd64.whl", hash = "sha256:3cac371c3b6302eb88bd1f33d43c7a92b5e553fdff14258b564f81e173a68c26", size = 940931, upload-time = "2026-05-07T08:29:15.604Z" }, ] +[[package]] +name = "referencing" +version = "0.37.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "attrs" }, + { name = "rpds-py" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/22/f5/df4e9027acead3ecc63e50fe1e36aca1523e1719559c499951bb4b53188f/referencing-0.37.0.tar.gz", hash = "sha256:44aefc3142c5b842538163acb373e24cce6632bd54bdb01b21ad5863489f50d8", size = 78036, upload-time = "2025-10-13T15:30:48.871Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2c/58/ca301544e1fa93ed4f80d724bf5b194f6e4b945841c5bfd555878eea9fcb/referencing-0.37.0-py3-none-any.whl", hash = "sha256:381329a9f99628c9069361716891d34ad94af76e461dcb0335825aecc7692231", size = 26766, upload-time = "2025-10-13T15:30:47.625Z" }, +] + [[package]] name = "rendercanvas" version = "2.6.3" @@ -789,6 +929,87 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/c1/b2/b06d7d89eef452ab50ad3e752604c09ec08c8eaf49484641e566a0e5141d/rendercanvas-2.6.3-py3-none-any.whl", hash = "sha256:c2defd04e3e29dcdf94b794e827045882925677bc48b04084f3b9696fd469018", size = 105194, upload-time = "2026-03-12T13:43:08.965Z" }, ] +[[package]] +name = "rpds-py" +version = "2026.6.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/aa/2a/9618a122aeb2a169a28b03889a2995fe297588964333d4a7d67bdf46e147/rpds_py-2026.6.3.tar.gz", hash = "sha256:1cebd1337c242e4ec2293e541f712b2da849b29f48f0c293684b71c0632625d4", size = 64051, upload-time = "2026-06-30T07:17:53.009Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a4/9e/b818ee580026ec578138e961027a68820c40afeb1ec8f6819b54fb99e196/rpds_py-2026.6.3-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:3cfe765c1da0072636ca06628261e0ea05688e160d5c8a03e0217c3854037223", size = 343012, upload-time = "2026-06-30T07:15:36.005Z" }, + { url = "https://files.pythonhosted.org/packages/f3/6b/686d9dc4359a8f163cfbbf89ee0b4e586431de22fe8248edb63a8cf50d49/rpds_py-2026.6.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:f4d78253f6996be4901669ad25319f842f740eccf4d58e3c7f3dd39e6dde1d8f", size = 338203, upload-time = "2026-06-30T07:15:37.462Z" }, + { url = "https://files.pythonhosted.org/packages/9e/9b/069aa329940f8207615e091f5eedbbd40e1e15eac68a0790fd05ccdf796c/rpds_py-2026.6.3-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:54f45a148e28767bf343d33a684693c70e451c6f4c0e9904709a723fafbdfc1f", size = 367984, upload-time = "2026-06-30T07:15:39.008Z" }, + { url = "https://files.pythonhosted.org/packages/14/db/34c203e4becff3703e4d3bc121842c00b8689197f398161203a880052f4e/rpds_py-2026.6.3-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:842e7b070435622248c7a2c44ae53fa1440e073cc3023bc919fed570884097a7", size = 374815, upload-time = "2026-06-30T07:15:40.253Z" }, + { url = "https://files.pythonhosted.org/packages/ee/7d/8071067d2cc453d916ad836e828c943f575e8a44612537759002a1e07381/rpds_py-2026.6.3-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8020133a74bd81b4572dd8e4be028a6b1ebcd70e6726edc3918008c08bee6ee6", size = 490545, upload-time = "2026-06-30T07:15:41.729Z" }, + { url = "https://files.pythonhosted.org/packages/a3/42/da06c5aa8f0484ff07f270787434204d9f4535e2f8c3b51ed402267e63c3/rpds_py-2026.6.3-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cdc7e35386f3847df728fbcb5e887e2d79c19e2fa1eba9e51b6621d23e3243af", size = 382828, upload-time = "2026-06-30T07:15:43.327Z" }, + { url = "https://files.pythonhosted.org/packages/57/d7/fe978efc2ae50abe48eb7464668ea99f53c010c60aeebb7b35ad27f23661/rpds_py-2026.6.3-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:acac386b453c2516111b50985d60ce46e7fadb5ea71ae7b25f4c946935bf27cf", size = 365678, upload-time = "2026-06-30T07:15:44.992Z" }, + { url = "https://files.pythonhosted.org/packages/69/9d/1d8922e1990b2a6eb532b6ff53d3e73d2b3bbffc84116c75826bee73dfc6/rpds_py-2026.6.3-cp313-cp313-manylinux_2_31_riscv64.whl", hash = "sha256:425560c6fa0415f27261727bb20bd097568485e5eb0c121f1949417d1c516885", size = 377811, upload-time = "2026-06-30T07:15:46.523Z" }, + { url = "https://files.pythonhosted.org/packages/b1/3d/198dceafb4fb034a6a47347e1b0735d34e0bd4a50be4e898d408ee66cb14/rpds_py-2026.6.3-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:a550fb4950a06dde3beb4721f5ad4b25bf4513784665b0a8522c792e2bd822a4", size = 395382, upload-time = "2026-06-30T07:15:47.955Z" }, + { url = "https://files.pythonhosted.org/packages/1f/f1/13968e49655d40b6b19d8b9140296bbc6f1d86b3f0f6c346cf9f1adddf4b/rpds_py-2026.6.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:4f4bca01b63096f606e095734dd56e74e175f94cfbf24ff3d63281cec61f7bb7", size = 543832, upload-time = "2026-06-30T07:15:49.33Z" }, + { url = "https://files.pythonhosted.org/packages/ac/ab/289bcb1b90bd3e40a2900c561fa0e2087345ecbb094f0b870f2345142b7c/rpds_py-2026.6.3-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:ccffae9a092a00deb7efd545fe5e2c33c33b88e7c054337e9a74c179347d0b7d", size = 611011, upload-time = "2026-06-30T07:15:50.847Z" }, + { url = "https://files.pythonhosted.org/packages/1e/16/5043105e679436ccfbc8e5e0dd2d663ed18a8b8113515fd06a5e5d77c83e/rpds_py-2026.6.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:1cf01971c4f2c5553b772a542e4aaf191789cd331bc2cd4ff0e6e65ba49e1e97", size = 572431, upload-time = "2026-06-30T07:15:52.394Z" }, + { url = "https://files.pythonhosted.org/packages/85/ed/adab103321c0a6565d5ae1c2998349bc3ee175b82ccc5ae8fc04cc413075/rpds_py-2026.6.3-cp313-cp313-win32.whl", hash = "sha256:8c3d1e9c15b9d51ca0391e13da1a25a0a4df3c58a37c9dc368e0736cf7f69df0", size = 201710, upload-time = "2026-06-30T07:15:53.894Z" }, + { url = "https://files.pythonhosted.org/packages/7b/ed/a03b09668e74e5dabbf2e211f6468e1820c0552f7b0500082da31841bf7b/rpds_py-2026.6.3-cp313-cp313-win_amd64.whl", hash = "sha256:9250a9a0a6fd4648b3f868da8d91a4c52b5811a62df58e753d50ae4454a36f80", size = 219454, upload-time = "2026-06-30T07:15:55.25Z" }, + { url = "https://files.pythonhosted.org/packages/27/17/b8642c12930b71bc2b25831f6708ccf0f75abcd11883932ec9ce54ba3a78/rpds_py-2026.6.3-cp313-cp313-win_arm64.whl", hash = "sha256:900a67df3fd1660b035a4761c4ce73c382ea6b35f90f9863c36c6fd8bf8b09bb", size = 215063, upload-time = "2026-06-30T07:15:56.573Z" }, + { url = "https://files.pythonhosted.org/packages/b6/36/7fbe9dcdaf857fb3f63c2a2284b62492d95f5e8334e947e5fb6e7f68c9be/rpds_py-2026.6.3-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:931908d9fc855d8f74783377822be318edb6dcb19e47169dc038f9a1bf60b06e", size = 344510, upload-time = "2026-06-30T07:15:57.921Z" }, + { url = "https://files.pythonhosted.org/packages/ba/54/f785cc3d3f60839ca57a5af4927a9f347b07b2799c373fc20f7949f87c7e/rpds_py-2026.6.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:d7469697dce35be237db177d42e2a2ee26e6dcc5fc052078a6fefabd288c6edd", size = 339495, upload-time = "2026-06-30T07:15:59.238Z" }, + { url = "https://files.pythonhosted.org/packages/63/ef/d4cdaf309e6b095b43597103cf8c0b951d6cca2acce68c474f75ec12e0c7/rpds_py-2026.6.3-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bcfbcf66006befb9fd2aeaa9e01feaf881b4dc330a02ba07d2322b1c11be7b5d", size = 369454, upload-time = "2026-06-30T07:16:01.021Z" }, + { url = "https://files.pythonhosted.org/packages/96/4a/9559a68b7ee15db09d7981212e8c2e219d2a1d6d4faa0391d813c3496a36/rpds_py-2026.6.3-cp314-cp314-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:847927daf4cffbd4e90e42bc890069897101edd015f956cb8721b3473372edda", size = 374583, upload-time = "2026-06-30T07:16:02.287Z" }, + { url = "https://files.pythonhosted.org/packages/ef/75/8964aa7d2c6e8ac43eba8eb6e6b0fdda1f46d39f2fc3e6aa9f2cb17f485d/rpds_py-2026.6.3-cp314-cp314-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:aca6c1ef08a82bfe327cc156da694660f599923e2e6665b6d81c9c2d0ac9ffc8", size = 492919, upload-time = "2026-06-30T07:16:03.723Z" }, + { url = "https://files.pythonhosted.org/packages/8f/97/6908094ac804115e65aedfd90f1b5fee4eebebd3f6c4cfc5419939267565/rpds_py-2026.6.3-cp314-cp314-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ae50181a047c871561212bb97f7932a2d45fb53e947bd9b57ebad85b529cbc53", size = 383725, upload-time = "2026-06-30T07:16:05.305Z" }, + { url = "https://files.pythonhosted.org/packages/d1/9c/0d1fdc2e7aba23e290d603bc494e97bd205bae262ce33c6b32a69768ed5e/rpds_py-2026.6.3-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dc319e5a1de4b6913aac94bf6a2f9e847371e0a140a43dd4991db1a09bc2d504", size = 367255, upload-time = "2026-06-30T07:16:07.086Z" }, + { url = "https://files.pythonhosted.org/packages/c4/fe/f0209ca4a9ed074bc8acb44dfd0e81c3122e94c9689f5645b7973a866719/rpds_py-2026.6.3-cp314-cp314-manylinux_2_31_riscv64.whl", hash = "sha256:e4316bf32babbed84e691e352faf967ce2f0f024174a8643c37c94a1080374fc", size = 379060, upload-time = "2026-06-30T07:16:08.525Z" }, + { url = "https://files.pythonhosted.org/packages/c6/8d/f1cc54c616b9d8897de8738aac148d20afca93f68187475fe194d09a71b9/rpds_py-2026.6.3-cp314-cp314-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:8c6e5a2f750cc71c3e3b11d71661f21d6f9bc6cebc6564b1466417a1ec03ec77", size = 395960, upload-time = "2026-06-30T07:16:09.989Z" }, + { url = "https://files.pythonhosted.org/packages/fb/04/aafff00f73aeca2945f734f1d483c64ab8f472d0864ab02377fd8e89c3b2/rpds_py-2026.6.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:4470ce197d4090875cf6affbf1f853338387428df97c4fb7b7106317b8214698", size = 545356, upload-time = "2026-06-30T07:16:11.816Z" }, + { url = "https://files.pythonhosted.org/packages/fd/cc/e229663b9e4ddac5a4acbe9085dd80a71af2a5d356b8b39d6bff233f24b0/rpds_py-2026.6.3-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:ea964164cc9afa72d4d9b23cc28dafae93693c0a53e0b42acbff15b22c3f9ddd", size = 612319, upload-time = "2026-06-30T07:16:13.586Z" }, + { url = "https://files.pythonhosted.org/packages/e3/7a/8a0e6d3e6cd066af108b71b43122c3fe158dd9eb86acac626593a2582eb1/rpds_py-2026.6.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:639c8929aa0afe81be836b04de888460d6bed38b9c54cfc18da8f6bfabf5af5d", size = 573508, upload-time = "2026-06-30T07:16:15.23Z" }, + { url = "https://files.pythonhosted.org/packages/87/03/2a69ab618a789cf6cf85c86bb844c62d090e700ab1a2aa676b3741b6c516/rpds_py-2026.6.3-cp314-cp314-win32.whl", hash = "sha256:882076c00c0a608b131187055ddc5ae29f2e7eaf870d6168980420d58528a5c8", size = 202504, upload-time = "2026-06-30T07:16:16.893Z" }, + { url = "https://files.pythonhosted.org/packages/85/62/a3892ba945f4e24c78f352e5de3c7620d8479f73f211406a97263d13c7d2/rpds_py-2026.6.3-cp314-cp314-win_amd64.whl", hash = "sha256:0be972be84cfcaf46c8c6edf690ca0f154ac17babf1f6a955a51579b34ad2dc5", size = 220380, upload-time = "2026-06-30T07:16:18.108Z" }, + { url = "https://files.pythonhosted.org/packages/3d/e7/c2bd44dc831931815ad11ebb5f430b5a0a4d3caa9de837107876c30c3432/rpds_py-2026.6.3-cp314-cp314-win_arm64.whl", hash = "sha256:2a9c6f195058cb45335e8cc3802745c603d716eb96bc9625950c1aac71c0c703", size = 215976, upload-time = "2026-06-30T07:16:19.654Z" }, + { url = "https://files.pythonhosted.org/packages/79/9c/fff7b74bce9a091ec9a012a03f9ff5f69364eaf9451060dfc4486da2ffdd/rpds_py-2026.6.3-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:f90938e92afda60266da758ee7d363447f7f0138c9559f9e1811629580582d90", size = 346840, upload-time = "2026-06-30T07:16:21.268Z" }, + { url = "https://files.pythonhosted.org/packages/e9/44/77bcb1168b33704908295533d27f10eb811e9e3e193e8993dc99572211d3/rpds_py-2026.6.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:ec829541c45bca16e61c7ae50c20501f213605beb75d1aba91a6ee37fbbb56a4", size = 340282, upload-time = "2026-06-30T07:16:22.875Z" }, + { url = "https://files.pythonhosted.org/packages/87/3c/7a9081c7c9e645b39efe19e4ffbeccd80add246327cd9b888aecffd72317/rpds_py-2026.6.3-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:afd70d95892096cdb26f15a00c45907b17817577aa8d1c76b2dcc2788391f9e9", size = 370403, upload-time = "2026-06-30T07:16:24.415Z" }, + { url = "https://files.pythonhosted.org/packages/f7/69/af47021eb7dad6ff3396cb001c08f0f3c4d06c20253f75be6421a59fe6b7/rpds_py-2026.6.3-cp314-cp314t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:29dfa0533a5d4c94d4dfa1b694fcb56c9c63aad8330ffdd816fd225d0a7a162f", size = 376055, upload-time = "2026-06-30T07:16:26.111Z" }, + { url = "https://files.pythonhosted.org/packages/81/fc/a3bcf517084396a6dd258c592567a3c011ba4557f2fde23dceaf26e74f2e/rpds_py-2026.6.3-cp314-cp314t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:af05d726809bff6b141be124d4c7ce998f9c9c7f30edb1f46c07aa103d540b41", size = 494419, upload-time = "2026-06-30T07:16:27.596Z" }, + { url = "https://files.pythonhosted.org/packages/c9/eb/13d529d1788135425c7bf207f8463458ca5d92e43f3f701365b83e9dffc1/rpds_py-2026.6.3-cp314-cp314t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9826217f048f620d9a712672818bf231442c1b35d96b227a07eabd11b4bb6945", size = 384848, upload-time = "2026-06-30T07:16:29.183Z" }, + { url = "https://files.pythonhosted.org/packages/8e/f4/b7ac49f30013aba8f7b9566b1dd07e81de95e708c1374b7bacc5b9bc5c9c/rpds_py-2026.6.3-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:536bceea4fa4acf7e1c61da2b5786304367c816c8895be71b8f537c480b0ea1f", size = 371369, upload-time = "2026-06-30T07:16:30.912Z" }, + { url = "https://files.pythonhosted.org/packages/31/86/6260bafa622f788b07ddec0e52d810305c8b9b0b8c27f58a2ab04bf62b4f/rpds_py-2026.6.3-cp314-cp314t-manylinux_2_31_riscv64.whl", hash = "sha256:bc0011654b91cc4fb2ae701bec0a0ba1e552c0714247fa7af6c59e0ccfa3a4e1", size = 379673, upload-time = "2026-06-30T07:16:32.486Z" }, + { url = "https://files.pythonhosted.org/packages/19/c3/03f1ee79a047b48daeca157c89a18509cde22b6b951d642b9b0af1be660a/rpds_py-2026.6.3-cp314-cp314t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:539d75de9e0d536c84ff18dfeb805398e58227001ce09231a26a08b9aed1ee0e", size = 397500, upload-time = "2026-06-30T07:16:34.471Z" }, + { url = "https://files.pythonhosted.org/packages/f0/95/8ed0cd8c377dca12aea498f119fe639fc474d1461545c39d2b5872eb1c0f/rpds_py-2026.6.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:166cf54d9f44fc6ceb53c7860258dde44a81406646de79f8ed3234fca3b6e538", size = 545978, upload-time = "2026-06-30T07:16:36.45Z" }, + { url = "https://files.pythonhosted.org/packages/d3/f2/0eb57f0eaa83f8fc152a7e03de968ab77e1f00732bebc892b190c6eebde7/rpds_py-2026.6.3-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:d34c20167764fbcf927194d532dd7e0c56772f0a5f943fa5ef9e9afbba8fb9db", size = 613350, upload-time = "2026-06-30T07:16:38.213Z" }, + { url = "https://files.pythonhosted.org/packages/5b/de/e0674bdbc3ef7634989b3f854c3f34bc1f587d36e5bfdc5c378d57034619/rpds_py-2026.6.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:ea7bb13b7c9a29791f87a0387ba7d3ad3a6d783d827e4d3f27b40a0ff44495e2", size = 576486, upload-time = "2026-06-30T07:16:39.797Z" }, + { url = "https://files.pythonhosted.org/packages/f2/f6/21101359743cd136ada781e8210a85769578422ba460672eea0e29739200/rpds_py-2026.6.3-cp314-cp314t-win32.whl", hash = "sha256:6de4744d05bd1aa1be4ed7ea1189e3979196808008113bbbf899a460966b925e", size = 201068, upload-time = "2026-06-30T07:16:41.316Z" }, + { url = "https://files.pythonhosted.org/packages/a6/b2/9574d4d44f7760c2aa32d92a0a4f41698e33f5b204a0bf5c9758f52c79d5/rpds_py-2026.6.3-cp314-cp314t-win_amd64.whl", hash = "sha256:c7b9a2f8f4d8e90af72571d3d495deebdd7e3c75451f5b41719aee166e940fc2", size = 220600, upload-time = "2026-06-30T07:16:43.091Z" }, + { url = "https://files.pythonhosted.org/packages/08/ae/f23a2697e6ee6340a578b0f136be6483657bef0c6f9497b752bb5c0964bb/rpds_py-2026.6.3-cp315-cp315-macosx_10_12_x86_64.whl", hash = "sha256:e059c5dde6452b44424bd1834557556c226b57781dee1227af23518459722b13", size = 344726, upload-time = "2026-06-30T07:16:44.5Z" }, + { url = "https://files.pythonhosted.org/packages/c3/63/e7b3a1a5358dd32c930a1062d8e15b67fd6e8922e81df9e91706d66ee5c8/rpds_py-2026.6.3-cp315-cp315-macosx_11_0_arm64.whl", hash = "sha256:2f7c26fbc5acd2522b95d4177fe4710ffd8e9b20529e703ffbf8db4d93903f05", size = 339587, upload-time = "2026-06-30T07:16:46.255Z" }, + { url = "https://files.pythonhosted.org/packages/ec/64/10a85681916ca55fffb91b0a211f84e34297c109243484dd6394660a8a7c/rpds_py-2026.6.3-cp315-cp315-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a3086b538543802f84c843911242db20447de00d8752dd0efc936dbcf02218ba", size = 369585, upload-time = "2026-06-30T07:16:48.101Z" }, + { url = "https://files.pythonhosted.org/packages/76/c2/baf95c7c38823e12ba34407c5f5767a89e5cf2233895e56f608167ae9493/rpds_py-2026.6.3-cp315-cp315-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:8f2e5c5ee828d42cb11760761c0af6507927bec42d0ad5458f97c9203b054617", size = 375479, upload-time = "2026-06-30T07:16:49.93Z" }, + { url = "https://files.pythonhosted.org/packages/6a/94/0aad06c72d65101e11d33528d438cda99a39ce0da99466e156158f2541d3/rpds_py-2026.6.3-cp315-cp315-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ed0c1e5d10cdc7135537988c74a0188da68e2f3c30813ba3744ab1e42e0480f9", size = 492418, upload-time = "2026-06-30T07:16:51.641Z" }, + { url = "https://files.pythonhosted.org/packages/b5/17/de3f5a479a1f056535d7489819639d8cd591ea6281d700390b43b1abd745/rpds_py-2026.6.3-cp315-cp315-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8c2642a7603ec0b16ed77da4555db3b4b472341904873788327c0b0d7b95f1bb", size = 384123, upload-time = "2026-06-30T07:16:53.622Z" }, + { url = "https://files.pythonhosted.org/packages/46/7d/bf09bd1b145bb2671c03e1e6d1ab8651858d90d8c7dfeadd85a37a934fd8/rpds_py-2026.6.3-cp315-cp315-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8e4320744c1ffdd95a603def63344bfab2d33edeab301c5007e7de9f9f5b3885", size = 367351, upload-time = "2026-06-30T07:16:55.241Z" }, + { url = "https://files.pythonhosted.org/packages/a3/ea/1bb734f314b8be319149ddee80b18bd41372bdcfbdf88d28131c0cd37719/rpds_py-2026.6.3-cp315-cp315-manylinux_2_31_riscv64.whl", hash = "sha256:a9f4645593036b81bbdb36b9c8e0ea0d1c3fee968c4d59db0344c14087ef143a", size = 378827, upload-time = "2026-06-30T07:16:56.841Z" }, + { url = "https://files.pythonhosted.org/packages/4b/93/d9611e5b25e26df9a3649813ed66193ace9347a7c7fc4ab7cf70e94851c0/rpds_py-2026.6.3-cp315-cp315-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:e55d236be29255554da47abe5c577637db7c24a02b8b46f0ca9524c855801868", size = 395966, upload-time = "2026-06-30T07:16:58.557Z" }, + { url = "https://files.pythonhosted.org/packages/c3/cb/99d77e16e5534ae1d90629bbe419ba6ee170833a6a85e3aa1cc41726fbbc/rpds_py-2026.6.3-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:24e9c5386e16669b674a69c156c8eeefcb578f3b3397b713b08e6d60f3c7b187", size = 545680, upload-time = "2026-06-30T07:17:00.164Z" }, + { url = "https://files.pythonhosted.org/packages/59/15/11a29755f790cef7a2f755e8e14f4f0c33f39489e1893a632a2eee59672b/rpds_py-2026.6.3-cp315-cp315-musllinux_1_2_i686.whl", hash = "sha256:c60924535c75f1566b6eb75b5c31a48a43fef04fa2d0d201acbad8a9969c6107", size = 611853, upload-time = "2026-06-30T07:17:01.962Z" }, + { url = "https://files.pythonhosted.org/packages/68/86/0c27547e21644da938fb530f7e1a8148dd24d02db07e7a5f2567a17ce710/rpds_py-2026.6.3-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:38a2fea2787428f811719ceb9114cb78964a3138838320c29ac39526c79c16ba", size = 573715, upload-time = "2026-06-30T07:17:03.693Z" }, + { url = "https://files.pythonhosted.org/packages/29/71/4d8fcf700931815594bce892255bbd973b94efaf0fc1932b0590df18d886/rpds_py-2026.6.3-cp315-cp315-win32.whl", hash = "sha256:d483fe17f01ad64b7bf7cc38fcefff1ca9fb83f8c2b2542b68f97ffe0611b369", size = 202864, upload-time = "2026-06-30T07:17:05.746Z" }, + { url = "https://files.pythonhosted.org/packages/eb/62/b577562de0edbb55b2be85ce5fd09c33e386b9b13eee09833af4240fd5c4/rpds_py-2026.6.3-cp315-cp315-win_amd64.whl", hash = "sha256:67e3a721ffc5d8d2210d3671872298c4a84e4b8035cfe42ffd7cde35d772b146", size = 220430, upload-time = "2026-06-30T07:17:07.471Z" }, + { url = "https://files.pythonhosted.org/packages/c8/95/d6d0b2509825141eef60669a5739eec88dbc6a48053d6c92993a5704defe/rpds_py-2026.6.3-cp315-cp315-win_arm64.whl", hash = "sha256:6e84adbcf4bf841aed8116a8264b9f50b4cb3e7bd89b516122e616ac56ca269e", size = 215877, upload-time = "2026-06-30T07:17:09.008Z" }, + { url = "https://files.pythonhosted.org/packages/b7/bf/f3ea278f0afd615c1d0f19cb69043a41526e2bb600c2b536eb192218eb27/rpds_py-2026.6.3-cp315-cp315t-macosx_10_12_x86_64.whl", hash = "sha256:ae6dd8f10bd17aad820876d24caec9efdafd80a318d16c0a48edb5e136902c6b", size = 346933, upload-time = "2026-06-30T07:17:10.762Z" }, + { url = "https://files.pythonhosted.org/packages/9d/29/9907bdf1c5346763cf10b7f6852aad86652168c259def904cbe0082c5864/rpds_py-2026.6.3-cp315-cp315t-macosx_11_0_arm64.whl", hash = "sha256:bdbd97738551fca3917c1bd7188bec1920bb520104f28e7e1007f9ceb17b7690", size = 340274, upload-time = "2026-06-30T07:17:12.266Z" }, + { url = "https://files.pythonhosted.org/packages/6f/2c/8e03767b5778ef25cebf74a7a91a2c3806f8eced4c92cb7406bbe060756d/rpds_py-2026.6.3-cp315-cp315t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8b95977e7211527ab0ba576e286d023389fbeeb32a6b7b771665d333c60e5342", size = 370763, upload-time = "2026-06-30T07:17:14.107Z" }, + { url = "https://files.pythonhosted.org/packages/2e/e1/df2a7e1ba2efd796af26194250b8d42c821b46592311595162af9ef0528d/rpds_py-2026.6.3-cp315-cp315t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:d15fde0e6fb0d88a60d221204873743e5d9f0b7d29165e62cd86d0413ad74ba6", size = 376467, upload-time = "2026-06-30T07:17:15.76Z" }, + { url = "https://files.pythonhosted.org/packages/6b/de/8a0814d1946af29cb068fb259aa8622f856df1d0bab58429448726b537f5/rpds_py-2026.6.3-cp315-cp315t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a136d453475ac0fcbda502ef1e6504bd28d6d904700915d278deeab0d00fe140", size = 496689, upload-time = "2026-06-30T07:17:17.308Z" }, + { url = "https://files.pythonhosted.org/packages/df/f3/f19e0c852ba13694f5a79f3b719331051573cb5693feacf8a88ffffc3a71/rpds_py-2026.6.3-cp315-cp315t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f826877d462181e5eb1c26a0026b8d0cab05d99844ecb6d8bf3627a2ca0c0442", size = 385340, upload-time = "2026-06-30T07:17:18.928Z" }, + { url = "https://files.pythonhosted.org/packages/e2/ae/7ec3a9d2d4351f99e37bcb06b6b6f954512646bfdbf9742e1de727865daf/rpds_py-2026.6.3-cp315-cp315t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:79486287de1730dbaff3dbd124d0ca4d2ef7f9d29bf2544f1f93c09b5bcbbd12", size = 372179, upload-time = "2026-06-30T07:17:20.539Z" }, + { url = "https://files.pythonhosted.org/packages/d3/ac/9cee911dff2aaa9a5a8354f6610bf2e6a616de9197c5fff4f54f82585f1e/rpds_py-2026.6.3-cp315-cp315t-manylinux_2_31_riscv64.whl", hash = "sha256:808345f53cb952433ca2816f1604ff3515608a81784954f38d4452acfe8e61d5", size = 379993, upload-time = "2026-06-30T07:17:22.212Z" }, + { url = "https://files.pythonhosted.org/packages/83/6b/7c2a07ba88d1e9a936612f7a5d067467ed03d971d5a06f7d309dff044a7e/rpds_py-2026.6.3-cp315-cp315t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:1967debc37f64f2c4dc90a7f563aec558b471966e12adcac4e1c4240496b6ebf", size = 398909, upload-time = "2026-06-30T07:17:23.66Z" }, + { url = "https://files.pythonhosted.org/packages/97/0b/776ffcb66783637b0031f6d58d6fb55913c8b5abf00aeecd46bf933fb477/rpds_py-2026.6.3-cp315-cp315t-musllinux_1_2_aarch64.whl", hash = "sha256:f0840b5b17057f7fd918b76183a4b5a0635f43e14eb2ce60dce1d4ee4707ea00", size = 546584, upload-time = "2026-06-30T07:17:25.264Z" }, + { url = "https://files.pythonhosted.org/packages/55/33/ba3bc04d7092bd553c9b2b195624992d2cc4f3de1f380b7b93cbee67bd79/rpds_py-2026.6.3-cp315-cp315t-musllinux_1_2_i686.whl", hash = "sha256:faa679d19a6696fd54259ad321251ad77a13e70e03dd834daa762a44fb6196ef", size = 614357, upload-time = "2026-06-30T07:17:26.888Z" }, + { url = "https://files.pythonhosted.org/packages/8b/71/14edf065f04630b1a8472f7653cad03f6c478bcf95ea0e6aed55451e33ea/rpds_py-2026.6.3-cp315-cp315t-musllinux_1_2_x86_64.whl", hash = "sha256:23a439f31ccbeff1574e24889128821d1f7917470e830cf6544dced1c662262a", size = 576533, upload-time = "2026-06-30T07:17:28.546Z" }, + { url = "https://files.pythonhosted.org/packages/ba/76/65002b08596c389105720a8c0d22298b8dc25a4baf89b2ce431343c8b1de/rpds_py-2026.6.3-cp315-cp315t-win32.whl", hash = "sha256:913ca42ccad3f8cc6e292b587ae8ae49c8c823e5dce51a736252fc7c7cdfa577", size = 201204, upload-time = "2026-06-30T07:17:30.193Z" }, + { url = "https://files.pythonhosted.org/packages/8c/97/d855d6b3c322d1f27e26f5241c42016b56cf01377ea8ed348285f54652f0/rpds_py-2026.6.3-cp315-cp315t-win_amd64.whl", hash = "sha256:ae3d4fe8c0b9213624fdce7279d70e3b148b682ca20719ebd193a23ebfa47324", size = 220719, upload-time = "2026-06-30T07:17:31.788Z" }, +] + [[package]] name = "rubicon-objc" version = "0.5.4" @@ -824,6 +1045,47 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/74/31/b0e29d572670dca3674eeee78e418f20bdf97fa8aa9ea71380885e175ca0/ruff-0.14.10-py3-none-win_arm64.whl", hash = "sha256:e51d046cf6dda98a4633b8a8a771451107413b0f07183b2bef03f075599e44e6", size = 13729839, upload-time = "2025-12-18T19:28:48.636Z" }, ] +[[package]] +name = "scipy" +version = "1.18.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "numpy" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/a7/25/c2700dfaf6442b4effaa91af24ebce5dc9d31bb4a69706313aae70d72cd0/scipy-1.18.0.tar.gz", hash = "sha256:67b2ad2ad54c72ca6d04975a9b2df8c3638c34ddd5b28738e94fc2b57929d378", size = 30774447, upload-time = "2026-06-19T15:01:43.456Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/05/52/9c0136c2de7ae0779b7b366447766cec6d9f0702c56bb8ffeb04c8fd3af4/scipy-1.18.0-cp313-cp313-macosx_10_15_x86_64.whl", hash = "sha256:09143f676d157d9f546d663504ef9c1becb819824f1afc018814176411942446", size = 31036107, upload-time = "2026-06-19T15:00:14.03Z" }, + { url = "https://files.pythonhosted.org/packages/02/73/0291a64843270f4efb86cdcf2ee0f2048631b65ec6b405398b2b4dbf11bf/scipy-1.18.0-cp313-cp313-macosx_12_0_arm64.whl", hash = "sha256:5efe260f69417b97ddae455bfb5a95e8359f7f66ad7fa9522a60feb66f169520", size = 28663303, upload-time = "2026-06-19T15:00:16.819Z" }, + { url = "https://files.pythonhosted.org/packages/d3/0f/10ffa0b697a572f4e0d48b92a88895d366422f019f723e7e14a84c050dac/scipy-1.18.0-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:68363b7eaacd8b5dd426df56d782cc156468ac79a127a1b87ca597d6e2e82197", size = 20404960, upload-time = "2026-06-19T15:00:19.635Z" }, + { url = "https://files.pythonhosted.org/packages/7e/d2/e896cea21ba8edd6c81d4c55b1ffcc717e79698dcbebf9641b4cfb4c6622/scipy-1.18.0-cp313-cp313-macosx_14_0_x86_64.whl", hash = "sha256:c5557d8be5da8e41353fcd4d21491fdbab83b062fc579e94dc09a7c8ab4f669b", size = 23034074, upload-time = "2026-06-19T15:00:22.107Z" }, + { url = "https://files.pythonhosted.org/packages/ea/b2/e83ea34279a52c03374477c74006256ec78df65fc877baa4617d6de1d202/scipy-1.18.0-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0d13bca67c096d89fb95ced0d8921807300fce0275643aef9533cc63a0773468", size = 33942038, upload-time = "2026-06-19T15:00:24.964Z" }, + { url = "https://files.pythonhosted.org/packages/f6/af/e8fe5fb136f51e2b01678b92cb4106d10d8cd68ec147ead2e7cb0ac75398/scipy-1.18.0-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a46f9273dbd0eb1cefba61c9b8648b4dfe3cbc14a080176f9a73e44b8336dc7f", size = 35266390, upload-time = "2026-06-19T15:00:28.059Z" }, + { url = "https://files.pythonhosted.org/packages/3a/49/2c5cbb907b56695fc67517811d1db234dfd83381a84814ec220aded2794d/scipy-1.18.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:5aba46108853ddfc77906b6557aac839d2b52e900c1d72a1180adaaab58d265f", size = 35551324, upload-time = "2026-06-19T15:00:31.014Z" }, + { url = "https://files.pythonhosted.org/packages/bb/73/eda39f7a2d306ff0ffc574afd13c0bbb6d10a603d9a413998ee269487a80/scipy-1.18.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:b6f758e35f12757b5d95c00bc6de2438e229c2664b7a92e96f205959d9f2dfa4", size = 37404785, upload-time = "2026-06-19T15:00:34.072Z" }, + { url = "https://files.pythonhosted.org/packages/b7/d2/ae881ee28d014f38e0ccbfd974a06a919ba9af34f1f74bf42b5301891d63/scipy-1.18.0-cp313-cp313-win_amd64.whl", hash = "sha256:1afac4a847207c7ff8efd321734a50b06d0280b3b2a2c0fc2f413101747ad7c7", size = 36554943, upload-time = "2026-06-19T15:00:36.903Z" }, + { url = "https://files.pythonhosted.org/packages/70/3a/21154e2d54eb3639c6bf4dbae2e531c68356bfe95990daa30df33b30d556/scipy-1.18.0-cp313-cp313-win_arm64.whl", hash = "sha256:c5dbddf60e58c2312316d097271a8e73d40eaf2eabfa4d95ed7d3695bbf2ce7b", size = 24350911, upload-time = "2026-06-19T15:00:40.062Z" }, + { url = "https://files.pythonhosted.org/packages/78/b5/915a19b3de2f7430062b509653563db1633ddbb6f021b06731521115d4e2/scipy-1.18.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:4c256ee70c0d1a8a2ace807e199ccd4e3f57037433842abb3fb36bc17eaa9578", size = 31036253, upload-time = "2026-06-19T15:00:43.216Z" }, + { url = "https://files.pythonhosted.org/packages/d7/88/b72def7262e150d16be13fca37a96481138d624e700340bc3362a7588929/scipy-1.18.0-cp314-cp314-macosx_12_0_arm64.whl", hash = "sha256:2ef3abc54a4ffc53765374b0d5728532dfdd2585ed23f6b11c206a1f0b1b9af8", size = 28673758, upload-time = "2026-06-19T15:00:46.663Z" }, + { url = "https://files.pythonhosted.org/packages/91/02/2e636a61a525632c373cf6a9c24442a3ffb79e364d38e98b32042964ac32/scipy-1.18.0-cp314-cp314-macosx_14_0_arm64.whl", hash = "sha256:f2a6af57bd9e4a75d70e4117e78a1bbee84f79ae3fbb6d0111005d6ebcc4cb8d", size = 20415514, upload-time = "2026-06-19T15:00:49.399Z" }, + { url = "https://files.pythonhosted.org/packages/c9/b6/2135974442f6aba159d9d39d774a1c8cb19947016725d69fecc685df45bf/scipy-1.18.0-cp314-cp314-macosx_14_0_x86_64.whl", hash = "sha256:3f1ac564d3bf6c03d861d2cd87a1bea0da2887136f7fb1bf519c05a8971452d6", size = 23034398, upload-time = "2026-06-19T15:00:51.941Z" }, + { url = "https://files.pythonhosted.org/packages/f6/e6/ba89ec5abf6ee9257c0d1ec985573f3ae32742c24bc03e016388a40b1b15/scipy-1.18.0-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:40395a5fcd1abee49a5c7aaa98c29db393eedc835138560a588c47ec16156690", size = 33998032, upload-time = "2026-06-19T15:00:54.838Z" }, + { url = "https://files.pythonhosted.org/packages/7f/c4/bc41eb19b0fd0db868f4132920879019318d80cc522ad8f2bca4611af808/scipy-1.18.0-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8ca01e8ae69f1b18e9a58d91afead31be3cef0dd905a10249dac559ee15460a0", size = 35283333, upload-time = "2026-06-19T15:00:58.152Z" }, + { url = "https://files.pythonhosted.org/packages/53/a4/cbdeef6eb3830a8462a9d4ada814de5fc984345cc9ecf17cbec51a036f1e/scipy-1.18.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:7a7f3b01647384dbc3a711e8c6778e0aabbe93959249fef5c7393396bcac0867", size = 35610216, upload-time = "2026-06-19T15:01:01.155Z" }, + { url = "https://files.pythonhosted.org/packages/80/4d/b2b82502b65f661d1b789c1665dcdf315d5f12194e06fc0b37946294ebae/scipy-1.18.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:6aa94e78ec192a30063a5e72e561c28af769dc311190b24fe91774eff1969709", size = 37418960, upload-time = "2026-06-19T15:01:04.155Z" }, + { url = "https://files.pythonhosted.org/packages/93/3e/902d836831474b0ab5a37d16404f7bc5fafd9efba632890e271ba952635f/scipy-1.18.0-cp314-cp314-win_amd64.whl", hash = "sha256:2d8bbdc6c817f5b4006a54d799d4f5bab6f910193cbb9a1ff310833d4d270f61", size = 37288845, upload-time = "2026-06-19T15:01:07.822Z" }, + { url = "https://files.pythonhosted.org/packages/b6/43/8d73b337a3bdb14daa0314f0434210747c02d79d729ce1777574a817dcf6/scipy-1.18.0-cp314-cp314-win_arm64.whl", hash = "sha256:18e9575f1569b2c54174e6159d32942e03731177f63dce7975f0a0c88d102f5b", size = 24988971, upload-time = "2026-06-19T15:01:11.076Z" }, + { url = "https://files.pythonhosted.org/packages/b4/b4/f11918b0508a2787031a0499a03fbe3546f3bb5ca05d01038c45b278c09a/scipy-1.18.0-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:f351e0dd702687d12a402b867a1b4146a256923e1c38317cbc472f6372b94707", size = 31399325, upload-time = "2026-06-19T15:01:13.723Z" }, + { url = "https://files.pythonhosted.org/packages/7b/d1/1f287b57c0ff0ee5185dff3946d92c8017d39b0e431f0ae79a3ff1859512/scipy-1.18.0-cp314-cp314t-macosx_12_0_arm64.whl", hash = "sha256:7c7a51b33ce387193c97f228320cf8e87361daa1bba750638677729598b3e677", size = 29092110, upload-time = "2026-06-19T15:01:16.908Z" }, + { url = "https://files.pythonhosted.org/packages/ff/1a/7b74eb6c392fdcb27d414c0e7558a6d0231eb3b6d73571f479bb81ea8794/scipy-1.18.0-cp314-cp314t-macosx_14_0_arm64.whl", hash = "sha256:84031d7b052a54fae2f8632e0ec802073d385476eb9a63079bce6e23ef9283d4", size = 20833811, upload-time = "2026-06-19T15:01:20.488Z" }, + { url = "https://files.pythonhosted.org/packages/7c/ad/f3941716320a7b9cb4d68734a903b45fe16eff5fb7da7e16f2e619304979/scipy-1.18.0-cp314-cp314t-macosx_14_0_x86_64.whl", hash = "sha256:56abf29a7c067dde59be8b9a22d606a4ea1b2f2a4b756d9d903c62818f5dacce", size = 23396644, upload-time = "2026-06-19T15:01:23.364Z" }, + { url = "https://files.pythonhosted.org/packages/22/22/1446b62ffe07f9719b7d9b1b6a4e05a772833ae8f441fe4c22c34c9b250f/scipy-1.18.0-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1ad44305cfa24b1ba5803cbbebf033590ccbac1aa5d612d727b785325ab408b0", size = 34079318, upload-time = "2026-06-19T15:01:26.002Z" }, + { url = "https://files.pythonhosted.org/packages/56/3b/b87da667098bb470fa30c7011b0ba351ee976dd395c78798c66e941665a3/scipy-1.18.0-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:945c1761b93f38d7f99ae81ae80c63e621471608c7eeead563f6df025585cd58", size = 35324320, upload-time = "2026-06-19T15:01:28.881Z" }, + { url = "https://files.pythonhosted.org/packages/f8/a1/c7932f91909759b0267f75fdea34e91309f96b895757534b76a90b6b4344/scipy-1.18.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:1a4441f15d620578772a49e5ab48c0ee1f7a0220e387110283062729136b2553", size = 35699541, upload-time = "2026-06-19T15:01:31.968Z" }, + { url = "https://files.pythonhosted.org/packages/f7/86/5185061a1fcc41d18c5dc2463969b3a3964b31d9ac67b2fb05d4c7ff7670/scipy-1.18.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:9aac6192fac56bf2ca534389d24623f07b39ff83317d58287285e7fbd622ff76", size = 37472480, upload-time = "2026-06-19T15:01:35.136Z" }, + { url = "https://files.pythonhosted.org/packages/31/8e/f04c68e39919a010d34f2ee1367fd705b0a25a02f609d755f0bfbc0a15fc/scipy-1.18.0-cp314-cp314t-win_amd64.whl", hash = "sha256:e40baea28ae7f5475c779741e2d90b1247c78531207b49c7030e698ff81cee3f", size = 37365390, upload-time = "2026-06-19T15:01:38.091Z" }, + { url = "https://files.pythonhosted.org/packages/d5/19/969dc072906c84dd0a3b05dcf57ea750936087d7873549e408b35cfc3f97/scipy-1.18.0-cp314-cp314t-win_arm64.whl", hash = "sha256:368e0a705903c466aa5f08eefb39e6b1b6b2d659e7352a31fd9e2438365be0f8", size = 25279661, upload-time = "2026-06-19T15:01:40.817Z" }, +] + [[package]] name = "setuptools" version = "82.0.1"