From bf83f12e093f2d0ccfd794dadb11c1f3b5c69146 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Wed, 2 Sep 2026 10:58:31 -0400 Subject: [PATCH 01/14] Start the #848 root-identity lane journal Co-Authored-By: Claude Fable 5.1 --- PROGRESS-root-identity-848.md | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 PROGRESS-root-identity-848.md diff --git a/PROGRESS-root-identity-848.md b/PROGRESS-root-identity-848.md new file mode 100644 index 000000000..ad49ad9b4 --- /dev/null +++ b/PROGRESS-root-identity-848.md @@ -0,0 +1,28 @@ +# PROGRESS — microcosm#848 root identity (branch `root-identity-848`) + +Lane journal for hash-pinning every raw microdata input and referencing its +Chronicle registration. (The root `PROGRESS.md` belongs to the ACS +predictor-release-join lane; this file follows the `PROGRESS-.md` +convention already used by `PROGRESS-graph-acceptance.md`.) Historical once +merged — see CLAUDE.md "Root journals are history, not state". + +## State + +Started 2026-09-02 from origin/main `d2b31496`. Investigation phase. + +## Done + +- Read CLAUDE.md, DESIGN.md, issue #848, chronicle#221, and the Chronicle + raw-microdata identity ADR (chronicle `origin/adr-raw-microdata-identity`). +- Inventoried every `*_microdata` artifact entry across + `build/{am,be,uk,us}/source_stages.json` and + `build/uk/hmrc_income_source_stages.json`. +- Confirmed Chronicle's live naming from `~/PolicyEngine/chronicle/db/data`: + `source_id` is a publisher slug, `package_id` is kebab-case and + publisher-prefixed (e.g. `hmrc` / `hmrc-spi-income-bands-2023-24`). + +## Next + +- Add `chronicle_artifact` validation to `source_manifest.py`. +- Add the fail-closed sha256 gate to the source runtime. +- Populate pins; write the pending allowlists; contract tests. From 459006197f109cacf2fe73a4ac69dabe30831a55 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Wed, 2 Sep 2026 11:21:04 -0400 Subject: [PATCH 02/14] Declare the raw-microdata identity contract in the source manifest loader Every artifact entry whose kind names microdata now has to say which exact bytes a stage reads and which single Chronicle registration witnesses them. source_manifest validates that shape at load: lowercase 64-hex sha256, a chronicle_artifact object carrying source_id/package_id/year/sha256/access (plus filename, required whenever access is public because only public registrations have an R2 object key), agreement between the entry's sha256 and the registration's, and no repeated locator inside a stage. Unknown access classes and unknown keys fail loudly rather than being ignored. Alongside the validation this adds the reader side the rest of microcosm#848 builds on: microdata_artifact_entries to enumerate the roots of a manifest, resolved_chronicle_registrations to collect the deduped registrations a build actually consumed, load_microdata_pin_allowlist for the per-country microdata_pins_pending.json ratchet, and audit_microdata_pins to report the entries that are neither pinned nor allowlisted. No manifest declares a registration yet, so this commit is inert on the real manifests. Co-Authored-By: Claude Fable 5.1 --- .../src/microcosm/build/source_manifest.py | 555 +++++++++++++++++- 1 file changed, 545 insertions(+), 10 deletions(-) diff --git a/packages/microcosm-build/src/microcosm/build/source_manifest.py b/packages/microcosm-build/src/microcosm/build/source_manifest.py index dc7c766db..0b65d632b 100644 --- a/packages/microcosm-build/src/microcosm/build/source_manifest.py +++ b/packages/microcosm-build/src/microcosm/build/source_manifest.py @@ -4,6 +4,15 @@ artifacts, required transformations, outputs, and validation requirements. The Python here is the shared interpreter contract only; it is intentionally not a country-specific donor loader. + +Raw microdata roots additionally carry an identity contract (microcosm#848, +Chronicle ADR "Raw microdata in Chronicle is identity, not content"): every +artifact entry whose ``kind`` names microdata declares the SHA-256 of the exact +file a stage reads and a ``chronicle_artifact`` reference to the one witnessed +Chronicle registration of that file. Entries that cannot be pinned yet are +listed, one row each, in the country's ``microdata_pins_pending.json`` +allowlist, which is a ratchet: it may shrink, never grow past its committed +baseline. """ from __future__ import annotations @@ -17,17 +26,30 @@ __all__ = [ "ALLOWED_SOURCE_OPERATION_KINDS", + "CHRONICLE_ACCESS_CLASSES", "FORBIDDEN_EXECUTABLE_LOADER_KEYS", "FORBIDDEN_EXECUTABLE_OPERATION_KINDS", "FORBIDDEN_SOURCE_DEPENDENCIES", + "MICRODATA_ARTIFACT_KINDS", + "MICRODATA_PIN_ALLOWLIST_FILENAME", + "EMPTY_MICRODATA_PIN_ALLOWLIST", + "ChronicleArtifactReference", + "MicrodataArtifactEntry", + "MicrodataPinAllowlist", + "MicrodataPinGap", + "MicrodataPinPendingEntry", "SourceManifest", "SourceOperationSpec", "SourceStageSpec", "SupportSpineManifest", "SupportSpineSourceSpec", "SupportSpineSpec", + "audit_microdata_pins", + "load_microdata_pin_allowlist", "load_source_manifest", "load_support_spine_manifest", + "microdata_artifact_entries", + "resolved_chronicle_registrations", ] @@ -188,6 +210,293 @@ ALLOWED_SUPPORT_SPINE_METHODS = frozenset({"pool_raw_asec_years"}) +# Artifact kinds whose bytes are raw microdata a build reads. Every entry of one +# of these kinds is a root of the build graph, so it carries a SHA-256 pin and a +# Chronicle registration reference, or an explicit allowlist row saying why not. +MICRODATA_ARTIFACT_KINDS = frozenset( + { + "licensed_microdata", + "private_microdata", + "public_microdata", + "restricted_microdata", + "versioned_derived_microdata", + } +) + +# Chronicle's closed access set. ``public`` is the only class whose bytes are +# archived; ``licensed`` and ``restricted`` registrations are hash-only and the +# bytes stay in the licensed environment the build already operates. +CHRONICLE_ACCESS_CLASSES = frozenset({"public", "licensed", "restricted"}) + +MICRODATA_PIN_ALLOWLIST_FILENAME = "microdata_pins_pending.json" + +_CHRONICLE_ARTIFACT_REQUIRED_KEYS = frozenset( + {"access", "package_id", "sha256", "source_id", "year"} +) +_CHRONICLE_ARTIFACT_OPTIONAL_KEYS = frozenset({"filename"}) +_MICRODATA_PIN_PENDING_KEYS = frozenset({"issue", "locator", "reason", "stage"}) +_LOWERCASE_SHA256 = re.compile(r"[0-9a-f]{64}") +_CHRONICLE_SLUG = re.compile(r"[a-z0-9]+(?:[_-][a-z0-9]+)*") + + +@dataclass(frozen=True) +class ChronicleArtifactReference: + """One witnessed Chronicle registration of a raw microdata file. + + ``access`` is Chronicle's closed class. Bytes exist in the raw bucket only + for ``public`` registrations; ``licensed`` and ``restricted`` ones are + hash-only by design, so :attr:`raw_object_key` is ``None`` for them. + """ + + source_id: str + package_id: str + year: int + sha256: str + access: str + filename: str = "" + + @classmethod + def from_mapping( + cls, raw: Mapping[str, Any], *, context: str + ) -> ChronicleArtifactReference: + keys = frozenset(raw) + missing = sorted(_CHRONICLE_ARTIFACT_REQUIRED_KEYS - keys) + if missing: + raise ValueError( + f"{context} chronicle_artifact is missing required key(s): {missing}." + ) + unknown = sorted( + keys - _CHRONICLE_ARTIFACT_REQUIRED_KEYS - _CHRONICLE_ARTIFACT_OPTIONAL_KEYS + ) + if unknown: + raise ValueError( + f"{context} chronicle_artifact declares unknown key(s): {unknown}." + ) + for key in ("source_id", "package_id"): + value = raw[key] + if not isinstance(value, str) or not _CHRONICLE_SLUG.fullmatch(value): + raise ValueError( + f"{context} chronicle_artifact {key!r} must be a lowercase " + f"slug, got {value!r}." + ) + year = raw["year"] + if not isinstance(year, int) or isinstance(year, bool): + raise ValueError( + f"{context} chronicle_artifact 'year' must be an integer, got {year!r}." + ) + sha256 = raw["sha256"] + if not isinstance(sha256, str) or not _LOWERCASE_SHA256.fullmatch(sha256): + raise ValueError( + f"{context} chronicle_artifact 'sha256' must be 64 lowercase hex " + f"characters, got {sha256!r}." + ) + access = raw["access"] + if access not in CHRONICLE_ACCESS_CLASSES: + raise ValueError( + f"{context} chronicle_artifact 'access' must be one of " + f"{sorted(CHRONICLE_ACCESS_CLASSES)}, got {access!r}." + ) + filename = raw.get("filename", "") + if not isinstance(filename, str): + raise ValueError( + f"{context} chronicle_artifact 'filename' must be a string." + ) + if access == "public" and not filename: + raise ValueError( + f"{context} chronicle_artifact declares public access without a " + "'filename'; the archived object key needs one." + ) + return cls( + source_id=raw["source_id"], + package_id=raw["package_id"], + year=year, + sha256=sha256, + access=access, + filename=filename, + ) + + @property + def raw_object_key(self) -> str | None: + """Content-addressed raw-bucket key, or ``None`` when no bytes exist.""" + + if self.access != "public": + return None + return ( + f"raw/{self.source_id}/{self.package_id}/{self.year}/" + f"{self.sha256}/{self.filename}" + ) + + def to_payload(self) -> dict[str, Any]: + """Canonical JSON-ready registration record for build manifests.""" + + payload: dict[str, Any] = { + "access": self.access, + "package_id": self.package_id, + "sha256": self.sha256, + "source_id": self.source_id, + "year": self.year, + } + if self.filename: + payload["filename"] = self.filename + key = self.raw_object_key + if key is not None: + payload["raw_object_key"] = key + return payload + + +@dataclass(frozen=True) +class MicrodataArtifactEntry: + """One raw microdata artifact entry, located by stage and locator.""" + + stage: str + locator: str + kind: str + artifact: Mapping[str, Any] + chronicle_artifact: ChronicleArtifactReference | None + + @property + def key(self) -> tuple[str, str]: + return (self.stage, self.locator) + + @property + def sha256(self) -> str | None: + value = self.artifact.get("sha256") + return value if isinstance(value, str) else None + + @property + def member_sha256(self) -> str | None: + value = self.artifact.get("member_sha256") + return value if isinstance(value, str) else None + + @property + def is_pinned(self) -> bool: + return self.sha256 is not None and self.chronicle_artifact is not None + + +@dataclass(frozen=True) +class MicrodataPinPendingEntry: + """One reviewed reason a microdata root is not pinned yet.""" + + stage: str + locator: str + reason: str + issue: str + + @property + def key(self) -> tuple[str, str]: + return (self.stage, self.locator) + + @classmethod + def from_mapping( + cls, raw: Mapping[str, Any], *, context: str + ) -> MicrodataPinPendingEntry: + keys = frozenset(raw) + if keys != _MICRODATA_PIN_PENDING_KEYS: + raise ValueError( + f"{context} pending row must declare exactly " + f"{sorted(_MICRODATA_PIN_PENDING_KEYS)}, got {sorted(keys)}." + ) + for key in sorted(_MICRODATA_PIN_PENDING_KEYS): + value = raw[key] + if not isinstance(value, str) or not value: + raise ValueError( + f"{context} pending row key {key!r} must be a non-empty string." + ) + return cls( + stage=raw["stage"], + locator=raw["locator"], + reason=raw["reason"], + issue=raw["issue"], + ) + + +@dataclass(frozen=True) +class MicrodataPinAllowlist: + """Country allowlist of microdata roots that are not pinned yet. + + ``baseline_count`` is the ratchet: the committed number of rows this country + is allowed to carry. Loading refuses a file whose row count exceeds it, so a + new unpinned root cannot land without either pinning something else or a + reviewed baseline change. + """ + + country: str + version: int + policy: str + baseline_count: int + pending: tuple[MicrodataPinPendingEntry, ...] + + @classmethod + def from_mapping(cls, raw: Mapping[str, Any]) -> MicrodataPinAllowlist: + country = raw.get("country") + version = raw.get("version") + policy = raw.get("policy", "") + baseline_count = raw.get("baseline_count") + if not isinstance(country, str) or not country: + raise ValueError("microdata pin allowlist requires a non-empty 'country'.") + if not isinstance(version, int) or isinstance(version, bool) or version < 1: + raise ValueError( + "microdata pin allowlist requires positive integer 'version'." + ) + if not isinstance(policy, str) or not policy: + raise ValueError("microdata pin allowlist requires a non-empty 'policy'.") + if ( + not isinstance(baseline_count, int) + or isinstance(baseline_count, bool) + or baseline_count < 0 + ): + raise ValueError( + "microdata pin allowlist requires a non-negative integer " + "'baseline_count'." + ) + context = f"{country} microdata pin allowlist" + pending = tuple( + MicrodataPinPendingEntry.from_mapping(row, context=context) + for row in _require_mapping_sequence(raw.get("pending", ())) + ) + keys = [row.key for row in pending] + duplicates = sorted({key for key in keys if keys.count(key) > 1}) + if duplicates: + raise ValueError(f"{context} repeats pending row(s): {duplicates}.") + if len(pending) > baseline_count: + raise ValueError( + f"{context} carries {len(pending)} pending row(s), above its " + f"committed baseline of {baseline_count}; the allowlist is a " + "ratchet and may only shrink." + ) + return cls( + country=country, + version=version, + policy=policy, + baseline_count=baseline_count, + pending=pending, + ) + + def row_map(self) -> Mapping[tuple[str, str], MicrodataPinPendingEntry]: + return {row.key: row for row in self.pending} + + +EMPTY_MICRODATA_PIN_ALLOWLIST = MicrodataPinAllowlist( + country="", + version=1, + policy="No allowlist file: every microdata root must be pinned.", + baseline_count=0, + pending=(), +) + + +@dataclass(frozen=True) +class MicrodataPinGap: + """One contract violation found by :func:`audit_microdata_pins`.""" + + stage: str + locator: str + problem: str + detail: str + + def message(self) -> str: + return f"{self.stage} / {self.locator}: {self.detail}" + @dataclass(frozen=True) class SourceOperationSpec: @@ -265,6 +574,9 @@ def from_mapping(cls, raw: Mapping[str, Any]) -> SourceStageSpec: raise ValueError("source stage 'notes' must be a string when provided.") _reject_executable_parameter_keys(raw, context=f"stage {raw['stage']!r}") _reject_incumbent_dependencies(raw, context=f"stage {raw['stage']!r}") + # Runs last so a stage that smuggles an executable loader is reported as + # that, not as a malformed microdata root. + _validate_microdata_artifacts(artifacts, stage=raw["stage"]) return cls( stage=raw["stage"], survey=raw["survey"], @@ -473,13 +785,134 @@ def from_mapping(cls, raw: Mapping[str, Any]) -> SupportSpineManifest: ) +def microdata_artifact_entries( + source: SourceManifest | SourceStageSpec | Mapping[str, Any], +) -> tuple[MicrodataArtifactEntry, ...]: + """Return every raw microdata artifact entry declared by ``source``. + + Entries are the roots of a build graph: the files a stage actually reads. + Each is located by ``(stage, locator)``, which is unique within a manifest. + + A raw manifest mapping is accepted as well as a loaded + :class:`SourceManifest`, because the frozen UK HMRC/SPI replay manifest is + read as JSON by its own contract rather than through the shared loader. + """ + + entries: list[MicrodataArtifactEntry] = [] + for stage, artifacts in _iter_stage_artifacts(source): + for artifact in artifacts: + if artifact.get("kind") not in MICRODATA_ARTIFACT_KINDS: + continue + entries.append(_microdata_artifact_entry(artifact, stage=stage)) + return tuple(entries) + + +def resolved_chronicle_registrations( + source: SourceManifest | SourceStageSpec | Mapping[str, Any], +) -> tuple[ChronicleArtifactReference, ...]: + """Return the distinct Chronicle registrations ``source`` resolves to. + + Several stages legitimately read the same file — the FRS ``adult`` tab feeds + five UK stages — and they all resolve to one registration, so the result is + deduplicated and ordered by ``(source_id, package_id, year, sha256)``. + """ + + registrations = { + entry.chronicle_artifact + for entry in microdata_artifact_entries(source) + if entry.chronicle_artifact is not None + } + return tuple( + sorted( + registrations, + key=lambda ref: (ref.source_id, ref.package_id, ref.year, ref.sha256), + ) + ) + + +def audit_microdata_pins( + source: SourceManifest | SourceStageSpec | Mapping[str, Any], + *, + allowlist: MicrodataPinAllowlist | None = None, +) -> tuple[MicrodataPinGap, ...]: + """Return every microdata root that is neither pinned nor allowlisted. + + A root is pinned when it declares both its own ``sha256`` and a + ``chronicle_artifact`` reference. Anything else must carry an allowlist row + naming the stage, locator, reason, and tracking issue. A row that names an + already-pinned root is itself a gap: stale rows would quietly inflate the + ratchet baseline. + """ + + rows = (allowlist or EMPTY_MICRODATA_PIN_ALLOWLIST).row_map() + entries = microdata_artifact_entries(source) + gaps: list[MicrodataPinGap] = [] + for entry in entries: + if entry.is_pinned: + if entry.key in rows: + gaps.append( + MicrodataPinGap( + stage=entry.stage, + locator=entry.locator, + problem="stale_allowlist_row", + detail=( + "is fully pinned but still carries a pending " + "allowlist row; remove the row so the ratchet " + "baseline can fall." + ), + ) + ) + continue + if entry.key in rows: + continue + if entry.sha256 is None: + detail = ( + f"{entry.kind} artifact declares no 'sha256' pin and has no " + "pending allowlist row." + ) + else: + detail = ( + f"{entry.kind} artifact is hash-pinned but declares no " + "'chronicle_artifact' registration and has no pending " + "allowlist row." + ) + gaps.append( + MicrodataPinGap( + stage=entry.stage, + locator=entry.locator, + problem="unpinned", + detail=detail, + ) + ) + known = {entry.key for entry in entries} + for key in sorted(rows): + if key not in known: + gaps.append( + MicrodataPinGap( + stage=key[0], + locator=key[1], + problem="orphan_allowlist_row", + detail=( + "pending allowlist row names no microdata artifact in " + "this manifest." + ), + ) + ) + return tuple(gaps) + + +def load_microdata_pin_allowlist(resource: Any) -> MicrodataPinAllowlist: + """Load and validate a country ``microdata_pins_pending.json`` allowlist.""" + + raw = json.loads(_read_manifest_text(resource)) + if not isinstance(raw, Mapping): + raise ValueError("microdata pin allowlist root must be a JSON object.") + return MicrodataPinAllowlist.from_mapping(raw) + + def load_source_manifest(resource: Any) -> SourceManifest: """Load and validate a source manifest from a path-like resource.""" - if hasattr(resource, "read_text"): - text = resource.read_text(encoding="utf-8") - else: - text = Path(resource).read_text(encoding="utf-8") - raw = json.loads(text) + raw = json.loads(_read_manifest_text(resource)) if not isinstance(raw, Mapping): raise ValueError("source manifest root must be a JSON object.") return SourceManifest.from_mapping(raw) @@ -487,16 +920,118 @@ def load_source_manifest(resource: Any) -> SourceManifest: def load_support_spine_manifest(resource: Any) -> SupportSpineManifest: """Load and validate a support-spine manifest from a path-like resource.""" - if hasattr(resource, "read_text"): - text = resource.read_text(encoding="utf-8") - else: - text = Path(resource).read_text(encoding="utf-8") - raw = json.loads(text) + raw = json.loads(_read_manifest_text(resource)) if not isinstance(raw, Mapping): raise ValueError("support-spine manifest root must be a JSON object.") return SupportSpineManifest.from_mapping(raw) +def _iter_stage_artifacts( + source: SourceManifest | SourceStageSpec | Mapping[str, Any], +) -> tuple[tuple[str, tuple[Mapping[str, Any], ...]], ...]: + if isinstance(source, SourceStageSpec): + return ((source.stage, source.artifacts),) + if isinstance(source, SourceManifest): + return tuple((stage.stage, stage.artifacts) for stage in source.stages) + if not isinstance(source, Mapping): + raise TypeError( + "expected a SourceManifest, SourceStageSpec, or raw manifest mapping, " + f"got {type(source).__name__}." + ) + stages = [] + for raw_stage in _require_mapping_sequence(source.get("stages", ())): + name = raw_stage.get("stage") + if not isinstance(name, str) or not name: + raise ValueError("raw source stage requires a non-empty 'stage'.") + stages.append( + (name, tuple(_require_mapping_sequence(raw_stage.get("artifacts", ())))) + ) + return tuple(stages) + + +def _validate_microdata_artifacts( + artifacts: Sequence[Mapping[str, Any]], *, stage: str +) -> None: + seen: set[str] = set() + for artifact in artifacts: + if artifact.get("kind") not in MICRODATA_ARTIFACT_KINDS: + continue + entry = _microdata_artifact_entry(artifact, stage=stage) + if entry.locator in seen: + raise ValueError( + f"stage {stage!r} repeats microdata locator {entry.locator!r}; " + "(stage, locator) identifies a raw input." + ) + seen.add(entry.locator) + + +def _microdata_artifact_entry( + artifact: Mapping[str, Any], *, stage: str +) -> MicrodataArtifactEntry: + kind = artifact["kind"] + locator = artifact.get("locator") + if not isinstance(locator, str) or not locator: + raise ValueError( + f"stage {stage!r} {kind} artifact requires a non-empty 'locator'." + ) + context = f"stage {stage!r} microdata artifact {locator!r}" + for key in ("sha256", "member_sha256"): + value = artifact.get(key) + if value is None: + continue + if not isinstance(value, str) or not _LOWERCASE_SHA256.fullmatch(value): + raise ValueError( + f"{context} key {key!r} must be 64 lowercase hex characters, " + f"got {value!r}." + ) + raw_reference = artifact.get("chronicle_artifact") + reference: ChronicleArtifactReference | None = None + if raw_reference is not None: + if not isinstance(raw_reference, Mapping): + raise ValueError(f"{context} 'chronicle_artifact' must be an object.") + reference = ChronicleArtifactReference.from_mapping( + raw_reference, context=context + ) + declared = artifact.get("sha256") + if not isinstance(declared, str): + raise ValueError( + f"{context} references a Chronicle registration without " + "declaring its own 'sha256'; the reference must witness the " + "exact bytes this stage reads." + ) + if reference.sha256 != declared: + raise ValueError( + f"{context} chronicle_artifact sha256 {reference.sha256} does " + f"not equal the artifact sha256 {declared}; a registration " + "witnesses one file." + ) + filename = artifact.get("filename") + if ( + isinstance(filename, str) + and filename + and reference.filename + and reference.filename != filename + ): + raise ValueError( + f"{context} chronicle_artifact filename " + f"{reference.filename!r} does not equal the artifact filename " + f"{filename!r}." + ) + return MicrodataArtifactEntry( + stage=stage, + locator=locator, + kind=kind, + artifact=artifact, + chronicle_artifact=reference, + ) + + +def _read_manifest_text(resource: Any) -> str: + if hasattr(resource, "read_text"): + return resource.read_text(encoding="utf-8") + return Path(resource).read_text(encoding="utf-8") + + def _require_mapping_sequence(raw: object) -> tuple[Mapping[str, Any], ...]: if not isinstance(raw, Sequence) or isinstance(raw, (str, bytes, bytearray)): raise ValueError("expected a list of objects.") From 0acfa658e591f661cb9b4b251d048ffbed119052 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Wed, 2 Sep 2026 11:31:30 -0400 Subject: [PATCH 03/14] Derive the Chronicle access class from the artifact kind and pin one registration per file MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three invariants turn the registration reference from free text into something a reviewer cannot get wrong: - access class is derived, not chosen: licensed_microdata registers as licensed, restricted_microdata and private_microdata as restricted (the conservative side — no bytes in any Chronicle store), public_microdata as public. versioned_derived_microdata has no mapping at all: a Microcosm-derived HDF5 is not a publisher release, so it belongs in the pending allowlist rather than in a registration. - the registration year must be one the entry already declares — exactly tax_year_start where present, otherwise any four-digit year in vintage. - entries sharing a SHA-256 must resolve to the same registration. The FRS adult tab feeds five UK stages and the SIPP public-use file feeds four US stages; identity belongs to the bytes, so one file cannot enter a build graph as two differently named roots. Also add load_country_microdata_pin_allowlist, which returns the empty zero-baseline allowlist for a country that ships no pending file. Co-Authored-By: Claude Fable 5.1 --- .../src/microcosm/build/source_manifest.py | 117 ++++++++++++++++++ .../spec_engine/schema/sources.schema.json | 76 ++++++++++++ 2 files changed, 193 insertions(+) diff --git a/packages/microcosm-build/src/microcosm/build/source_manifest.py b/packages/microcosm-build/src/microcosm/build/source_manifest.py index 0b65d632b..43e44f479 100644 --- a/packages/microcosm-build/src/microcosm/build/source_manifest.py +++ b/packages/microcosm-build/src/microcosm/build/source_manifest.py @@ -21,11 +21,13 @@ import re from collections.abc import Mapping, Sequence from dataclasses import dataclass, field +from importlib.resources import files from pathlib import Path from typing import Any __all__ = [ "ALLOWED_SOURCE_OPERATION_KINDS", + "CHRONICLE_ACCESS_BY_ARTIFACT_KIND", "CHRONICLE_ACCESS_CLASSES", "FORBIDDEN_EXECUTABLE_LOADER_KEYS", "FORBIDDEN_EXECUTABLE_OPERATION_KINDS", @@ -45,6 +47,7 @@ "SupportSpineSourceSpec", "SupportSpineSpec", "audit_microdata_pins", + "load_country_microdata_pin_allowlist", "load_microdata_pin_allowlist", "load_source_manifest", "load_support_spine_manifest", @@ -228,6 +231,21 @@ # bytes stay in the licensed environment the build already operates. CHRONICLE_ACCESS_CLASSES = frozenset({"public", "licensed", "restricted"}) +# Microcosm's artifact kinds already state redistributability, so the Chronicle +# access class a root registers under is derived from its kind rather than +# chosen per entry. ``private_microdata`` maps to ``restricted`` — the +# conservative side, no bytes in any Chronicle store — because a caller-supplied +# local input is by definition not something Chronicle may redistribute. +# ``versioned_derived_microdata`` has no mapping: a Microcosm-derived HDF5 is +# not a publisher release, so such a root is allowlisted rather than registered +# until a publisher artifact exists behind it. +CHRONICLE_ACCESS_BY_ARTIFACT_KIND = { + "licensed_microdata": "licensed", + "private_microdata": "restricted", + "public_microdata": "public", + "restricted_microdata": "restricted", +} + MICRODATA_PIN_ALLOWLIST_FILENAME = "microdata_pins_pending.json" _CHRONICLE_ARTIFACT_REQUIRED_KEYS = frozenset( @@ -237,6 +255,7 @@ _MICRODATA_PIN_PENDING_KEYS = frozenset({"issue", "locator", "reason", "stage"}) _LOWERCASE_SHA256 = re.compile(r"[0-9a-f]{64}") _CHRONICLE_SLUG = re.compile(r"[a-z0-9]+(?:[_-][a-z0-9]+)*") +_FOUR_DIGIT_YEAR = re.compile(r"(? SourceManifest: raise ValueError(f"duplicate source stage spec(s): {duplicates}.") _reject_executable_parameter_keys(raw, context=f"{country} source manifest") _reject_incumbent_dependencies(raw, context=f"{country} source manifest") + _assert_one_registration_per_file( + [ + _microdata_artifact_entry(artifact, stage=stage.stage) + for stage in stages + for artifact in stage.artifacts + if artifact.get("kind") in MICRODATA_ARTIFACT_KINDS + ] + ) return cls(country=country, version=version, policy=policy, stages=stages) def stage_map(self) -> Mapping[str, SourceStageSpec]: @@ -804,6 +831,7 @@ def microdata_artifact_entries( if artifact.get("kind") not in MICRODATA_ARTIFACT_KINDS: continue entries.append(_microdata_artifact_entry(artifact, stage=stage)) + _assert_one_registration_per_file(entries) return tuple(entries) @@ -910,6 +938,27 @@ def load_microdata_pin_allowlist(resource: Any) -> MicrodataPinAllowlist: return MicrodataPinAllowlist.from_mapping(raw) +def load_country_microdata_pin_allowlist(country: str) -> MicrodataPinAllowlist: + """Load a country's pending allowlist, or the empty one when it has none. + + A country with no allowlist file has nothing pending: every microdata root + it declares is pinned, and its ratchet baseline is zero. + """ + + resource = files(f"microcosm.build.{country}").joinpath( + MICRODATA_PIN_ALLOWLIST_FILENAME + ) + if not resource.is_file(): + return EMPTY_MICRODATA_PIN_ALLOWLIST + allowlist = load_microdata_pin_allowlist(resource) + if allowlist.country != country: + raise ValueError( + f"{country} {MICRODATA_PIN_ALLOWLIST_FILENAME} declares country " + f"{allowlist.country!r}." + ) + return allowlist + + def load_source_manifest(resource: Any) -> SourceManifest: """Load and validate a source manifest from a path-like resource.""" raw = json.loads(_read_manifest_text(resource)) @@ -1017,6 +1066,26 @@ def _microdata_artifact_entry( f"{reference.filename!r} does not equal the artifact filename " f"{filename!r}." ) + expected_access = CHRONICLE_ACCESS_BY_ARTIFACT_KIND.get(kind) + if expected_access is None: + raise ValueError( + f"{context} kind {kind!r} has no Chronicle access class; a " + "Microcosm-derived artifact is not a publisher release and " + "belongs in the pending allowlist, not in a registration." + ) + if reference.access != expected_access: + raise ValueError( + f"{context} kind {kind!r} registers under Chronicle access " + f"{expected_access!r}, but the reference declares " + f"{reference.access!r}." + ) + declared_years = _declared_vintage_years(artifact) + if declared_years and reference.year not in declared_years: + raise ValueError( + f"{context} chronicle_artifact year {reference.year} is not one " + f"of the years this entry declares ({sorted(declared_years)}); " + "a registration names the vintage the stage reads." + ) return MicrodataArtifactEntry( stage=stage, locator=locator, @@ -1026,6 +1095,54 @@ def _microdata_artifact_entry( ) +def _assert_one_registration_per_file( + entries: Sequence[MicrodataArtifactEntry], +) -> None: + """Refuse two registrations for one file. + + Several stages read the same bytes — the FRS ``adult`` tab feeds five UK + stages, and the SIPP public-use file feeds four US stages. Identity is a + property of the bytes, so entries sharing a SHA-256 must resolve to the same + Chronicle registration; otherwise one file would enter a build graph as two + differently named roots. + """ + + by_sha: dict[str, tuple[MicrodataArtifactEntry, ChronicleArtifactReference]] = {} + for entry in entries: + reference = entry.chronicle_artifact + if reference is None or entry.sha256 is None: + continue + first = by_sha.setdefault(entry.sha256, (entry, reference)) + if first[1] != reference: + raise ValueError( + f"stage {entry.stage!r} artifact {entry.locator!r} and stage " + f"{first[0].stage!r} artifact {first[0].locator!r} share " + f"SHA-256 {entry.sha256} but declare different Chronicle " + "registrations; one file has one registration." + ) + + +def _declared_vintage_years(artifact: Mapping[str, Any]) -> frozenset[int]: + """Years this artifact entry itself declares, for registration agreement. + + ``tax_year_start`` is exact when present, so it is the only admissible + year. Otherwise every four-digit year written into ``vintage`` is + admissible: a vintage such as ``"2023 ASEC / 2022 income reference year"`` + legitimately names both the survey year the file is published under and the + income year it measures, and the registration may name either. An entry + that declares neither constrains nothing here — its year is still fixed by + the same-file agreement check across the manifest. + """ + + tax_year_start = artifact.get("tax_year_start") + if isinstance(tax_year_start, int) and not isinstance(tax_year_start, bool): + return frozenset({tax_year_start}) + vintage = artifact.get("vintage") + if not isinstance(vintage, str): + return frozenset() + return frozenset(int(match) for match in _FOUR_DIGIT_YEAR.findall(vintage)) + + def _read_manifest_text(resource: Any) -> str: if hasattr(resource, "read_text"): return resource.read_text(encoding="utf-8") diff --git a/packages/microcosm-build/src/microcosm/build/spec_engine/schema/sources.schema.json b/packages/microcosm-build/src/microcosm/build/spec_engine/schema/sources.schema.json index c9f3ad80f..33e1d976e 100644 --- a/packages/microcosm-build/src/microcosm/build/spec_engine/schema/sources.schema.json +++ b/packages/microcosm-build/src/microcosm/build/spec_engine/schema/sources.schema.json @@ -68,6 +68,61 @@ } }, "$defs": { + "chronicle_artifact": { + "type": "object", + "description": "The single Chronicle registration that witnesses the exact bytes this entry pins. Only public registrations archive the bytes, at raw/{source_id}/{package_id}/{year}/{sha256}/{filename}; licensed and restricted registrations record identity alone.", + "additionalProperties": false, + "required": [ + "access", + "package_id", + "sha256", + "source_id", + "year" + ], + "properties": { + "source_id": { + "type": "string", + "pattern": "^[a-z0-9]+([_-][a-z0-9]+)*$" + }, + "package_id": { + "type": "string", + "pattern": "^[a-z0-9]+([_-][a-z0-9]+)*$" + }, + "year": { + "type": "integer" + }, + "sha256": { + "$ref": "defs.schema.json#/$defs/sha256" + }, + "filename": { + "type": "string", + "minLength": 1 + }, + "access": { + "enum": [ + "public", + "licensed", + "restricted" + ] + } + }, + "allOf": [ + { + "if": { + "properties": { + "access": { + "const": "public" + } + } + }, + "then": { + "required": [ + "filename" + ] + } + } + ] + }, "stage_asset": { "type": "object", "additionalProperties": false, @@ -4149,6 +4204,9 @@ }, "vintage": { "type": "string" + }, + "chronicle_artifact": { + "$ref": "#/$defs/chronicle_artifact" } }, "required": [ @@ -4261,6 +4319,9 @@ }, "vintage": { "type": "string" + }, + "chronicle_artifact": { + "$ref": "#/$defs/chronicle_artifact" } }, "required": [ @@ -4649,6 +4710,12 @@ }, "vintage": { "type": "string" + }, + "chronicle_artifact": { + "$ref": "#/$defs/chronicle_artifact" + }, + "sha256": { + "$ref": "defs.schema.json#/$defs/sha256" } }, "required": [ @@ -5247,6 +5314,9 @@ }, "vintage": { "type": "string" + }, + "chronicle_artifact": { + "$ref": "#/$defs/chronicle_artifact" } }, "required": [ @@ -5385,6 +5455,9 @@ }, "doi": { "type": "string" + }, + "chronicle_artifact": { + "$ref": "#/$defs/chronicle_artifact" } } }, @@ -7366,6 +7439,9 @@ }, "vintage": { "type": "string" + }, + "chronicle_artifact": { + "$ref": "#/$defs/chronicle_artifact" } } }, From 45d5af520ed76c40c88f868f646c3ea5c312d0b7 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Wed, 2 Sep 2026 12:25:24 -0400 Subject: [PATCH 04/14] Pin every UK raw microdata root to a Chronicle registration and list the rest MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The UK side is now fully witnessed: all 29 microdata entries in uk/source_stages.json plus the frozen HMRC/SPI replay manifest carry a chronicle_artifact, so the UK ratchet baseline is zero. The US side pins the two roots that name a publisher-distributed file with a verified hash — the 2023 CPS ASEC archive and the SCF 2022 summary extract — and lists the other 37 with a reviewed reason. Registration ids follow Chronicle's live db/data convention (source_id a publisher slug, package_id kebab-case and publisher-prefixed) and one package per distributed file, because a Chronicle manifest holds one file per (package, year): dwp/dwp-frs-2024-25- licensed 14 FRS 2024-25 tabs, SN 9563 ons/ons-was-round-8-household restricted WAS round 8 EUL ons/ons-lcfs-2023-24-{household,person} restricted ons/ons-etb-1977-2024-household restricted hmrc/hmrc-spi-public-use-tape-2022-23 restricted SN 9422 census_cps/census-cps-asec-2023 public asecpub23csv.zip federal_reserve/federal-reserve-scf-2022-summary-extract public The pending allowlist is one file for the whole workspace, build/microdata_pins_pending.json, with a country tag per row, so the ratchet is a single reviewable number (39) rather than one per country. Rows say what specifically blocks each pin: 24 US entries name a pooled ASEC column universe rather than one file, 6 pin the retired pipeline's Hugging Face mirror rather than the Census release, 2 are Microcosm-derived HDF5s with no publisher release behind them, and the rest are named individually. sources.schema.json gains the chronicle_artifact definition, with filename and access annotated operational to match how the artifact-level keys of those names are already classified. Byte pins over the changed manifests move with them: the US stage_asset digest and its two test copies, the UK frozen replay digest, the regenerated UK release_input_coverage_manifest, and the four spec_sha256 vectors that attest the schema set. Co-Authored-By: Claude Fable 5.1 --- .../build/microdata_pins_pending.json | 280 ++++++++++++++++++ .../src/microcosm/build/source_manifest.py | 68 ++--- .../src/microcosm/build/source_runtime.py | 238 ++++++++++++++- .../spec_engine/schema/sources.schema.json | 6 +- .../build/uk/hmrc_income_source_stages.json | 8 + .../src/microcosm/build/uk/source_stages.json | 232 +++++++++++++++ .../src/microcosm/build/uk/spec/sources.yaml | 203 +++++++++++++ .../src/microcosm/build/us/source_stages.json | 16 + .../src/microcosm/build/us/spec/sources.yaml | 16 +- .../tests/test_spec_engine_country_bundles.py | 6 +- .../tests/test_spec_engine_loader.py | 2 +- .../tests/test_uk_source_stages.py | 2 +- .../tests/test_us_bundle_core_contracts.py | 2 +- .../tests/test_us_spec_bundle.py | 2 +- tools/generate_us_bundle_from_constants.py | 2 +- 15 files changed, 1034 insertions(+), 49 deletions(-) create mode 100644 packages/microcosm-build/src/microcosm/build/microdata_pins_pending.json diff --git a/packages/microcosm-build/src/microcosm/build/microdata_pins_pending.json b/packages/microcosm-build/src/microcosm/build/microdata_pins_pending.json new file mode 100644 index 000000000..007478bd7 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/microdata_pins_pending.json @@ -0,0 +1,280 @@ +{ + "version": 1, + "policy": "Raw microdata roots that no Chronicle registration witnesses yet (microcosm#848). One reviewed row per root, and the file is a ratchet: a row leaves when its root is pinned, but the row count may never exceed baseline_count. Adding an unpinned root therefore costs a pin somewhere else or a reviewed baseline change. A country absent from this file has every microdata root pinned.", + "baseline_count": 39, + "pending": [ + { + "country": "am", + "stage": "load_populace_us_support_pool", + "locator": "policyengine/populace-us", + "reason": "Armenia is a spec-only package. The policyengine/populace-us support pool has no certified revision, filename, or SHA-256 yet; the entry's own integrity_note makes those artifact coordinates a harvest item.", + "issue": "PolicyEngine/microcosm#848" + }, + { + "country": "be", + "stage": "silc_load", + "locator": "Statbel BE-SILC scientific-use files: D (household register), R (personal register), H (household data), P (personal data)", + "reason": "Statbel BE-SILC scientific-use files (D, R, H, P) are restricted and not yet provisioned for a build; the entry names four files with no hashes. Registration is hash-only and needs the provisioned copies first.", + "issue": "PolicyEngine/microcosm#848" + }, + { + "country": "us", + "stage": "puf_tax_detail", + "locator": "IRS SOI Public Use File, including source-year adjusted gross income E00100, weight S006, total interest paid deduction E19200, state and local tax refunds E00700, educator expense E03220, alimony income E00800, alimony expense E03500, domestic-production deduction E03240, casualty-loss E20500, unreimbursed employee business expense E20400, farm operations E02100, farm rent E27200, Form 4952 elected investment income E58990, collectibles gain E24518, and unrecaptured section 1250 gain E24515", + "reason": "IRS SOI Public Use File. The entry names the 2015 PUF field set with no distributed-file hash, and the IRS withdrew the file, so a registration must witness the licensed copy the build actually reads.", + "issue": "PolicyEngine/microcosm#848" + }, + { + "country": "us", + "stage": "puf_tax_detail", + "locator": "release://policyengine/irs-soi-puf/1.8.0/puf_2024.h5; qbi_simulation_version=1 materialized at archived commit 42ed5d45c56df80d754fbe24cce21cfeb8d05cbe", + "reason": "puf_2024.h5 is a versioned_derived_microdata artifact materialized by the retired pipeline at commit 42ed5d45c56df80d754fbe24cce21cfeb8d05cbe, not a publisher release, so it has no Chronicle access class. Its identity must resolve through the registration of the 2015 PUF behind it.", + "issue": "PolicyEngine/microcosm#848" + }, + { + "country": "us", + "stage": "education_inputs", + "locator": "IRS SOI Public Use File E03230 and optional E87530 tuition fields", + "reason": "IRS SOI Public Use File. The entry names the 2015 PUF field set with no distributed-file hash, and the IRS withdrew the file, so a registration must witness the licensed copy the build actually reads.", + "issue": "PolicyEngine/microcosm#848" + }, + { + "country": "us", + "stage": "education_inputs", + "locator": "Census CPS ASEC person ED_VAL educational-assistance field", + "reason": "Stage reads CPS ASEC columns pooled across the build-selected income years, so the entry names a column universe rather than one distributed file and no single SHA-256 witnesses it. Pinning needs one artifact entry per pooled ASEC survey-year archive; the 2023, 2024, and 2025 zips are already hash-pinned in us_runtime/education_assistance_source.py.", + "issue": "PolicyEngine/microcosm#848" + }, + { + "country": "us", + "stage": "retirement_contributions", + "locator": "Census CPS ASEC person RETCB_VAL, WSAL_VAL, and SEMP_VAL fields", + "reason": "Stage reads CPS ASEC columns pooled across the build-selected income years, so the entry names a column universe rather than one distributed file and no single SHA-256 witnesses it. Pinning needs one artifact entry per pooled ASEC survey-year archive; the 2023, 2024, and 2025 zips are already hash-pinned in us_runtime/education_assistance_source.py.", + "issue": "PolicyEngine/microcosm#848" + }, + { + "country": "us", + "stage": "childcare_inputs", + "locator": "Census CPS ASEC replicated SPM-unit SPM_CHILDCAREXPNS field", + "reason": "Stage reads CPS ASEC columns pooled across the build-selected income years, so the entry names a column universe rather than one distributed file and no single SHA-256 witnesses it. Pinning needs one artifact entry per pooled ASEC survey-year archive; the 2023, 2024, and 2025 zips are already hash-pinned in us_runtime/education_assistance_source.py.", + "issue": "PolicyEngine/microcosm#848" + }, + { + "country": "us", + "stage": "adult_care_inputs", + "locator": "Census CPS ASEC person PEDISDRS self-care difficulty item and the measured SPM-unit childcare expense distribution", + "reason": "Stage reads CPS ASEC columns pooled across the build-selected income years, so the entry names a column universe rather than one distributed file and no single SHA-256 witnesses it. Pinning needs one artifact entry per pooled ASEC survey-year archive; the 2023, 2024, and 2025 zips are already hash-pinned in us_runtime/education_assistance_source.py.", + "issue": "PolicyEngine/microcosm#848" + }, + { + "country": "us", + "stage": "energy_subsidy", + "locator": "Census CPS ASEC replicated SPM-unit SPM_ENGVAL field", + "reason": "Stage reads CPS ASEC columns pooled across the build-selected income years, so the entry names a column universe rather than one distributed file and no single SHA-256 witnesses it. Pinning needs one artifact entry per pooled ASEC survey-year archive; the 2023, 2024, and 2025 zips are already hash-pinned in us_runtime/education_assistance_source.py.", + "issue": "PolicyEngine/microcosm#848" + }, + { + "country": "us", + "stage": "child_support_inputs", + "locator": "Census CPS ASEC person CSP_VAL child support received and CHSP_VAL annual child support paid fields", + "reason": "Stage reads CPS ASEC columns pooled across the build-selected income years, so the entry names a column universe rather than one distributed file and no single SHA-256 witnesses it. Pinning needs one artifact entry per pooled ASEC survey-year archive; the 2023, 2024, and 2025 zips are already hash-pinned in us_runtime/education_assistance_source.py.", + "issue": "PolicyEngine/microcosm#848" + }, + { + "country": "us", + "stage": "disability_benefits_input", + "locator": "Census CPS ASEC person DIS_VAL1, DIS_SC1, DIS_VAL2, and DIS_SC2 fields", + "reason": "Stage reads CPS ASEC columns pooled across the build-selected income years, so the entry names a column universe rather than one distributed file and no single SHA-256 witnesses it. Pinning needs one artifact entry per pooled ASEC survey-year archive; the 2023, 2024, and 2025 zips are already hash-pinned in us_runtime/education_assistance_source.py.", + "issue": "PolicyEngine/microcosm#848" + }, + { + "country": "us", + "stage": "workers_compensation_input", + "locator": "Census CPS ASEC person annual workers' compensation amount WC_VAL", + "reason": "Stage reads CPS ASEC columns pooled across the build-selected income years, so the entry names a column universe rather than one distributed file and no single SHA-256 witnesses it. Pinning needs one artifact entry per pooled ASEC survey-year archive; the 2023, 2024, and 2025 zips are already hash-pinned in us_runtime/education_assistance_source.py.", + "issue": "PolicyEngine/microcosm#848" + }, + { + "country": "us", + "stage": "scf_wealth", + "locator": "https://www.federalreserve.gov/econres/files/scf2022s.zip", + "reason": "Full-file SHA-256 pending one network-enabled provisioning fetch, as the entry's own integrity_note records; the summary-extract hashes are a different artifact and must not be reused here.", + "issue": "PolicyEngine/microcosm#848" + }, + { + "country": "us", + "stage": "scf_wealth", + "locator": "Census SIPP 2023 public-use file; immutable Hugging Face mirror revision 21280dca5995e978d706740a8a4b9b7860cfd7b6", + "reason": "The declared SHA-256 witnesses the retired pipeline's Hugging Face mirror revision 21280dca5995e978d706740a8a4b9b7860cfd7b6, not the Census-distributed SIPP 2023 release. Chronicle registers the publisher artifact, whose own hash needs one network-enabled fetch.", + "issue": "PolicyEngine/microcosm#848" + }, + { + "country": "us", + "stage": "ssi_disability_criteria", + "locator": "Census SIPP 2023 public-use file; immutable mirror revision 21280dca5995e978d706740a8a4b9b7860cfd7b6", + "reason": "The declared SHA-256 witnesses the retired pipeline's Hugging Face mirror revision 21280dca5995e978d706740a8a4b9b7860cfd7b6, not the Census-distributed SIPP 2023 release. Chronicle registers the publisher artifact, whose own hash needs one network-enabled fetch.", + "issue": "PolicyEngine/microcosm#848" + }, + { + "country": "us", + "stage": "sipp_head_start", + "locator": "Census SIPP 2023 public-use file; immutable mirror revision 21280dca5995e978d706740a8a4b9b7860cfd7b6", + "reason": "The declared SHA-256 witnesses the retired pipeline's Hugging Face mirror revision 21280dca5995e978d706740a8a4b9b7860cfd7b6, not the Census-distributed SIPP 2023 release. Chronicle registers the publisher artifact, whose own hash needs one network-enabled fetch.", + "issue": "PolicyEngine/microcosm#848" + }, + { + "country": "us", + "stage": "ssi_take_up", + "locator": "CPS ASEC annual SSI amount SSI_VAL, used only as the reported-recipient true anchor", + "reason": "Stage reads CPS ASEC columns pooled across the build-selected income years, so the entry names a column universe rather than one distributed file and no single SHA-256 witnesses it. Pinning needs one artifact entry per pooled ASEC survey-year archive; the 2023, 2024, and 2025 zips are already hash-pinned in us_runtime/education_assistance_source.py.", + "issue": "PolicyEngine/microcosm#848" + }, + { + "country": "us", + "stage": "sipp_tips", + "locator": "Census SIPP 2023 public-use slim tip extract; immutable mirror revision 21280dca5995e978d706740a8a4b9b7860cfd7b6", + "reason": "The declared SHA-256 witnesses pu2023_slim.csv on the retired pipeline's Hugging Face mirror \u2014 a PolicyEngine-derived column subset of SIPP 2023 with no publisher release of its own to register. Registration must reference the Census-distributed SIPP 2023 file the subset came from.", + "issue": "PolicyEngine/microcosm#848" + }, + { + "country": "us", + "stage": "org_wages", + "locator": "jan24pub through dec24pub; HRMIS 4 and 8", + "reason": "Reads twelve CPS basic monthly files (jan24pub through dec24pub). The entry pins the generated cache's content hash, not the twelve distributed archives, so registration needs one entry per monthly file.", + "issue": "PolicyEngine/microcosm#848" + }, + { + "country": "us", + "stage": "meps_esi_premiums", + "locator": "Census CPS ASEC NOW_OWNGRP, NOW_HIPAID, NOW_GRPFTYP, and PHIP_VAL fields; the three NOW_* fields are absent from the current hermetic HDF inputs", + "reason": "Stage reads CPS ASEC columns pooled across the build-selected income years, so the entry names a column universe rather than one distributed file and no single SHA-256 witnesses it. Pinning needs one artifact entry per pooled ASEC survey-year archive; the 2023, 2024, and 2025 zips are already hash-pinned in us_runtime/education_assistance_source.py.", + "issue": "PolicyEngine/microcosm#848" + }, + { + "country": "us", + "stage": "prior_year_income", + "locator": "Census CPS ASEC person files identified by source_year and PERIDNUM", + "reason": "Stage reads CPS ASEC columns pooled across the build-selected income years, so the entry names a column universe rather than one distributed file and no single SHA-256 witnesses it. Pinning needs one artifact entry per pooled ASEC survey-year archive; the 2023, 2024, and 2025 zips are already hash-pinned in us_runtime/education_assistance_source.py.", + "issue": "PolicyEngine/microcosm#848" + }, + { + "country": "us", + "stage": "immigration_status", + "locator": "Census CPS ASEC person file citizenship (PRCITSHP), entry year (PEINUSYR), nativity (PENATVTY), and program-participation fields", + "reason": "Stage reads CPS ASEC columns pooled across the build-selected income years, so the entry names a column universe rather than one distributed file and no single SHA-256 witnesses it. Pinning needs one artifact entry per pooled ASEC survey-year archive; the 2023, 2024, and 2025 zips are already hash-pinned in us_runtime/education_assistance_source.py.", + "issue": "PolicyEngine/microcosm#848" + }, + { + "country": "us", + "stage": "hours_worked", + "locator": "Census CPS ASEC person file usual weekly hours (HRSWK), reference-week hours (A_HRS1), and weeks worked (WKSWORK)", + "reason": "Stage reads CPS ASEC columns pooled across the build-selected income years, so the entry names a column universe rather than one distributed file and no single SHA-256 witnesses it. Pinning needs one artifact entry per pooled ASEC survey-year archive; the 2023, 2024, and 2025 zips are already hash-pinned in us_runtime/education_assistance_source.py.", + "issue": "PolicyEngine/microcosm#848" + }, + { + "country": "us", + "stage": "snap_take_up", + "locator": "Census CPS ASEC SPM unit reported SNAP subsidy (SPM_SNAPSUB)", + "reason": "Stage reads CPS ASEC columns pooled across the build-selected income years, so the entry names a column universe rather than one distributed file and no single SHA-256 witnesses it. Pinning needs one artifact entry per pooled ASEC survey-year archive; the 2023, 2024, and 2025 zips are already hash-pinned in us_runtime/education_assistance_source.py.", + "issue": "PolicyEngine/microcosm#848" + }, + { + "country": "us", + "stage": "snap_state_take_up", + "locator": "Census CPS ASEC SPM unit reported SNAP subsidy (SPM_SNAPSUB)", + "reason": "Stage reads CPS ASEC columns pooled across the build-selected income years, so the entry names a column universe rather than one distributed file and no single SHA-256 witnesses it. Pinning needs one artifact entry per pooled ASEC survey-year archive; the 2023, 2024, and 2025 zips are already hash-pinned in us_runtime/education_assistance_source.py.", + "issue": "PolicyEngine/microcosm#848" + }, + { + "country": "us", + "stage": "relationship_inputs", + "locator": "Census CPS ASEC person household sequence (PH_SEQ), within-household person sequence (P_SEQ), and marital status (A_MARITL)", + "reason": "Stage reads CPS ASEC columns pooled across the build-selected income years, so the entry names a column universe rather than one distributed file and no single SHA-256 witnesses it. Pinning needs one artifact entry per pooled ASEC survey-year archive; the 2023, 2024, and 2025 zips are already hash-pinned in us_runtime/education_assistance_source.py.", + "issue": "PolicyEngine/microcosm#848" + }, + { + "country": "us", + "stage": "medicare_take_up_input", + "locator": "SHA-locked Census CPS ASEC person Medicare coverage last year (MCARE)", + "reason": "Stage reads CPS ASEC columns pooled across the build-selected income years, so the entry names a column universe rather than one distributed file and no single SHA-256 witnesses it. Pinning needs one artifact entry per pooled ASEC survey-year archive; the 2023, 2024, and 2025 zips are already hash-pinned in us_runtime/education_assistance_source.py.", + "issue": "PolicyEngine/microcosm#848" + }, + { + "country": "us", + "stage": "retirement_distributions", + "locator": "Census CPS ASEC person retirement-distribution account codes and amounts (DST_SC1/DST_VAL1, DST_SC2/DST_VAL2, DST_SC1_YNG/DST_VAL1_YNG, DST_SC2_YNG/DST_VAL2_YNG)", + "reason": "Stage reads CPS ASEC columns pooled across the build-selected income years, so the entry names a column universe rather than one distributed file and no single SHA-256 witnesses it. Pinning needs one artifact entry per pooled ASEC survey-year archive; the 2023, 2024, and 2025 zips are already hash-pinned in us_runtime/education_assistance_source.py.", + "issue": "PolicyEngine/microcosm#848" + }, + { + "country": "us", + "stage": "eligibility_inputs", + "locator": "Census CPS ASEC person file disability-difficulty items (PEDISDRS/PEDISEAR/PEDISEYE/PEDISOUT/PEDISPHY/PEDISREM), school enrollment (A_HSCOL, A_FTPT), parent line pointers (PEPAR1, PEPAR2), veterans' payments (VET_VAL), and reported SSI (SSI_VAL)", + "reason": "Stage reads CPS ASEC columns pooled across the build-selected income years, so the entry names a column universe rather than one distributed file and no single SHA-256 witnesses it. Pinning needs one artifact entry per pooled ASEC survey-year archive; the 2023, 2024, and 2025 zips are already hash-pinned in us_runtime/education_assistance_source.py.", + "issue": "PolicyEngine/microcosm#848" + }, + { + "country": "us", + "stage": "pregnancy", + "locator": "Census CPS ASEC person file sex (A_SEX) and age (A_AGE)", + "reason": "Stage reads CPS ASEC columns pooled across the build-selected income years, so the entry names a column universe rather than one distributed file and no single SHA-256 witnesses it. Pinning needs one artifact entry per pooled ASEC survey-year archive; the 2023, 2024, and 2025 zips are already hash-pinned in us_runtime/education_assistance_source.py.", + "issue": "PolicyEngine/microcosm#848" + }, + { + "country": "us", + "stage": "snap_abawd_discretionary_exemption", + "locator": "Census CPS ASEC person file age (A_AGE)", + "reason": "Stage reads CPS ASEC columns pooled across the build-selected income years, so the entry names a column universe rather than one distributed file and no single SHA-256 witnesses it. Pinning needs one artifact entry per pooled ASEC survey-year archive; the 2023, 2024, and 2025 zips are already hash-pinned in us_runtime/education_assistance_source.py.", + "issue": "PolicyEngine/microcosm#848" + }, + { + "country": "us", + "stage": "acs_rent", + "locator": "SHA-locked Census CPS ASEC person and household tables", + "reason": "Locator names SHA-locked CPS ASEC person and household tables pooled over 2022-2024 rather than one distributed file; the same per-archive split as the other ASEC column entries is required before it can be pinned.", + "issue": "PolicyEngine/microcosm#848" + }, + { + "country": "us", + "stage": "acs_rent", + "locator": "Census ACS 2022 PUMS processed person/household arrays (acs_2022.h5)", + "reason": "acs_2022.h5 is hash-pinned but is processed ACS 2022 PUMS arrays produced by the retired pipeline, not a publisher release, so versioned_derived_microdata has no Chronicle access class. The official person and household PUMS zips it derives from are named in the entry and unhashed.", + "issue": "PolicyEngine/microcosm#848" + }, + { + "country": "us", + "stage": "vehicle_assets", + "locator": "Census SIPP 2023 public-use file; immutable mirror revision 21280dca5995e978d706740a8a4b9b7860cfd7b6", + "reason": "The declared SHA-256 witnesses the retired pipeline's Hugging Face mirror revision 21280dca5995e978d706740a8a4b9b7860cfd7b6, not the Census-distributed SIPP 2023 release. Chronicle registers the publisher artifact, whose own hash needs one network-enabled fetch.", + "issue": "PolicyEngine/microcosm#848" + }, + { + "country": "us", + "stage": "voluntary_filing_input", + "locator": "Census SIPP 2023 public-use file; immutable mirror revision 21280dca5995e978d706740a8a4b9b7860cfd7b6", + "reason": "The declared SHA-256 witnesses the retired pipeline's Hugging Face mirror revision 21280dca5995e978d706740a8a4b9b7860cfd7b6, not the Census-distributed SIPP 2023 release. Chronicle registers the publisher artifact, whose own hash needs one network-enabled fetch.", + "issue": "PolicyEngine/microcosm#848" + }, + { + "country": "us", + "stage": "aca_marketplace_inputs", + "locator": "CPS ASEC current coverage and health premium fields", + "reason": "Stage reads CPS ASEC columns pooled across the build-selected income years, so the entry names a column universe rather than one distributed file and no single SHA-256 witnesses it. Pinning needs one artifact entry per pooled ASEC survey-year archive; the 2023, 2024, and 2025 zips are already hash-pinned in us_runtime/education_assistance_source.py.", + "issue": "PolicyEngine/microcosm#848" + }, + { + "country": "us", + "stage": "medicaid_take_up", + "locator": "CPS ASEC current Medicaid coverage at interview (has_medicaid_health_coverage_at_interview)", + "reason": "Stage reads CPS ASEC columns pooled across the build-selected income years, so the entry names a column universe rather than one distributed file and no single SHA-256 witnesses it. Pinning needs one artifact entry per pooled ASEC survey-year archive; the 2023, 2024, and 2025 zips are already hash-pinned in us_runtime/education_assistance_source.py.", + "issue": "PolicyEngine/microcosm#848" + }, + { + "country": "us", + "stage": "other_health_insurance_premiums", + "locator": "Census CPS ASEC person PHIP_VAL reported premium, carried as health_insurance_premiums_without_medicare_part_b", + "reason": "Stage reads CPS ASEC columns pooled across the build-selected income years, so the entry names a column universe rather than one distributed file and no single SHA-256 witnesses it. Pinning needs one artifact entry per pooled ASEC survey-year archive; the 2023, 2024, and 2025 zips are already hash-pinned in us_runtime/education_assistance_source.py.", + "issue": "PolicyEngine/microcosm#848" + } + ] +} diff --git a/packages/microcosm-build/src/microcosm/build/source_manifest.py b/packages/microcosm-build/src/microcosm/build/source_manifest.py index 43e44f479..f6aec2131 100644 --- a/packages/microcosm-build/src/microcosm/build/source_manifest.py +++ b/packages/microcosm-build/src/microcosm/build/source_manifest.py @@ -47,11 +47,11 @@ "SupportSpineSourceSpec", "SupportSpineSpec", "audit_microdata_pins", - "load_country_microdata_pin_allowlist", "load_microdata_pin_allowlist", "load_source_manifest", "load_support_spine_manifest", "microdata_artifact_entries", + "packaged_microdata_pin_allowlist", "resolved_chronicle_registrations", ] @@ -252,7 +252,9 @@ {"access", "package_id", "sha256", "source_id", "year"} ) _CHRONICLE_ARTIFACT_OPTIONAL_KEYS = frozenset({"filename"}) -_MICRODATA_PIN_PENDING_KEYS = frozenset({"issue", "locator", "reason", "stage"}) +_MICRODATA_PIN_PENDING_KEYS = frozenset( + {"country", "issue", "locator", "reason", "stage"} +) _LOWERCASE_SHA256 = re.compile(r"[0-9a-f]{64}") _CHRONICLE_SLUG = re.compile(r"[a-z0-9]+(?:[_-][a-z0-9]+)*") _FOUR_DIGIT_YEAR = re.compile(r"(? bool: class MicrodataPinPendingEntry: """One reviewed reason a microdata root is not pinned yet.""" + country: str stage: str locator: str reason: str @@ -422,6 +425,7 @@ def from_mapping( f"{context} pending row key {key!r} must be a non-empty string." ) return cls( + country=raw["country"], stage=raw["stage"], locator=raw["locator"], reason=raw["reason"], @@ -431,15 +435,16 @@ def from_mapping( @dataclass(frozen=True) class MicrodataPinAllowlist: - """Country allowlist of microdata roots that are not pinned yet. - - ``baseline_count`` is the ratchet: the committed number of rows this country - is allowed to carry. Loading refuses a file whose row count exceeds it, so a - new unpinned root cannot land without either pinning something else or a - reviewed baseline change. + """The one allowlist of microdata roots that are not pinned yet. + + There is a single file for the whole workspace rather than one per country, + so the ratchet is a single number a reviewer can watch. ``baseline_count`` + is that ratchet: the committed number of rows the repository is allowed to + carry. Loading refuses a file whose row count exceeds it, so a new unpinned + root cannot land without either pinning an existing one or a reviewed + baseline change. """ - country: str version: int policy: str baseline_count: int @@ -447,12 +452,9 @@ class MicrodataPinAllowlist: @classmethod def from_mapping(cls, raw: Mapping[str, Any]) -> MicrodataPinAllowlist: - country = raw.get("country") version = raw.get("version") policy = raw.get("policy", "") baseline_count = raw.get("baseline_count") - if not isinstance(country, str) or not country: - raise ValueError("microdata pin allowlist requires a non-empty 'country'.") if not isinstance(version, int) or isinstance(version, bool) or version < 1: raise ValueError( "microdata pin allowlist requires positive integer 'version'." @@ -468,12 +470,12 @@ def from_mapping(cls, raw: Mapping[str, Any]) -> MicrodataPinAllowlist: "microdata pin allowlist requires a non-negative integer " "'baseline_count'." ) - context = f"{country} microdata pin allowlist" + context = "microdata pin allowlist" pending = tuple( MicrodataPinPendingEntry.from_mapping(row, context=context) for row in _require_mapping_sequence(raw.get("pending", ())) ) - keys = [row.key for row in pending] + keys = [(row.country, *row.key) for row in pending] duplicates = sorted({key for key in keys if keys.count(key) > 1}) if duplicates: raise ValueError(f"{context} repeats pending row(s): {duplicates}.") @@ -484,19 +486,25 @@ def from_mapping(cls, raw: Mapping[str, Any]) -> MicrodataPinAllowlist: "ratchet and may only shrink." ) return cls( - country=country, version=version, policy=policy, baseline_count=baseline_count, pending=pending, ) - def row_map(self) -> Mapping[tuple[str, str], MicrodataPinPendingEntry]: - return {row.key: row for row in self.pending} + def for_country(self, country: str) -> tuple[MicrodataPinPendingEntry, ...]: + """Return the rows that excuse ``country``'s unpinned roots.""" + + return tuple(row for row in self.pending if row.country == country) + + def row_map( + self, country: str | None = None + ) -> Mapping[tuple[str, str], MicrodataPinPendingEntry]: + rows = self.pending if country is None else self.for_country(country) + return {row.key: row for row in rows} EMPTY_MICRODATA_PIN_ALLOWLIST = MicrodataPinAllowlist( - country="", version=1, policy="No allowlist file: every microdata root must be pinned.", baseline_count=0, @@ -872,7 +880,8 @@ def audit_microdata_pins( ratchet baseline. """ - rows = (allowlist or EMPTY_MICRODATA_PIN_ALLOWLIST).row_map() + country = source.country if isinstance(source, SourceManifest) else None + rows = (allowlist or EMPTY_MICRODATA_PIN_ALLOWLIST).row_map(country) entries = microdata_artifact_entries(source) gaps: list[MicrodataPinGap] = [] for entry in entries: @@ -938,25 +947,12 @@ def load_microdata_pin_allowlist(resource: Any) -> MicrodataPinAllowlist: return MicrodataPinAllowlist.from_mapping(raw) -def load_country_microdata_pin_allowlist(country: str) -> MicrodataPinAllowlist: - """Load a country's pending allowlist, or the empty one when it has none. - - A country with no allowlist file has nothing pending: every microdata root - it declares is pinned, and its ratchet baseline is zero. - """ +def packaged_microdata_pin_allowlist() -> MicrodataPinAllowlist: + """Load the one packaged allowlist of not-yet-pinned microdata roots.""" - resource = files(f"microcosm.build.{country}").joinpath( - MICRODATA_PIN_ALLOWLIST_FILENAME + return load_microdata_pin_allowlist( + files("microcosm.build").joinpath(MICRODATA_PIN_ALLOWLIST_FILENAME) ) - if not resource.is_file(): - return EMPTY_MICRODATA_PIN_ALLOWLIST - allowlist = load_microdata_pin_allowlist(resource) - if allowlist.country != country: - raise ValueError( - f"{country} {MICRODATA_PIN_ALLOWLIST_FILENAME} declares country " - f"{allowlist.country!r}." - ) - return allowlist def load_source_manifest(resource: Any) -> SourceManifest: diff --git a/packages/microcosm-build/src/microcosm/build/source_runtime.py b/packages/microcosm-build/src/microcosm/build/source_runtime.py index 59ddac0df..dd4b5fd48 100644 --- a/packages/microcosm-build/src/microcosm/build/source_runtime.py +++ b/packages/microcosm-build/src/microcosm/build/source_runtime.py @@ -4,25 +4,46 @@ generic envelope of that plan: table reads, operation dispatch, and explicit stop points for staged/cached builds. Operation implementations are injected by shared runtimes, not named inside manifests. + +It also owns the fail-closed root-identity gate (microcosm#848): before a build +reads a raw microdata root it hashes the file it was handed and refuses to +continue unless the bytes are the ones the manifest pins and a Chronicle +registration witnesses. Where a producing run already recorded per-source pins — +the ASEC raw-stage checkpoint does — the recorded pins are cross-checked against +the manifest instead of re-hashing gigabytes. """ from __future__ import annotations -from collections.abc import Callable, Mapping +import hashlib +from collections.abc import Callable, Mapping, Sequence from dataclasses import dataclass, field +from pathlib import Path from typing import Any import pandas as pd -from microcosm.build.source_manifest import SourceOperationSpec, SourceStageSpec +from microcosm.build.source_manifest import ( + ChronicleArtifactReference, + MicrodataArtifactEntry, + SourceManifest, + SourceOperationSpec, + SourceStageSpec, + microdata_artifact_entries, +) __all__ = [ + "MicrodataFileVerification", + "MicrodataIdentityError", "SourceOperationHandler", "SourceRuntimeConfig", "SourceRuntimeContext", "SourceRuntimeError", "UnsupportedSourceOperationError", "run_source_stage", + "sha256_file", + "verify_microdata_files", + "verify_recorded_microdata_pins", ] @@ -75,6 +96,16 @@ class UnsupportedSourceOperationError(SourceRuntimeError): """Raised when a manifest operation has no injected runtime handler.""" +class MicrodataIdentityError(SourceRuntimeError): + """Raised when a raw microdata root is not the file the manifest pins. + + This is the fail-closed root-identity gate (microcosm#848). A build reads + raw microdata only after the bytes on disk are shown to be the registered + ones; there is no warn-and-continue path, because every downstream artifact + would otherwise claim a provenance it does not have. + """ + + def run_source_stage( stage: SourceStageSpec, *, @@ -151,3 +182,206 @@ def _run_read_table( if not isinstance(table, str) or not table: raise SourceRuntimeError("read_table operation requires a non-empty table.") return context.read_table(table) + + +@dataclass(frozen=True) +class MicrodataFileVerification: + """One raw-microdata root checked against its manifest pin.""" + + stage: str + locator: str + path: Path + key: str + expected_sha256: str + actual_sha256: str + registration: ChronicleArtifactReference | None + + @property + def matched(self) -> bool: + return self.expected_sha256 == self.actual_sha256 + + def to_payload(self) -> dict[str, Any]: + payload: dict[str, Any] = { + "stage": self.stage, + "locator": self.locator, + "key": self.key, + "sha256": self.actual_sha256, + } + if self.registration is not None: + payload["chronicle_artifact"] = self.registration.to_payload() + return payload + + +def sha256_file(path: str | Path, *, chunk_size: int = 1 << 20) -> str: + """Return the SHA-256 of a file, streamed so large microdata fits memory.""" + + digest = hashlib.sha256() + with Path(path).open("rb") as stream: + for chunk in iter(lambda: stream.read(chunk_size), b""): + digest.update(chunk) + return digest.hexdigest() + + +def verify_microdata_files( + source: SourceManifest | SourceStageSpec | Mapping[str, Any], + files: Mapping[str, str | Path], + *, + chunk_size: int = 1 << 20, +) -> tuple[MicrodataFileVerification, ...]: + """Hash caller-supplied microdata roots and refuse any that is not the pin. + + ``files`` maps a manifest key to the local file a build was handed. A key + resolves against every microdata entry's ``locator`` and its ``filename``, + because a caller-supplied private input declares the real name under + ``filename`` while its locator is the placeholder + ``"caller-supplied local input"``. + + Raises: + MicrodataIdentityError: If a key names no pinned microdata root, if a + named file is missing, or if any file's bytes are not the pinned + ones. The message carries the publisher, vintage, locator, expected + and actual digests, so an operator can tell a wrong vintage from a + corrupted download without rerunning the build. + """ + + entries = microdata_artifact_entries(source) + by_key: dict[str, list[MicrodataArtifactEntry]] = {} + for entry in entries: + if entry.sha256 is None: + continue + for key in _entry_keys(entry): + by_key.setdefault(key, []).append(entry) + + verifications: list[MicrodataFileVerification] = [] + failures: list[str] = [] + for key in sorted(files): + matches = by_key.get(key) + if not matches: + raise MicrodataIdentityError( + f"{key!r} names no hash-pinned microdata artifact in this " + f"manifest; pinned keys are {sorted(by_key)}." + ) + path = Path(files[key]) + if not path.is_file(): + raise MicrodataIdentityError( + f"{key!r} was supplied as {path}, which is not a file." + ) + actual = sha256_file(path, chunk_size=chunk_size) + for entry in matches: + expected = entry.sha256 + assert expected is not None # guarded when by_key was built + verification = MicrodataFileVerification( + stage=entry.stage, + locator=entry.locator, + path=path, + key=key, + expected_sha256=expected, + actual_sha256=actual, + registration=entry.chronicle_artifact, + ) + verifications.append(verification) + if not verification.matched: + failures.append(_identity_failure(verification, entry)) + if failures: + raise MicrodataIdentityError( + "Raw microdata identity check failed; the build reads different " + "bytes than the manifest pins:\n" + "\n".join(failures) + ) + return tuple(verifications) + + +def verify_recorded_microdata_pins( + source: SourceManifest | SourceStageSpec | Mapping[str, Any], + pins: Sequence[Mapping[str, Any]], + *, + context: str, +) -> tuple[ChronicleArtifactReference, ...]: + """Cross-check pins a checkpoint already recorded against the manifest. + + The ASEC raw-stage checkpoint records a ``sha256``/``member_sha256`` pin per + source file it consumed. Re-hashing those archives would cost gigabytes of + reads for a digest the producing run already computed, so this compares the + recorded pins against the manifest instead and returns the registrations + they resolve to. + + Raises: + MicrodataIdentityError: If a recorded pin matches no manifest entry, or + matches one whose archive or member digest differs. + """ + + entries = [ + entry + for entry in microdata_artifact_entries(source) + if entry.sha256 is not None + ] + resolved: list[ChronicleArtifactReference] = [] + failures: list[str] = [] + for index, pin in enumerate(pins): + locator = pin.get("locator") + sha256 = pin.get("sha256") + member_sha256 = pin.get("member_sha256") + matches = [entry for entry in entries if entry.locator == locator] + if not matches: + failures.append( + f"pin[{index}] locator {locator!r} names no hash-pinned " + "microdata artifact in this manifest." + ) + continue + for entry in matches: + if entry.sha256 != sha256: + failures.append( + f"pin[{index}] {locator!r} recorded sha256 {sha256!r}; " + f"stage {entry.stage!r} pins {entry.sha256}." + ) + continue + if ( + entry.member_sha256 is not None + and member_sha256 is not None + and entry.member_sha256 != member_sha256 + ): + failures.append( + f"pin[{index}] {locator!r} recorded member_sha256 " + f"{member_sha256!r}; stage {entry.stage!r} pins " + f"{entry.member_sha256}." + ) + continue + if entry.chronicle_artifact is not None: + resolved.append(entry.chronicle_artifact) + if failures: + raise MicrodataIdentityError( + f"{context}: recorded raw-microdata pins disagree with the source " + "manifest:\n" + "\n".join(failures) + ) + return tuple( + sorted( + set(resolved), + key=lambda ref: (ref.source_id, ref.package_id, ref.year, ref.sha256), + ) + ) + + +def _entry_keys(entry: MicrodataArtifactEntry) -> tuple[str, ...]: + filename = entry.artifact.get("filename") + keys = [entry.locator] + if isinstance(filename, str) and filename and filename != entry.locator: + keys.append(filename) + return tuple(keys) + + +def _identity_failure( + verification: MicrodataFileVerification, + entry: MicrodataArtifactEntry, +) -> str: + registration = entry.chronicle_artifact + publisher = ( + f"{registration.source_id}/{registration.package_id}" + if registration is not None + else "unregistered" + ) + vintage = entry.artifact.get("vintage") + return ( + f" stage {entry.stage!r} artifact {entry.locator!r} " + f"(publisher {publisher}, vintage {vintage!r}) supplied as " + f"{verification.path}: expected SHA-256 " + f"{verification.expected_sha256}, got {verification.actual_sha256}." + ) diff --git a/packages/microcosm-build/src/microcosm/build/spec_engine/schema/sources.schema.json b/packages/microcosm-build/src/microcosm/build/spec_engine/schema/sources.schema.json index 33e1d976e..03254333f 100644 --- a/packages/microcosm-build/src/microcosm/build/spec_engine/schema/sources.schema.json +++ b/packages/microcosm-build/src/microcosm/build/spec_engine/schema/sources.schema.json @@ -96,14 +96,16 @@ }, "filename": { "type": "string", - "minLength": 1 + "minLength": 1, + "x-spec-surface": "operational" }, "access": { "enum": [ "public", "licensed", "restricted" - ] + ], + "x-spec-surface": "operational" } }, "allOf": [ diff --git a/packages/microcosm-build/src/microcosm/build/uk/hmrc_income_source_stages.json b/packages/microcosm-build/src/microcosm/build/uk/hmrc_income_source_stages.json index 8e8635ae4..f606482d1 100644 --- a/packages/microcosm-build/src/microcosm/build/uk/hmrc_income_source_stages.json +++ b/packages/microcosm-build/src/microcosm/build/uk/hmrc_income_source_stages.json @@ -28,6 +28,14 @@ "doi": "10.5255/UKDA-SN-9422-1", "filename": "put2223uk.tab", "sha256": "5ef829461060c91a2a47be59ad541d9b519fc3976d66ca80d4920f711bb96f66", + "chronicle_artifact": { + "source_id": "hmrc", + "package_id": "hmrc-spi-public-use-tape-2022-23", + "year": 2022, + "sha256": "5ef829461060c91a2a47be59ad541d9b519fc3976d66ca80d4920f711bb96f66", + "filename": "put2223uk.tab", + "access": "restricted" + }, "size_bytes": 141323762, "reviewed_source": "PolicyEngine licensed copy from policyengine/policyengine-uk-data-private on Hugging Face, spi_2022_23.zip", "access": "private_local_input", diff --git a/packages/microcosm-build/src/microcosm/build/uk/source_stages.json b/packages/microcosm-build/src/microcosm/build/uk/source_stages.json index c403cf81b..9cac16ff4 100644 --- a/packages/microcosm-build/src/microcosm/build/uk/source_stages.json +++ b/packages/microcosm-build/src/microcosm/build/uk/source_stages.json @@ -17,6 +17,14 @@ "vintage": "2024_25", "locator": "accounts.tab", "sha256": "fa7871eb45cad0db5fd05ede454ced60405d2f9c598651ea5acea5c91a6ff52f", + "chronicle_artifact": { + "source_id": "dwp", + "package_id": "dwp-frs-2024-25-accounts", + "year": 2024, + "sha256": "fa7871eb45cad0db5fd05ede454ced60405d2f9c598651ea5acea5c91a6ff52f", + "filename": "accounts.tab", + "access": "licensed" + }, "size_bytes": 1812923, "runtime_sha256_required": true, "tax_year_start": 2024, @@ -31,6 +39,14 @@ "vintage": "2024_25", "locator": "adult.tab", "sha256": "4eaea0809a7ccca0fddeb98e358771e4a6e5ebb81b21c4fac4070fc9d227658d", + "chronicle_artifact": { + "source_id": "dwp", + "package_id": "dwp-frs-2024-25-adult", + "year": 2024, + "sha256": "4eaea0809a7ccca0fddeb98e358771e4a6e5ebb81b21c4fac4070fc9d227658d", + "filename": "adult.tab", + "access": "licensed" + }, "size_bytes": 34885825, "runtime_sha256_required": true, "tax_year_start": 2024, @@ -45,6 +61,14 @@ "vintage": "2024_25", "locator": "benefits.tab", "sha256": "f6ad22b408a13e2239c04b0d076a36418dcf5dd89a8c60daa792c4d735b911d3", + "chronicle_artifact": { + "source_id": "dwp", + "package_id": "dwp-frs-2024-25-benefits", + "year": 2024, + "sha256": "f6ad22b408a13e2239c04b0d076a36418dcf5dd89a8c60daa792c4d735b911d3", + "filename": "benefits.tab", + "access": "licensed" + }, "size_bytes": 2362329, "runtime_sha256_required": true, "tax_year_start": 2024, @@ -59,6 +83,14 @@ "vintage": "2024_25", "locator": "benunit.tab", "sha256": "66b894624498316d19b6259e287a607e98ed3daacc9be3d3e9067d32b8e09a5a", + "chronicle_artifact": { + "source_id": "dwp", + "package_id": "dwp-frs-2024-25-benunit", + "year": 2024, + "sha256": "66b894624498316d19b6259e287a607e98ed3daacc9be3d3e9067d32b8e09a5a", + "filename": "benunit.tab", + "access": "licensed" + }, "size_bytes": 13986782, "runtime_sha256_required": true, "tax_year_start": 2024, @@ -73,6 +105,14 @@ "vintage": "2024_25", "locator": "child.tab", "sha256": "88ec53fc52eea4374864bbc74219d551f4b4c5f54a220bf9607c7a6289719aa5", + "chronicle_artifact": { + "source_id": "dwp", + "package_id": "dwp-frs-2024-25-child", + "year": 2024, + "sha256": "88ec53fc52eea4374864bbc74219d551f4b4c5f54a220bf9607c7a6289719aa5", + "filename": "child.tab", + "access": "licensed" + }, "size_bytes": 2753961, "runtime_sha256_required": true, "tax_year_start": 2024, @@ -87,6 +127,14 @@ "vintage": "2024_25", "locator": "chldcare.tab", "sha256": "7ccd3f92f299a1f49b24063188177cdb8a958d8bcd753fc3d74dadda6ad04023", + "chronicle_artifact": { + "source_id": "dwp", + "package_id": "dwp-frs-2024-25-chldcare", + "year": 2024, + "sha256": "7ccd3f92f299a1f49b24063188177cdb8a958d8bcd753fc3d74dadda6ad04023", + "filename": "chldcare.tab", + "access": "licensed" + }, "size_bytes": 275878, "runtime_sha256_required": true, "tax_year_start": 2024, @@ -101,6 +149,14 @@ "vintage": "2024_25", "locator": "extchild.tab", "sha256": "c661379a4aa5079ce482b1f98f0bfb9157ad9b3ba4eb10739b61846f9c9548e4", + "chronicle_artifact": { + "source_id": "dwp", + "package_id": "dwp-frs-2024-25-extchild", + "year": 2024, + "sha256": "c661379a4aa5079ce482b1f98f0bfb9157ad9b3ba4eb10739b61846f9c9548e4", + "filename": "extchild.tab", + "access": "licensed" + }, "size_bytes": 15150, "runtime_sha256_required": true, "tax_year_start": 2024, @@ -115,6 +171,14 @@ "vintage": "2024_25", "locator": "househol.tab", "sha256": "2b93b6aed49e1591d6f5360736b4aee3a11ef506b276435f4d2c011a8afbb6a5", + "chronicle_artifact": { + "source_id": "dwp", + "package_id": "dwp-frs-2024-25-househol", + "year": 2024, + "sha256": "2b93b6aed49e1591d6f5360736b4aee3a11ef506b276435f4d2c011a8afbb6a5", + "filename": "househol.tab", + "access": "licensed" + }, "size_bytes": 12108606, "runtime_sha256_required": true, "tax_year_start": 2024, @@ -129,6 +193,14 @@ "vintage": "2024_25", "locator": "job.tab", "sha256": "eb7faf7ada3a3851cb2afb83e2983f8907ffeec897cfbe01e56cb0dfefa853e2", + "chronicle_artifact": { + "source_id": "dwp", + "package_id": "dwp-frs-2024-25-job", + "year": 2024, + "sha256": "eb7faf7ada3a3851cb2afb83e2983f8907ffeec897cfbe01e56cb0dfefa853e2", + "filename": "job.tab", + "access": "licensed" + }, "size_bytes": 10518760, "runtime_sha256_required": true, "tax_year_start": 2024, @@ -143,6 +215,14 @@ "vintage": "2024_25", "locator": "maint.tab", "sha256": "e7a8d6f47cab7bf9db9bfd7b3ad5ebe5830ec75245d065dcf8654c7c20b97a7d", + "chronicle_artifact": { + "source_id": "dwp", + "package_id": "dwp-frs-2024-25-maint", + "year": 2024, + "sha256": "e7a8d6f47cab7bf9db9bfd7b3ad5ebe5830ec75245d065dcf8654c7c20b97a7d", + "filename": "maint.tab", + "access": "licensed" + }, "size_bytes": 13993, "runtime_sha256_required": true, "tax_year_start": 2024, @@ -157,6 +237,14 @@ "vintage": "2024_25", "locator": "mortgage.tab", "sha256": "6a08f6846970dfdc544a7efc8a93fed4f3210d872cd2d160dfb14ca8d92d5ed0", + "chronicle_artifact": { + "source_id": "dwp", + "package_id": "dwp-frs-2024-25-mortgage", + "year": 2024, + "sha256": "6a08f6846970dfdc544a7efc8a93fed4f3210d872cd2d160dfb14ca8d92d5ed0", + "filename": "mortgage.tab", + "access": "licensed" + }, "size_bytes": 600552, "runtime_sha256_required": true, "tax_year_start": 2024, @@ -171,6 +259,14 @@ "vintage": "2024_25", "locator": "oddjob.tab", "sha256": "dfff1baf71a3de05f3a2fcf0c01a3995df5657f242cd7846aa61f6cc27a1cead", + "chronicle_artifact": { + "source_id": "dwp", + "package_id": "dwp-frs-2024-25-oddjob", + "year": 2024, + "sha256": "dfff1baf71a3de05f3a2fcf0c01a3995df5657f242cd7846aa61f6cc27a1cead", + "filename": "oddjob.tab", + "access": "licensed" + }, "size_bytes": 5339, "runtime_sha256_required": true, "tax_year_start": 2024, @@ -185,6 +281,14 @@ "vintage": "2024_25", "locator": "penprov.tab", "sha256": "9e53de0dc969baec000b3cd68387f0f2dfb3f678732e408de175e0a1d6e3fdc1", + "chronicle_artifact": { + "source_id": "dwp", + "package_id": "dwp-frs-2024-25-penprov", + "year": 2024, + "sha256": "9e53de0dc969baec000b3cd68387f0f2dfb3f678732e408de175e0a1d6e3fdc1", + "filename": "penprov.tab", + "access": "licensed" + }, "size_bytes": 513614, "runtime_sha256_required": true, "tax_year_start": 2024, @@ -199,6 +303,14 @@ "vintage": "2024_25", "locator": "pension.tab", "sha256": "2b9be1eb6583cc8916fc06294be27e6217f2aea73da24b97b3226293f6a6ec24", + "chronicle_artifact": { + "source_id": "dwp", + "package_id": "dwp-frs-2024-25-pension", + "year": 2024, + "sha256": "2b9be1eb6583cc8916fc06294be27e6217f2aea73da24b97b3226293f6a6ec24", + "filename": "pension.tab", + "access": "licensed" + }, "size_bytes": 1232411, "runtime_sha256_required": true, "tax_year_start": 2024, @@ -369,6 +481,14 @@ "vintage": "2024_25", "locator": "adult.tab", "sha256": "4eaea0809a7ccca0fddeb98e358771e4a6e5ebb81b21c4fac4070fc9d227658d", + "chronicle_artifact": { + "source_id": "dwp", + "package_id": "dwp-frs-2024-25-adult", + "year": 2024, + "sha256": "4eaea0809a7ccca0fddeb98e358771e4a6e5ebb81b21c4fac4070fc9d227658d", + "filename": "adult.tab", + "access": "licensed" + }, "size_bytes": 34885825, "runtime_sha256_required": true, "tax_year_start": 2024, @@ -414,6 +534,14 @@ "vintage": "2024_25", "locator": "househol.tab", "sha256": "2b93b6aed49e1591d6f5360736b4aee3a11ef506b276435f4d2c011a8afbb6a5", + "chronicle_artifact": { + "source_id": "dwp", + "package_id": "dwp-frs-2024-25-househol", + "year": 2024, + "sha256": "2b93b6aed49e1591d6f5360736b4aee3a11ef506b276435f4d2c011a8afbb6a5", + "filename": "househol.tab", + "access": "licensed" + }, "size_bytes": 12108606, "runtime_sha256_required": true, "tax_year_start": 2024, @@ -492,6 +620,14 @@ "vintage": "2024_25", "locator": "adult.tab", "sha256": "4eaea0809a7ccca0fddeb98e358771e4a6e5ebb81b21c4fac4070fc9d227658d", + "chronicle_artifact": { + "source_id": "dwp", + "package_id": "dwp-frs-2024-25-adult", + "year": 2024, + "sha256": "4eaea0809a7ccca0fddeb98e358771e4a6e5ebb81b21c4fac4070fc9d227658d", + "filename": "adult.tab", + "access": "licensed" + }, "size_bytes": 34885825, "runtime_sha256_required": true, "tax_year_start": 2024, @@ -506,6 +642,14 @@ "vintage": "2024_25", "locator": "child.tab", "sha256": "88ec53fc52eea4374864bbc74219d551f4b4c5f54a220bf9607c7a6289719aa5", + "chronicle_artifact": { + "source_id": "dwp", + "package_id": "dwp-frs-2024-25-child", + "year": 2024, + "sha256": "88ec53fc52eea4374864bbc74219d551f4b4c5f54a220bf9607c7a6289719aa5", + "filename": "child.tab", + "access": "licensed" + }, "size_bytes": 2753961, "runtime_sha256_required": true, "tax_year_start": 2024, @@ -568,6 +712,14 @@ "vintage": "2024_25", "locator": "adult.tab", "sha256": "4eaea0809a7ccca0fddeb98e358771e4a6e5ebb81b21c4fac4070fc9d227658d", + "chronicle_artifact": { + "source_id": "dwp", + "package_id": "dwp-frs-2024-25-adult", + "year": 2024, + "sha256": "4eaea0809a7ccca0fddeb98e358771e4a6e5ebb81b21c4fac4070fc9d227658d", + "filename": "adult.tab", + "access": "licensed" + }, "size_bytes": 34885825, "runtime_sha256_required": true, "tax_year_start": 2024, @@ -884,6 +1036,14 @@ "doi": "10.5255/UKDA-SN-7215-20", "filename": "was_round_8_hhold_eul_may_2025_230525.tab", "sha256": "18b3eb980c02c99f3d8a3254af859bee31682b2bdc11703877677292b3ce9374", + "chronicle_artifact": { + "source_id": "ons", + "package_id": "ons-was-round-8-household", + "year": 2018, + "sha256": "18b3eb980c02c99f3d8a3254af859bee31682b2bdc11703877677292b3ce9374", + "filename": "was_round_8_hhold_eul_may_2025_230525.tab", + "access": "restricted" + }, "size_bytes": 39073613, "access": "private_local_input", "locator": "caller-supplied local input", @@ -1094,6 +1254,14 @@ "vintage": "2023_24", "locator": "dvhh_ukanon_v2_2023.tab", "sha256": "6e78f0914be38e63853165486d641cbd790753cc471086210c6f672bfa18ca72", + "chronicle_artifact": { + "source_id": "ons", + "package_id": "ons-lcfs-2023-24-household", + "year": 2023, + "sha256": "6e78f0914be38e63853165486d641cbd790753cc471086210c6f672bfa18ca72", + "filename": "dvhh_ukanon_v2_2023.tab", + "access": "restricted" + }, "size_bytes": 22812887, "runtime_sha256_required": true, "filename": "dvhh_ukanon_v2_2023.tab" @@ -1105,6 +1273,14 @@ "vintage": "2023_24", "locator": "dvper_ukanon_202324_2023.tab", "sha256": "f32d54d83cdecf023f0ac73530be3a99372099b596e0106a56eae42a64929e50", + "chronicle_artifact": { + "source_id": "ons", + "package_id": "ons-lcfs-2023-24-person", + "year": 2023, + "sha256": "f32d54d83cdecf023f0ac73530be3a99372099b596e0106a56eae42a64929e50", + "filename": "dvper_ukanon_202324_2023.tab", + "access": "restricted" + }, "size_bytes": 6545146, "runtime_sha256_required": true, "filename": "dvper_ukanon_202324_2023.tab" @@ -1116,6 +1292,14 @@ "vintage": "2018_20", "locator": "caller-supplied local input", "sha256": "18b3eb980c02c99f3d8a3254af859bee31682b2bdc11703877677292b3ce9374", + "chronicle_artifact": { + "source_id": "ons", + "package_id": "ons-was-round-8-household", + "year": 2018, + "sha256": "18b3eb980c02c99f3d8a3254af859bee31682b2bdc11703877677292b3ce9374", + "filename": "was_round_8_hhold_eul_may_2025_230525.tab", + "access": "restricted" + }, "size_bytes": 39073613, "runtime_sha256_required": true, "filename": "was_round_8_hhold_eul_may_2025_230525.tab" @@ -1340,6 +1524,14 @@ "vintage": "1977_24", "locator": "householdv2_1977-2024.tab", "sha256": "d0e94ebc92e85ca1b9fb3a7353dcaf41db2c5110c9f07c7793dc8c0b695250d8", + "chronicle_artifact": { + "source_id": "ons", + "package_id": "ons-etb-1977-2024-household", + "year": 1977, + "sha256": "d0e94ebc92e85ca1b9fb3a7353dcaf41db2c5110c9f07c7793dc8c0b695250d8", + "filename": "householdv2_1977-2024.tab", + "access": "restricted" + }, "size_bytes": 216967663, "runtime_sha256_required": true, "filename": "householdv2_1977-2024.tab" @@ -1414,6 +1606,14 @@ "vintage": "1977_24", "locator": "householdv2_1977-2024.tab", "sha256": "d0e94ebc92e85ca1b9fb3a7353dcaf41db2c5110c9f07c7793dc8c0b695250d8", + "chronicle_artifact": { + "source_id": "ons", + "package_id": "ons-etb-1977-2024-household", + "year": 1977, + "sha256": "d0e94ebc92e85ca1b9fb3a7353dcaf41db2c5110c9f07c7793dc8c0b695250d8", + "filename": "householdv2_1977-2024.tab", + "access": "restricted" + }, "size_bytes": 216967663, "runtime_sha256_required": true, "filename": "householdv2_1977-2024.tab" @@ -1535,6 +1735,14 @@ "vintage": "2024_25", "locator": "adult.tab", "sha256": "4eaea0809a7ccca0fddeb98e358771e4a6e5ebb81b21c4fac4070fc9d227658d", + "chronicle_artifact": { + "source_id": "dwp", + "package_id": "dwp-frs-2024-25-adult", + "year": 2024, + "sha256": "4eaea0809a7ccca0fddeb98e358771e4a6e5ebb81b21c4fac4070fc9d227658d", + "filename": "adult.tab", + "access": "licensed" + }, "size_bytes": 34885825, "runtime_sha256_required": true, "tax_year_start": 2024, @@ -1549,6 +1757,14 @@ "vintage": "2024_25", "locator": "benefits.tab", "sha256": "f6ad22b408a13e2239c04b0d076a36418dcf5dd89a8c60daa792c4d735b911d3", + "chronicle_artifact": { + "source_id": "dwp", + "package_id": "dwp-frs-2024-25-benefits", + "year": 2024, + "sha256": "f6ad22b408a13e2239c04b0d076a36418dcf5dd89a8c60daa792c4d735b911d3", + "filename": "benefits.tab", + "access": "licensed" + }, "size_bytes": 2362329, "runtime_sha256_required": true, "tax_year_start": 2024, @@ -1745,6 +1961,14 @@ "doi": "10.5255/UKDA-SN-9422-1", "filename": "put2223uk.tab", "sha256": "5ef829461060c91a2a47be59ad541d9b519fc3976d66ca80d4920f711bb96f66", + "chronicle_artifact": { + "source_id": "hmrc", + "package_id": "hmrc-spi-public-use-tape-2022-23", + "year": 2022, + "sha256": "5ef829461060c91a2a47be59ad541d9b519fc3976d66ca80d4920f711bb96f66", + "filename": "put2223uk.tab", + "access": "restricted" + }, "size_bytes": 141323762, "reviewed_source": "PolicyEngine licensed UKDS mirror (private Hugging Face repository), spi_2022_23.zip", "access": "private_local_input", @@ -3085,6 +3309,14 @@ "doi": "10.5255/UKDA-SN-9422-1", "filename": "put2223uk.tab", "sha256": "5ef829461060c91a2a47be59ad541d9b519fc3976d66ca80d4920f711bb96f66", + "chronicle_artifact": { + "source_id": "hmrc", + "package_id": "hmrc-spi-public-use-tape-2022-23", + "year": 2022, + "sha256": "5ef829461060c91a2a47be59ad541d9b519fc3976d66ca80d4920f711bb96f66", + "filename": "put2223uk.tab", + "access": "restricted" + }, "size_bytes": 141323762, "reviewed_source": "PolicyEngine licensed UKDS mirror (private Hugging Face repository), spi_2022_23.zip", "access": "private_local_input", diff --git a/packages/microcosm-build/src/microcosm/build/uk/spec/sources.yaml b/packages/microcosm-build/src/microcosm/build/uk/spec/sources.yaml index ca30dbece..28cfa6518 100644 --- a/packages/microcosm-build/src/microcosm/build/uk/spec/sources.yaml +++ b/packages/microcosm-build/src/microcosm/build/uk/spec/sources.yaml @@ -27,6 +27,13 @@ stages: vintage: '2024_25' locator: accounts.tab sha256: fa7871eb45cad0db5fd05ede454ced60405d2f9c598651ea5acea5c91a6ff52f + chronicle_artifact: + source_id: "dwp" + package_id: "dwp-frs-2024-25-accounts" + year: 2024 + sha256: "fa7871eb45cad0db5fd05ede454ced60405d2f9c598651ea5acea5c91a6ff52f" + filename: "accounts.tab" + access: "licensed" size_bytes: 1812923 runtime_sha256_required: true tax_year_start: 2024 @@ -39,6 +46,13 @@ stages: vintage: '2024_25' locator: adult.tab sha256: 4eaea0809a7ccca0fddeb98e358771e4a6e5ebb81b21c4fac4070fc9d227658d + chronicle_artifact: + source_id: "dwp" + package_id: "dwp-frs-2024-25-adult" + year: 2024 + sha256: "4eaea0809a7ccca0fddeb98e358771e4a6e5ebb81b21c4fac4070fc9d227658d" + filename: "adult.tab" + access: "licensed" size_bytes: 34885825 runtime_sha256_required: true tax_year_start: 2024 @@ -51,6 +65,13 @@ stages: vintage: '2024_25' locator: benefits.tab sha256: f6ad22b408a13e2239c04b0d076a36418dcf5dd89a8c60daa792c4d735b911d3 + chronicle_artifact: + source_id: "dwp" + package_id: "dwp-frs-2024-25-benefits" + year: 2024 + sha256: "f6ad22b408a13e2239c04b0d076a36418dcf5dd89a8c60daa792c4d735b911d3" + filename: "benefits.tab" + access: "licensed" size_bytes: 2362329 runtime_sha256_required: true tax_year_start: 2024 @@ -63,6 +84,13 @@ stages: vintage: '2024_25' locator: benunit.tab sha256: 66b894624498316d19b6259e287a607e98ed3daacc9be3d3e9067d32b8e09a5a + chronicle_artifact: + source_id: "dwp" + package_id: "dwp-frs-2024-25-benunit" + year: 2024 + sha256: "66b894624498316d19b6259e287a607e98ed3daacc9be3d3e9067d32b8e09a5a" + filename: "benunit.tab" + access: "licensed" size_bytes: 13986782 runtime_sha256_required: true tax_year_start: 2024 @@ -75,6 +103,13 @@ stages: vintage: '2024_25' locator: child.tab sha256: 88ec53fc52eea4374864bbc74219d551f4b4c5f54a220bf9607c7a6289719aa5 + chronicle_artifact: + source_id: "dwp" + package_id: "dwp-frs-2024-25-child" + year: 2024 + sha256: "88ec53fc52eea4374864bbc74219d551f4b4c5f54a220bf9607c7a6289719aa5" + filename: "child.tab" + access: "licensed" size_bytes: 2753961 runtime_sha256_required: true tax_year_start: 2024 @@ -87,6 +122,13 @@ stages: vintage: '2024_25' locator: chldcare.tab sha256: 7ccd3f92f299a1f49b24063188177cdb8a958d8bcd753fc3d74dadda6ad04023 + chronicle_artifact: + source_id: "dwp" + package_id: "dwp-frs-2024-25-chldcare" + year: 2024 + sha256: "7ccd3f92f299a1f49b24063188177cdb8a958d8bcd753fc3d74dadda6ad04023" + filename: "chldcare.tab" + access: "licensed" size_bytes: 275878 runtime_sha256_required: true tax_year_start: 2024 @@ -99,6 +141,13 @@ stages: vintage: '2024_25' locator: extchild.tab sha256: c661379a4aa5079ce482b1f98f0bfb9157ad9b3ba4eb10739b61846f9c9548e4 + chronicle_artifact: + source_id: "dwp" + package_id: "dwp-frs-2024-25-extchild" + year: 2024 + sha256: "c661379a4aa5079ce482b1f98f0bfb9157ad9b3ba4eb10739b61846f9c9548e4" + filename: "extchild.tab" + access: "licensed" size_bytes: 15150 runtime_sha256_required: true tax_year_start: 2024 @@ -111,6 +160,13 @@ stages: vintage: '2024_25' locator: househol.tab sha256: 2b93b6aed49e1591d6f5360736b4aee3a11ef506b276435f4d2c011a8afbb6a5 + chronicle_artifact: + source_id: "dwp" + package_id: "dwp-frs-2024-25-househol" + year: 2024 + sha256: "2b93b6aed49e1591d6f5360736b4aee3a11ef506b276435f4d2c011a8afbb6a5" + filename: "househol.tab" + access: "licensed" size_bytes: 12108606 runtime_sha256_required: true tax_year_start: 2024 @@ -123,6 +179,13 @@ stages: vintage: '2024_25' locator: job.tab sha256: eb7faf7ada3a3851cb2afb83e2983f8907ffeec897cfbe01e56cb0dfefa853e2 + chronicle_artifact: + source_id: "dwp" + package_id: "dwp-frs-2024-25-job" + year: 2024 + sha256: "eb7faf7ada3a3851cb2afb83e2983f8907ffeec897cfbe01e56cb0dfefa853e2" + filename: "job.tab" + access: "licensed" size_bytes: 10518760 runtime_sha256_required: true tax_year_start: 2024 @@ -135,6 +198,13 @@ stages: vintage: '2024_25' locator: maint.tab sha256: e7a8d6f47cab7bf9db9bfd7b3ad5ebe5830ec75245d065dcf8654c7c20b97a7d + chronicle_artifact: + source_id: "dwp" + package_id: "dwp-frs-2024-25-maint" + year: 2024 + sha256: "e7a8d6f47cab7bf9db9bfd7b3ad5ebe5830ec75245d065dcf8654c7c20b97a7d" + filename: "maint.tab" + access: "licensed" size_bytes: 13993 runtime_sha256_required: true tax_year_start: 2024 @@ -147,6 +217,13 @@ stages: vintage: '2024_25' locator: mortgage.tab sha256: 6a08f6846970dfdc544a7efc8a93fed4f3210d872cd2d160dfb14ca8d92d5ed0 + chronicle_artifact: + source_id: "dwp" + package_id: "dwp-frs-2024-25-mortgage" + year: 2024 + sha256: "6a08f6846970dfdc544a7efc8a93fed4f3210d872cd2d160dfb14ca8d92d5ed0" + filename: "mortgage.tab" + access: "licensed" size_bytes: 600552 runtime_sha256_required: true tax_year_start: 2024 @@ -159,6 +236,13 @@ stages: vintage: '2024_25' locator: oddjob.tab sha256: dfff1baf71a3de05f3a2fcf0c01a3995df5657f242cd7846aa61f6cc27a1cead + chronicle_artifact: + source_id: "dwp" + package_id: "dwp-frs-2024-25-oddjob" + year: 2024 + sha256: "dfff1baf71a3de05f3a2fcf0c01a3995df5657f242cd7846aa61f6cc27a1cead" + filename: "oddjob.tab" + access: "licensed" size_bytes: 5339 runtime_sha256_required: true tax_year_start: 2024 @@ -171,6 +255,13 @@ stages: vintage: '2024_25' locator: penprov.tab sha256: 9e53de0dc969baec000b3cd68387f0f2dfb3f678732e408de175e0a1d6e3fdc1 + chronicle_artifact: + source_id: "dwp" + package_id: "dwp-frs-2024-25-penprov" + year: 2024 + sha256: "9e53de0dc969baec000b3cd68387f0f2dfb3f678732e408de175e0a1d6e3fdc1" + filename: "penprov.tab" + access: "licensed" size_bytes: 513614 runtime_sha256_required: true tax_year_start: 2024 @@ -183,6 +274,13 @@ stages: vintage: '2024_25' locator: pension.tab sha256: 2b9be1eb6583cc8916fc06294be27e6217f2aea73da24b97b3226293f6a6ec24 + chronicle_artifact: + source_id: "dwp" + package_id: "dwp-frs-2024-25-pension" + year: 2024 + sha256: "2b9be1eb6583cc8916fc06294be27e6217f2aea73da24b97b3226293f6a6ec24" + filename: "pension.tab" + access: "licensed" size_bytes: 1232411 runtime_sha256_required: true tax_year_start: 2024 @@ -333,6 +431,13 @@ stages: vintage: '2024_25' locator: adult.tab sha256: 4eaea0809a7ccca0fddeb98e358771e4a6e5ebb81b21c4fac4070fc9d227658d + chronicle_artifact: + source_id: "dwp" + package_id: "dwp-frs-2024-25-adult" + year: 2024 + sha256: "4eaea0809a7ccca0fddeb98e358771e4a6e5ebb81b21c4fac4070fc9d227658d" + filename: "adult.tab" + access: "licensed" size_bytes: 34885825 runtime_sha256_required: true tax_year_start: 2024 @@ -366,6 +471,13 @@ stages: vintage: '2024_25' locator: househol.tab sha256: 2b93b6aed49e1591d6f5360736b4aee3a11ef506b276435f4d2c011a8afbb6a5 + chronicle_artifact: + source_id: "dwp" + package_id: "dwp-frs-2024-25-househol" + year: 2024 + sha256: "2b93b6aed49e1591d6f5360736b4aee3a11ef506b276435f4d2c011a8afbb6a5" + filename: "househol.tab" + access: "licensed" size_bytes: 12108606 runtime_sha256_required: true tax_year_start: 2024 @@ -423,6 +535,13 @@ stages: vintage: '2024_25' locator: adult.tab sha256: 4eaea0809a7ccca0fddeb98e358771e4a6e5ebb81b21c4fac4070fc9d227658d + chronicle_artifact: + source_id: "dwp" + package_id: "dwp-frs-2024-25-adult" + year: 2024 + sha256: "4eaea0809a7ccca0fddeb98e358771e4a6e5ebb81b21c4fac4070fc9d227658d" + filename: "adult.tab" + access: "licensed" size_bytes: 34885825 runtime_sha256_required: true tax_year_start: 2024 @@ -435,6 +554,13 @@ stages: vintage: '2024_25' locator: child.tab sha256: 88ec53fc52eea4374864bbc74219d551f4b4c5f54a220bf9607c7a6289719aa5 + chronicle_artifact: + source_id: "dwp" + package_id: "dwp-frs-2024-25-child" + year: 2024 + sha256: "88ec53fc52eea4374864bbc74219d551f4b4c5f54a220bf9607c7a6289719aa5" + filename: "child.tab" + access: "licensed" size_bytes: 2753961 runtime_sha256_required: true tax_year_start: 2024 @@ -482,6 +608,13 @@ stages: vintage: '2024_25' locator: adult.tab sha256: 4eaea0809a7ccca0fddeb98e358771e4a6e5ebb81b21c4fac4070fc9d227658d + chronicle_artifact: + source_id: "dwp" + package_id: "dwp-frs-2024-25-adult" + year: 2024 + sha256: "4eaea0809a7ccca0fddeb98e358771e4a6e5ebb81b21c4fac4070fc9d227658d" + filename: "adult.tab" + access: "licensed" size_bytes: 34885825 runtime_sha256_required: true tax_year_start: 2024 @@ -705,6 +838,13 @@ stages: doi: 10.5255/UKDA-SN-7215-20 filename: was_round_8_hhold_eul_may_2025_230525.tab sha256: 18b3eb980c02c99f3d8a3254af859bee31682b2bdc11703877677292b3ce9374 + chronicle_artifact: + source_id: "ons" + package_id: "ons-was-round-8-household" + year: 2018 + sha256: "18b3eb980c02c99f3d8a3254af859bee31682b2bdc11703877677292b3ce9374" + filename: "was_round_8_hhold_eul_may_2025_230525.tab" + access: "restricted" size_bytes: 39073613 access: private_local_input locator: caller-supplied local input @@ -874,6 +1014,13 @@ stages: vintage: '2023_24' locator: dvhh_ukanon_v2_2023.tab sha256: 6e78f0914be38e63853165486d641cbd790753cc471086210c6f672bfa18ca72 + chronicle_artifact: + source_id: "ons" + package_id: "ons-lcfs-2023-24-household" + year: 2023 + sha256: "6e78f0914be38e63853165486d641cbd790753cc471086210c6f672bfa18ca72" + filename: "dvhh_ukanon_v2_2023.tab" + access: "restricted" size_bytes: 22812887 runtime_sha256_required: true filename: dvhh_ukanon_v2_2023.tab @@ -883,6 +1030,13 @@ stages: vintage: '2023_24' locator: dvper_ukanon_202324_2023.tab sha256: f32d54d83cdecf023f0ac73530be3a99372099b596e0106a56eae42a64929e50 + chronicle_artifact: + source_id: "ons" + package_id: "ons-lcfs-2023-24-person" + year: 2023 + sha256: "f32d54d83cdecf023f0ac73530be3a99372099b596e0106a56eae42a64929e50" + filename: "dvper_ukanon_202324_2023.tab" + access: "restricted" size_bytes: 6545146 runtime_sha256_required: true filename: dvper_ukanon_202324_2023.tab @@ -892,6 +1046,13 @@ stages: vintage: '2018_20' locator: caller-supplied local input sha256: 18b3eb980c02c99f3d8a3254af859bee31682b2bdc11703877677292b3ce9374 + chronicle_artifact: + source_id: "ons" + package_id: "ons-was-round-8-household" + year: 2018 + sha256: "18b3eb980c02c99f3d8a3254af859bee31682b2bdc11703877677292b3ce9374" + filename: "was_round_8_hhold_eul_may_2025_230525.tab" + access: "restricted" size_bytes: 39073613 runtime_sha256_required: true filename: was_round_8_hhold_eul_may_2025_230525.tab @@ -1070,6 +1231,13 @@ stages: vintage: '1977_24' locator: householdv2_1977-2024.tab sha256: d0e94ebc92e85ca1b9fb3a7353dcaf41db2c5110c9f07c7793dc8c0b695250d8 + chronicle_artifact: + source_id: "ons" + package_id: "ons-etb-1977-2024-household" + year: 1977 + sha256: "d0e94ebc92e85ca1b9fb3a7353dcaf41db2c5110c9f07c7793dc8c0b695250d8" + filename: "householdv2_1977-2024.tab" + access: "restricted" size_bytes: 216967663 runtime_sha256_required: true filename: householdv2_1977-2024.tab @@ -1123,6 +1291,13 @@ stages: vintage: '1977_24' locator: householdv2_1977-2024.tab sha256: d0e94ebc92e85ca1b9fb3a7353dcaf41db2c5110c9f07c7793dc8c0b695250d8 + chronicle_artifact: + source_id: "ons" + package_id: "ons-etb-1977-2024-household" + year: 1977 + sha256: "d0e94ebc92e85ca1b9fb3a7353dcaf41db2c5110c9f07c7793dc8c0b695250d8" + filename: "householdv2_1977-2024.tab" + access: "restricted" size_bytes: 216967663 runtime_sha256_required: true filename: householdv2_1977-2024.tab @@ -1216,6 +1391,13 @@ stages: vintage: '2024_25' locator: adult.tab sha256: 4eaea0809a7ccca0fddeb98e358771e4a6e5ebb81b21c4fac4070fc9d227658d + chronicle_artifact: + source_id: "dwp" + package_id: "dwp-frs-2024-25-adult" + year: 2024 + sha256: "4eaea0809a7ccca0fddeb98e358771e4a6e5ebb81b21c4fac4070fc9d227658d" + filename: "adult.tab" + access: "licensed" size_bytes: 34885825 runtime_sha256_required: true tax_year_start: 2024 @@ -1228,6 +1410,13 @@ stages: vintage: '2024_25' locator: benefits.tab sha256: f6ad22b408a13e2239c04b0d076a36418dcf5dd89a8c60daa792c4d735b911d3 + chronicle_artifact: + source_id: "dwp" + package_id: "dwp-frs-2024-25-benefits" + year: 2024 + sha256: "f6ad22b408a13e2239c04b0d076a36418dcf5dd89a8c60daa792c4d735b911d3" + filename: "benefits.tab" + access: "licensed" size_bytes: 2362329 runtime_sha256_required: true tax_year_start: 2024 @@ -1380,6 +1569,13 @@ stages: doi: 10.5255/UKDA-SN-9422-1 filename: put2223uk.tab sha256: 5ef829461060c91a2a47be59ad541d9b519fc3976d66ca80d4920f711bb96f66 + chronicle_artifact: + source_id: "hmrc" + package_id: "hmrc-spi-public-use-tape-2022-23" + year: 2022 + sha256: "5ef829461060c91a2a47be59ad541d9b519fc3976d66ca80d4920f711bb96f66" + filename: "put2223uk.tab" + access: "restricted" size_bytes: 141323762 reviewed_source: PolicyEngine licensed UKDS mirror (private Hugging Face repository), spi_2022_23.zip access: private_local_input @@ -2446,6 +2642,13 @@ stages: doi: 10.5255/UKDA-SN-9422-1 filename: put2223uk.tab sha256: 5ef829461060c91a2a47be59ad541d9b519fc3976d66ca80d4920f711bb96f66 + chronicle_artifact: + source_id: "hmrc" + package_id: "hmrc-spi-public-use-tape-2022-23" + year: 2022 + sha256: "5ef829461060c91a2a47be59ad541d9b519fc3976d66ca80d4920f711bb96f66" + filename: "put2223uk.tab" + access: "restricted" size_bytes: 141323762 reviewed_source: PolicyEngine licensed UKDS mirror (private Hugging Face repository), spi_2022_23.zip access: private_local_input diff --git a/packages/microcosm-build/src/microcosm/build/us/source_stages.json b/packages/microcosm-build/src/microcosm/build/us/source_stages.json index 5304c4352..d1c416056 100644 --- a/packages/microcosm-build/src/microcosm/build/us/source_stages.json +++ b/packages/microcosm-build/src/microcosm/build/us/source_stages.json @@ -739,6 +739,14 @@ "vintage": "2023 ASEC / 2022 income reference year", "locator": "https://www2.census.gov/programs-surveys/cps/datasets/2023/march/asecpub23csv.zip", "sha256": "d2e000250782adfbdd7f29c82b66d866591a30f0d330496698ec19f9c784ce11", + "chronicle_artifact": { + "source_id": "census_cps", + "package_id": "census-cps-asec-2023", + "year": 2023, + "sha256": "d2e000250782adfbdd7f29c82b66d866591a30f0d330496698ec19f9c784ce11", + "filename": "asecpub23csv.zip", + "access": "public" + }, "size_bytes": 150165063, "member": "pppub23.csv", "member_size_bytes": 281065733, @@ -866,6 +874,14 @@ "locator": "https://www.federalreserve.gov/econres/files/scfp2022s.zip", "member": "rscfp2022.dta", "sha256": "3bb4d890ae2463ff6039ec7692e375f544dd98a55a37ca2cb2340354b9cc9d80", + "chronicle_artifact": { + "source_id": "federal_reserve", + "package_id": "federal-reserve-scf-2022-summary-extract", + "year": 2022, + "sha256": "3bb4d890ae2463ff6039ec7692e375f544dd98a55a37ca2cb2340354b9cc9d80", + "filename": "scfp2022s.zip", + "access": "public" + }, "member_sha256": "6b8dd2d935a76ed225ddebc80fb2db22a467f0c80d9a1acaa67b4584aa4bafd1" }, { diff --git a/packages/microcosm-build/src/microcosm/build/us/spec/sources.yaml b/packages/microcosm-build/src/microcosm/build/us/spec/sources.yaml index a160533f4..a008dc877 100644 --- a/packages/microcosm-build/src/microcosm/build/us/spec/sources.yaml +++ b/packages/microcosm-build/src/microcosm/build/us/spec/sources.yaml @@ -99,7 +99,7 @@ sources: stage_asset: id: source_stages path: microcosm.build.us/source_stages.json - sha256: dc58a0d700f0add7b658cec774df6e9587303beb58a1f432a35a18dcd1ac4097 + sha256: 062586b78813461a49df66ad99afd7aa3b0fcfd4f42b305d80a83575bc27ad71 stage_manifest: version: 1 country: us @@ -840,6 +840,13 @@ stages: vintage: 2023 ASEC / 2022 income reference year locator: https://www2.census.gov/programs-surveys/cps/datasets/2023/march/asecpub23csv.zip sha256: d2e000250782adfbdd7f29c82b66d866591a30f0d330496698ec19f9c784ce11 + chronicle_artifact: + source_id: census_cps + package_id: census-cps-asec-2023 + year: 2023 + sha256: d2e000250782adfbdd7f29c82b66d866591a30f0d330496698ec19f9c784ce11 + filename: asecpub23csv.zip + access: public size_bytes: 150165063 member: pppub23.csv member_size_bytes: 281065733 @@ -964,6 +971,13 @@ stages: locator: https://www.federalreserve.gov/econres/files/scfp2022s.zip member: rscfp2022.dta sha256: 3bb4d890ae2463ff6039ec7692e375f544dd98a55a37ca2cb2340354b9cc9d80 + chronicle_artifact: + source_id: federal_reserve + package_id: federal-reserve-scf-2022-summary-extract + year: 2022 + sha256: 3bb4d890ae2463ff6039ec7692e375f544dd98a55a37ca2cb2340354b9cc9d80 + filename: scfp2022s.zip + access: public member_sha256: 6b8dd2d935a76ed225ddebc80fb2db22a467f0c80d9a1acaa67b4584aa4bafd1 - kind: public_microdata format: stata_in_zip diff --git a/packages/microcosm-build/tests/test_spec_engine_country_bundles.py b/packages/microcosm-build/tests/test_spec_engine_country_bundles.py index 225af1c0d..ba0bf50d1 100644 --- a/packages/microcosm-build/tests/test_spec_engine_country_bundles.py +++ b/packages/microcosm-build/tests/test_spec_engine_country_bundles.py @@ -25,7 +25,7 @@ "spine", "vintages", } -AM_SPEC_SHA256 = "659b6baf5ebbd71fb7786ec4c4d49df565b2bddabeb868a9385ed226c56880f9" +AM_SPEC_SHA256 = "6792caa8280649b0c20265ee4b4e8dc0d4061e9d7476e12515b135c84e8f0dc1" @pytest.mark.parametrize( @@ -45,7 +45,7 @@ ), ( "be", - "7062e38f4d623553fb0604380a8dac0edacb6261c155b6e31fc38ef7c0f1c57c", + "8f31a6b4c6f0ae8b65d55ef12995c459b47048080fbad54208747fa3c3d11028", { "household.household_id", "person.person_id", @@ -55,7 +55,7 @@ ), ( "uk", - "8e6f5861362e1e23b650c1d2937995bebee573bb9f1cab51fc8e1fef2cef8a9c", + "b3a23e6ea34525b35ac7235d2e01263eaea0d7ba9edac57820e0939258941826", { "benunit.benunit_id", "household.household_id", diff --git a/packages/microcosm-build/tests/test_spec_engine_loader.py b/packages/microcosm-build/tests/test_spec_engine_loader.py index a77c90f6e..0aaba605f 100644 --- a/packages/microcosm-build/tests/test_spec_engine_loader.py +++ b/packages/microcosm-build/tests/test_spec_engine_loader.py @@ -236,7 +236,7 @@ def test_semantic_hash_has_golden_vector_and_surface_separation(tmp_path) -> Non # Pin the domain separator, normalization rules, schema-set receipt, and # exact normative projection as one reviewable golden vector. assert first.spec_sha256 == ( - "9f5b372796f2638378125d97ef5150be4c1f4cba9147b44973e4cd6a5f52f10a" + "d78bf9e7678cbba15953a535a19f5c1b6b5accffc0cacb5533854c01ea9a50b7" ) second_root = _rich_minimal(tmp_path / "xy", note="second", store="local:b") diff --git a/packages/microcosm-build/tests/test_uk_source_stages.py b/packages/microcosm-build/tests/test_uk_source_stages.py index 67c4bf59b..c2f4b73b6 100644 --- a/packages/microcosm-build/tests/test_uk_source_stages.py +++ b/packages/microcosm-build/tests/test_uk_source_stages.py @@ -83,7 +83,7 @@ "hmrc_spi_income", ] FROZEN_SOURCE_STAGES_SHA256 = ( - "c0341af7166ae3a85a3c1164e7d9e880c4b4aec122f1a8fa90c73b46c596e1ea" + "2b6a74d33686d5a18f531a680c30330f7e5923af6fca97ac548c0785626f04aa" ) diff --git a/packages/microcosm-build/tests/test_us_bundle_core_contracts.py b/packages/microcosm-build/tests/test_us_bundle_core_contracts.py index 1ab095883..adf8d31cd 100644 --- a/packages/microcosm-build/tests/test_us_bundle_core_contracts.py +++ b/packages/microcosm-build/tests/test_us_bundle_core_contracts.py @@ -129,7 +129,7 @@ def test_source_surface_classification_is_complete() -> None: } assert normative["stage_asset"] == { "id": "source_stages", - "sha256": "dc58a0d700f0add7b658cec774df6e9587303beb58a1f432a35a18dcd1ac4097", + "sha256": "062586b78813461a49df66ad99afd7aa3b0fcfd4f42b305d80a83575bc27ad71", } assert operational["stage_asset"] == { "path": "microcosm.build.us/source_stages.json" diff --git a/packages/microcosm-build/tests/test_us_spec_bundle.py b/packages/microcosm-build/tests/test_us_spec_bundle.py index 892ece523..b6eb21bf6 100644 --- a/packages/microcosm-build/tests/test_us_spec_bundle.py +++ b/packages/microcosm-build/tests/test_us_spec_bundle.py @@ -190,7 +190,7 @@ def _load_generator_module(): LEGACY_COMPATIBILITY_SHA256 = { "source_stages.json": ( - "dc58a0d700f0add7b658cec774df6e9587303beb58a1f432a35a18dcd1ac4097" + "062586b78813461a49df66ad99afd7aa3b0fcfd4f42b305d80a83575bc27ad71" ), "support_spine.json": ( "68f37dc6ae6e0cde7ebccb53f88dd4a800e63456f838fa214ff98d1db8d815be" diff --git a/tools/generate_us_bundle_from_constants.py b/tools/generate_us_bundle_from_constants.py index 465bd4a4f..23f66e41b 100644 --- a/tools/generate_us_bundle_from_constants.py +++ b/tools/generate_us_bundle_from_constants.py @@ -122,7 +122,7 @@ def _domain_builders() -> dict[str, Callable[[], object]]: # frozen files, is the forward YAML -> legacy-payload path. FROZEN_LEGACY_RESOURCE_SHA256 = { "source_stages.json": ( - "dc58a0d700f0add7b658cec774df6e9587303beb58a1f432a35a18dcd1ac4097" + "062586b78813461a49df66ad99afd7aa3b0fcfd4f42b305d80a83575bc27ad71" ), "support_spine.json": ( "68f37dc6ae6e0cde7ebccb53f88dd4a800e63456f838fa214ff98d1db8d815be" From 92cddc6ce70fd3ec571bfa7ad214bf7f67d32d4f Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Wed, 2 Sep 2026 12:33:17 -0400 Subject: [PATCH 05/14] Refuse to build from raw microdata whose bytes are not the registered ones MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit source_runtime gains the fail-closed root-identity gate. verify_microdata_files hashes each raw microdata file a run was handed and refuses to continue unless the bytes are the pinned ones; the refusal names the publisher, the vintage, the locator, the path, and both digests, so an operator can tell a reissued vintage from a corrupted download without rerunning anything. Keys resolve against both the locator and the declared filename, because a caller-supplied private input's locator is the placeholder "caller-supplied local input". verify_recorded_microdata_pins is the cheap path for a producing run that already hashed its inputs. The ASEC raw-stage checkpoint records a sha256/member_sha256 pin per archive, so load_asec_raw_stage_checkpoint now cross-checks those against the US manifest instead of re-reading hundreds of megabytes per archive. Disagreement is fatal; absence is not — a locator the manifest declares no pin for is reported as unregistered, which is exactly what microdata_pins_pending.json records, and the checkpoint metadata contract is unchanged. Both builds record what they resolved. build_uk_frs_spine verifies every FRS tab and caller-supplied licensed input before any stage reads a table and writes the resulting registrations into the spine sidecar; build_us_fiscal_refresh_release writes a microdata_registrations block beside the existing Chronicle consumer-artifact pin, carrying both the resolved registrations and the pending rows, so a release manifest states its own unwitnessed surface. test_microdata_root_identity is the contract: pinned-or-listed for all four countries and the frozen UK replay manifest, the allowlist ratchet and its stale/orphan row failures, the derived access class, the public-only R2 key, the registration/pin agreement, and both gate paths including the exact diagnosable message. Co-Authored-By: Claude Fable 5.1 --- .../src/microcosm/build/source_runtime.py | 77 ++- .../build/us_runtime/asec_checkpoint.py | 42 ++ .../tests/test_microdata_root_identity.py | 559 ++++++++++++++++++ tools/build_uk_frs_spine.py | 71 +++ tools/build_us_fiscal_refresh_release.py | 39 ++ 5 files changed, 772 insertions(+), 16 deletions(-) create mode 100644 packages/microcosm-build/tests/test_microdata_root_identity.py diff --git a/packages/microcosm-build/src/microcosm/build/source_runtime.py b/packages/microcosm-build/src/microcosm/build/source_runtime.py index dd4b5fd48..407919478 100644 --- a/packages/microcosm-build/src/microcosm/build/source_runtime.py +++ b/packages/microcosm-build/src/microcosm/build/source_runtime.py @@ -35,6 +35,7 @@ __all__ = [ "MicrodataFileVerification", "MicrodataIdentityError", + "RecordedPinAudit", "SourceOperationHandler", "SourceRuntimeConfig", "SourceRuntimeContext", @@ -43,6 +44,7 @@ "run_source_stage", "sha256_file", "verify_microdata_files", + "verified_chronicle_registrations", "verify_recorded_microdata_pins", ] @@ -290,23 +292,38 @@ def verify_microdata_files( return tuple(verifications) +@dataclass(frozen=True) +class RecordedPinAudit: + """What a producing run's recorded source pins resolve to.""" + + resolved: tuple[ChronicleArtifactReference, ...] + unregistered: tuple[str, ...] + + def to_payload(self) -> dict[str, Any]: + return { + "chronicle_artifacts": [ref.to_payload() for ref in self.resolved], + "unregistered_locators": list(self.unregistered), + } + + def verify_recorded_microdata_pins( source: SourceManifest | SourceStageSpec | Mapping[str, Any], pins: Sequence[Mapping[str, Any]], *, context: str, -) -> tuple[ChronicleArtifactReference, ...]: +) -> RecordedPinAudit: """Cross-check pins a checkpoint already recorded against the manifest. The ASEC raw-stage checkpoint records a ``sha256``/``member_sha256`` pin per - source file it consumed. Re-hashing those archives would cost gigabytes of - reads for a digest the producing run already computed, so this compares the - recorded pins against the manifest instead and returns the registrations - they resolve to. - - Raises: - MicrodataIdentityError: If a recorded pin matches no manifest entry, or - matches one whose archive or member digest differs. + source archive it consumed. Re-hashing those archives would cost gigabytes + of reads for digests the producing run already computed, so this compares + the recorded pins against the manifest instead. + + Disagreement is fatal: a recorded pin whose locator the manifest pins to + different bytes stops the build. Absence is not — a locator the manifest + declares no pin for is reported as unregistered, because that is exactly the + state ``microdata_pins_pending.json`` records, and this check must not + invent a registration the repository has not made. """ entries = [ @@ -315,6 +332,7 @@ def verify_recorded_microdata_pins( if entry.sha256 is not None ] resolved: list[ChronicleArtifactReference] = [] + unregistered: list[str] = [] failures: list[str] = [] for index, pin in enumerate(pins): locator = pin.get("locator") @@ -322,15 +340,13 @@ def verify_recorded_microdata_pins( member_sha256 = pin.get("member_sha256") matches = [entry for entry in entries if entry.locator == locator] if not matches: - failures.append( - f"pin[{index}] locator {locator!r} names no hash-pinned " - "microdata artifact in this manifest." - ) + if isinstance(locator, str) and locator not in unregistered: + unregistered.append(locator) continue for entry in matches: if entry.sha256 != sha256: failures.append( - f"pin[{index}] {locator!r} recorded sha256 {sha256!r}; " + f" pin[{index}] {locator!r} recorded sha256 {sha256!r}; " f"stage {entry.stage!r} pins {entry.sha256}." ) continue @@ -340,7 +356,7 @@ def verify_recorded_microdata_pins( and entry.member_sha256 != member_sha256 ): failures.append( - f"pin[{index}] {locator!r} recorded member_sha256 " + f" pin[{index}] {locator!r} recorded member_sha256 " f"{member_sha256!r}; stage {entry.stage!r} pins " f"{entry.member_sha256}." ) @@ -352,9 +368,38 @@ def verify_recorded_microdata_pins( f"{context}: recorded raw-microdata pins disagree with the source " "manifest:\n" + "\n".join(failures) ) + return RecordedPinAudit( + resolved=tuple( + sorted( + set(resolved), + key=lambda ref: ( + ref.source_id, + ref.package_id, + ref.year, + ref.sha256, + ), + ) + ), + unregistered=tuple(sorted(unregistered)), + ) + + +def verified_chronicle_registrations( + verifications: Sequence[MicrodataFileVerification], +) -> tuple[ChronicleArtifactReference, ...]: + """The distinct registrations a set of verified files resolves to. + + This is the receipt a build records: not every registration the manifest + declares, but the ones behind the files this run actually read and hashed. + """ + return tuple( sorted( - set(resolved), + { + verification.registration + for verification in verifications + if verification.registration is not None + }, key=lambda ref: (ref.source_id, ref.package_id, ref.year, ref.sha256), ) ) diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/asec_checkpoint.py b/packages/microcosm-build/src/microcosm/build/us_runtime/asec_checkpoint.py index 0c6ebe0fa..882f21761 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/asec_checkpoint.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/asec_checkpoint.py @@ -12,7 +12,9 @@ import re from collections.abc import Mapping +from importlib.resources import files from pathlib import Path +from typing import Any import numpy as np import pandas as pd @@ -24,6 +26,11 @@ frame_identity, ) from microcosm.build.serialization_dtypes import canonicalize_frame_string_dtypes +from microcosm.build.source_manifest import load_source_manifest +from microcosm.build.source_runtime import ( + RecordedPinAudit, + verify_recorded_microdata_pins, +) from microcosm.build.us_runtime.operator_boundary import ( assert_operator_free_source_frame, ) @@ -177,6 +184,10 @@ def load_asec_raw_stage_checkpoint( label=f"ASEC raw-stage checkpoint {checkpoint_path}", ) _validate_raw_stage_source_columns(loaded.frame, path=checkpoint_path) + _cross_check_recorded_source_pins( + metadata["raw_source_mappings"], + path=checkpoint_path, + ) source_construction_identity = FrameIdentity.from_payload( metadata["source_construction_identity"], label="ASEC raw-stage source-construction identity", @@ -400,6 +411,37 @@ def _validate_raw_source_mappings(mappings: object, *, path: Path) -> None: ) +def _cross_check_recorded_source_pins( + mappings: Mapping[str, Any], + *, + path: Path, +) -> RecordedPinAudit: + """Reconcile the checkpoint's recorded source pins with the US manifest. + + Root identity (microcosm#848). The producing run already hashed each ASEC + archive it read and wrote the digests into ``source_pins``, so this reads + those pins rather than re-hashing hundreds of megabytes per archive. A pin + the manifest pins to different bytes stops the load; a locator the manifest + declares no pin for is reported, because that is the state + ``microdata_pins_pending.json`` records rather than a fault of this + checkpoint. The metadata contract is unchanged: this is a gate, and the + registrations it resolves are recorded by the release builder, not smuggled + into the checkpoint binding. + """ + + manifest = load_source_manifest( + files("microcosm.build.us").joinpath("source_stages.json") + ) + pins = [ + pin for column in sorted(mappings) for pin in mappings[column]["source_pins"] + ] + return verify_recorded_microdata_pins( + manifest, + pins, + context=f"ASEC raw-stage checkpoint {path}", + ) + + def _validate_raw_stage_source_columns(frame: Frame, *, path: Path) -> None: person = frame.table("person") missing = sorted(_RAW_STAGE_REQUIRED_PERSON_COLUMNS - set(person)) diff --git a/packages/microcosm-build/tests/test_microdata_root_identity.py b/packages/microcosm-build/tests/test_microdata_root_identity.py new file mode 100644 index 000000000..251e6590c --- /dev/null +++ b/packages/microcosm-build/tests/test_microdata_root_identity.py @@ -0,0 +1,559 @@ +"""Every raw microdata root is pinned to a Chronicle registration or listed. + +The contract (microcosm#848, Chronicle ADR "Raw microdata in Chronicle is +identity, not content") is that a build graph has no anonymous roots. A source +manifest entry whose ``kind`` names microdata either declares the SHA-256 of +the exact file a stage reads plus the one Chronicle registration that witnesses +it, or it carries a reviewed row in ``microdata_pins_pending.json`` saying what +blocks the pin. The allowlist is a ratchet, so the unwitnessed surface can only +shrink. +""" + +from __future__ import annotations + +import hashlib +import json +from importlib.resources import files +from pathlib import Path + +import pytest + +from microcosm.build.source_manifest import ( + CHRONICLE_ACCESS_BY_ARTIFACT_KIND, + MICRODATA_ARTIFACT_KINDS, + MICRODATA_PIN_ALLOWLIST_FILENAME, + ChronicleArtifactReference, + SourceManifest, + audit_microdata_pins, + load_microdata_pin_allowlist, + microdata_artifact_entries, + packaged_microdata_pin_allowlist, + resolved_chronicle_registrations, +) +from microcosm.build.source_runtime import ( + MicrodataIdentityError, + sha256_file, + verified_chronicle_registrations, + verify_microdata_files, + verify_recorded_microdata_pins, +) + +COUNTRIES = ("am", "be", "uk", "us") + +# The committed ratchet. A change that raises this number is a deliberate +# decision to add an unwitnessed build root and must be argued in review; a +# change that lowers it is a pin landing. +COMMITTED_PENDING_BASELINE = 39 + +#: Countries with no pending rows at all: every microdata root is witnessed. +FULLY_PINNED_COUNTRIES = ("uk",) + + +def _manifest(country: str) -> SourceManifest: + from microcosm.build.source_manifest import load_source_manifest + + return load_source_manifest( + files(f"microcosm.build.{country}").joinpath("source_stages.json") + ) + + +def _frozen_uk_replay_manifest() -> dict: + return json.loads( + files("microcosm.build.uk") + .joinpath("hmrc_income_source_stages.json") + .read_text(encoding="utf-8") + ) + + +def _sha(value: str) -> str: + return hashlib.sha256(value.encode("utf-8")).hexdigest() + + +class TestPinnedOrListed: + @pytest.mark.parametrize("country", COUNTRIES) + def test_every_microdata_root_is_pinned_or_listed(self, country: str) -> None: + gaps = audit_microdata_pins( + _manifest(country), allowlist=packaged_microdata_pin_allowlist() + ) + + assert [gap.message() for gap in gaps] == [] + + def test_the_frozen_uk_replay_manifest_is_pinned_too(self) -> None: + entries = microdata_artifact_entries(_frozen_uk_replay_manifest()) + + assert entries + assert all(entry.is_pinned for entry in entries) + + @pytest.mark.parametrize("country", FULLY_PINNED_COUNTRIES) + def test_fully_pinned_countries_carry_no_pending_rows(self, country: str) -> None: + allowlist = packaged_microdata_pin_allowlist() + entries = microdata_artifact_entries(_manifest(country)) + + assert allowlist.for_country(country) == () + assert entries + assert all(entry.is_pinned for entry in entries) + + +class TestAllowlistRatchet: + def test_the_allowlist_never_exceeds_its_committed_baseline(self) -> None: + allowlist = packaged_microdata_pin_allowlist() + + assert allowlist.baseline_count <= COMMITTED_PENDING_BASELINE + assert len(allowlist.pending) <= allowlist.baseline_count + + def test_every_pending_row_names_a_live_unpinned_root(self) -> None: + allowlist = packaged_microdata_pin_allowlist() + unpinned = { + (country, entry.stage, entry.locator) + for country in COUNTRIES + for entry in microdata_artifact_entries(_manifest(country)) + if not entry.is_pinned + } + + listed = {(row.country, row.stage, row.locator) for row in allowlist.pending} + assert listed == unpinned + + def test_every_pending_row_states_a_reason_and_an_issue(self) -> None: + for row in packaged_microdata_pin_allowlist().pending: + assert row.issue.startswith("PolicyEngine/microcosm#") + # A reason has to say what blocks the pin, not merely that one is + # missing; a one-liner would let a row outlive its cause unnoticed. + assert len(row.reason) > 80 + + def test_a_row_count_above_the_baseline_is_refused(self) -> None: + raw = json.loads( + files("microcosm.build") + .joinpath(MICRODATA_PIN_ALLOWLIST_FILENAME) + .read_text(encoding="utf-8") + ) + raw["baseline_count"] = len(raw["pending"]) - 1 + + with pytest.raises(ValueError, match="ratchet and may only shrink"): + load_microdata_pin_allowlist(_written(raw)) + + def test_a_stale_row_for_an_already_pinned_root_is_a_gap(self) -> None: + manifest = _manifest("uk") + entry = microdata_artifact_entries(manifest)[0] + allowlist = load_microdata_pin_allowlist( + _written( + { + "version": 1, + "policy": "test", + "baseline_count": 1, + "pending": [ + { + "country": "uk", + "stage": entry.stage, + "locator": entry.locator, + "reason": "stale", + "issue": "PolicyEngine/microcosm#848", + } + ], + } + ) + ) + + gaps = audit_microdata_pins(manifest, allowlist=allowlist) + + assert [gap.problem for gap in gaps] == ["stale_allowlist_row"] + + def test_a_row_naming_no_artifact_is_a_gap(self) -> None: + allowlist = load_microdata_pin_allowlist( + _written( + { + "version": 1, + "policy": "test", + "baseline_count": 1, + "pending": [ + { + "country": "uk", + "stage": "frs_spine", + "locator": "nonexistent.tab", + "reason": "orphan", + "issue": "PolicyEngine/microcosm#848", + } + ], + } + ) + ) + + gaps = audit_microdata_pins(_manifest("uk"), allowlist=allowlist) + + assert [gap.problem for gap in gaps] == ["orphan_allowlist_row"] + + +class TestRegistrationValidation: + def test_access_class_is_derived_from_the_artifact_kind(self) -> None: + for country in COUNTRIES: + for entry in microdata_artifact_entries(_manifest(country)): + reference = entry.chronicle_artifact + if reference is None: + continue + assert reference.access == CHRONICLE_ACCESS_BY_ARTIFACT_KIND[entry.kind] + + def test_only_public_registrations_have_raw_object_keys(self) -> None: + for country in COUNTRIES: + for reference in resolved_chronicle_registrations(_manifest(country)): + key = reference.raw_object_key + if reference.access == "public": + assert key == ( + f"raw/{reference.source_id}/{reference.package_id}/" + f"{reference.year}/{reference.sha256}/{reference.filename}" + ) + else: + assert key is None + + def test_registration_sha256_equals_the_artifact_pin(self) -> None: + for country in COUNTRIES: + for entry in microdata_artifact_entries(_manifest(country)): + if entry.chronicle_artifact is None: + continue + assert entry.chronicle_artifact.sha256 == entry.sha256 + + def test_one_registration_per_distinct_file(self) -> None: + for country in COUNTRIES: + by_sha: dict[str, ChronicleArtifactReference] = {} + for entry in microdata_artifact_entries(_manifest(country)): + reference = entry.chronicle_artifact + if reference is None or entry.sha256 is None: + continue + assert by_sha.setdefault(entry.sha256, reference) == reference + + def test_a_registration_witnessing_other_bytes_is_refused(self) -> None: + with pytest.raises(ValueError, match="does not equal the artifact sha256"): + SourceManifest.from_mapping( + _stage_with_artifact( + { + "kind": "public_microdata", + "locator": "example.zip", + "vintage": "2023", + "sha256": _sha("a"), + "chronicle_artifact": { + "source_id": "census_cps", + "package_id": "census-cps-asec-2023", + "year": 2023, + "sha256": _sha("b"), + "filename": "example.zip", + "access": "public", + }, + } + ) + ) + + def test_an_unknown_access_class_is_refused(self) -> None: + with pytest.raises(ValueError, match="'access' must be one of"): + SourceManifest.from_mapping( + _stage_with_artifact( + _artifact(access="open", sha256=_sha("a")), + ) + ) + + def test_an_access_class_the_kind_does_not_imply_is_refused(self) -> None: + with pytest.raises(ValueError, match="registers under Chronicle access"): + SourceManifest.from_mapping( + _stage_with_artifact(_artifact(access="licensed", sha256=_sha("a"))) + ) + + def test_a_public_registration_without_a_filename_is_refused(self) -> None: + artifact = _artifact(sha256=_sha("a")) + del artifact["chronicle_artifact"]["filename"] + + with pytest.raises(ValueError, match="public access without a"): + SourceManifest.from_mapping(_stage_with_artifact(artifact)) + + def test_a_registration_without_an_artifact_pin_is_refused(self) -> None: + artifact = _artifact(sha256=_sha("a")) + del artifact["sha256"] + + with pytest.raises(ValueError, match="without .*declaring its own"): + SourceManifest.from_mapping(_stage_with_artifact(artifact)) + + def test_an_uppercase_digest_is_refused(self) -> None: + with pytest.raises(ValueError, match="64 lowercase hex characters"): + SourceManifest.from_mapping( + _stage_with_artifact( + { + "kind": "public_microdata", + "locator": "example.zip", + "vintage": "2023", + "sha256": _sha("a").upper(), + } + ) + ) + + def test_a_year_the_entry_does_not_declare_is_refused(self) -> None: + artifact = _artifact(sha256=_sha("a")) + artifact["chronicle_artifact"]["year"] = 1999 + + with pytest.raises(ValueError, match="is not one of the years"): + SourceManifest.from_mapping(_stage_with_artifact(artifact)) + + def test_a_derived_artifact_cannot_carry_a_registration(self) -> None: + artifact = _artifact(sha256=_sha("a")) + artifact["kind"] = "versioned_derived_microdata" + + with pytest.raises(ValueError, match="no Chronicle access class"): + SourceManifest.from_mapping(_stage_with_artifact(artifact)) + + def test_two_registrations_for_one_file_are_refused(self) -> None: + first = _artifact(sha256=_sha("a")) + second = json.loads(json.dumps(first)) + second["chronicle_artifact"]["package_id"] = "census-cps-asec-2023-other" + + with pytest.raises(ValueError, match="one file has one registration"): + SourceManifest.from_mapping( + _stage_with_artifact(first, second, locators=("a.zip", "b.zip")) + ) + + +class TestFailClosedGate: + def test_matching_bytes_pass_and_resolve_their_registration( + self, tmp_path: Path + ) -> None: + payload = b"asec bytes" + path = tmp_path / "example.zip" + path.write_bytes(payload) + manifest = SourceManifest.from_mapping( + _stage_with_artifact(_artifact(sha256=hashlib.sha256(payload).hexdigest())) + ) + + verifications = verify_microdata_files(manifest, {"example.zip": path}) + + assert [check.matched for check in verifications] == [True] + assert verifications[0].actual_sha256 == hashlib.sha256(payload).hexdigest() + registrations = verified_chronicle_registrations(verifications) + assert [reference.package_id for reference in registrations] == [ + "census-cps-asec-2023" + ] + + def test_different_bytes_stop_the_build_with_a_diagnosable_message( + self, tmp_path: Path + ) -> None: + path = tmp_path / "example.zip" + path.write_bytes(b"a reissued vintage") + manifest = SourceManifest.from_mapping( + _stage_with_artifact(_artifact(sha256=_sha("the pinned release"))) + ) + + with pytest.raises(MicrodataIdentityError) as error: + verify_microdata_files(manifest, {"example.zip": path}) + + message = str(error.value) + assert "census_cps/census-cps-asec-2023" in message + assert "'2023'" in message + assert "example.zip" in message + assert _sha("the pinned release") in message + assert sha256_file(path) in message + + def test_a_key_naming_no_pinned_root_is_refused(self, tmp_path: Path) -> None: + path = tmp_path / "stranger.zip" + path.write_bytes(b"x") + manifest = SourceManifest.from_mapping( + _stage_with_artifact(_artifact(sha256=_sha("a"))) + ) + + with pytest.raises(MicrodataIdentityError, match="names no hash-pinned"): + verify_microdata_files(manifest, {"stranger.zip": path}) + + def test_a_missing_file_is_refused(self, tmp_path: Path) -> None: + manifest = SourceManifest.from_mapping( + _stage_with_artifact(_artifact(sha256=_sha("a"))) + ) + + with pytest.raises(MicrodataIdentityError, match="which is not a file"): + verify_microdata_files(manifest, {"example.zip": tmp_path / "absent.zip"}) + + def test_a_caller_supplied_input_resolves_by_its_declared_filename( + self, tmp_path: Path + ) -> None: + payload = b"licensed tab" + path = tmp_path / "put2223uk.tab" + path.write_bytes(payload) + artifact = { + "kind": "private_microdata", + "locator": "caller-supplied local input", + "filename": "put2223uk.tab", + "vintage": "2022-23", + "tax_year_start": 2022, + "sha256": hashlib.sha256(payload).hexdigest(), + "chronicle_artifact": { + "source_id": "hmrc", + "package_id": "hmrc-spi-public-use-tape-2022-23", + "year": 2022, + "sha256": hashlib.sha256(payload).hexdigest(), + "filename": "put2223uk.tab", + "access": "restricted", + }, + } + manifest = SourceManifest.from_mapping(_stage_with_artifact(artifact)) + + verifications = verify_microdata_files(manifest, {"put2223uk.tab": path}) + + assert [check.matched for check in verifications] == [True] + + def test_the_uk_manifest_gate_accepts_the_real_pins(self, tmp_path: Path) -> None: + # Every UK root is hash-pinned, so a synthetic file whose bytes hash to + # a declared pin is impossible to fabricate; instead assert the gate + # resolves keys for all of them and rejects the tampered one. + manifest = _manifest("uk") + entries = microdata_artifact_entries(manifest) + adult = next(entry for entry in entries if entry.locator == "adult.tab") + path = tmp_path / "adult.tab" + path.write_bytes(b"not the licensed tab") + + with pytest.raises(MicrodataIdentityError) as error: + verify_microdata_files(manifest, {"adult.tab": path}) + + message = str(error.value) + assert message.count("adult.tab") >= 5 # every stage that reads it + assert adult.sha256 in message + assert "dwp/dwp-frs-2024-25-adult" in message + + +class TestRecordedPinCrossCheck: + def test_agreeing_pins_resolve_to_their_registrations(self) -> None: + manifest = _manifest("us") + entry = next( + candidate + for candidate in microdata_artifact_entries(manifest) + if candidate.stage == "weeks_unemployed_input" + ) + + audit = verify_recorded_microdata_pins( + manifest, + [ + { + "locator": entry.locator, + "sha256": entry.sha256, + "member_sha256": entry.member_sha256, + } + ], + context="test", + ) + + assert [reference.package_id for reference in audit.resolved] == [ + "census-cps-asec-2023" + ] + assert audit.unregistered == () + + def test_a_disagreeing_archive_digest_stops_the_build(self) -> None: + manifest = _manifest("us") + entry = next( + candidate + for candidate in microdata_artifact_entries(manifest) + if candidate.stage == "weeks_unemployed_input" + ) + + with pytest.raises(MicrodataIdentityError, match="recorded sha256"): + verify_recorded_microdata_pins( + manifest, + [{"locator": entry.locator, "sha256": _sha("other")}], + context="test", + ) + + def test_a_disagreeing_member_digest_stops_the_build(self) -> None: + manifest = _manifest("us") + entry = next( + candidate + for candidate in microdata_artifact_entries(manifest) + if candidate.stage == "weeks_unemployed_input" + ) + + with pytest.raises(MicrodataIdentityError, match="recorded member_sha256"): + verify_recorded_microdata_pins( + manifest, + [ + { + "locator": entry.locator, + "sha256": entry.sha256, + "member_sha256": _sha("other member"), + } + ], + context="test", + ) + + def test_an_unregistered_locator_is_reported_not_refused(self) -> None: + # The pooled ASEC archives for later survey years have no manifest + # entry yet; that is the allowlist's business, not a build failure. + audit = verify_recorded_microdata_pins( + _manifest("us"), + [ + { + "locator": ( + "https://www2.census.gov/programs-surveys/cps/datasets/" + "2025/march/asecpub25csv.zip" + ), + "sha256": _sha("later vintage"), + } + ], + context="test", + ) + + assert audit.resolved == () + assert audit.unregistered == ( + "https://www2.census.gov/programs-surveys/cps/datasets/2025/march/" + "asecpub25csv.zip", + ) + + +def _artifact(*, sha256: str, access: str = "public") -> dict: + return { + "kind": "public_microdata", + "locator": "example.zip", + "vintage": "2023", + "sha256": sha256, + "chronicle_artifact": { + "source_id": "census_cps", + "package_id": "census-cps-asec-2023", + "year": 2023, + "sha256": sha256, + "filename": "example.zip", + "access": access, + }, + } + + +def _stage_with_artifact( + *artifacts: dict, locators: tuple[str, ...] | None = None +) -> dict: + prepared = [] + for index, artifact in enumerate(artifacts): + entry = json.loads(json.dumps(artifact)) + if locators is not None: + entry["locator"] = locators[index] + prepared.append(entry) + return { + "version": 1, + "country": "xx", + "policy": "test manifest", + "stages": [ + { + "stage": "example_stage", + "survey": "Example", + "source": "https://example.invalid/", + "grain": "person", + "artifacts": prepared, + "operations": [{"kind": "read_table", "table": "example"}], + "outputs": ["example_output"], + } + ], + } + + +def _written(payload: dict) -> Path: + import tempfile + + handle = tempfile.NamedTemporaryFile( # noqa: SIM115 - path outlives the handle + "w", suffix=".json", delete=False, encoding="utf-8" + ) + with handle: + json.dump(payload, handle) + return Path(handle.name) + + +def test_every_microdata_kind_is_covered_by_the_contract() -> None: + # A new microdata kind must decide its Chronicle access class or be + # explicitly derived-only; it must not silently escape the audit. + undecided = MICRODATA_ARTIFACT_KINDS - set(CHRONICLE_ACCESS_BY_ARTIFACT_KIND) + + assert undecided == {"versioned_derived_microdata"} diff --git a/tools/build_uk_frs_spine.py b/tools/build_uk_frs_spine.py index f2c9a24dd..28a10e077 100644 --- a/tools/build_uk_frs_spine.py +++ b/tools/build_uk_frs_spine.py @@ -38,6 +38,11 @@ write_error_receipt, ) from microcosm.build.plan import StageRecord +from microcosm.build.source_manifest import MICRODATA_ARTIFACT_KINDS +from microcosm.build.source_runtime import ( + verified_chronicle_registrations, + verify_microdata_files, +) from microcosm.build.uk_runtime.age_tail import UKAgeTailStageTransform from microcosm.build.uk_runtime.battery_bindings import UK_GATE_REGISTRY from microcosm.build.uk_runtime.calibration_run import ( @@ -337,6 +342,54 @@ def _stage_artifact_pins(stage) -> dict[str, dict[str, object]]: } +#: Which caller-supplied path carries each private microdata role. FRS tabs +#: come out of ``--frs-raw-dir`` by table name; every other licensed root +#: arrives as its own flag, so the root-identity gate can name the file it +#: hashed when a pin fails. +_PRIVATE_ROLE_ARGUMENTS = { + "was_qrf_donor": "was_tab", + "was_bridge_donor": "was_tab", + "lcfs_household_tab": "lcfs_hh_tab", + "lcfs_person_tab": "lcfs_person_tab", + "etb_household_tab": "etb_tab", + "qrf_donor": "spi_tab", +} + + +def _microdata_files(stages, args) -> dict[str, Path]: + """Map manifest microdata keys to the local files this run was handed. + + Keys are what :func:`verify_microdata_files` resolves against: a tab's + locator for FRS tables, and the declared ``filename`` for caller-supplied + private inputs, whose locator is the placeholder + ``"caller-supplied local input"``. + """ + + files: dict[str, Path] = {} + for stage in stages: + for artifact in stage.artifacts: + if artifact.get("kind") not in MICRODATA_ARTIFACT_KINDS: + continue + table = artifact.get("table") + if table is not None: + locator = str(artifact["locator"]) + files[locator] = args.frs_raw_dir / locator + continue + role = str(artifact.get("role") or "") + attribute = _PRIVATE_ROLE_ARGUMENTS.get(role) + if attribute is None: + raise ValueError( + f"stage {stage.stage!r} declares private microdata role " + f"{role!r} with no caller-supplied path; the root-identity " + "gate cannot verify it." + ) + supplied = getattr(args, attribute) + if supplied is None: + continue + files[str(artifact["filename"])] = supplied + return files + + def _resource_pins(stages, spec) -> dict[str, str]: """Country-package resources the selected stages declare as inputs. @@ -564,6 +617,7 @@ def _build_sidecar( artifact_pins, resource_pins: dict[str, str], input_artifact_pins: dict[str, dict[str, object]], + microdata_registrations: list[dict[str, object]], hmrc_replay: dict[str, object], stochastic_contract_sha256: str, frs_vintage: str, @@ -583,6 +637,9 @@ def _build_sidecar( "artifact_pins": artifact_pins, "resource_pins": resource_pins, "input_artifact_pins": input_artifact_pins, + # Root identity (microcosm#848): the Chronicle registrations behind the + # raw microdata files this run hashed and accepted. + "microdata_registrations": microdata_registrations, "hmrc_replay": hmrc_replay, "stage_artifact_pins": { stage.stage: _stage_artifact_pins(stage) for stage in stages @@ -1060,6 +1117,19 @@ def main(argv: list[str] | None = None) -> int: state.input_pins_digest = role_pins_digest( _role_pins({**artifact_pins, **input_artifact_pins}) ) + # Root identity (microcosm#848): hash every raw microdata file this run + # was handed and refuse to continue unless the bytes are the ones the + # manifest pins and a Chronicle registration witnesses. This runs before + # any stage reads a table, so a wrong vintage or a corrupted download + # cannot reach a build artifact. + microdata_verifications = verify_microdata_files( + spec.sources, _microdata_files(stages, args) + ) + microdata_registrations = [ + reference.to_payload() + for reference in verified_chronicle_registrations(microdata_verifications) + ] + append_phase(state, "microdata_identity_verified") run_config = { "pipeline": _PIPELINE, "stages": list(stage_names), @@ -1345,6 +1415,7 @@ def main(argv: list[str] | None = None) -> int: artifact_pins=artifact_pins, resource_pins=resource_pins, input_artifact_pins=input_artifact_pins, + microdata_registrations=microdata_registrations, hmrc_replay=replay_binding, stochastic_contract_sha256=stochastic_contract.resource_sha256, frs_vintage=frs_release.vintage, diff --git a/tools/build_us_fiscal_refresh_release.py b/tools/build_us_fiscal_refresh_release.py index 8c1ed2c75..692edbbfe 100644 --- a/tools/build_us_fiscal_refresh_release.py +++ b/tools/build_us_fiscal_refresh_release.py @@ -63,6 +63,10 @@ target_profile_coverage_gate, ) from microcosm.build.ledger_artifact import load_ledger_consumer_artifact +from microcosm.build.source_manifest import ( + packaged_microdata_pin_allowlist, + resolved_chronicle_registrations, +) from microcosm.build.source_runtime import SourceRuntimeConfig, run_source_stage from microcosm.build.staging import DEFAULT_STAGING_PREFIX, StagingTelemetry from microcosm.build.us_runtime import ( @@ -1796,6 +1800,36 @@ def _copy_base_h5_for_local_audit( return destination +def _microdata_registration_receipt() -> dict[str, object]: + """The build's raw-microdata root identity, as a release-manifest block. + + ``chronicle_artifacts`` are the registrations the US source manifest + resolves to — each one a witnessed statement that a named publisher release + with that SHA-256 is what this build started from. ``pending`` mirrors the + reviewed allowlist rows for the roots no registration witnesses yet, so a + reader of the manifest sees the unwitnessed roots without having to know the + allowlist exists (microcosm#848). + """ + + allowlist = packaged_microdata_pin_allowlist() + return { + "chronicle_artifacts": [ + reference.to_payload() + for reference in resolved_chronicle_registrations(US_SOURCE_MANIFEST) + ], + "pending": [ + { + "stage": row.stage, + "locator": row.locator, + "reason": row.reason, + "issue": row.issue, + } + for row in allowlist.for_country("us") + ], + "pending_baseline_count": allowlist.baseline_count, + } + + def _runtime_versions() -> dict[str, str]: packages = ( "microcosm-build", @@ -7629,6 +7663,11 @@ def _build_manifests( "runtime": runtime, "timing": timing_payload, "ledger_artifact": dict(ledger_artifact) if ledger_artifact else None, + # Root identity (microcosm#848): the Chronicle registrations that + # witness this build's raw microdata roots, and the roots no + # registration witnesses yet, next to the Chronicle consumer-artifact + # pin above. + "microdata_registrations": _microdata_registration_receipt(), **( {"exact_k_ladder": dict(exact_k_ladder)} if exact_k_ladder is not None From 02742357edc43011b69cab6527da675ecefb502c Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Wed, 2 Sep 2026 12:33:57 -0400 Subject: [PATCH 06/14] Record the root-identity rule in DESIGN.md and add the changelog fragment DESIGN.md's process rules gain "build graphs have no anonymous roots": what a microdata root must declare, why licensed and restricted releases register hash-only, that the build gate is fail-closed, and that the pending allowlist is a ratchet. Co-Authored-By: Claude Fable 5.1 --- DESIGN.md | 12 ++++++++++++ changelog.d/848-root-identity.added.md | 1 + 2 files changed, 13 insertions(+) create mode 100644 changelog.d/848-root-identity.added.md diff --git a/DESIGN.md b/DESIGN.md index 7dd1f921f..ac2bd1dfe 100644 --- a/DESIGN.md +++ b/DESIGN.md @@ -196,6 +196,18 @@ extension, not a rewrite (the kernel must grow these hooks before then). 4. **Stage manifests are load-bearing.** Every pipeline stage reads/writes a versioned artifact with invariant checks; A/B experiments re-run one stage against cached upstreams, not whole builds. +5. **Build graphs have no anonymous roots.** Every raw microdata input a + country manifest declares carries the SHA-256 of the exact file a stage + reads and a `chronicle_artifact` reference to the one Chronicle + registration that witnesses it — publisher, vintage, checksum, licence, + and an access class from `{public, licensed, restricted}`. Chronicle + archives bytes only for redistributable public-use files; licensed and + restricted releases are registered hash-only, so the identity is shared + without the bytes ever leaving the environment that is allowed to hold + them. The build gate is fail-closed: bytes that do not match the pin stop + the run. Roots that cannot be pinned yet live, one reviewed row each, in + `microcosm/build/microdata_pins_pending.json`, whose row count is a + ratchet that may only fall. ## Naming diff --git a/changelog.d/848-root-identity.added.md b/changelog.d/848-root-identity.added.md new file mode 100644 index 000000000..4b0f61e93 --- /dev/null +++ b/changelog.d/848-root-identity.added.md @@ -0,0 +1 @@ +Every raw microdata input a country source manifest declares now carries the SHA-256 of the exact file its stage reads plus a `chronicle_artifact` reference to the Chronicle registration that witnesses it, a fail-closed gate refuses to build from bytes that do not match the pin, and the roots that cannot be pinned yet are listed one reviewed row each in a ratcheted `microdata_pins_pending.json`. From 607718eb3d7a15e597d1980d18e06baceff7b5bf Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Wed, 2 Sep 2026 12:34:34 -0400 Subject: [PATCH 07/14] Bring the #848 lane journal up to the implemented contract Records what was built, the Chronicle registration ids chosen and why, the 39-row ratchet baseline, and the note chronicle#221 needs: the FRS the UK build reads is 2024-25 / SN 9563, not 2023-24. Co-Authored-By: Claude Fable 5.1 --- PROGRESS-root-identity-848.md | 83 ++++++++++++++++++++++++++++++----- 1 file changed, 71 insertions(+), 12 deletions(-) diff --git a/PROGRESS-root-identity-848.md b/PROGRESS-root-identity-848.md index ad49ad9b4..a1280eb5d 100644 --- a/PROGRESS-root-identity-848.md +++ b/PROGRESS-root-identity-848.md @@ -8,21 +8,80 @@ merged — see CLAUDE.md "Root journals are history, not state". ## State -Started 2026-09-02 from origin/main `d2b31496`. Investigation phase. +Started 2026-09-02 from origin/main `d2b31496`. Implementation complete; +verification and PR in progress. Nothing merged. + +## Contract implemented + +Every artifact entry whose `kind` names microdata declares the SHA-256 of the +exact file its stage reads plus a `chronicle_artifact` +`{source_id, package_id, year, sha256, filename, access}`, or carries one +reviewed row in `packages/microcosm-build/src/microcosm/build/microdata_pins_pending.json`. + +Three invariants make the reference hard to get wrong: + +- the access class is **derived from the kind** (`licensed_microdata` → + `licensed`, `private_microdata`/`restricted_microdata` → `restricted`, + `public_microdata` → `public`, `versioned_derived_microdata` → no class, so + it must be allowlisted); +- the registration year must be one the entry already declares + (`tax_year_start` exactly, else a four-digit year in `vintage`); +- entries sharing a SHA-256 must resolve to the same registration. + +## Naming chosen (for chronicle#221) + +Chronicle's live `db/data/*/manifest.yaml` uses a publisher-slug `source_id` +and a kebab-case, publisher-prefixed `package_id` +(`hmrc` / `hmrc-spi-income-bands-2023-24`), and `files[year]` holds exactly one +file per (package, year). So one package per distributed file: + +| release | source_id | package_id | access | +| --- | --- | --- | --- | +| FRS 2024-25 (UKDS SN 9563), 14 tabs | `dwp` | `dwp-frs-2024-25-` | licensed | +| WAS round 8 household EUL (SN 7215) | `ons` | `ons-was-round-8-household` | restricted | +| LCFS 2023-24 household / person | `ons` | `ons-lcfs-2023-24-{household,person}` | restricted | +| ETB 1977-2024 household | `ons` | `ons-etb-1977-2024-household` | restricted | +| SPI Public Use Tape 2022-23 (SN 9422) | `hmrc` | `hmrc-spi-public-use-tape-2022-23` | restricted | +| CPS ASEC 2023 archive | `census_cps` | `census-cps-asec-2023` | public | +| SCF 2022 summary extract | `federal_reserve` | `federal-reserve-scf-2022-summary-extract` | public | + +Note for chronicle#221: the FRS the UK build reads is **2024-25 / SN 9563** +(`tax_year_start: 2024`), not 2023-24, and the UK manifest carries **21** +licensed references over 14 distinct tabs. ## Done -- Read CLAUDE.md, DESIGN.md, issue #848, chronicle#221, and the Chronicle - raw-microdata identity ADR (chronicle `origin/adr-raw-microdata-identity`). -- Inventoried every `*_microdata` artifact entry across - `build/{am,be,uk,us}/source_stages.json` and - `build/uk/hmrc_income_source_stages.json`. -- Confirmed Chronicle's live naming from `~/PolicyEngine/chronicle/db/data`: - `source_id` is a publisher slug, `package_id` is kebab-case and - publisher-prefixed (e.g. `hmrc` / `hmrc-spi-income-bands-2023-24`). +- Read CLAUDE.md, DESIGN.md, the Chronicle raw-microdata identity ADR + (chronicle `origin/adr-raw-microdata-identity`) and chronicle#221's lane log. +- `source_manifest.py`: registration validation, the three invariants, + `microdata_artifact_entries`, `resolved_chronicle_registrations`, + `audit_microdata_pins`, and the `MicrodataPinAllowlist` ratchet loader. +- `sources.schema.json`: `chronicle_artifact` definition, with `filename` and + `access` annotated `operational` to match the artifact-level keys of those + names. +- Populated registrations: **31 pinned** entries (29 UK + the frozen UK replay + manifest + 2 US); UK is fully pinned with a zero baseline. +- One shared allowlist with **39** rows (37 US, 1 BE, 1 AM), country-tagged, so + the ratchet is a single number. A per-country file would have added a + resource to the AM and BE packages and moved their goldens, which the lane + brief forbids. +- `source_runtime.py`: fail-closed `verify_microdata_files` (hashes, refuses, + names publisher/vintage/locator/expected/actual) and + `verify_recorded_microdata_pins` (cross-checks a producing run's recorded + pins; fatal on disagreement, reports absence). +- Wired: `build_uk_frs_spine` verifies every tab and licensed input before any + stage reads a table and records the registrations in its sidecar; + `load_asec_raw_stage_checkpoint` cross-checks the recorded ASEC pins; + `build_us_fiscal_refresh_release` writes `microdata_registrations` next to + the Chronicle consumer-artifact pin. +- `test_microdata_root_identity.py`: 36 contract tests (`shared-spec` group, + `tools/ci_test_groups.py --verify` green). +- Re-pinned the byte pins that legitimately moved with the manifests: the US + `stage_asset` digest and its two test copies, the UK frozen replay digest, + the regenerated `uk/release_input_coverage_manifest.json`, and four + `spec_sha256` vectors that attest the schema set. +- DESIGN.md process rule 5; `changelog.d/848-root-identity.added.md`. ## Next -- Add `chronicle_artifact` validation to `source_manifest.py`. -- Add the fail-closed sha256 gate to the source runtime. -- Populate pins; write the pending allowlists; contract tests. +- Full `uv run pytest` and `uv run ruff check .`, then push and open the PR. From 7f7be2ced8b3412e2336f684781df8a67c7367d6 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Wed, 2 Sep 2026 12:35:30 -0400 Subject: [PATCH 08/14] Refuse an ambiguous microdata key instead of picking one of its pins "caller-supplied local input" is the locator every UK private root shares, so resolving a supplied file by that key would compare one file against the WAS, LCFS, ETB, and SPI pins at once and report three spurious mismatches. The gate now names the ambiguity and points at the declared filename, which is what the UK builder already keys by. Co-Authored-By: Claude Fable 5.1 --- .../src/microcosm/build/source_runtime.py | 12 ++++++++++++ .../tests/test_microdata_root_identity.py | 14 ++++++++++++++ 2 files changed, 26 insertions(+) diff --git a/packages/microcosm-build/src/microcosm/build/source_runtime.py b/packages/microcosm-build/src/microcosm/build/source_runtime.py index 407919478..ed361a57d 100644 --- a/packages/microcosm-build/src/microcosm/build/source_runtime.py +++ b/packages/microcosm-build/src/microcosm/build/source_runtime.py @@ -263,6 +263,18 @@ def verify_microdata_files( f"{key!r} names no hash-pinned microdata artifact in this " f"manifest; pinned keys are {sorted(by_key)}." ) + pinned = {entry.sha256 for entry in matches} + if len(pinned) > 1: + # The placeholder locator "caller-supplied local input" is shared by + # every private root, so it names several distinct files. One file + # cannot satisfy several pins; the caller must key by filename. + raise MicrodataIdentityError( + f"{key!r} is ambiguous: it names " + f"{len(matches)} microdata artifacts pinned to " + f"{len(pinned)} different files " + f"({sorted(entry.locator for entry in matches)}). Supply the " + "declared filename instead." + ) path = Path(files[key]) if not path.is_file(): raise MicrodataIdentityError( diff --git a/packages/microcosm-build/tests/test_microdata_root_identity.py b/packages/microcosm-build/tests/test_microdata_root_identity.py index 251e6590c..c8af2268e 100644 --- a/packages/microcosm-build/tests/test_microdata_root_identity.py +++ b/packages/microcosm-build/tests/test_microdata_root_identity.py @@ -391,6 +391,20 @@ def test_a_caller_supplied_input_resolves_by_its_declared_filename( assert [check.matched for check in verifications] == [True] + def test_the_shared_placeholder_locator_is_refused_as_ambiguous( + self, tmp_path: Path + ) -> None: + # "caller-supplied local input" is the locator of every UK private + # root, so it names several distinct files; one file cannot satisfy + # several pins and the gate must say so rather than pick one. + path = tmp_path / "something.tab" + path.write_bytes(b"x") + + with pytest.raises(MicrodataIdentityError, match="is ambiguous"): + verify_microdata_files( + _manifest("uk"), {"caller-supplied local input": path} + ) + def test_the_uk_manifest_gate_accepts_the_real_pins(self, tmp_path: Path) -> None: # Every UK root is hash-pinned, so a synthetic file whose bytes hash to # a declared pin is impossible to fabricate; instead assert the gate From a3130e3ac6154f0a7acc6e988560732466573af5 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Wed, 2 Sep 2026 12:45:53 -0400 Subject: [PATCH 09/14] Re-pin the manifest and spec digests after rebasing onto main Main moved the UK source manifest and the release input-coverage manifest, so the byte pins over both had to be recomputed on the rebased tree rather than carried across: the regenerated uk/release_input_coverage_manifest.json and the four spec_sha256 vectors that attest the schema set. No contract changed. Co-Authored-By: Claude Fable 5.1 --- .../uk/release_input_coverage_manifest.json | 28 +++++++++---------- .../tests/test_spec_engine_country_bundles.py | 6 ++-- .../tests/test_spec_engine_loader.py | 2 +- 3 files changed, 18 insertions(+), 18 deletions(-) diff --git a/packages/microcosm-build/src/microcosm/build/uk/release_input_coverage_manifest.json b/packages/microcosm-build/src/microcosm/build/uk/release_input_coverage_manifest.json index cc4d78e88..7ccc2731d 100644 --- a/packages/microcosm-build/src/microcosm/build/uk/release_input_coverage_manifest.json +++ b/packages/microcosm-build/src/microcosm/build/uk/release_input_coverage_manifest.json @@ -473,7 +473,7 @@ "capital_gains" ], "source_manifest": "source_stages.json", - "source_manifest_sha256": "1128013c99cd3592c06adab25cd0efa7f92667765445e2c6370c6c867a4a3517", + "source_manifest_sha256": "b13a7ea430498e4ca1a5ea2a5f9edf8995f079690dda76250cb3bcaa368b94ca", "source_vintages": { "source": "HMRC Capital Gains Tax statistics, July 2025, Table 2.1a", "survey": "HMRC Capital Gains Tax statistics Table 2.1a and Advani-Summers capital-gains incidence" @@ -496,7 +496,7 @@ "capital_gains" ], "source_manifest": "source_stages.json", - "source_manifest_sha256": "1128013c99cd3592c06adab25cd0efa7f92667765445e2c6370c6c867a4a3517", + "source_manifest_sha256": "b13a7ea430498e4ca1a5ea2a5f9edf8995f079690dda76250cb3bcaa368b94ca", "source_vintages": { "source": "Advani and Summers (2020), Capital Gains and UK Inequality, CAGE Working Paper 465", "survey": "Family Resources Survey 2024-25, SPI synthetic support, and Advani-Summers capital-gains incidence" @@ -525,7 +525,7 @@ "required_mass_change_reason": "E5 source-stage transform preserves household rows and typed household weights; total household mass is conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "1128013c99cd3592c06adab25cd0efa7f92667765445e2c6370c6c867a4a3517", + "source_manifest_sha256": "b13a7ea430498e4ca1a5ea2a5f9edf8995f079690dda76250cb3bcaa368b94ca", "source_vintages": { "source": "UK Data Service SN 8856 Effects of Taxes and Benefits household tab, DfT rail fare index, and public NHS activity/cost table.", "survey": "Effects of Taxes and Benefits 1977-2024 and NHS age-gender public table" @@ -545,7 +545,7 @@ "required_mass_change_reason": "E5 source-stage transform preserves household rows and typed household weights; total household mass is conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "1128013c99cd3592c06adab25cd0efa7f92667765445e2c6370c6c867a4a3517", + "source_manifest_sha256": "b13a7ea430498e4ca1a5ea2a5f9edf8995f079690dda76250cb3bcaa368b94ca", "source_vintages": { "source": "UK Data Service SN 8856 Effects of Taxes and Benefits household tab and cited VAT anchor resource.", "survey": "Effects of Taxes and Benefits 1977-2024" @@ -576,7 +576,7 @@ "superseded_by": { "reason": "The FRS spine build executes hmrc_cgt_gains_spine, which applies the same HMRC Table 3 amounts redraw directly in source_stages.json before calibration.", "source_manifest": "source_stages.json", - "source_manifest_sha256": "1128013c99cd3592c06adab25cd0efa7f92667765445e2c6370c6c867a4a3517", + "source_manifest_sha256": "b13a7ea430498e4ca1a5ea2a5f9edf8995f079690dda76250cb3bcaa368b94ca", "stage": "hmrc_cgt_gains_spine" } }, @@ -596,7 +596,7 @@ "capital_gains" ], "source_manifest": "source_stages.json", - "source_manifest_sha256": "1128013c99cd3592c06adab25cd0efa7f92667765445e2c6370c6c867a4a3517", + "source_manifest_sha256": "b13a7ea430498e4ca1a5ea2a5f9edf8995f079690dda76250cb3bcaa368b94ca", "source_vintages": { "hmrc_surface": "2023-24", "mapped_build_period": "2024" @@ -610,7 +610,7 @@ "base_candidate_tier": "frs", "calibration_permitted": false, "canonical_source_manifest": "source_stages.json", - "canonical_source_manifest_sha256": "1128013c99cd3592c06adab25cd0efa7f92667765445e2c6370c6c867a4a3517", + "canonical_source_manifest_sha256": "b13a7ea430498e4ca1a5ea2a5f9edf8995f079690dda76250cb3bcaa368b94ca", "effective_mass_requirements": { "charitable_investment_gifts": { "mass_share_denominator": "all_person_effective_mass", @@ -680,7 +680,7 @@ "full_frs_tei_band_unavailable" ], "source_manifest": "hmrc_income_source_stages.json", - "source_manifest_sha256": "c0341af7166ae3a85a3c1164e7d9e880c4b4aec122f1a8fa90c73b46c596e1ea", + "source_manifest_sha256": "2b6a74d33686d5a18f531a680c30330f7e5923af6fca97ac548c0785626f04aa", "source_vintages": { "hmrc_surface": "2023-24", "mapped_build_period": "2024", @@ -693,7 +693,7 @@ "superseded_by": { "reason": "The FRS spine build executes hmrc_spi_income_spine, which supersedes the June retained-leaves/hmrc_spi_income pair inside source_stages.json.", "source_manifest": "source_stages.json", - "source_manifest_sha256": "1128013c99cd3592c06adab25cd0efa7f92667765445e2c6370c6c867a4a3517", + "source_manifest_sha256": "b13a7ea430498e4ca1a5ea2a5f9edf8995f079690dda76250cb3bcaa368b94ca", "stage": "hmrc_spi_income_spine" } }, @@ -727,7 +727,7 @@ "required_mass_change_reason": "E5 source-stage transform preserves household rows and typed household weights; total household mass is conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "1128013c99cd3592c06adab25cd0efa7f92667765445e2c6370c6c867a4a3517", + "source_manifest_sha256": "b13a7ea430498e4ca1a5ea2a5f9edf8995f079690dda76250cb3bcaa368b94ca", "source_vintages": { "source": "UK Data Service SN 9468 Living Costs and Food Survey 2023-24 household/person tabs, NEED 2023 headline energy tables, Ofgem Q2 2026 unit rates, and WAS round-8 bridge donor.", "survey": "Living Costs and Food Survey 2023-24" @@ -748,7 +748,7 @@ "property_wealth" ], "source_manifest": "source_stages.json", - "source_manifest_sha256": "1128013c99cd3592c06adab25cd0efa7f92667765445e2c6370c6c867a4a3517", + "source_manifest_sha256": "b13a7ea430498e4ca1a5ea2a5f9edf8995f079690dda76250cb3bcaa368b94ca", "source_vintages": { "source": "MHCLG dwellings and ONS UK House Price Index December 2025 regional average prices.", "survey": "Public regional property reference" @@ -772,7 +772,7 @@ "employee_pension_contributions" ], "source_manifest": "source_stages.json", - "source_manifest_sha256": "1128013c99cd3592c06adab25cd0efa7f92667765445e2c6370c6c867a4a3517", + "source_manifest_sha256": "b13a7ea430498e4ca1a5ea2a5f9edf8995f079690dda76250cb3bcaa368b94ca", "source_vintages": { "source": "HMRC, Salary sacrifice reform for pension contributions effective from 6 April 2029", "survey": "Family Resources Survey 2024-25 salary-sacrifice respondents and HMRC salary-sacrifice reform analysis" @@ -794,7 +794,7 @@ "student_loan_plan" ], "source_manifest": "source_stages.json", - "source_manifest_sha256": "1128013c99cd3592c06adab25cd0efa7f92667765445e2c6370c6c867a4a3517", + "source_manifest_sha256": "b13a7ea430498e4ca1a5ea2a5f9edf8995f079690dda76250cb3bcaa368b94ca", "source_vintages": { "source": "Explore Education Statistics Table 6a, Higher education total", "survey": "Family Resources Survey 2024-25 and Student Loans Company borrower forecasts for England" @@ -827,7 +827,7 @@ "required_mass_change_reason": "E5 source-stage transform preserves household rows and typed household weights; total household mass is conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "1128013c99cd3592c06adab25cd0efa7f92667765445e2c6370c6c867a4a3517", + "source_manifest_sha256": "b13a7ea430498e4ca1a5ea2a5f9edf8995f079690dda76250cb3bcaa368b94ca", "source_vintages": { "source": "Office for National Statistics Wealth and Assets Survey, UK Data Service SN 7215, DOI 10.5255/UKDA-SN-7215-20; local licensed 2006-22 household tab.", "survey": "Wealth and Assets Survey round 8" diff --git a/packages/microcosm-build/tests/test_spec_engine_country_bundles.py b/packages/microcosm-build/tests/test_spec_engine_country_bundles.py index ba0bf50d1..52de9bb37 100644 --- a/packages/microcosm-build/tests/test_spec_engine_country_bundles.py +++ b/packages/microcosm-build/tests/test_spec_engine_country_bundles.py @@ -25,7 +25,7 @@ "spine", "vintages", } -AM_SPEC_SHA256 = "6792caa8280649b0c20265ee4b4e8dc0d4061e9d7476e12515b135c84e8f0dc1" +AM_SPEC_SHA256 = "4d2476f8ec232296fb46aa8a24748fbcb091dfa243f31de007a7ae50b54159a7" @pytest.mark.parametrize( @@ -45,7 +45,7 @@ ), ( "be", - "8f31a6b4c6f0ae8b65d55ef12995c459b47048080fbad54208747fa3c3d11028", + "205338168141e5ea95a7428acbfdf0cf4a31a4b01d43a9d37098a10e24830427", { "household.household_id", "person.person_id", @@ -55,7 +55,7 @@ ), ( "uk", - "b3a23e6ea34525b35ac7235d2e01263eaea0d7ba9edac57820e0939258941826", + "cb9baeea7de036c0eb4038b5ed080b33e1abdb0abe949c04f9d28eb91dc5ae7c", { "benunit.benunit_id", "household.household_id", diff --git a/packages/microcosm-build/tests/test_spec_engine_loader.py b/packages/microcosm-build/tests/test_spec_engine_loader.py index 0aaba605f..11be8d933 100644 --- a/packages/microcosm-build/tests/test_spec_engine_loader.py +++ b/packages/microcosm-build/tests/test_spec_engine_loader.py @@ -236,7 +236,7 @@ def test_semantic_hash_has_golden_vector_and_surface_separation(tmp_path) -> Non # Pin the domain separator, normalization rules, schema-set receipt, and # exact normative projection as one reviewable golden vector. assert first.spec_sha256 == ( - "d78bf9e7678cbba15953a535a19f5c1b6b5accffc0cacb5533854c01ea9a50b7" + "fcde1de85fd79c6386ad73facdf49136202b771634cce1c5e6880c3bfbac5abb" ) second_root = _rich_minimal(tmp_path / "xy", note="second", store="local:b") From 24cc32ab6366f0dd375f81d00c1beb94a318cccc Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Wed, 2 Sep 2026 12:59:59 -0400 Subject: [PATCH 10/14] Read the pinned digest without an assert in the identity gate python -O strips asserts, so a bare assert is the wrong guard inside a fail-closed check. The ambiguity check just above already proves every match shares one pinned digest, so read that value directly and raise if it is somehow absent. Co-Authored-By: Claude Fable 5.1 --- .../src/microcosm/build/source_runtime.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/packages/microcosm-build/src/microcosm/build/source_runtime.py b/packages/microcosm-build/src/microcosm/build/source_runtime.py index ed361a57d..acad27bc4 100644 --- a/packages/microcosm-build/src/microcosm/build/source_runtime.py +++ b/packages/microcosm-build/src/microcosm/build/source_runtime.py @@ -281,9 +281,15 @@ def verify_microdata_files( f"{key!r} was supplied as {path}, which is not a file." ) actual = sha256_file(path, chunk_size=chunk_size) + # ``by_key`` only ever holds pinned entries, so this is the one digest + # every match shares; read it without an assert, which -O would strip + # out of a fail-closed gate. + expected = next(iter(pinned)) + if expected is None: # pragma: no cover - defensive + raise MicrodataIdentityError( + f"{key!r} resolved to an unpinned microdata artifact." + ) for entry in matches: - expected = entry.sha256 - assert expected is not None # guarded when by_key was built verification = MicrodataFileVerification( stage=entry.stage, locator=entry.locator, From eaf9216c170e7b13b1eb92143e25f858ad9e5511 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Wed, 2 Sep 2026 13:00:56 -0400 Subject: [PATCH 11/14] Scope the shared allowlist by country when auditing a raw manifest mapping MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The allowlist spans every country, so auditing the frozen UK replay manifest — read as JSON rather than through the loader — would have reported all 37 US rows as orphans of it. A SourceManifest names its own country; anything else must now be told, and the audit takes an explicit country override. Co-Authored-By: Claude Fable 5.1 --- .../src/microcosm/build/source_manifest.py | 8 +++++++- .../tests/test_microdata_root_identity.py | 14 ++++++++++++++ 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/packages/microcosm-build/src/microcosm/build/source_manifest.py b/packages/microcosm-build/src/microcosm/build/source_manifest.py index f6aec2131..e9d3da35d 100644 --- a/packages/microcosm-build/src/microcosm/build/source_manifest.py +++ b/packages/microcosm-build/src/microcosm/build/source_manifest.py @@ -870,6 +870,7 @@ def audit_microdata_pins( source: SourceManifest | SourceStageSpec | Mapping[str, Any], *, allowlist: MicrodataPinAllowlist | None = None, + country: str | None = None, ) -> tuple[MicrodataPinGap, ...]: """Return every microdata root that is neither pinned nor allowlisted. @@ -878,9 +879,14 @@ def audit_microdata_pins( naming the stage, locator, reason, and tracking issue. A row that names an already-pinned root is itself a gap: stale rows would quietly inflate the ratchet baseline. + + The allowlist is shared across countries, so its rows are scoped before the + orphan check runs: a :class:`SourceManifest` names its own country, and a + stage spec or raw manifest mapping must be told which one with ``country``. """ - country = source.country if isinstance(source, SourceManifest) else None + if country is None and isinstance(source, SourceManifest): + country = source.country rows = (allowlist or EMPTY_MICRODATA_PIN_ALLOWLIST).row_map(country) entries = microdata_artifact_entries(source) gaps: list[MicrodataPinGap] = [] diff --git a/packages/microcosm-build/tests/test_microdata_root_identity.py b/packages/microcosm-build/tests/test_microdata_root_identity.py index c8af2268e..ee72e38d2 100644 --- a/packages/microcosm-build/tests/test_microdata_root_identity.py +++ b/packages/microcosm-build/tests/test_microdata_root_identity.py @@ -84,6 +84,20 @@ def test_the_frozen_uk_replay_manifest_is_pinned_too(self) -> None: assert entries assert all(entry.is_pinned for entry in entries) + def test_a_raw_manifest_mapping_scopes_rows_by_the_country_it_is_told( + self, + ) -> None: + # The allowlist spans every country, so auditing a raw mapping without + # a country would report the 37 US rows as orphans of the UK replay + # manifest. + gaps = audit_microdata_pins( + _frozen_uk_replay_manifest(), + allowlist=packaged_microdata_pin_allowlist(), + country="uk", + ) + + assert [gap.message() for gap in gaps] == [] + @pytest.mark.parametrize("country", FULLY_PINNED_COUNTRIES) def test_fully_pinned_countries_carry_no_pending_rows(self, country: str) -> None: allowlist = packaged_microdata_pin_allowlist() From 24c16b518543733aaa12cb3927d36a83032c7e57 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Wed, 2 Sep 2026 13:17:06 -0400 Subject: [PATCH 12/14] Match the UK bundle's quoting to the generated US one and re-pin the spec digests microcosm.build.source_runtime is a _DIRECT_KERNEL_MODULE in spec_engine/seeds.py, so its source is attested by every country's spec_sha256: the am, be, uk, and loader golden vectors move whenever the gate's code changes, by design. This re-pins them once against the final code, and drops the quotes the registration blocks in uk/spec/sources.yaml were written with so they read like the surrounding entries and like the generator's output in us/spec/sources.yaml. Co-Authored-By: Claude Fable 5.1 --- .../src/microcosm/build/uk/spec/sources.yaml | 290 +++++++++--------- .../tests/test_spec_engine_country_bundles.py | 6 +- .../tests/test_spec_engine_loader.py | 2 +- 3 files changed, 149 insertions(+), 149 deletions(-) diff --git a/packages/microcosm-build/src/microcosm/build/uk/spec/sources.yaml b/packages/microcosm-build/src/microcosm/build/uk/spec/sources.yaml index 28cfa6518..c1e27d5b8 100644 --- a/packages/microcosm-build/src/microcosm/build/uk/spec/sources.yaml +++ b/packages/microcosm-build/src/microcosm/build/uk/spec/sources.yaml @@ -28,12 +28,12 @@ stages: locator: accounts.tab sha256: fa7871eb45cad0db5fd05ede454ced60405d2f9c598651ea5acea5c91a6ff52f chronicle_artifact: - source_id: "dwp" - package_id: "dwp-frs-2024-25-accounts" + source_id: dwp + package_id: dwp-frs-2024-25-accounts year: 2024 - sha256: "fa7871eb45cad0db5fd05ede454ced60405d2f9c598651ea5acea5c91a6ff52f" - filename: "accounts.tab" - access: "licensed" + sha256: fa7871eb45cad0db5fd05ede454ced60405d2f9c598651ea5acea5c91a6ff52f + filename: accounts.tab + access: licensed size_bytes: 1812923 runtime_sha256_required: true tax_year_start: 2024 @@ -47,12 +47,12 @@ stages: locator: adult.tab sha256: 4eaea0809a7ccca0fddeb98e358771e4a6e5ebb81b21c4fac4070fc9d227658d chronicle_artifact: - source_id: "dwp" - package_id: "dwp-frs-2024-25-adult" + source_id: dwp + package_id: dwp-frs-2024-25-adult year: 2024 - sha256: "4eaea0809a7ccca0fddeb98e358771e4a6e5ebb81b21c4fac4070fc9d227658d" - filename: "adult.tab" - access: "licensed" + sha256: 4eaea0809a7ccca0fddeb98e358771e4a6e5ebb81b21c4fac4070fc9d227658d + filename: adult.tab + access: licensed size_bytes: 34885825 runtime_sha256_required: true tax_year_start: 2024 @@ -66,12 +66,12 @@ stages: locator: benefits.tab sha256: f6ad22b408a13e2239c04b0d076a36418dcf5dd89a8c60daa792c4d735b911d3 chronicle_artifact: - source_id: "dwp" - package_id: "dwp-frs-2024-25-benefits" + source_id: dwp + package_id: dwp-frs-2024-25-benefits year: 2024 - sha256: "f6ad22b408a13e2239c04b0d076a36418dcf5dd89a8c60daa792c4d735b911d3" - filename: "benefits.tab" - access: "licensed" + sha256: f6ad22b408a13e2239c04b0d076a36418dcf5dd89a8c60daa792c4d735b911d3 + filename: benefits.tab + access: licensed size_bytes: 2362329 runtime_sha256_required: true tax_year_start: 2024 @@ -85,12 +85,12 @@ stages: locator: benunit.tab sha256: 66b894624498316d19b6259e287a607e98ed3daacc9be3d3e9067d32b8e09a5a chronicle_artifact: - source_id: "dwp" - package_id: "dwp-frs-2024-25-benunit" + source_id: dwp + package_id: dwp-frs-2024-25-benunit year: 2024 - sha256: "66b894624498316d19b6259e287a607e98ed3daacc9be3d3e9067d32b8e09a5a" - filename: "benunit.tab" - access: "licensed" + sha256: 66b894624498316d19b6259e287a607e98ed3daacc9be3d3e9067d32b8e09a5a + filename: benunit.tab + access: licensed size_bytes: 13986782 runtime_sha256_required: true tax_year_start: 2024 @@ -104,12 +104,12 @@ stages: locator: child.tab sha256: 88ec53fc52eea4374864bbc74219d551f4b4c5f54a220bf9607c7a6289719aa5 chronicle_artifact: - source_id: "dwp" - package_id: "dwp-frs-2024-25-child" + source_id: dwp + package_id: dwp-frs-2024-25-child year: 2024 - sha256: "88ec53fc52eea4374864bbc74219d551f4b4c5f54a220bf9607c7a6289719aa5" - filename: "child.tab" - access: "licensed" + sha256: 88ec53fc52eea4374864bbc74219d551f4b4c5f54a220bf9607c7a6289719aa5 + filename: child.tab + access: licensed size_bytes: 2753961 runtime_sha256_required: true tax_year_start: 2024 @@ -123,12 +123,12 @@ stages: locator: chldcare.tab sha256: 7ccd3f92f299a1f49b24063188177cdb8a958d8bcd753fc3d74dadda6ad04023 chronicle_artifact: - source_id: "dwp" - package_id: "dwp-frs-2024-25-chldcare" + source_id: dwp + package_id: dwp-frs-2024-25-chldcare year: 2024 - sha256: "7ccd3f92f299a1f49b24063188177cdb8a958d8bcd753fc3d74dadda6ad04023" - filename: "chldcare.tab" - access: "licensed" + sha256: 7ccd3f92f299a1f49b24063188177cdb8a958d8bcd753fc3d74dadda6ad04023 + filename: chldcare.tab + access: licensed size_bytes: 275878 runtime_sha256_required: true tax_year_start: 2024 @@ -142,12 +142,12 @@ stages: locator: extchild.tab sha256: c661379a4aa5079ce482b1f98f0bfb9157ad9b3ba4eb10739b61846f9c9548e4 chronicle_artifact: - source_id: "dwp" - package_id: "dwp-frs-2024-25-extchild" + source_id: dwp + package_id: dwp-frs-2024-25-extchild year: 2024 - sha256: "c661379a4aa5079ce482b1f98f0bfb9157ad9b3ba4eb10739b61846f9c9548e4" - filename: "extchild.tab" - access: "licensed" + sha256: c661379a4aa5079ce482b1f98f0bfb9157ad9b3ba4eb10739b61846f9c9548e4 + filename: extchild.tab + access: licensed size_bytes: 15150 runtime_sha256_required: true tax_year_start: 2024 @@ -161,12 +161,12 @@ stages: locator: househol.tab sha256: 2b93b6aed49e1591d6f5360736b4aee3a11ef506b276435f4d2c011a8afbb6a5 chronicle_artifact: - source_id: "dwp" - package_id: "dwp-frs-2024-25-househol" + source_id: dwp + package_id: dwp-frs-2024-25-househol year: 2024 - sha256: "2b93b6aed49e1591d6f5360736b4aee3a11ef506b276435f4d2c011a8afbb6a5" - filename: "househol.tab" - access: "licensed" + sha256: 2b93b6aed49e1591d6f5360736b4aee3a11ef506b276435f4d2c011a8afbb6a5 + filename: househol.tab + access: licensed size_bytes: 12108606 runtime_sha256_required: true tax_year_start: 2024 @@ -180,12 +180,12 @@ stages: locator: job.tab sha256: eb7faf7ada3a3851cb2afb83e2983f8907ffeec897cfbe01e56cb0dfefa853e2 chronicle_artifact: - source_id: "dwp" - package_id: "dwp-frs-2024-25-job" + source_id: dwp + package_id: dwp-frs-2024-25-job year: 2024 - sha256: "eb7faf7ada3a3851cb2afb83e2983f8907ffeec897cfbe01e56cb0dfefa853e2" - filename: "job.tab" - access: "licensed" + sha256: eb7faf7ada3a3851cb2afb83e2983f8907ffeec897cfbe01e56cb0dfefa853e2 + filename: job.tab + access: licensed size_bytes: 10518760 runtime_sha256_required: true tax_year_start: 2024 @@ -199,12 +199,12 @@ stages: locator: maint.tab sha256: e7a8d6f47cab7bf9db9bfd7b3ad5ebe5830ec75245d065dcf8654c7c20b97a7d chronicle_artifact: - source_id: "dwp" - package_id: "dwp-frs-2024-25-maint" + source_id: dwp + package_id: dwp-frs-2024-25-maint year: 2024 - sha256: "e7a8d6f47cab7bf9db9bfd7b3ad5ebe5830ec75245d065dcf8654c7c20b97a7d" - filename: "maint.tab" - access: "licensed" + sha256: e7a8d6f47cab7bf9db9bfd7b3ad5ebe5830ec75245d065dcf8654c7c20b97a7d + filename: maint.tab + access: licensed size_bytes: 13993 runtime_sha256_required: true tax_year_start: 2024 @@ -218,12 +218,12 @@ stages: locator: mortgage.tab sha256: 6a08f6846970dfdc544a7efc8a93fed4f3210d872cd2d160dfb14ca8d92d5ed0 chronicle_artifact: - source_id: "dwp" - package_id: "dwp-frs-2024-25-mortgage" + source_id: dwp + package_id: dwp-frs-2024-25-mortgage year: 2024 - sha256: "6a08f6846970dfdc544a7efc8a93fed4f3210d872cd2d160dfb14ca8d92d5ed0" - filename: "mortgage.tab" - access: "licensed" + sha256: 6a08f6846970dfdc544a7efc8a93fed4f3210d872cd2d160dfb14ca8d92d5ed0 + filename: mortgage.tab + access: licensed size_bytes: 600552 runtime_sha256_required: true tax_year_start: 2024 @@ -237,12 +237,12 @@ stages: locator: oddjob.tab sha256: dfff1baf71a3de05f3a2fcf0c01a3995df5657f242cd7846aa61f6cc27a1cead chronicle_artifact: - source_id: "dwp" - package_id: "dwp-frs-2024-25-oddjob" + source_id: dwp + package_id: dwp-frs-2024-25-oddjob year: 2024 - sha256: "dfff1baf71a3de05f3a2fcf0c01a3995df5657f242cd7846aa61f6cc27a1cead" - filename: "oddjob.tab" - access: "licensed" + sha256: dfff1baf71a3de05f3a2fcf0c01a3995df5657f242cd7846aa61f6cc27a1cead + filename: oddjob.tab + access: licensed size_bytes: 5339 runtime_sha256_required: true tax_year_start: 2024 @@ -256,12 +256,12 @@ stages: locator: penprov.tab sha256: 9e53de0dc969baec000b3cd68387f0f2dfb3f678732e408de175e0a1d6e3fdc1 chronicle_artifact: - source_id: "dwp" - package_id: "dwp-frs-2024-25-penprov" + source_id: dwp + package_id: dwp-frs-2024-25-penprov year: 2024 - sha256: "9e53de0dc969baec000b3cd68387f0f2dfb3f678732e408de175e0a1d6e3fdc1" - filename: "penprov.tab" - access: "licensed" + sha256: 9e53de0dc969baec000b3cd68387f0f2dfb3f678732e408de175e0a1d6e3fdc1 + filename: penprov.tab + access: licensed size_bytes: 513614 runtime_sha256_required: true tax_year_start: 2024 @@ -275,12 +275,12 @@ stages: locator: pension.tab sha256: 2b9be1eb6583cc8916fc06294be27e6217f2aea73da24b97b3226293f6a6ec24 chronicle_artifact: - source_id: "dwp" - package_id: "dwp-frs-2024-25-pension" + source_id: dwp + package_id: dwp-frs-2024-25-pension year: 2024 - sha256: "2b9be1eb6583cc8916fc06294be27e6217f2aea73da24b97b3226293f6a6ec24" - filename: "pension.tab" - access: "licensed" + sha256: 2b9be1eb6583cc8916fc06294be27e6217f2aea73da24b97b3226293f6a6ec24 + filename: pension.tab + access: licensed size_bytes: 1232411 runtime_sha256_required: true tax_year_start: 2024 @@ -432,12 +432,12 @@ stages: locator: adult.tab sha256: 4eaea0809a7ccca0fddeb98e358771e4a6e5ebb81b21c4fac4070fc9d227658d chronicle_artifact: - source_id: "dwp" - package_id: "dwp-frs-2024-25-adult" + source_id: dwp + package_id: dwp-frs-2024-25-adult year: 2024 - sha256: "4eaea0809a7ccca0fddeb98e358771e4a6e5ebb81b21c4fac4070fc9d227658d" - filename: "adult.tab" - access: "licensed" + sha256: 4eaea0809a7ccca0fddeb98e358771e4a6e5ebb81b21c4fac4070fc9d227658d + filename: adult.tab + access: licensed size_bytes: 34885825 runtime_sha256_required: true tax_year_start: 2024 @@ -472,12 +472,12 @@ stages: locator: househol.tab sha256: 2b93b6aed49e1591d6f5360736b4aee3a11ef506b276435f4d2c011a8afbb6a5 chronicle_artifact: - source_id: "dwp" - package_id: "dwp-frs-2024-25-househol" + source_id: dwp + package_id: dwp-frs-2024-25-househol year: 2024 - sha256: "2b93b6aed49e1591d6f5360736b4aee3a11ef506b276435f4d2c011a8afbb6a5" - filename: "househol.tab" - access: "licensed" + sha256: 2b93b6aed49e1591d6f5360736b4aee3a11ef506b276435f4d2c011a8afbb6a5 + filename: househol.tab + access: licensed size_bytes: 12108606 runtime_sha256_required: true tax_year_start: 2024 @@ -536,12 +536,12 @@ stages: locator: adult.tab sha256: 4eaea0809a7ccca0fddeb98e358771e4a6e5ebb81b21c4fac4070fc9d227658d chronicle_artifact: - source_id: "dwp" - package_id: "dwp-frs-2024-25-adult" + source_id: dwp + package_id: dwp-frs-2024-25-adult year: 2024 - sha256: "4eaea0809a7ccca0fddeb98e358771e4a6e5ebb81b21c4fac4070fc9d227658d" - filename: "adult.tab" - access: "licensed" + sha256: 4eaea0809a7ccca0fddeb98e358771e4a6e5ebb81b21c4fac4070fc9d227658d + filename: adult.tab + access: licensed size_bytes: 34885825 runtime_sha256_required: true tax_year_start: 2024 @@ -555,12 +555,12 @@ stages: locator: child.tab sha256: 88ec53fc52eea4374864bbc74219d551f4b4c5f54a220bf9607c7a6289719aa5 chronicle_artifact: - source_id: "dwp" - package_id: "dwp-frs-2024-25-child" + source_id: dwp + package_id: dwp-frs-2024-25-child year: 2024 - sha256: "88ec53fc52eea4374864bbc74219d551f4b4c5f54a220bf9607c7a6289719aa5" - filename: "child.tab" - access: "licensed" + sha256: 88ec53fc52eea4374864bbc74219d551f4b4c5f54a220bf9607c7a6289719aa5 + filename: child.tab + access: licensed size_bytes: 2753961 runtime_sha256_required: true tax_year_start: 2024 @@ -609,12 +609,12 @@ stages: locator: adult.tab sha256: 4eaea0809a7ccca0fddeb98e358771e4a6e5ebb81b21c4fac4070fc9d227658d chronicle_artifact: - source_id: "dwp" - package_id: "dwp-frs-2024-25-adult" + source_id: dwp + package_id: dwp-frs-2024-25-adult year: 2024 - sha256: "4eaea0809a7ccca0fddeb98e358771e4a6e5ebb81b21c4fac4070fc9d227658d" - filename: "adult.tab" - access: "licensed" + sha256: 4eaea0809a7ccca0fddeb98e358771e4a6e5ebb81b21c4fac4070fc9d227658d + filename: adult.tab + access: licensed size_bytes: 34885825 runtime_sha256_required: true tax_year_start: 2024 @@ -839,12 +839,12 @@ stages: filename: was_round_8_hhold_eul_may_2025_230525.tab sha256: 18b3eb980c02c99f3d8a3254af859bee31682b2bdc11703877677292b3ce9374 chronicle_artifact: - source_id: "ons" - package_id: "ons-was-round-8-household" + source_id: ons + package_id: ons-was-round-8-household year: 2018 - sha256: "18b3eb980c02c99f3d8a3254af859bee31682b2bdc11703877677292b3ce9374" - filename: "was_round_8_hhold_eul_may_2025_230525.tab" - access: "restricted" + sha256: 18b3eb980c02c99f3d8a3254af859bee31682b2bdc11703877677292b3ce9374 + filename: was_round_8_hhold_eul_may_2025_230525.tab + access: restricted size_bytes: 39073613 access: private_local_input locator: caller-supplied local input @@ -1015,12 +1015,12 @@ stages: locator: dvhh_ukanon_v2_2023.tab sha256: 6e78f0914be38e63853165486d641cbd790753cc471086210c6f672bfa18ca72 chronicle_artifact: - source_id: "ons" - package_id: "ons-lcfs-2023-24-household" + source_id: ons + package_id: ons-lcfs-2023-24-household year: 2023 - sha256: "6e78f0914be38e63853165486d641cbd790753cc471086210c6f672bfa18ca72" - filename: "dvhh_ukanon_v2_2023.tab" - access: "restricted" + sha256: 6e78f0914be38e63853165486d641cbd790753cc471086210c6f672bfa18ca72 + filename: dvhh_ukanon_v2_2023.tab + access: restricted size_bytes: 22812887 runtime_sha256_required: true filename: dvhh_ukanon_v2_2023.tab @@ -1031,12 +1031,12 @@ stages: locator: dvper_ukanon_202324_2023.tab sha256: f32d54d83cdecf023f0ac73530be3a99372099b596e0106a56eae42a64929e50 chronicle_artifact: - source_id: "ons" - package_id: "ons-lcfs-2023-24-person" + source_id: ons + package_id: ons-lcfs-2023-24-person year: 2023 - sha256: "f32d54d83cdecf023f0ac73530be3a99372099b596e0106a56eae42a64929e50" - filename: "dvper_ukanon_202324_2023.tab" - access: "restricted" + sha256: f32d54d83cdecf023f0ac73530be3a99372099b596e0106a56eae42a64929e50 + filename: dvper_ukanon_202324_2023.tab + access: restricted size_bytes: 6545146 runtime_sha256_required: true filename: dvper_ukanon_202324_2023.tab @@ -1047,12 +1047,12 @@ stages: locator: caller-supplied local input sha256: 18b3eb980c02c99f3d8a3254af859bee31682b2bdc11703877677292b3ce9374 chronicle_artifact: - source_id: "ons" - package_id: "ons-was-round-8-household" + source_id: ons + package_id: ons-was-round-8-household year: 2018 - sha256: "18b3eb980c02c99f3d8a3254af859bee31682b2bdc11703877677292b3ce9374" - filename: "was_round_8_hhold_eul_may_2025_230525.tab" - access: "restricted" + sha256: 18b3eb980c02c99f3d8a3254af859bee31682b2bdc11703877677292b3ce9374 + filename: was_round_8_hhold_eul_may_2025_230525.tab + access: restricted size_bytes: 39073613 runtime_sha256_required: true filename: was_round_8_hhold_eul_may_2025_230525.tab @@ -1232,12 +1232,12 @@ stages: locator: householdv2_1977-2024.tab sha256: d0e94ebc92e85ca1b9fb3a7353dcaf41db2c5110c9f07c7793dc8c0b695250d8 chronicle_artifact: - source_id: "ons" - package_id: "ons-etb-1977-2024-household" + source_id: ons + package_id: ons-etb-1977-2024-household year: 1977 - sha256: "d0e94ebc92e85ca1b9fb3a7353dcaf41db2c5110c9f07c7793dc8c0b695250d8" - filename: "householdv2_1977-2024.tab" - access: "restricted" + sha256: d0e94ebc92e85ca1b9fb3a7353dcaf41db2c5110c9f07c7793dc8c0b695250d8 + filename: householdv2_1977-2024.tab + access: restricted size_bytes: 216967663 runtime_sha256_required: true filename: householdv2_1977-2024.tab @@ -1292,12 +1292,12 @@ stages: locator: householdv2_1977-2024.tab sha256: d0e94ebc92e85ca1b9fb3a7353dcaf41db2c5110c9f07c7793dc8c0b695250d8 chronicle_artifact: - source_id: "ons" - package_id: "ons-etb-1977-2024-household" + source_id: ons + package_id: ons-etb-1977-2024-household year: 1977 - sha256: "d0e94ebc92e85ca1b9fb3a7353dcaf41db2c5110c9f07c7793dc8c0b695250d8" - filename: "householdv2_1977-2024.tab" - access: "restricted" + sha256: d0e94ebc92e85ca1b9fb3a7353dcaf41db2c5110c9f07c7793dc8c0b695250d8 + filename: householdv2_1977-2024.tab + access: restricted size_bytes: 216967663 runtime_sha256_required: true filename: householdv2_1977-2024.tab @@ -1392,12 +1392,12 @@ stages: locator: adult.tab sha256: 4eaea0809a7ccca0fddeb98e358771e4a6e5ebb81b21c4fac4070fc9d227658d chronicle_artifact: - source_id: "dwp" - package_id: "dwp-frs-2024-25-adult" + source_id: dwp + package_id: dwp-frs-2024-25-adult year: 2024 - sha256: "4eaea0809a7ccca0fddeb98e358771e4a6e5ebb81b21c4fac4070fc9d227658d" - filename: "adult.tab" - access: "licensed" + sha256: 4eaea0809a7ccca0fddeb98e358771e4a6e5ebb81b21c4fac4070fc9d227658d + filename: adult.tab + access: licensed size_bytes: 34885825 runtime_sha256_required: true tax_year_start: 2024 @@ -1411,12 +1411,12 @@ stages: locator: benefits.tab sha256: f6ad22b408a13e2239c04b0d076a36418dcf5dd89a8c60daa792c4d735b911d3 chronicle_artifact: - source_id: "dwp" - package_id: "dwp-frs-2024-25-benefits" + source_id: dwp + package_id: dwp-frs-2024-25-benefits year: 2024 - sha256: "f6ad22b408a13e2239c04b0d076a36418dcf5dd89a8c60daa792c4d735b911d3" - filename: "benefits.tab" - access: "licensed" + sha256: f6ad22b408a13e2239c04b0d076a36418dcf5dd89a8c60daa792c4d735b911d3 + filename: benefits.tab + access: licensed size_bytes: 2362329 runtime_sha256_required: true tax_year_start: 2024 @@ -1570,12 +1570,12 @@ stages: filename: put2223uk.tab sha256: 5ef829461060c91a2a47be59ad541d9b519fc3976d66ca80d4920f711bb96f66 chronicle_artifact: - source_id: "hmrc" - package_id: "hmrc-spi-public-use-tape-2022-23" + source_id: hmrc + package_id: hmrc-spi-public-use-tape-2022-23 year: 2022 - sha256: "5ef829461060c91a2a47be59ad541d9b519fc3976d66ca80d4920f711bb96f66" - filename: "put2223uk.tab" - access: "restricted" + sha256: 5ef829461060c91a2a47be59ad541d9b519fc3976d66ca80d4920f711bb96f66 + filename: put2223uk.tab + access: restricted size_bytes: 141323762 reviewed_source: PolicyEngine licensed UKDS mirror (private Hugging Face repository), spi_2022_23.zip access: private_local_input @@ -2643,12 +2643,12 @@ stages: filename: put2223uk.tab sha256: 5ef829461060c91a2a47be59ad541d9b519fc3976d66ca80d4920f711bb96f66 chronicle_artifact: - source_id: "hmrc" - package_id: "hmrc-spi-public-use-tape-2022-23" + source_id: hmrc + package_id: hmrc-spi-public-use-tape-2022-23 year: 2022 - sha256: "5ef829461060c91a2a47be59ad541d9b519fc3976d66ca80d4920f711bb96f66" - filename: "put2223uk.tab" - access: "restricted" + sha256: 5ef829461060c91a2a47be59ad541d9b519fc3976d66ca80d4920f711bb96f66 + filename: put2223uk.tab + access: restricted size_bytes: 141323762 reviewed_source: PolicyEngine licensed UKDS mirror (private Hugging Face repository), spi_2022_23.zip access: private_local_input diff --git a/packages/microcosm-build/tests/test_spec_engine_country_bundles.py b/packages/microcosm-build/tests/test_spec_engine_country_bundles.py index 52de9bb37..e049ea131 100644 --- a/packages/microcosm-build/tests/test_spec_engine_country_bundles.py +++ b/packages/microcosm-build/tests/test_spec_engine_country_bundles.py @@ -25,7 +25,7 @@ "spine", "vintages", } -AM_SPEC_SHA256 = "4d2476f8ec232296fb46aa8a24748fbcb091dfa243f31de007a7ae50b54159a7" +AM_SPEC_SHA256 = "2748a7d3f8f29ae94c4f30b3854a632d9fb13f2036c711fa8a09afb07f7893db" @pytest.mark.parametrize( @@ -45,7 +45,7 @@ ), ( "be", - "205338168141e5ea95a7428acbfdf0cf4a31a4b01d43a9d37098a10e24830427", + "41f2f236916dd423adfbcabfeb69936092ac16feddf7d51a320d92248a4677c6", { "household.household_id", "person.person_id", @@ -55,7 +55,7 @@ ), ( "uk", - "cb9baeea7de036c0eb4038b5ed080b33e1abdb0abe949c04f9d28eb91dc5ae7c", + "4cd516675378c009d5a0b12b97801cdf284da1cf87b33f42f2f99a43268a8c13", { "benunit.benunit_id", "household.household_id", diff --git a/packages/microcosm-build/tests/test_spec_engine_loader.py b/packages/microcosm-build/tests/test_spec_engine_loader.py index 11be8d933..cba63aea6 100644 --- a/packages/microcosm-build/tests/test_spec_engine_loader.py +++ b/packages/microcosm-build/tests/test_spec_engine_loader.py @@ -236,7 +236,7 @@ def test_semantic_hash_has_golden_vector_and_surface_separation(tmp_path) -> Non # Pin the domain separator, normalization rules, schema-set receipt, and # exact normative projection as one reviewable golden vector. assert first.spec_sha256 == ( - "fcde1de85fd79c6386ad73facdf49136202b771634cce1c5e6880c3bfbac5abb" + "729257f82354886f123610429c81f6d9c7015759ac596f3302f8c6da9efafb91" ) second_root = _rich_minimal(tmp_path / "xy", note="second", store="local:b") From b90e550609957aea7e9dfb84bdd06ff79732a5e0 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Wed, 2 Sep 2026 13:17:44 -0400 Subject: [PATCH 13/14] Note in the lane journal that source_runtime source is spec-attested Anyone editing the gate again will move four spec_sha256 vectors with it; say so once rather than let the next editor rediscover it. Co-Authored-By: Claude Fable 5.1 --- PROGRESS-root-identity-848.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/PROGRESS-root-identity-848.md b/PROGRESS-root-identity-848.md index a1280eb5d..1ea70178b 100644 --- a/PROGRESS-root-identity-848.md +++ b/PROGRESS-root-identity-848.md @@ -82,6 +82,14 @@ licensed references over 14 distinct tabs. `spec_sha256` vectors that attest the schema set. - DESIGN.md process rule 5; `changelog.d/848-root-identity.added.md`. +## Gotcha for the next editor of this lane + +`microcosm.build.source_runtime` is a `_DIRECT_KERNEL_MODULE` in +`spec_engine/seeds.py`, so its **source** is attested by every country's +`spec_sha256`. Any edit to the gate's code moves the `am`, `be`, `uk`, and +loader golden vectors, even when no manifest changed. Re-pin those four last, +after the code is final, or you will chase them repeatedly. + ## Next - Full `uv run pytest` and `uv run ruff check .`, then push and open the PR. From e7f5cba48881350e5cd09becdca674907359c422 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Thu, 3 Sep 2026 09:21:19 -0400 Subject: [PATCH 14/14] Reference one Chronicle package per publisher release Consumer references now use the package-per-release convention shared with chronicle#227: dwp/dwp-frs-2024-25 (year 2024, 14 tabs), hmrc/hmrc-spi-public-use-tape-2022-23 (year 2022), ons/ons-was-round-8, ons/ons-lcfs-2023-24, ons/ons-etb-1977-2024, with files distinguished by filename and sha256 inside a package. Spec digests re-pinned; the root-identity contract tests pass unchanged. Co-Authored-By: Claude Fable 5.1 --- PROGRESS-root-identity-848.md | 83 ++++++++++++++++--- .../uk/release_input_coverage_manifest.json | 26 +++--- .../src/microcosm/build/uk/source_stages.json | 54 ++++++------ .../src/microcosm/build/uk/spec/sources.yaml | 54 ++++++------ .../tests/test_microdata_root_identity.py | 2 +- .../tests/test_spec_engine_country_bundles.py | 2 +- 6 files changed, 139 insertions(+), 82 deletions(-) diff --git a/PROGRESS-root-identity-848.md b/PROGRESS-root-identity-848.md index 1ea70178b..183a29a2a 100644 --- a/PROGRESS-root-identity-848.md +++ b/PROGRESS-root-identity-848.md @@ -8,8 +8,57 @@ merged — see CLAUDE.md "Root journals are history, not state". ## State -Started 2026-09-02 from origin/main `d2b31496`. Implementation complete; -verification and PR in progress. Nothing merged. +Started 2026-09-02 from origin/main `d2b31496`. Implementation complete; PR +#853 open. 2026-09-02 (later the same day): realigned the Chronicle package +granularity per the cross-PR convention ruling on #853 (see "Convention +realignment" below). Verification green; PR body updated; nothing merged. + +## Convention realignment (2026-09-02, cross-PR ruling on #853) + +The "Naming chosen" table below documented **one package per distributed +file**. `MaxGhenis` posted a cross-PR ruling on #853 reconciling this with +chronicle#227: Chronicle's `kind: microdata_release` manifests hold a list +under `files[year]`, so **one package per publisher release** is correct — +multiple files share a `package_id`, disambiguated by the existing +`filename` + `sha256` fields already on every `chronicle_artifact` block. +Changed: + +| release | old `package_id` | new `package_id` | +| --- | --- | --- | +| FRS 2024-25, 14 tabs | `dwp-frs-2024-25-` | `dwp-frs-2024-25` | +| WAS round 8 household EUL | `ons-was-round-8-household` | `ons-was-round-8` | +| LCFS 2023-24 household / person | `ons-lcfs-2023-24-{household,person}` | `ons-lcfs-2023-24` | +| ETB 1977-2024 household | `ons-etb-1977-2024-household` | `ons-etb-1977-2024` | + +SPI PUT 2022-23 (`hmrc-spi-public-use-tape-2022-23`) and the two US packages +were already one-package-per-release and are unchanged. + +**No `ons-lcfs-2018-20` package exists or was created.** The ruling comment's +"LCFS 2023-24 and 2018-20" phrase describes two *vintages appearing under the +`lcfs_consumption` stage's artifacts*, not two LCFS releases: the `2018_20` +vintage tag belongs to the `was_bridge_donor` artifact, which is the same WAS +round-8 file (identical sha256) as the `was_qrf_donor` artifact in the +`was_wealth` stage. Both already shared `package_id` before this change and +both now resolve to `ons-was-round-8` — giving the bridge donor a separate +`ons-lcfs-2018-20` id would have violated invariant 3 (same sha256 must +resolve to the same registration) by registering one file's bytes under two +different packages. Verified directly: only 19 distinct UK sha256 values +carry registrations, matching the ruling's "19 hashes today" count, and no +sha256 resolves to two different `ChronicleArtifactReference` tuples. + +Mechanically: `source_manifest.py`'s invariants are keyed on `sha256`, never +on `package_id` uniqueness, so no code, schema, or tooling changed — only +`uk/spec/sources.yaml`, its generated JSON mirror `uk/source_stages.json` +(hand-kept in lockstep; no UK bundle generator exists the way +`generate_us_bundle_from_constants.py` exists for US), +`release_input_coverage_manifest.json` (regenerated via +`tools/build_uk_release_input_coverage_manifest.py`), the one hardcoded +`"dwp/dwp-frs-2024-25-adult"` string in +`test_microdata_root_identity.py:438`, and the UK `spec_sha256` vector in +`test_spec_engine_country_bundles.py` (AM, BE, and the loader golden vector +were unaffected, since no schema/kernel-module code moved — only UK manifest +data did). `DESIGN.md` process rule 5 is convention-agnostic and needed no +edit. ## Contract implemented @@ -28,26 +77,32 @@ Three invariants make the reference hard to get wrong: (`tax_year_start` exactly, else a four-digit year in `vintage`); - entries sharing a SHA-256 must resolve to the same registration. -## Naming chosen (for chronicle#221) +## Naming chosen (for chronicle#221 / chronicle#227) — superseded, see above + +Original (2026-09-02 morning) choice, kept here for history — **superseded +same-day** by the "Convention realignment" section above: Chronicle's live `db/data/*/manifest.yaml` uses a publisher-slug `source_id` and a kebab-case, publisher-prefixed `package_id` (`hmrc` / `hmrc-spi-income-bands-2023-24`), and `files[year]` holds exactly one -file per (package, year). So one package per distributed file: +file per (package, year). So one package per distributed file — ~~this +granularity was wrong; Chronicle's `files[year]` list is precisely what lets +several distributed files share one registration~~: -| release | source_id | package_id | access | +| release | source_id | package_id (current) | access | | --- | --- | --- | --- | -| FRS 2024-25 (UKDS SN 9563), 14 tabs | `dwp` | `dwp-frs-2024-25-` | licensed | -| WAS round 8 household EUL (SN 7215) | `ons` | `ons-was-round-8-household` | restricted | -| LCFS 2023-24 household / person | `ons` | `ons-lcfs-2023-24-{household,person}` | restricted | -| ETB 1977-2024 household | `ons` | `ons-etb-1977-2024-household` | restricted | +| FRS 2024-25 (UKDS SN 9563), 14 tabs | `dwp` | `dwp-frs-2024-25` | licensed | +| WAS round 8 household EUL (SN 7215) | `ons` | `ons-was-round-8` | restricted | +| LCFS 2023-24 household / person | `ons` | `ons-lcfs-2023-24` | restricted | +| ETB 1977-2024 household | `ons` | `ons-etb-1977-2024` | restricted | | SPI Public Use Tape 2022-23 (SN 9422) | `hmrc` | `hmrc-spi-public-use-tape-2022-23` | restricted | | CPS ASEC 2023 archive | `census_cps` | `census-cps-asec-2023` | public | | SCF 2022 summary extract | `federal_reserve` | `federal-reserve-scf-2022-summary-extract` | public | -Note for chronicle#221: the FRS the UK build reads is **2024-25 / SN 9563** -(`tax_year_start: 2024`), not 2023-24, and the UK manifest carries **21** -licensed references over 14 distinct tabs. +Note for chronicle#221/#227: the FRS the UK build reads is **2024-25 / SN +9563** (`tax_year_start: 2024`), not 2023-24, and the UK manifest carries +**21** licensed references over 14 distinct tabs, now consolidated to **5** +UK package registrations (plus 2 US) over **19** distinct UK sha256 values. ## Done @@ -92,4 +147,6 @@ after the code is final, or you will chase them repeatedly. ## Next -- Full `uv run pytest` and `uv run ruff check .`, then push and open the PR. +- Push the convention-realignment commits, edit the #853 PR body to state the + package-per-release convention and the chosen ONS ids, and let CI run. + Do not merge. diff --git a/packages/microcosm-build/src/microcosm/build/uk/release_input_coverage_manifest.json b/packages/microcosm-build/src/microcosm/build/uk/release_input_coverage_manifest.json index 7ccc2731d..c0b466267 100644 --- a/packages/microcosm-build/src/microcosm/build/uk/release_input_coverage_manifest.json +++ b/packages/microcosm-build/src/microcosm/build/uk/release_input_coverage_manifest.json @@ -473,7 +473,7 @@ "capital_gains" ], "source_manifest": "source_stages.json", - "source_manifest_sha256": "b13a7ea430498e4ca1a5ea2a5f9edf8995f079690dda76250cb3bcaa368b94ca", + "source_manifest_sha256": "8033ccc70fc2bf61f4656055bc3dd6e9b17c5fddb196575d05acfe871aca5421", "source_vintages": { "source": "HMRC Capital Gains Tax statistics, July 2025, Table 2.1a", "survey": "HMRC Capital Gains Tax statistics Table 2.1a and Advani-Summers capital-gains incidence" @@ -496,7 +496,7 @@ "capital_gains" ], "source_manifest": "source_stages.json", - "source_manifest_sha256": "b13a7ea430498e4ca1a5ea2a5f9edf8995f079690dda76250cb3bcaa368b94ca", + "source_manifest_sha256": "8033ccc70fc2bf61f4656055bc3dd6e9b17c5fddb196575d05acfe871aca5421", "source_vintages": { "source": "Advani and Summers (2020), Capital Gains and UK Inequality, CAGE Working Paper 465", "survey": "Family Resources Survey 2024-25, SPI synthetic support, and Advani-Summers capital-gains incidence" @@ -525,7 +525,7 @@ "required_mass_change_reason": "E5 source-stage transform preserves household rows and typed household weights; total household mass is conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "b13a7ea430498e4ca1a5ea2a5f9edf8995f079690dda76250cb3bcaa368b94ca", + "source_manifest_sha256": "8033ccc70fc2bf61f4656055bc3dd6e9b17c5fddb196575d05acfe871aca5421", "source_vintages": { "source": "UK Data Service SN 8856 Effects of Taxes and Benefits household tab, DfT rail fare index, and public NHS activity/cost table.", "survey": "Effects of Taxes and Benefits 1977-2024 and NHS age-gender public table" @@ -545,7 +545,7 @@ "required_mass_change_reason": "E5 source-stage transform preserves household rows and typed household weights; total household mass is conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "b13a7ea430498e4ca1a5ea2a5f9edf8995f079690dda76250cb3bcaa368b94ca", + "source_manifest_sha256": "8033ccc70fc2bf61f4656055bc3dd6e9b17c5fddb196575d05acfe871aca5421", "source_vintages": { "source": "UK Data Service SN 8856 Effects of Taxes and Benefits household tab and cited VAT anchor resource.", "survey": "Effects of Taxes and Benefits 1977-2024" @@ -576,7 +576,7 @@ "superseded_by": { "reason": "The FRS spine build executes hmrc_cgt_gains_spine, which applies the same HMRC Table 3 amounts redraw directly in source_stages.json before calibration.", "source_manifest": "source_stages.json", - "source_manifest_sha256": "b13a7ea430498e4ca1a5ea2a5f9edf8995f079690dda76250cb3bcaa368b94ca", + "source_manifest_sha256": "8033ccc70fc2bf61f4656055bc3dd6e9b17c5fddb196575d05acfe871aca5421", "stage": "hmrc_cgt_gains_spine" } }, @@ -596,7 +596,7 @@ "capital_gains" ], "source_manifest": "source_stages.json", - "source_manifest_sha256": "b13a7ea430498e4ca1a5ea2a5f9edf8995f079690dda76250cb3bcaa368b94ca", + "source_manifest_sha256": "8033ccc70fc2bf61f4656055bc3dd6e9b17c5fddb196575d05acfe871aca5421", "source_vintages": { "hmrc_surface": "2023-24", "mapped_build_period": "2024" @@ -610,7 +610,7 @@ "base_candidate_tier": "frs", "calibration_permitted": false, "canonical_source_manifest": "source_stages.json", - "canonical_source_manifest_sha256": "b13a7ea430498e4ca1a5ea2a5f9edf8995f079690dda76250cb3bcaa368b94ca", + "canonical_source_manifest_sha256": "8033ccc70fc2bf61f4656055bc3dd6e9b17c5fddb196575d05acfe871aca5421", "effective_mass_requirements": { "charitable_investment_gifts": { "mass_share_denominator": "all_person_effective_mass", @@ -693,7 +693,7 @@ "superseded_by": { "reason": "The FRS spine build executes hmrc_spi_income_spine, which supersedes the June retained-leaves/hmrc_spi_income pair inside source_stages.json.", "source_manifest": "source_stages.json", - "source_manifest_sha256": "b13a7ea430498e4ca1a5ea2a5f9edf8995f079690dda76250cb3bcaa368b94ca", + "source_manifest_sha256": "8033ccc70fc2bf61f4656055bc3dd6e9b17c5fddb196575d05acfe871aca5421", "stage": "hmrc_spi_income_spine" } }, @@ -727,7 +727,7 @@ "required_mass_change_reason": "E5 source-stage transform preserves household rows and typed household weights; total household mass is conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "b13a7ea430498e4ca1a5ea2a5f9edf8995f079690dda76250cb3bcaa368b94ca", + "source_manifest_sha256": "8033ccc70fc2bf61f4656055bc3dd6e9b17c5fddb196575d05acfe871aca5421", "source_vintages": { "source": "UK Data Service SN 9468 Living Costs and Food Survey 2023-24 household/person tabs, NEED 2023 headline energy tables, Ofgem Q2 2026 unit rates, and WAS round-8 bridge donor.", "survey": "Living Costs and Food Survey 2023-24" @@ -748,7 +748,7 @@ "property_wealth" ], "source_manifest": "source_stages.json", - "source_manifest_sha256": "b13a7ea430498e4ca1a5ea2a5f9edf8995f079690dda76250cb3bcaa368b94ca", + "source_manifest_sha256": "8033ccc70fc2bf61f4656055bc3dd6e9b17c5fddb196575d05acfe871aca5421", "source_vintages": { "source": "MHCLG dwellings and ONS UK House Price Index December 2025 regional average prices.", "survey": "Public regional property reference" @@ -772,7 +772,7 @@ "employee_pension_contributions" ], "source_manifest": "source_stages.json", - "source_manifest_sha256": "b13a7ea430498e4ca1a5ea2a5f9edf8995f079690dda76250cb3bcaa368b94ca", + "source_manifest_sha256": "8033ccc70fc2bf61f4656055bc3dd6e9b17c5fddb196575d05acfe871aca5421", "source_vintages": { "source": "HMRC, Salary sacrifice reform for pension contributions effective from 6 April 2029", "survey": "Family Resources Survey 2024-25 salary-sacrifice respondents and HMRC salary-sacrifice reform analysis" @@ -794,7 +794,7 @@ "student_loan_plan" ], "source_manifest": "source_stages.json", - "source_manifest_sha256": "b13a7ea430498e4ca1a5ea2a5f9edf8995f079690dda76250cb3bcaa368b94ca", + "source_manifest_sha256": "8033ccc70fc2bf61f4656055bc3dd6e9b17c5fddb196575d05acfe871aca5421", "source_vintages": { "source": "Explore Education Statistics Table 6a, Higher education total", "survey": "Family Resources Survey 2024-25 and Student Loans Company borrower forecasts for England" @@ -827,7 +827,7 @@ "required_mass_change_reason": "E5 source-stage transform preserves household rows and typed household weights; total household mass is conserved.", "rewrites": [], "source_manifest": "source_stages.json", - "source_manifest_sha256": "b13a7ea430498e4ca1a5ea2a5f9edf8995f079690dda76250cb3bcaa368b94ca", + "source_manifest_sha256": "8033ccc70fc2bf61f4656055bc3dd6e9b17c5fddb196575d05acfe871aca5421", "source_vintages": { "source": "Office for National Statistics Wealth and Assets Survey, UK Data Service SN 7215, DOI 10.5255/UKDA-SN-7215-20; local licensed 2006-22 household tab.", "survey": "Wealth and Assets Survey round 8" diff --git a/packages/microcosm-build/src/microcosm/build/uk/source_stages.json b/packages/microcosm-build/src/microcosm/build/uk/source_stages.json index 9cac16ff4..43216fa18 100644 --- a/packages/microcosm-build/src/microcosm/build/uk/source_stages.json +++ b/packages/microcosm-build/src/microcosm/build/uk/source_stages.json @@ -19,7 +19,7 @@ "sha256": "fa7871eb45cad0db5fd05ede454ced60405d2f9c598651ea5acea5c91a6ff52f", "chronicle_artifact": { "source_id": "dwp", - "package_id": "dwp-frs-2024-25-accounts", + "package_id": "dwp-frs-2024-25", "year": 2024, "sha256": "fa7871eb45cad0db5fd05ede454ced60405d2f9c598651ea5acea5c91a6ff52f", "filename": "accounts.tab", @@ -41,7 +41,7 @@ "sha256": "4eaea0809a7ccca0fddeb98e358771e4a6e5ebb81b21c4fac4070fc9d227658d", "chronicle_artifact": { "source_id": "dwp", - "package_id": "dwp-frs-2024-25-adult", + "package_id": "dwp-frs-2024-25", "year": 2024, "sha256": "4eaea0809a7ccca0fddeb98e358771e4a6e5ebb81b21c4fac4070fc9d227658d", "filename": "adult.tab", @@ -63,7 +63,7 @@ "sha256": "f6ad22b408a13e2239c04b0d076a36418dcf5dd89a8c60daa792c4d735b911d3", "chronicle_artifact": { "source_id": "dwp", - "package_id": "dwp-frs-2024-25-benefits", + "package_id": "dwp-frs-2024-25", "year": 2024, "sha256": "f6ad22b408a13e2239c04b0d076a36418dcf5dd89a8c60daa792c4d735b911d3", "filename": "benefits.tab", @@ -85,7 +85,7 @@ "sha256": "66b894624498316d19b6259e287a607e98ed3daacc9be3d3e9067d32b8e09a5a", "chronicle_artifact": { "source_id": "dwp", - "package_id": "dwp-frs-2024-25-benunit", + "package_id": "dwp-frs-2024-25", "year": 2024, "sha256": "66b894624498316d19b6259e287a607e98ed3daacc9be3d3e9067d32b8e09a5a", "filename": "benunit.tab", @@ -107,7 +107,7 @@ "sha256": "88ec53fc52eea4374864bbc74219d551f4b4c5f54a220bf9607c7a6289719aa5", "chronicle_artifact": { "source_id": "dwp", - "package_id": "dwp-frs-2024-25-child", + "package_id": "dwp-frs-2024-25", "year": 2024, "sha256": "88ec53fc52eea4374864bbc74219d551f4b4c5f54a220bf9607c7a6289719aa5", "filename": "child.tab", @@ -129,7 +129,7 @@ "sha256": "7ccd3f92f299a1f49b24063188177cdb8a958d8bcd753fc3d74dadda6ad04023", "chronicle_artifact": { "source_id": "dwp", - "package_id": "dwp-frs-2024-25-chldcare", + "package_id": "dwp-frs-2024-25", "year": 2024, "sha256": "7ccd3f92f299a1f49b24063188177cdb8a958d8bcd753fc3d74dadda6ad04023", "filename": "chldcare.tab", @@ -151,7 +151,7 @@ "sha256": "c661379a4aa5079ce482b1f98f0bfb9157ad9b3ba4eb10739b61846f9c9548e4", "chronicle_artifact": { "source_id": "dwp", - "package_id": "dwp-frs-2024-25-extchild", + "package_id": "dwp-frs-2024-25", "year": 2024, "sha256": "c661379a4aa5079ce482b1f98f0bfb9157ad9b3ba4eb10739b61846f9c9548e4", "filename": "extchild.tab", @@ -173,7 +173,7 @@ "sha256": "2b93b6aed49e1591d6f5360736b4aee3a11ef506b276435f4d2c011a8afbb6a5", "chronicle_artifact": { "source_id": "dwp", - "package_id": "dwp-frs-2024-25-househol", + "package_id": "dwp-frs-2024-25", "year": 2024, "sha256": "2b93b6aed49e1591d6f5360736b4aee3a11ef506b276435f4d2c011a8afbb6a5", "filename": "househol.tab", @@ -195,7 +195,7 @@ "sha256": "eb7faf7ada3a3851cb2afb83e2983f8907ffeec897cfbe01e56cb0dfefa853e2", "chronicle_artifact": { "source_id": "dwp", - "package_id": "dwp-frs-2024-25-job", + "package_id": "dwp-frs-2024-25", "year": 2024, "sha256": "eb7faf7ada3a3851cb2afb83e2983f8907ffeec897cfbe01e56cb0dfefa853e2", "filename": "job.tab", @@ -217,7 +217,7 @@ "sha256": "e7a8d6f47cab7bf9db9bfd7b3ad5ebe5830ec75245d065dcf8654c7c20b97a7d", "chronicle_artifact": { "source_id": "dwp", - "package_id": "dwp-frs-2024-25-maint", + "package_id": "dwp-frs-2024-25", "year": 2024, "sha256": "e7a8d6f47cab7bf9db9bfd7b3ad5ebe5830ec75245d065dcf8654c7c20b97a7d", "filename": "maint.tab", @@ -239,7 +239,7 @@ "sha256": "6a08f6846970dfdc544a7efc8a93fed4f3210d872cd2d160dfb14ca8d92d5ed0", "chronicle_artifact": { "source_id": "dwp", - "package_id": "dwp-frs-2024-25-mortgage", + "package_id": "dwp-frs-2024-25", "year": 2024, "sha256": "6a08f6846970dfdc544a7efc8a93fed4f3210d872cd2d160dfb14ca8d92d5ed0", "filename": "mortgage.tab", @@ -261,7 +261,7 @@ "sha256": "dfff1baf71a3de05f3a2fcf0c01a3995df5657f242cd7846aa61f6cc27a1cead", "chronicle_artifact": { "source_id": "dwp", - "package_id": "dwp-frs-2024-25-oddjob", + "package_id": "dwp-frs-2024-25", "year": 2024, "sha256": "dfff1baf71a3de05f3a2fcf0c01a3995df5657f242cd7846aa61f6cc27a1cead", "filename": "oddjob.tab", @@ -283,7 +283,7 @@ "sha256": "9e53de0dc969baec000b3cd68387f0f2dfb3f678732e408de175e0a1d6e3fdc1", "chronicle_artifact": { "source_id": "dwp", - "package_id": "dwp-frs-2024-25-penprov", + "package_id": "dwp-frs-2024-25", "year": 2024, "sha256": "9e53de0dc969baec000b3cd68387f0f2dfb3f678732e408de175e0a1d6e3fdc1", "filename": "penprov.tab", @@ -305,7 +305,7 @@ "sha256": "2b9be1eb6583cc8916fc06294be27e6217f2aea73da24b97b3226293f6a6ec24", "chronicle_artifact": { "source_id": "dwp", - "package_id": "dwp-frs-2024-25-pension", + "package_id": "dwp-frs-2024-25", "year": 2024, "sha256": "2b9be1eb6583cc8916fc06294be27e6217f2aea73da24b97b3226293f6a6ec24", "filename": "pension.tab", @@ -483,7 +483,7 @@ "sha256": "4eaea0809a7ccca0fddeb98e358771e4a6e5ebb81b21c4fac4070fc9d227658d", "chronicle_artifact": { "source_id": "dwp", - "package_id": "dwp-frs-2024-25-adult", + "package_id": "dwp-frs-2024-25", "year": 2024, "sha256": "4eaea0809a7ccca0fddeb98e358771e4a6e5ebb81b21c4fac4070fc9d227658d", "filename": "adult.tab", @@ -536,7 +536,7 @@ "sha256": "2b93b6aed49e1591d6f5360736b4aee3a11ef506b276435f4d2c011a8afbb6a5", "chronicle_artifact": { "source_id": "dwp", - "package_id": "dwp-frs-2024-25-househol", + "package_id": "dwp-frs-2024-25", "year": 2024, "sha256": "2b93b6aed49e1591d6f5360736b4aee3a11ef506b276435f4d2c011a8afbb6a5", "filename": "househol.tab", @@ -622,7 +622,7 @@ "sha256": "4eaea0809a7ccca0fddeb98e358771e4a6e5ebb81b21c4fac4070fc9d227658d", "chronicle_artifact": { "source_id": "dwp", - "package_id": "dwp-frs-2024-25-adult", + "package_id": "dwp-frs-2024-25", "year": 2024, "sha256": "4eaea0809a7ccca0fddeb98e358771e4a6e5ebb81b21c4fac4070fc9d227658d", "filename": "adult.tab", @@ -644,7 +644,7 @@ "sha256": "88ec53fc52eea4374864bbc74219d551f4b4c5f54a220bf9607c7a6289719aa5", "chronicle_artifact": { "source_id": "dwp", - "package_id": "dwp-frs-2024-25-child", + "package_id": "dwp-frs-2024-25", "year": 2024, "sha256": "88ec53fc52eea4374864bbc74219d551f4b4c5f54a220bf9607c7a6289719aa5", "filename": "child.tab", @@ -714,7 +714,7 @@ "sha256": "4eaea0809a7ccca0fddeb98e358771e4a6e5ebb81b21c4fac4070fc9d227658d", "chronicle_artifact": { "source_id": "dwp", - "package_id": "dwp-frs-2024-25-adult", + "package_id": "dwp-frs-2024-25", "year": 2024, "sha256": "4eaea0809a7ccca0fddeb98e358771e4a6e5ebb81b21c4fac4070fc9d227658d", "filename": "adult.tab", @@ -1038,7 +1038,7 @@ "sha256": "18b3eb980c02c99f3d8a3254af859bee31682b2bdc11703877677292b3ce9374", "chronicle_artifact": { "source_id": "ons", - "package_id": "ons-was-round-8-household", + "package_id": "ons-was-round-8", "year": 2018, "sha256": "18b3eb980c02c99f3d8a3254af859bee31682b2bdc11703877677292b3ce9374", "filename": "was_round_8_hhold_eul_may_2025_230525.tab", @@ -1256,7 +1256,7 @@ "sha256": "6e78f0914be38e63853165486d641cbd790753cc471086210c6f672bfa18ca72", "chronicle_artifact": { "source_id": "ons", - "package_id": "ons-lcfs-2023-24-household", + "package_id": "ons-lcfs-2023-24", "year": 2023, "sha256": "6e78f0914be38e63853165486d641cbd790753cc471086210c6f672bfa18ca72", "filename": "dvhh_ukanon_v2_2023.tab", @@ -1275,7 +1275,7 @@ "sha256": "f32d54d83cdecf023f0ac73530be3a99372099b596e0106a56eae42a64929e50", "chronicle_artifact": { "source_id": "ons", - "package_id": "ons-lcfs-2023-24-person", + "package_id": "ons-lcfs-2023-24", "year": 2023, "sha256": "f32d54d83cdecf023f0ac73530be3a99372099b596e0106a56eae42a64929e50", "filename": "dvper_ukanon_202324_2023.tab", @@ -1294,7 +1294,7 @@ "sha256": "18b3eb980c02c99f3d8a3254af859bee31682b2bdc11703877677292b3ce9374", "chronicle_artifact": { "source_id": "ons", - "package_id": "ons-was-round-8-household", + "package_id": "ons-was-round-8", "year": 2018, "sha256": "18b3eb980c02c99f3d8a3254af859bee31682b2bdc11703877677292b3ce9374", "filename": "was_round_8_hhold_eul_may_2025_230525.tab", @@ -1526,7 +1526,7 @@ "sha256": "d0e94ebc92e85ca1b9fb3a7353dcaf41db2c5110c9f07c7793dc8c0b695250d8", "chronicle_artifact": { "source_id": "ons", - "package_id": "ons-etb-1977-2024-household", + "package_id": "ons-etb-1977-2024", "year": 1977, "sha256": "d0e94ebc92e85ca1b9fb3a7353dcaf41db2c5110c9f07c7793dc8c0b695250d8", "filename": "householdv2_1977-2024.tab", @@ -1608,7 +1608,7 @@ "sha256": "d0e94ebc92e85ca1b9fb3a7353dcaf41db2c5110c9f07c7793dc8c0b695250d8", "chronicle_artifact": { "source_id": "ons", - "package_id": "ons-etb-1977-2024-household", + "package_id": "ons-etb-1977-2024", "year": 1977, "sha256": "d0e94ebc92e85ca1b9fb3a7353dcaf41db2c5110c9f07c7793dc8c0b695250d8", "filename": "householdv2_1977-2024.tab", @@ -1737,7 +1737,7 @@ "sha256": "4eaea0809a7ccca0fddeb98e358771e4a6e5ebb81b21c4fac4070fc9d227658d", "chronicle_artifact": { "source_id": "dwp", - "package_id": "dwp-frs-2024-25-adult", + "package_id": "dwp-frs-2024-25", "year": 2024, "sha256": "4eaea0809a7ccca0fddeb98e358771e4a6e5ebb81b21c4fac4070fc9d227658d", "filename": "adult.tab", @@ -1759,7 +1759,7 @@ "sha256": "f6ad22b408a13e2239c04b0d076a36418dcf5dd89a8c60daa792c4d735b911d3", "chronicle_artifact": { "source_id": "dwp", - "package_id": "dwp-frs-2024-25-benefits", + "package_id": "dwp-frs-2024-25", "year": 2024, "sha256": "f6ad22b408a13e2239c04b0d076a36418dcf5dd89a8c60daa792c4d735b911d3", "filename": "benefits.tab", diff --git a/packages/microcosm-build/src/microcosm/build/uk/spec/sources.yaml b/packages/microcosm-build/src/microcosm/build/uk/spec/sources.yaml index c1e27d5b8..3de6eda2a 100644 --- a/packages/microcosm-build/src/microcosm/build/uk/spec/sources.yaml +++ b/packages/microcosm-build/src/microcosm/build/uk/spec/sources.yaml @@ -29,7 +29,7 @@ stages: sha256: fa7871eb45cad0db5fd05ede454ced60405d2f9c598651ea5acea5c91a6ff52f chronicle_artifact: source_id: dwp - package_id: dwp-frs-2024-25-accounts + package_id: dwp-frs-2024-25 year: 2024 sha256: fa7871eb45cad0db5fd05ede454ced60405d2f9c598651ea5acea5c91a6ff52f filename: accounts.tab @@ -48,7 +48,7 @@ stages: sha256: 4eaea0809a7ccca0fddeb98e358771e4a6e5ebb81b21c4fac4070fc9d227658d chronicle_artifact: source_id: dwp - package_id: dwp-frs-2024-25-adult + package_id: dwp-frs-2024-25 year: 2024 sha256: 4eaea0809a7ccca0fddeb98e358771e4a6e5ebb81b21c4fac4070fc9d227658d filename: adult.tab @@ -67,7 +67,7 @@ stages: sha256: f6ad22b408a13e2239c04b0d076a36418dcf5dd89a8c60daa792c4d735b911d3 chronicle_artifact: source_id: dwp - package_id: dwp-frs-2024-25-benefits + package_id: dwp-frs-2024-25 year: 2024 sha256: f6ad22b408a13e2239c04b0d076a36418dcf5dd89a8c60daa792c4d735b911d3 filename: benefits.tab @@ -86,7 +86,7 @@ stages: sha256: 66b894624498316d19b6259e287a607e98ed3daacc9be3d3e9067d32b8e09a5a chronicle_artifact: source_id: dwp - package_id: dwp-frs-2024-25-benunit + package_id: dwp-frs-2024-25 year: 2024 sha256: 66b894624498316d19b6259e287a607e98ed3daacc9be3d3e9067d32b8e09a5a filename: benunit.tab @@ -105,7 +105,7 @@ stages: sha256: 88ec53fc52eea4374864bbc74219d551f4b4c5f54a220bf9607c7a6289719aa5 chronicle_artifact: source_id: dwp - package_id: dwp-frs-2024-25-child + package_id: dwp-frs-2024-25 year: 2024 sha256: 88ec53fc52eea4374864bbc74219d551f4b4c5f54a220bf9607c7a6289719aa5 filename: child.tab @@ -124,7 +124,7 @@ stages: sha256: 7ccd3f92f299a1f49b24063188177cdb8a958d8bcd753fc3d74dadda6ad04023 chronicle_artifact: source_id: dwp - package_id: dwp-frs-2024-25-chldcare + package_id: dwp-frs-2024-25 year: 2024 sha256: 7ccd3f92f299a1f49b24063188177cdb8a958d8bcd753fc3d74dadda6ad04023 filename: chldcare.tab @@ -143,7 +143,7 @@ stages: sha256: c661379a4aa5079ce482b1f98f0bfb9157ad9b3ba4eb10739b61846f9c9548e4 chronicle_artifact: source_id: dwp - package_id: dwp-frs-2024-25-extchild + package_id: dwp-frs-2024-25 year: 2024 sha256: c661379a4aa5079ce482b1f98f0bfb9157ad9b3ba4eb10739b61846f9c9548e4 filename: extchild.tab @@ -162,7 +162,7 @@ stages: sha256: 2b93b6aed49e1591d6f5360736b4aee3a11ef506b276435f4d2c011a8afbb6a5 chronicle_artifact: source_id: dwp - package_id: dwp-frs-2024-25-househol + package_id: dwp-frs-2024-25 year: 2024 sha256: 2b93b6aed49e1591d6f5360736b4aee3a11ef506b276435f4d2c011a8afbb6a5 filename: househol.tab @@ -181,7 +181,7 @@ stages: sha256: eb7faf7ada3a3851cb2afb83e2983f8907ffeec897cfbe01e56cb0dfefa853e2 chronicle_artifact: source_id: dwp - package_id: dwp-frs-2024-25-job + package_id: dwp-frs-2024-25 year: 2024 sha256: eb7faf7ada3a3851cb2afb83e2983f8907ffeec897cfbe01e56cb0dfefa853e2 filename: job.tab @@ -200,7 +200,7 @@ stages: sha256: e7a8d6f47cab7bf9db9bfd7b3ad5ebe5830ec75245d065dcf8654c7c20b97a7d chronicle_artifact: source_id: dwp - package_id: dwp-frs-2024-25-maint + package_id: dwp-frs-2024-25 year: 2024 sha256: e7a8d6f47cab7bf9db9bfd7b3ad5ebe5830ec75245d065dcf8654c7c20b97a7d filename: maint.tab @@ -219,7 +219,7 @@ stages: sha256: 6a08f6846970dfdc544a7efc8a93fed4f3210d872cd2d160dfb14ca8d92d5ed0 chronicle_artifact: source_id: dwp - package_id: dwp-frs-2024-25-mortgage + package_id: dwp-frs-2024-25 year: 2024 sha256: 6a08f6846970dfdc544a7efc8a93fed4f3210d872cd2d160dfb14ca8d92d5ed0 filename: mortgage.tab @@ -238,7 +238,7 @@ stages: sha256: dfff1baf71a3de05f3a2fcf0c01a3995df5657f242cd7846aa61f6cc27a1cead chronicle_artifact: source_id: dwp - package_id: dwp-frs-2024-25-oddjob + package_id: dwp-frs-2024-25 year: 2024 sha256: dfff1baf71a3de05f3a2fcf0c01a3995df5657f242cd7846aa61f6cc27a1cead filename: oddjob.tab @@ -257,7 +257,7 @@ stages: sha256: 9e53de0dc969baec000b3cd68387f0f2dfb3f678732e408de175e0a1d6e3fdc1 chronicle_artifact: source_id: dwp - package_id: dwp-frs-2024-25-penprov + package_id: dwp-frs-2024-25 year: 2024 sha256: 9e53de0dc969baec000b3cd68387f0f2dfb3f678732e408de175e0a1d6e3fdc1 filename: penprov.tab @@ -276,7 +276,7 @@ stages: sha256: 2b9be1eb6583cc8916fc06294be27e6217f2aea73da24b97b3226293f6a6ec24 chronicle_artifact: source_id: dwp - package_id: dwp-frs-2024-25-pension + package_id: dwp-frs-2024-25 year: 2024 sha256: 2b9be1eb6583cc8916fc06294be27e6217f2aea73da24b97b3226293f6a6ec24 filename: pension.tab @@ -433,7 +433,7 @@ stages: sha256: 4eaea0809a7ccca0fddeb98e358771e4a6e5ebb81b21c4fac4070fc9d227658d chronicle_artifact: source_id: dwp - package_id: dwp-frs-2024-25-adult + package_id: dwp-frs-2024-25 year: 2024 sha256: 4eaea0809a7ccca0fddeb98e358771e4a6e5ebb81b21c4fac4070fc9d227658d filename: adult.tab @@ -473,7 +473,7 @@ stages: sha256: 2b93b6aed49e1591d6f5360736b4aee3a11ef506b276435f4d2c011a8afbb6a5 chronicle_artifact: source_id: dwp - package_id: dwp-frs-2024-25-househol + package_id: dwp-frs-2024-25 year: 2024 sha256: 2b93b6aed49e1591d6f5360736b4aee3a11ef506b276435f4d2c011a8afbb6a5 filename: househol.tab @@ -537,7 +537,7 @@ stages: sha256: 4eaea0809a7ccca0fddeb98e358771e4a6e5ebb81b21c4fac4070fc9d227658d chronicle_artifact: source_id: dwp - package_id: dwp-frs-2024-25-adult + package_id: dwp-frs-2024-25 year: 2024 sha256: 4eaea0809a7ccca0fddeb98e358771e4a6e5ebb81b21c4fac4070fc9d227658d filename: adult.tab @@ -556,7 +556,7 @@ stages: sha256: 88ec53fc52eea4374864bbc74219d551f4b4c5f54a220bf9607c7a6289719aa5 chronicle_artifact: source_id: dwp - package_id: dwp-frs-2024-25-child + package_id: dwp-frs-2024-25 year: 2024 sha256: 88ec53fc52eea4374864bbc74219d551f4b4c5f54a220bf9607c7a6289719aa5 filename: child.tab @@ -610,7 +610,7 @@ stages: sha256: 4eaea0809a7ccca0fddeb98e358771e4a6e5ebb81b21c4fac4070fc9d227658d chronicle_artifact: source_id: dwp - package_id: dwp-frs-2024-25-adult + package_id: dwp-frs-2024-25 year: 2024 sha256: 4eaea0809a7ccca0fddeb98e358771e4a6e5ebb81b21c4fac4070fc9d227658d filename: adult.tab @@ -840,7 +840,7 @@ stages: sha256: 18b3eb980c02c99f3d8a3254af859bee31682b2bdc11703877677292b3ce9374 chronicle_artifact: source_id: ons - package_id: ons-was-round-8-household + package_id: ons-was-round-8 year: 2018 sha256: 18b3eb980c02c99f3d8a3254af859bee31682b2bdc11703877677292b3ce9374 filename: was_round_8_hhold_eul_may_2025_230525.tab @@ -1016,7 +1016,7 @@ stages: sha256: 6e78f0914be38e63853165486d641cbd790753cc471086210c6f672bfa18ca72 chronicle_artifact: source_id: ons - package_id: ons-lcfs-2023-24-household + package_id: ons-lcfs-2023-24 year: 2023 sha256: 6e78f0914be38e63853165486d641cbd790753cc471086210c6f672bfa18ca72 filename: dvhh_ukanon_v2_2023.tab @@ -1032,7 +1032,7 @@ stages: sha256: f32d54d83cdecf023f0ac73530be3a99372099b596e0106a56eae42a64929e50 chronicle_artifact: source_id: ons - package_id: ons-lcfs-2023-24-person + package_id: ons-lcfs-2023-24 year: 2023 sha256: f32d54d83cdecf023f0ac73530be3a99372099b596e0106a56eae42a64929e50 filename: dvper_ukanon_202324_2023.tab @@ -1048,7 +1048,7 @@ stages: sha256: 18b3eb980c02c99f3d8a3254af859bee31682b2bdc11703877677292b3ce9374 chronicle_artifact: source_id: ons - package_id: ons-was-round-8-household + package_id: ons-was-round-8 year: 2018 sha256: 18b3eb980c02c99f3d8a3254af859bee31682b2bdc11703877677292b3ce9374 filename: was_round_8_hhold_eul_may_2025_230525.tab @@ -1233,7 +1233,7 @@ stages: sha256: d0e94ebc92e85ca1b9fb3a7353dcaf41db2c5110c9f07c7793dc8c0b695250d8 chronicle_artifact: source_id: ons - package_id: ons-etb-1977-2024-household + package_id: ons-etb-1977-2024 year: 1977 sha256: d0e94ebc92e85ca1b9fb3a7353dcaf41db2c5110c9f07c7793dc8c0b695250d8 filename: householdv2_1977-2024.tab @@ -1293,7 +1293,7 @@ stages: sha256: d0e94ebc92e85ca1b9fb3a7353dcaf41db2c5110c9f07c7793dc8c0b695250d8 chronicle_artifact: source_id: ons - package_id: ons-etb-1977-2024-household + package_id: ons-etb-1977-2024 year: 1977 sha256: d0e94ebc92e85ca1b9fb3a7353dcaf41db2c5110c9f07c7793dc8c0b695250d8 filename: householdv2_1977-2024.tab @@ -1393,7 +1393,7 @@ stages: sha256: 4eaea0809a7ccca0fddeb98e358771e4a6e5ebb81b21c4fac4070fc9d227658d chronicle_artifact: source_id: dwp - package_id: dwp-frs-2024-25-adult + package_id: dwp-frs-2024-25 year: 2024 sha256: 4eaea0809a7ccca0fddeb98e358771e4a6e5ebb81b21c4fac4070fc9d227658d filename: adult.tab @@ -1412,7 +1412,7 @@ stages: sha256: f6ad22b408a13e2239c04b0d076a36418dcf5dd89a8c60daa792c4d735b911d3 chronicle_artifact: source_id: dwp - package_id: dwp-frs-2024-25-benefits + package_id: dwp-frs-2024-25 year: 2024 sha256: f6ad22b408a13e2239c04b0d076a36418dcf5dd89a8c60daa792c4d735b911d3 filename: benefits.tab diff --git a/packages/microcosm-build/tests/test_microdata_root_identity.py b/packages/microcosm-build/tests/test_microdata_root_identity.py index ee72e38d2..8dbf65e66 100644 --- a/packages/microcosm-build/tests/test_microdata_root_identity.py +++ b/packages/microcosm-build/tests/test_microdata_root_identity.py @@ -435,7 +435,7 @@ def test_the_uk_manifest_gate_accepts_the_real_pins(self, tmp_path: Path) -> Non message = str(error.value) assert message.count("adult.tab") >= 5 # every stage that reads it assert adult.sha256 in message - assert "dwp/dwp-frs-2024-25-adult" in message + assert "dwp/dwp-frs-2024-25" in message class TestRecordedPinCrossCheck: diff --git a/packages/microcosm-build/tests/test_spec_engine_country_bundles.py b/packages/microcosm-build/tests/test_spec_engine_country_bundles.py index e049ea131..3bef262fa 100644 --- a/packages/microcosm-build/tests/test_spec_engine_country_bundles.py +++ b/packages/microcosm-build/tests/test_spec_engine_country_bundles.py @@ -55,7 +55,7 @@ ), ( "uk", - "4cd516675378c009d5a0b12b97801cdf284da1cf87b33f42f2f99a43268a8c13", + "b02da562662b1966dcdae55a82b9f7570e4baf800011b93da1a5cec5e61d57d8", { "benunit.benunit_id", "household.household_id",