From bb72d82e192619b2ac091c0c53052bc84806c138 Mon Sep 17 00:00:00 2001 From: LoadingALIAS Date: Mon, 20 Jul 2026 11:37:32 -0400 Subject: [PATCH] ci: repair weekly CT and fuzz gates Bind compiler API inventories to required targets and exclude discarded GNU linker sections from live symbol evidence. Keep keyed BLAKE3 fuzz equality on the production owner type. --- ct.toml | 14 +++- fuzz/target_impls/hash_blake3_keyed.rs | 4 +- scripts/ct/evidence_validation_test.py | 47 ++++++++++++++ scripts/ct/symbolize_linked_binary.py | 13 ++++ scripts/ct/validate.py | 88 ++++++++++++++++++-------- 5 files changed, 137 insertions(+), 29 deletions(-) diff --git a/ct.toml b/ct.toml index 36aff6a0..3ad2a56e 100644 --- a/ct.toml +++ b/ct.toml @@ -77,8 +77,6 @@ compiler_api_prefixes = [ "rscrypto::secret", "rscrypto::traits", ] -compiler_api_item_count = 2248 -compiler_api_sha256 = "1179eb9727d21ccfbecbc64bef9e54b2ddb98739991edc1ca1460cd806829ea8" [[public_len_comparison]] id = "kmac.verify_expected_output" @@ -2593,6 +2591,8 @@ name = "x86_64-unknown-linux-gnu" group = "linux" backend = "llvm" linker = "platform-default-unpinned" +compiler_api_item_count = 2253 +compiler_api_sha256 = "3911999a7a2d68a4d0dd5e0604f0d8cb78b04c340c36fc39e82ad658f348c9f0" claim = "ct-intended" physical_timing = "required" binsec = "required" @@ -2603,6 +2603,8 @@ name = "aarch64-unknown-linux-gnu" group = "linux" backend = "llvm" linker = "platform-default-unpinned" +compiler_api_item_count = 2248 +compiler_api_sha256 = "1179eb9727d21ccfbecbc64bef9e54b2ddb98739991edc1ca1460cd806829ea8" claim = "ct-intended" physical_timing = "required" binsec = "required" @@ -2661,6 +2663,8 @@ name = "aarch64-apple-darwin" group = "macos" backend = "llvm" linker = "apple-ld-unpinned" +compiler_api_item_count = 2248 +compiler_api_sha256 = "1179eb9727d21ccfbecbc64bef9e54b2ddb98739991edc1ca1460cd806829ea8" claim = "ct-intended" physical_timing = "required" binsec = "unsupported" @@ -2684,6 +2688,8 @@ name = "s390x-unknown-linux-gnu" group = "ibm" backend = "llvm" linker = "platform-default-unpinned" +compiler_api_item_count = 2242 +compiler_api_sha256 = "93ae78986ed4ad9b10aa847b8120eceeba8dc72a545d3e4fea581a8c939df5a9" claim = "ct-intended" physical_timing = "required" binsec = "unsupported" @@ -2695,6 +2701,8 @@ name = "powerpc64le-unknown-linux-gnu" group = "ibm" backend = "llvm" linker = "platform-default-unpinned" +compiler_api_item_count = 2242 +compiler_api_sha256 = "35dd3622aa03f55488161e245664b40f7245518d187ed4759d97772d2ecd2ff6" claim = "ct-intended" physical_timing = "required" binsec = "unsupported" @@ -2706,6 +2714,8 @@ name = "riscv64gc-unknown-linux-gnu" group = "linux" backend = "llvm" linker = "platform-default-unpinned" +compiler_api_item_count = 2244 +compiler_api_sha256 = "4da5379ebd04c77e14b8fcc449393c8322af346cf6b80f399e3511fbac8c3cc1" claim = "ct-intended" physical_timing = "required" binsec = "unsupported" diff --git a/fuzz/target_impls/hash_blake3_keyed.rs b/fuzz/target_impls/hash_blake3_keyed.rs index c581dbc9..af530704 100644 --- a/fuzz/target_impls/hash_blake3_keyed.rs +++ b/fuzz/target_impls/hash_blake3_keyed.rs @@ -1,4 +1,4 @@ -use rscrypto::{Blake3, Digest}; +use rscrypto::{Blake3, Blake3KeyedHash, Digest}; use rscrypto_fuzz::{FuzzInput, assert_xof_prefix, some_or_return}; pub fn run(data: &[u8]) { @@ -13,7 +13,7 @@ pub fn run(data: &[u8]) { let mut h = Blake3::new_keyed(&key); h.update(a); h.update(b); - let got = h.finalize(); + let got = Blake3KeyedHash::from_bytes(h.finalize()); assert_eq!(expected, got, "keyed blake3: chunk-split mismatch"); // Property: reset preserves keyed mode diff --git a/scripts/ct/evidence_validation_test.py b/scripts/ct/evidence_validation_test.py index 481673be..0465343e 100644 --- a/scripts/ct/evidence_validation_test.py +++ b/scripts/ct/evidence_validation_test.py @@ -6,6 +6,7 @@ import tempfile from pathlib import Path +import validate as manifest_validation from asm_heuristics import apply_public_operand_rules from provenance import codegen_value from symbolize_linked_binary import Symbol, parse_indirect_symbols, parse_link_map, symbolize @@ -89,6 +90,29 @@ def equality_fixture() -> tuple[dict, dict, dict]: return ct, evidence, heuristics +def manifest_errors(mutate) -> list[str]: + root = Path(__file__).resolve().parents[2] + manifest = manifest_validation.load_toml(root / "ct.toml") + mutate(manifest) + selected_target = "aarch64-apple-darwin" + selected = next(target for target in manifest["target"] if target["name"] == selected_target) + + original_load = manifest_validation.load_toml + original_snapshot = manifest_validation.compiler_public_api_snapshot + manifest_validation.load_toml = lambda _path: manifest + manifest_validation.compiler_public_api_snapshot = lambda _root, _target, _prefixes, _errors: ( + selected["compiler_api_item_count"], + selected["compiler_api_sha256"], + ) + try: + errors: list[str] = [] + manifest_validation.validate_manifest(root, selected_target, errors, []) + return errors + finally: + manifest_validation.load_toml = original_load + manifest_validation.compiler_public_api_snapshot = original_snapshot + + def main() -> None: assert codegen_value(["-C", "target-cpu=native", "-C", "target-cpu=x86-64"], "target-cpu") == "x86-64" @@ -99,6 +123,21 @@ def main() -> None: expect_failure(lambda: records_by_name([{"name": "same"}, {"name": "same"}], "fixture")) expect_failure(lambda: records_by_path([{"path": "same"}, {"path": "same"}], "fixture")) + + def duplicate_target(manifest) -> None: + manifest["target"].append(dict(manifest["target"][0])) + + assert "duplicate target x86_64-unknown-linux-gnu" in manifest_errors(duplicate_target) + + def required_target_without_snapshot(manifest) -> None: + target = next(row for row in manifest["target"] if row["name"] == "x86_64-unknown-linux-musl") + target["physical_timing"] = "required" + + assert ( + "target x86_64-unknown-linux-musl requires a compiler public-API snapshot for release evidence" + in manifest_errors(required_target_without_snapshot) + ) + with tempfile.TemporaryDirectory() as temporary: temporary_path = Path(temporary) hashes = temporary_path / "hashes.txt" @@ -116,6 +155,14 @@ def main() -> None: gnu_link_map = temporary_path / "gnu-link-map.txt" gnu_link_map.write_text( + "Discarded input sections\n" + "\n" + " .text.ct_entry_discarded\n" + " 0x0000000000000000 0x20 input.o\n" + "\n" + "Memory Configuration\n" + "\n" + "Linker script and memory map\n" " .text.ct_entry_owner_eq_16\n" " 0x0000000000002000 0x10 input.o\n" " .text._RNv_test\n" diff --git a/scripts/ct/symbolize_linked_binary.py b/scripts/ct/symbolize_linked_binary.py index 86e040f0..d683f1b4 100644 --- a/scripts/ct/symbolize_linked_binary.py +++ b/scripts/ct/symbolize_linked_binary.py @@ -65,7 +65,20 @@ def parse_link_map(path: Path, known_names: dict[str, str]) -> list[Symbol]: gnu_wrapped_section = re.compile(r"^\s*(\.text\.[^ ]+)\s*$") gnu_wrapped_address = re.compile(r"^\s*0x([0-9a-fA-F]+)\s+0x([0-9a-fA-F]+)(?:\s+.*)?$") pending_gnu_section: str | None = None + in_discarded_sections = False for line in path.read_text(errors="replace").splitlines(): + marker = line.strip() + if marker == "Discarded input sections": + in_discarded_sections = True + pending_gnu_section = None + continue + if marker in {"Memory Configuration", "Linker script and memory map"}: + in_discarded_sections = False + pending_gnu_section = None + continue + if in_discarded_sections: + continue + match = table_row.match(line) if match is not None: pending_gnu_section = None diff --git a/scripts/ct/validate.py b/scripts/ct/validate.py index a1855fac..b644a5f0 100755 --- a/scripts/ct/validate.py +++ b/scripts/ct/validate.py @@ -218,7 +218,12 @@ def dudect_registered_benches(root: Path, errors: list[str]) -> set[str]: return set(re.findall(r"\(\s*([A-Za-z0-9_]+),\s*Some\(", body)) -def compiler_public_api_snapshot(root: Path, prefixes: tuple[str, ...], errors: list[str]) -> tuple[int, str] | None: +def compiler_public_api_snapshot( + root: Path, + target: str, + prefixes: tuple[str, ...], + errors: list[str], +) -> tuple[int, str] | None: """Return the audited public security API count and digest from rustdoc JSON.""" target_dir = root / "target" / "ct-api-inventory" command = [ @@ -227,6 +232,8 @@ def compiler_public_api_snapshot(root: Path, prefixes: tuple[str, ...], errors: "--quiet", "--lib", "--all-features", + "--target", + target, "--target-dir", str(target_dir), "--", @@ -242,7 +249,7 @@ def compiler_public_api_snapshot(root: Path, prefixes: tuple[str, ...], errors: fail(errors, f"compiler public-API inventory failed{suffix}") return None - rustdoc_path = target_dir / "doc" / "rscrypto.json" + rustdoc_path = target_dir / target / "doc" / "rscrypto.json" if not rustdoc_path.is_file(): fail(errors, f"compiler public-API inventory missing {rustdoc_path}") return None @@ -301,7 +308,7 @@ def compiler_public_api_snapshot(root: Path, prefixes: tuple[str, ...], errors: return len(rows), hashlib.sha256(encoded).hexdigest() -def validate_manifest(root: Path, errors: list[str], warnings: list[str]) -> dict: +def validate_manifest(root: Path, selected_target: str, errors: list[str], warnings: list[str]) -> dict: ct_path = root / "ct.toml" matrix_path = root / ".config" / "target-matrix.json" if not ct_path.exists(): @@ -314,8 +321,17 @@ def validate_manifest(root: Path, errors: list[str], warnings: list[str]) -> dic ct = load_toml(ct_path) matrix = json.loads(matrix_path.read_text()) + target_rows = ct.get("target", []) + target_by_name: dict[str, dict] = {} + for target in target_rows: + name = target.get("name", "") + if name in target_by_name: + fail(errors, f"duplicate target {name or ''}") + else: + target_by_name[name] = target + expected_targets = matrix_targets(matrix) - actual_targets = {target.get("name", "") for target in ct.get("target", [])} + actual_targets = set(target_by_name) missing = sorted(expected_targets - actual_targets) extra = sorted(actual_targets - expected_targets) if missing: @@ -323,7 +339,7 @@ def validate_manifest(root: Path, errors: list[str], warnings: list[str]) -> dic if extra: fail(errors, f"ct.toml has target(s) not in target matrix: {', '.join(extra)}") - for target in ct.get("target", []): + for target in target_rows: name = target.get("name", "") claim = target.get("claim") if claim not in VALID_CLAIMS: @@ -345,6 +361,20 @@ def validate_manifest(root: Path, errors: list[str], warnings: list[str]) -> dic fail(errors, f"target {name} binsec unsupported requires binsec_reason") if target.get("binsec") == "required" and "linux" not in name: fail(errors, f"target {name} requires BINSEC but is not a Linux target") + compiler_api_item_count = target.get("compiler_api_item_count") + compiler_api_sha256 = target.get("compiler_api_sha256") + if (compiler_api_item_count is None) != (compiler_api_sha256 is None): + fail(errors, f"target {name} compiler public-API snapshot must provide both count and digest") + elif target.get("physical_timing") == "required" and compiler_api_item_count is None: + fail(errors, f"target {name} requires a compiler public-API snapshot for release evidence") + if compiler_api_item_count is not None and ( + isinstance(compiler_api_item_count, bool) + or not isinstance(compiler_api_item_count, int) + or compiler_api_item_count <= 0 + ): + fail(errors, f"target {name} compiler_api_item_count must be a positive integer") + if compiler_api_sha256 is not None and not re.fullmatch(r"[0-9a-f]{64}", str(compiler_api_sha256)): + fail(errors, f"target {name} compiler_api_sha256 must be a lowercase SHA-256 digest") harness_sections = ct.get("harness", []) all_harness_symbols = {symbol for harness in harness_sections for symbol in harness.get("symbols", [])} @@ -684,18 +714,27 @@ def validate_manifest(root: Path, errors: list[str], warnings: list[str]) -> dic elif len(prefixes) != len(set(prefixes)): fail(errors, "operation_inventory compiler_api_prefixes must not contain duplicates") else: - snapshot = compiler_public_api_snapshot(root, tuple(prefixes), errors) - if snapshot is not None: - actual_count, actual_sha256 = snapshot - expected_count = inventory.get("compiler_api_item_count") - expected_sha256 = inventory.get("compiler_api_sha256") - if actual_count != expected_count: - fail(errors, f"compiler public-API inventory expected {expected_count} items, got {actual_count}") - if actual_sha256 != expected_sha256: - fail( - errors, - "compiler public-API inventory digest changed; audit and classify the public surface before updating ct.toml", - ) + target_inventory = target_by_name.get(selected_target) + if target_inventory is None: + fail(errors, f"compiler public-API inventory target is not declared: {selected_target}") + elif target_inventory.get("compiler_api_item_count") is None or target_inventory.get("compiler_api_sha256") is None: + fail(errors, f"target {selected_target} has no compiler public-API snapshot; evidence remains unsupported") + else: + snapshot = compiler_public_api_snapshot(root, selected_target, tuple(prefixes), errors) + if snapshot is not None: + actual_count, actual_sha256 = snapshot + expected_count = target_inventory.get("compiler_api_item_count") + expected_sha256 = target_inventory.get("compiler_api_sha256") + if actual_count != expected_count: + fail( + errors, + f"compiler public-API inventory for {selected_target} expected {expected_count} items, got {actual_count}", + ) + if actual_sha256 != expected_sha256: + fail( + errors, + f"compiler public-API inventory digest changed for {selected_target}; audit and classify the public surface before updating ct.toml", + ) operation_ids: set[str] = set() operation_api: dict[str, str] = {} @@ -1394,16 +1433,15 @@ def main() -> int: errors: list[str] = [] warnings: list[str] = [] - ct = validate_manifest(root, errors, warnings) + target = args.target + if target is None: + verbose = subprocess.check_output(["rustc", "-vV"], text=True) + target = next(line.split(":", 1)[1].strip() for line in verbose.splitlines() if line.startswith("host:")) + + ct = validate_manifest(root, target, errors, warnings) if args.strict_coverage: - validate_strict_coverage(ct, errors, args.target) + validate_strict_coverage(ct, errors, target) if not args.manifest_only: - target = args.target - if target is None: - import subprocess - - target = subprocess.check_output(["rustc", "-vV"], text=True) - target = next(line.split(":", 1)[1].strip() for line in target.splitlines() if line.startswith("host:")) validate_artifacts(root, target, args.profile, ct, errors, warnings) if args.require_dudect: validate_dudect(root, target, args.profile, ct, errors, warnings)