From 18f6f2ce957c8082d100f9e7fd54292ac6aa643c Mon Sep 17 00:00:00 2001 From: CodeRiskTools Date: Sat, 25 Jul 2026 06:43:16 +0000 Subject: [PATCH] feat: publish explicitly partial OSV snapshot (243,381 records) with signed release artifacts - Add scripts/finalize_partial_osv_vulndb.py to finalize a verified, interrupted OSV prefix as profile=osv-partial / completeness=partial-osv-prefix. - Finalizer uses isolated working copy, re-derives the prefix into a disposable SQLite, and requires identical compact-v1 normalized state. - Make verifier descriptor-pinned and support partial profile contract. - Bind bootstrap to expected profile contract; separate release vs installed database SHA-256 reporting. - Add regression tests for replacement-inode-safe cleanup, partial finalization, checksum tampering and profile binding. - Update docs and README to disclose partial coverage and growth path. - Update DEFAULT_GLOBAL_OSV_RELEASE artifact URLs to v3.1.0 partial release. --- CHANGELOG.md | 4 +- README.md | 12 +- docs/GLOBAL-OSV-SNAPSHOT.md | 43 +-- docs/RELEASE-NOTES-3.1.0.md | 8 +- scripts/finalize_partial_osv_vulndb.py | 439 +++++++++++++++++++++++++ scripts/package_global_osv_release.py | 5 + scripts/verify_global_osv_vulndb.py | 129 ++++++-- src/__main__.py | 4 +- src/vulnerability/global_bootstrap.py | 83 +++-- tests/test_global_bootstrap.py | 48 +++ tests/test_global_osv_snapshot.py | 150 +++++++++ 11 files changed, 853 insertions(+), 72 deletions(-) create mode 100644 scripts/finalize_partial_osv_vulndb.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 63855c7..da4f876 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,14 +13,14 @@ All notable changes to `coderisktools-scanner` are documented here. - added bounded public feed adapters and explicit provenance/quality reports without claiming full-feed coverage; - added a real partial `seed` snapshot with 187 advisories, 378 affected-package rows and seven represented OSV ecosystems; - added signed, pinned seed bootstrap and a separate explicit `--profile seed --apply` activation command. -- added a streamed, signed global OSV SQLite ZIP bootstrap that installs and activates the pinned database on first default vulnerability scan; +- added a streamed, signed partial OSV-prefix SQLite ZIP bootstrap that installs and activates the pinned database on first default vulnerability scan; the manifest discloses included and available source counts; - added `vuln-db bootstrap-global` and `vuln scan --no-bootstrap` controls. ### Fixed - directory self-scan now skips SQLite database artifacts, preventing the real seed from tripping the scanner byte cap in CI; - bootstrap now verifies the detached Ed25519 manifest envelope, exact database SHA-256, SQLite integrity, foreign keys, snapshot identity and manifest counts before atomic installation. -- global ZIP bootstrap verifies the compressed asset digest, one-member extraction contract, expanded database digest, compact provenance manifest and staged snapshot before local activation. +- partial OSV ZIP bootstrap verifies the compressed asset digest, one-member extraction contract, expanded database digest, compact provenance manifest and staged snapshot before local activation; it remains explicitly non-complete. ### Seed boundary diff --git a/README.md b/README.md index 925da8b..ec4f007 100644 --- a/README.md +++ b/README.md @@ -245,9 +245,13 @@ secret-scanner vuln inventory \ The sidecar schema is `coderisktools.vulnerability.external-evidence-provenance` v1 and includes `source_id`, `source_format`, `source_sha256`, timezone-aware `collected_at`, `collector` and `tool_version`. The source digest and tool identity must match the evidence report. Tampered or mismatched sidecars fail closed. The adapter also validates tool version, bounded arrays, package identity, PURL and aliases. It does not run OSV-Scanner. -## 4. Local vulnerability scanning +## 4. Local vulnerability scanning — a real database from the first run -The vulnerability path uses a local SQLite database with an active snapshot. If `--database` is omitted and the default database is absent, the first run downloads, verifies, extracts and activates the pinned signed global OSV snapshot: +CodeRiskTools starts with a real, signed OSV database rather than invented fixtures. Release `v3.1.0` bootstraps an explicitly partial snapshot containing approximately **243,000 real OSV records** (the signed release manifest is authoritative for the exact count). This gives dependency scanning useful coverage immediately after installation without placing a gigabyte-scale SQLite file in Git history. + +The database is designed to grow. `secret-scanner vuln-db update --full` can build a larger snapshot from configured sources, and `--database FILE` can select an independently supplied, verified database. Future signed release snapshots can therefore expand to substantially larger record counts without changing the matching format. Every UI and report must retain the profile, completeness and source counts: absence of a match in a partial snapshot is never proof that a dependency is safe. + +If `--database` is omitted and the default database is absent, the first run downloads, verifies, extracts and activates the pinned signed partial OSV snapshot: ```bash secret-scanner vuln scan \ @@ -271,7 +275,7 @@ secret-scanner vuln scan \ Options: - `--root DIR` — local repository root; -- `--database FILE` — local regular SQLite database; defaults to `~/.local/share/coderisktools/vuln-db/global-osv.sqlite`; +- `--database FILE` — local regular SQLite database; defaults to `~/.local/share/coderisktools/vuln-db/osv-partial.sqlite`; - `--no-bootstrap` — reject a missing default database instead of downloading it; - `--format {json,sarif,markdown,html,csv}`; - `--output FILE` — write the report atomically; @@ -279,7 +283,7 @@ Options: - `--vex FILE` — local OpenVEX or CycloneDX VEX; - `--suppressions FILE` — strict local suppression document. -After first-use bootstrap, matching is offline, active-snapshot-only and read-only. The database path cannot be a symlink, URL or non-regular file. Manual installation is available as `secret-scanner vuln-db bootstrap-global`. +After first-use bootstrap, matching is offline, active-snapshot-only and read-only. The pinned `v3.1.0` default is explicitly labeled `osv-partial` / `partial-osv-prefix`; an unmatched dependency is not proof of safety or complete OSV coverage. The database path cannot be a symlink, URL or non-regular file. Manual installation is available as `secret-scanner vuln-db bootstrap-global`. ## 5. OSV feed import diff --git a/docs/GLOBAL-OSV-SNAPSHOT.md b/docs/GLOBAL-OSV-SNAPSHOT.md index d46e992..4d7cedf 100644 --- a/docs/GLOBAL-OSV-SNAPSHOT.md +++ b/docs/GLOBAL-OSV-SNAPSHOT.md @@ -1,16 +1,17 @@ -# Global OSV SQLite snapshot +# OSV SQLite snapshots -The global OSV builder imports the pinned OSV `all.zip` directly into a temporary SQLite database without extracting the archive to disk. +The scanner supports both a complete import of a pinned OSV source archive and an explicitly partial, verified filename-sorted prefix. Release `v3.1.0` ships the latter to keep bootstrap size and build requirements bounded. ## Scope and naming -A successful artifact is labeled: +The shipped artifact is labeled: -- `profile: global-osv` -- `completeness: full-osv-source` +- `profile: osv-partial` +- `completeness: partial-osv-prefix` - `production_full_database: false` +- source status: `bounded-prefix` -`full-osv-source` means that every accepted JSON member from the pinned global OSV archive was processed. It does **not** mean complete Core coverage, complete vulnerability coverage, or proof that an unmatched component is safe. GHSA, KEV, EPSS, NVD, distro feeds, and other enrichments have separate provenance and completeness requirements. +Its manifest records both the exact included record count and `available_records` from the pinned archive. This snapshot must **not** be interpreted as Full, Core, Complete, or proof that an unmatched component is safe. The separate complete-import builder retains `profile: global-osv` / `completeness: full-osv-source`, but that larger artifact is not the `v3.1.0` default. GHSA, KEV, EPSS, NVD, distro feeds, and other enrichments have separate provenance and completeness requirements. ## Space-efficient evidence mode @@ -36,24 +37,28 @@ python scripts/build_global_osv_vulndb.py \ --snapshot-id global-osv-YYYY-MM-DD ``` -The builder: +The complete-source builder imports bounded batches and requires every member. To finalize a previously interrupted but internally consistent prefix as the partial profile, use the dedicated command. It re-hashes the full pinned archive, verifies every included `id + canonical JSON SHA-256`, then stages an isolated private copy; the interrupted input database is never modified: -1. rejects symlinks, unsafe paths, duplicate members, encrypted files, non-JSON payloads, oversized members, and oversized expanded archives; -2. verifies the pinned archive SHA-256 before import; -3. imports bounded batches with no archive extraction; -4. rejects the build if the configured import-error threshold is exceeded; -5. checks SQLite integrity and foreign keys; -6. generates a bounded-memory `compact-v1` content digest; -7. publishes manifest and checksum first and the SQLite readiness artifact last with no-overwrite hard links; -8. leaves the embedded snapshot **staged**, never active. +```bash +python scripts/finalize_partial_osv_vulndb.py \ + --archive /path/to/all.zip \ + --source-manifest /path/to/manifest.json \ + --partial-database /path/to/interrupted.sqlite \ + --output /path/to/coderisktools-vulndb-osv-partial.sqlite \ + --manifest-output /path/to/coderisktools-vulndb-osv-partial.manifest.json \ + --sha256-output /path/to/coderisktools-vulndb-osv-partial.sqlite.sha256 \ + --snapshot-id osv-partial-YYYY-MM-DD +``` + +Both paths verify the pinned archive SHA-256, validate ZIP member safety and provenance counts, check SQLite integrity and foreign keys, generate a `compact-v1` content digest, publish through no-overwrite links with a readiness marker last, and leave the embedded snapshot **staged**. The complete-source builder imports bounded batches and requires the whole source. The partial finalizer requires an existing 230,000–400,000-record prefix, verifies every included source-record digest against filename-sorted archive members, independently re-imports that prefix into a disposable SQLite database, and requires an identical normalized `compact-v1` projection with zero import errors before publication. ## GitHub Release ZIP and first-run installation The repository never stores the multi-gigabyte database in Git history. Release `v3.1.0` publishes: -- `coderisktools-vulndb-global-osv-2026-07-23.sqlite.zip`; -- `coderisktools-vulndb-global-osv-2026-07-23.manifest.json`; -- `coderisktools-vulndb-global-osv-2026-07-23.manifest.sig.json`. +- `coderisktools-vulndb-osv-partial-2026-07-23.sqlite.zip`; +- `coderisktools-vulndb-osv-partial-2026-07-23.manifest.json`; +- `coderisktools-vulndb-osv-partial-2026-07-23.manifest.sig.json`. The ZIP contains exactly one SQLite member. On the first `vuln scan` invocation, when the default database path does not exist, the scanner: @@ -69,7 +74,7 @@ The ZIP contains exactly one SQLite member. On the first `vuln scan` invocation, Default location: ```text -~/.local/share/coderisktools/vuln-db/global-osv.sqlite +~/.local/share/coderisktools/vuln-db/osv-partial.sqlite ``` Manual bootstrap: diff --git a/docs/RELEASE-NOTES-3.1.0.md b/docs/RELEASE-NOTES-3.1.0.md index 2e6d513..5af7289 100644 --- a/docs/RELEASE-NOTES-3.1.0.md +++ b/docs/RELEASE-NOTES-3.1.0.md @@ -1,6 +1,6 @@ # CodeRiskTools Scanner 3.1.0 — release notes -CodeRiskTools Scanner 3.1.0 adds a controlled, local-first vulnerability database workflow, a small verified **partial seed**, and a pinned signed global OSV SQLite ZIP for first-use installation. +CodeRiskTools Scanner 3.1.0 adds a controlled, local-first vulnerability database workflow, a small verified **partial seed**, and a pinned, signed **partial OSV prefix** SQLite ZIP for first-use installation. The release database is not Full/Core/Complete; its manifest exposes included and available source counts. ## What is included @@ -12,8 +12,8 @@ CodeRiskTools Scanner 3.1.0 adds a controlled, local-first vulnerability databas - signed pinned bootstrap that installs seed as staged only; - explicit `vuln-db activate --profile seed --apply` activation; - real lodash `4.17.15` end-to-end matching evidence with stable fingerprints. -- streamed first-use global database bootstrap with ZIP/database SHA-256, Ed25519, SQLite integrity, foreign-key and compact-manifest verification; -- automatic installation to `~/.local/share/coderisktools/vuln-db/global-osv.sqlite` when the default database is missing; +- streamed first-use partial OSV database bootstrap with ZIP/database SHA-256, Ed25519, SQLite integrity, foreign-key and compact-manifest verification; +- automatic installation to `~/.local/share/coderisktools/vuln-db/osv-partial.sqlite` when the default database is missing; - `vuln-db bootstrap-global` for an explicit installation and `vuln scan --no-bootstrap` to disable automatic network bootstrap. ## Release assets @@ -24,7 +24,7 @@ CodeRiskTools Scanner 3.1.0 adds a controlled, local-first vulnerability databas - SHA-256 sidecar; - Ed25519 signed manifest envelope (`.sig` JSON); - public release keyring. -- global OSV single-SQLite ZIP, detached manifest and Ed25519 signed manifest envelope. +- partial OSV prefix single-SQLite ZIP, detached manifest and Ed25519 signed manifest envelope. ## Verified seed facts diff --git a/scripts/finalize_partial_osv_vulndb.py b/scripts/finalize_partial_osv_vulndb.py new file mode 100644 index 0000000..770eadf --- /dev/null +++ b/scripts/finalize_partial_osv_vulndb.py @@ -0,0 +1,439 @@ +#!/usr/bin/env python3 +"""Finalize an interrupted, verified OSV prefix as an explicitly partial snapshot.""" +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import secrets +import stat +import sys +import zipfile +from pathlib import Path +from typing import Any + +if __package__ in {None, ""}: + sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from scripts.build_global_osv_vulndb import _read_regular_json, _sha256_descriptor +from src.vulnerability.canonical import canonical_json_bytes +from src.vulnerability.database import VulnerabilityDatabase +from src.vulnerability.full_snapshot import _validate_member, validate_zip_central_directory + + +def _sha256_stream(stream: Any) -> str: + digest = hashlib.sha256() + stream.seek(0) + while chunk := stream.read(1024 * 1024): + digest.update(chunk) + stream.seek(0) + return "sha256:" + digest.hexdigest() + + +def _identity(file_stat: os.stat_result) -> tuple[int, int, int]: + return file_stat.st_dev, file_stat.st_ino, file_stat.st_size + + +def _create_private_at(directory_descriptor: int, prefix: str) -> tuple[str, int]: + for _ in range(128): + name = f".{prefix}.{secrets.token_hex(12)}.tmp" + try: + descriptor = os.open( + name, + os.O_RDWR | os.O_CREAT | os.O_EXCL | getattr(os, "O_CLOEXEC", 0) | getattr(os, "O_NOFOLLOW", 0), + 0o600, + dir_fd=directory_descriptor, + ) + return name, descriptor + except FileExistsError: + continue + raise FileExistsError("could not allocate private temporary artifact") + + +def _write_private_at(directory_descriptor: int, prefix: str, content: bytes) -> tuple[str, os.stat_result]: + name, descriptor = _create_private_at(directory_descriptor, prefix) + try: + os.write(descriptor, content) + os.fsync(descriptor) + return name, os.fstat(descriptor) + except BaseException: + try: + os.unlink(name, dir_fd=directory_descriptor) + except FileNotFoundError: + pass + raise + finally: + os.close(descriptor) + + +def _link_owned_at(directory_descriptor: int, source: str, destination: str, source_identity: os.stat_result) -> os.stat_result: + os.link(source, destination, src_dir_fd=directory_descriptor, dst_dir_fd=directory_descriptor, follow_symlinks=False) + destination_identity = os.stat(destination, dir_fd=directory_descriptor, follow_symlinks=False) + if _identity(destination_identity) != _identity(source_identity): + _unlink_owned_at(directory_descriptor, destination, source_identity) + raise OSError("published artifact identity mismatch") + return destination_identity + + +def _unlink_owned_at(directory_descriptor: int, name: str, expected: os.stat_result) -> bool: + try: + current = os.stat(name, dir_fd=directory_descriptor, follow_symlinks=False) + except FileNotFoundError: + return False + if _identity(current) != _identity(expected): + return False + os.unlink(name, dir_fd=directory_descriptor) + return True + + +def finalize_partial_osv_snapshot( + archive: Path, + source_manifest: Path, + partial_database: Path, + output: Path, + manifest_output: Path, + sha256_output: Path, + *, + snapshot_id: str, + minimum_records: int = 230_000, + maximum_records: int = 400_000, + maximum_database_bytes: int = 8_500_000_000, + reserve_free_bytes: int = 512 * 1024 * 1024, +) -> dict[str, Any]: + """Validate the exact filename-sorted archive prefix, stage it, and publish atomically.""" + ready_output = output.with_suffix(output.suffix + ".ready.json") + if not snapshot_id.startswith("osv-partial-") or len(snapshot_id) > 128: + raise ValueError("partial snapshot_id must start with osv-partial-") + if minimum_records <= 0 or maximum_records < minimum_records: + raise ValueError("partial record bounds are invalid") + if maximum_database_bytes <= 0 or reserve_free_bytes <= 0: + raise ValueError("partial database byte limits are invalid") + output.parent.mkdir(parents=True, exist_ok=True) + if len({path.parent.resolve() for path in (partial_database, output, manifest_output, sha256_output, ready_output)}) != 1: + raise ValueError("partial input and outputs must share one publication directory") + if partial_database in {output, manifest_output, sha256_output, ready_output}: + raise ValueError("partial input database must be separate from release outputs") + for path in (output, manifest_output, sha256_output, ready_output): + if path.exists() or path.is_symlink(): + raise FileExistsError(f"refusing to overwrite output: {path}") + + source = _read_regular_json(source_manifest) + expected_digest = source.get("sha256") + available_records = source.get("records") + expected_uncompressed = source.get("uncompressed_bytes") + if ( + not isinstance(expected_digest, str) + or len(expected_digest) != 71 + or not expected_digest.startswith("sha256:") + or type(available_records) is not int + or available_records <= maximum_records + or type(expected_uncompressed) is not int + or expected_uncompressed <= 0 + ): + raise ValueError("source manifest lacks valid full-source provenance") + + database_flags = os.O_RDONLY | getattr(os, "O_CLOEXEC", 0) | getattr(os, "O_NOFOLLOW", 0) + archive_flags = os.O_RDONLY | getattr(os, "O_CLOEXEC", 0) | getattr(os, "O_NOFOLLOW", 0) + directory_descriptor = os.open( + output.parent, + os.O_RDONLY | getattr(os, "O_DIRECTORY", 0) | getattr(os, "O_CLOEXEC", 0) | getattr(os, "O_NOFOLLOW", 0), + ) + original_database_descriptor = os.open(partial_database.name, database_flags, dir_fd=directory_descriptor) + archive_descriptor = os.open(archive, archive_flags) + working_name, database_descriptor = _create_private_at(directory_descriptor, output.name + ".working") + verification_name, verification_descriptor = _create_private_at(directory_descriptor, output.name + ".rederived") + temporary_manifest: tuple[str, os.stat_result] | None = None + temporary_sha: tuple[str, os.stat_result] | None = None + temporary_ready: tuple[str, os.stat_result] | None = None + published: list[tuple[str, os.stat_result]] = [] + temporary_guards: list[int] = [] + try: + database_identity = os.fstat(original_database_descriptor) + archive_identity = os.fstat(archive_descriptor) + if not stat.S_ISREG(database_identity.st_mode) or database_identity.st_size <= 0 or database_identity.st_size > maximum_database_bytes: + raise ValueError("partial database must be a bounded regular file") + if not stat.S_ISREG(archive_identity.st_mode) or archive_identity.st_size <= 0: + raise ValueError("OSV archive must be a regular file") + if _identity(database_identity) != _identity(os.stat(partial_database.name, dir_fd=directory_descriptor, follow_symlinks=False)): + raise OSError("partial database changed while being claimed") + if _identity(archive_identity) != _identity(os.stat(archive, follow_symlinks=False)): + raise OSError("OSV archive changed while being claimed") + filesystem = os.fstatvfs(directory_descriptor) + free_bytes = filesystem.f_bavail * filesystem.f_frsize + if free_bytes < 2 * database_identity.st_size + reserve_free_bytes: + raise OSError("insufficient space for isolated working and re-derived databases") + with os.fdopen(os.dup(original_database_descriptor), "rb") as input_stream, os.fdopen(os.dup(database_descriptor), "wb") as target: + while chunk := input_stream.read(1024 * 1024): + target.write(chunk) + target.flush() + os.fsync(target.fileno()) + os.lseek(database_descriptor, 0, os.SEEK_SET) + + with ( + VulnerabilityDatabase.from_file_descriptor(database_descriptor, readonly=False) as database, + VulnerabilityDatabase.from_file_descriptor(verification_descriptor, readonly=False, initialize_schema=True) as rederived, + ): + integrity = database.integrity_check() + foreign_key_errors = len(database.connection.execute("PRAGMA foreign_key_check").fetchall()) + if integrity != "ok" or foreign_key_errors: + raise ValueError("partial SQLite failed integrity or foreign-key checks") + if database.connection.execute("SELECT COUNT(*) FROM snapshots").fetchone()[0] or database.active_snapshot() is not None: + raise ValueError("partial SQLite must not already contain a snapshot") + rows = database.connection.execute( + "SELECT native_record_id, content_digest FROM source_records WHERE source_id = 'osv'" + ).fetchall() + prefix_digests = {str(row["native_record_id"]): str(row["content_digest"]) for row in rows} + record_count = len(rows) + total_source_records = int(database.connection.execute("SELECT COUNT(*) FROM source_records").fetchone()[0]) + if ( + record_count < minimum_records + or record_count > maximum_records + or record_count >= available_records + or len(prefix_digests) != record_count + or database.advisory_count() != record_count + or total_source_records != record_count + ): + raise ValueError("partial SQLite record identity/count contract failed") + malformed_digest = database.connection.execute( + "SELECT COUNT(*) FROM source_records WHERE source_id != 'osv' " + "OR length(native_record_id) > 512 OR content_digest NOT GLOB 'sha256:[0-9a-f]*' " + "OR length(content_digest) != 71" + ).fetchone()[0] + non_digest_only = database.connection.execute( + "SELECT COUNT(*) FROM source_records WHERE json_extract(record_json, '$._payload_omitted') IS NOT 1" + ).fetchone()[0] + if malformed_digest or non_digest_only: + raise ValueError("partial SQLite source evidence contract failed") + + with os.fdopen(os.dup(archive_descriptor), "rb") as archive_stream: + if _sha256_stream(archive_stream) != expected_digest: + raise ValueError("OSV archive SHA-256 does not match source provenance") + declared_members, _ = validate_zip_central_directory(archive_stream, max_members=1_000_000) + with zipfile.ZipFile(archive_stream) as zipped: + infos = zipped.infolist() + if len(infos) != declared_members: + raise ValueError("ZIP central-directory count changed during open") + infos.sort(key=lambda info: info.filename) + total_uncompressed = 0 + payload_count = 0 + names: set[str] = set() + for info in infos: + _validate_member(info, max_member_bytes=24 * 1024 * 1024) + if info.filename in names: + raise ValueError("OSV ZIP contains duplicate member names") + names.add(info.filename) + total_uncompressed += info.file_size + if not info.is_dir(): + payload_count += 1 + if payload_count != available_records or total_uncompressed != expected_uncompressed: + raise ValueError("OSV archive count/size does not match source provenance") + seen = 0 + derivation_batch: list[dict[str, Any]] = [] + for info in infos: + if info.is_dir(): + continue + seen += 1 + if seen > record_count: + break + raw = zipped.read(info) + record = json.loads(raw) + if not isinstance(record, dict): + raise ValueError("OSV prefix record must be an object") + advisory_id = record.get("id") + source_record = record.get("_source_record", record) + if not isinstance(advisory_id, str) or not advisory_id or not isinstance(source_record, dict): + raise ValueError("OSV prefix record identity is invalid") + actual_digest = "sha256:" + hashlib.sha256(canonical_json_bytes(source_record)).hexdigest() + if prefix_digests.pop(advisory_id, None) != actual_digest: + raise ValueError("partial SQLite is not the exact canonical OSV archive prefix") + derivation_batch.append(record) + if len(derivation_batch) >= 1_000: + stats = rederived.import_osv_records( + derivation_batch, + source_record_mode="digest-only", + correlate_aliases=False, + record_savepoints=False, + ) + if stats.errors or stats.advisories_imported != len(derivation_batch): + detail = stats.errors[0] if stats.errors else f"imported {stats.advisories_imported}/{len(derivation_batch)}" + raise ValueError(f"clean OSV prefix re-derivation failed: {detail}") + derivation_batch.clear() + if seen != record_count + 1 or prefix_digests: + raise ValueError("partial SQLite prefix boundary is inconsistent") + if derivation_batch: + stats = rederived.import_osv_records( + derivation_batch, + source_record_mode="digest-only", + correlate_aliases=False, + record_savepoints=False, + ) + if stats.errors or stats.advisories_imported != len(derivation_batch): + detail = stats.errors[0] if stats.errors else f"imported {stats.advisories_imported}/{len(derivation_batch)}" + raise ValueError(f"clean OSV prefix re-derivation failed: {detail}") + + database.correlate_aliases() + rederived.correlate_aliases() + existing_projection = database.build_compact_snapshot_manifest({"derivation": "existing"}) + rederived_projection = rederived.build_compact_snapshot_manifest({"derivation": "existing"}) + for key in ("advisory_count", "affected_package_count", "content_digest", "manifest_version"): + if existing_projection.get(key) != rederived_projection.get(key): + raise ValueError("partial SQLite normalized state does not match a clean OSV prefix derivation") + unmapped_advisories = int(database.connection.execute( + "SELECT COUNT(*) FROM advisories a WHERE NOT EXISTS " + "(SELECT 1 FROM affected_packages p WHERE p.advisory_id = a.id)" + ).fetchone()[0]) + database.connection.execute( + "INSERT INTO source_snapshots " + "(snapshot_id, source_id, content_digest, observed_at, record_count, status, metadata_json) " + "VALUES (?, 'osv-global', ?, NULL, ?, 'bounded-prefix', ?)", + ( + snapshot_id, + expected_digest, + record_count, + json.dumps({ + "archive_bytes": archive_identity.st_size, + "available_records": available_records, + "prefix_order": "zip-member-filename-ascending", + "source_record_mode": "digest-only", + "url": source.get("url"), + }, sort_keys=True, separators=(",", ":")), + ), + ) + database.connection.execute( + "INSERT INTO quality_metrics(snapshot_id, metric_name, metric_value, details_json) VALUES (?, 'import_errors', 0, '{}')", + (snapshot_id,), + ) + database.connection.execute( + "INSERT INTO quality_metrics(snapshot_id, metric_name, metric_value, details_json) VALUES (?, 'unmapped_advisories', ?, '{}')", + (snapshot_id, float(unmapped_advisories)), + ) + sources = { + "osv-global": { + "archive_bytes": archive_identity.st_size, + "available_records": available_records, + "declared_uncompressed_bytes": expected_uncompressed, + "members_seen": record_count, + "prefix_order": "zip-member-filename-ascending", + "records": record_count, + "sha256": expected_digest, + "status": "bounded-prefix", + "url": source.get("url"), + } + } + provenance = { + "profile": "osv-partial", + "completeness": "partial-osv-prefix", + "production_full_database": False, + "snapshot_id": snapshot_id, + "source_digest": expected_digest, + "source_record_mode": "digest-only", + "sources": sources, + } + manifest = database.build_compact_snapshot_manifest(provenance) + manifest.update(provenance) + manifest["quality"] = { + "foreign_key_errors": foreign_key_errors, + "import_errors": 0, + "integrity_check": integrity, + "unmapped_advisories": unmapped_advisories, + } + database.stage_snapshot(snapshot_id, expected_digest, manifest) + + finalized_database_identity = os.fstat(database_descriptor) + if _identity(finalized_database_identity) != _identity(os.stat(working_name, dir_fd=directory_descriptor, follow_symlinks=False)): + raise OSError("working database changed during finalization") + filesystem = os.fstatvfs(directory_descriptor) + if filesystem.f_bavail * filesystem.f_frsize < reserve_free_bytes: + raise OSError("free disk reserve exhausted before publication") + os.fsync(database_descriptor) + database_digest = _sha256_descriptor(database_descriptor) + database_bytes = os.fstat(database_descriptor).st_size + manifest["database_bytes"] = database_bytes + manifest["database_sha256"] = database_digest + manifest_bytes = (json.dumps(manifest, indent=2, sort_keys=True) + "\n").encode("utf-8") + sha_bytes = f"{database_digest.removeprefix('sha256:')} {output.name}\n".encode("ascii") + ready = { + "schema": "coderisktools.vulnerability.release-set-ready.v1", + "snapshot_id": snapshot_id, + "database": {"name": output.name, "bytes": database_bytes, "sha256": database_digest}, + "manifest": {"name": manifest_output.name, "sha256": "sha256:" + hashlib.sha256(manifest_bytes).hexdigest()}, + "checksum": {"name": sha256_output.name, "sha256": "sha256:" + hashlib.sha256(sha_bytes).hexdigest()}, + } + temporary_manifest = _write_private_at(directory_descriptor, manifest_output.name, manifest_bytes) + temporary_sha = _write_private_at(directory_descriptor, sha256_output.name, sha_bytes) + temporary_ready = _write_private_at( + directory_descriptor, + ready_output.name, + (json.dumps(ready, indent=2, sort_keys=True) + "\n").encode("utf-8"), + ) + for temporary in (temporary_manifest, temporary_sha, temporary_ready): + temporary_guards.append(os.open( + temporary[0], + os.O_RDONLY | getattr(os, "O_CLOEXEC", 0) | getattr(os, "O_NOFOLLOW", 0), + dir_fd=directory_descriptor, + )) + manifest_identity = _link_owned_at(directory_descriptor, temporary_manifest[0], manifest_output.name, temporary_manifest[1]) + published.append((manifest_output.name, manifest_identity)) + sha_identity = _link_owned_at(directory_descriptor, temporary_sha[0], sha256_output.name, temporary_sha[1]) + published.append((sha256_output.name, sha_identity)) + database_output_identity = _link_owned_at(directory_descriptor, working_name, output.name, finalized_database_identity) + published.append((output.name, database_output_identity)) + os.fsync(directory_descriptor) + ready_identity = _link_owned_at(directory_descriptor, temporary_ready[0], ready_output.name, temporary_ready[1]) + published.append((ready_output.name, ready_identity)) + os.fsync(directory_descriptor) + for temporary in (temporary_manifest, temporary_sha, temporary_ready): + _unlink_owned_at(directory_descriptor, temporary[0], temporary[1]) + os.fsync(directory_descriptor) + return manifest + except BaseException: + for name, identity in reversed(published): + _unlink_owned_at(directory_descriptor, name, identity) + for temporary in (temporary_manifest, temporary_sha, temporary_ready): + if temporary is not None: + _unlink_owned_at(directory_descriptor, temporary[0], temporary[1]) + raise + finally: + working_identity = os.fstat(database_descriptor) + os.close(database_descriptor) + _unlink_owned_at(directory_descriptor, working_name, working_identity) + verification_identity = os.fstat(verification_descriptor) + os.close(verification_descriptor) + _unlink_owned_at(directory_descriptor, verification_name, verification_identity) + for guard in temporary_guards: + os.close(guard) + os.close(archive_descriptor) + os.close(original_database_descriptor) + os.close(directory_descriptor) + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--archive", type=Path, required=True) + parser.add_argument("--source-manifest", type=Path, required=True) + parser.add_argument("--partial-database", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--manifest-output", type=Path, required=True) + parser.add_argument("--sha256-output", type=Path, required=True) + parser.add_argument("--snapshot-id", required=True) + parser.add_argument("--minimum-records", type=int, default=230_000) + parser.add_argument("--maximum-records", type=int, default=400_000) + args = parser.parse_args() + manifest = finalize_partial_osv_snapshot( + args.archive, + args.source_manifest, + args.partial_database, + args.output, + args.manifest_output, + args.sha256_output, + snapshot_id=args.snapshot_id, + minimum_records=args.minimum_records, + maximum_records=args.maximum_records, + ) + print(json.dumps(manifest, indent=2, sort_keys=True)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/package_global_osv_release.py b/scripts/package_global_osv_release.py index cbad1d4..c4bd3d3 100644 --- a/scripts/package_global_osv_release.py +++ b/scripts/package_global_osv_release.py @@ -6,12 +6,17 @@ import hashlib import json import os +import shutil import stat +import sys import tempfile import zipfile from pathlib import Path from typing import Any +if __package__ in {None, ""}: + sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + from scripts.verify_global_osv_vulndb import _read_bounded_regular, verify_global_osv_snapshot from src.vulnerability.manifest_signing import sign_manifest diff --git a/scripts/verify_global_osv_vulndb.py b/scripts/verify_global_osv_vulndb.py index c135330..4931288 100644 --- a/scripts/verify_global_osv_vulndb.py +++ b/scripts/verify_global_osv_vulndb.py @@ -7,9 +7,13 @@ import json import os import stat +import sys from pathlib import Path from typing import Any +if __package__ in {None, ""}: + sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + from src.vulnerability.database import VulnerabilityDatabase _MAX_METADATA_BYTES = 2 * 1024 * 1024 @@ -37,6 +41,27 @@ def _read_bounded_regular(path: Path, label: str) -> bytes: os.close(descriptor) +def _read_bounded_at(directory_descriptor: int, name: str, label: str) -> bytes: + if Path(name).name != name: + raise ValueError(f"{label} name is invalid") + descriptor = os.open(name, os.O_RDONLY | getattr(os, "O_CLOEXEC", 0) | getattr(os, "O_NOFOLLOW", 0), dir_fd=directory_descriptor) + try: + file_stat = os.fstat(descriptor) + if not stat.S_ISREG(file_stat.st_mode) or file_stat.st_size <= 0 or file_stat.st_size > _MAX_METADATA_BYTES: + raise ValueError(f"{label} must be a bounded regular file") + data = bytearray() + while len(data) < file_stat.st_size: + chunk = os.read(descriptor, min(1024 * 1024, file_stat.st_size - len(data))) + if not chunk: + break + data.extend(chunk) + if len(data) != file_stat.st_size or os.read(descriptor, 1): + raise ValueError(f"{label} changed while being read") + return bytes(data) + finally: + os.close(descriptor) + + def _sha256(path: Path) -> str: digest = hashlib.sha256() with path.open("rb") as stream: @@ -48,31 +73,59 @@ def _sha256(path: Path) -> str: return "sha256:" + digest.hexdigest() -def verify_global_osv_snapshot(database_path: Path, manifest_path: Path, *, minimum_records: int = 1) -> dict[str, Any]: - for path, label in ((database_path, "database"), (manifest_path, "manifest")): - if path.is_symlink() or not path.is_file(): - raise ValueError(f"{label} must be a regular non-symlink file") - if minimum_records <= 0: - raise ValueError("minimum_records must be positive") - manifest_raw = _read_bounded_regular(manifest_path, "manifest") +def _sha256_descriptor(descriptor: int) -> str: + digest = hashlib.sha256() + os.lseek(descriptor, 0, os.SEEK_SET) + while chunk := os.read(descriptor, 1024 * 1024): + digest.update(chunk) + os.lseek(descriptor, 0, os.SEEK_SET) + return "sha256:" + digest.hexdigest() + + +def _verify_global_osv_snapshot_pinned( + database_path: Path, + manifest_path: Path, + directory_descriptor: int, + database_descriptor: int, + database_stat: os.stat_result, + *, + minimum_records: int, +) -> dict[str, Any]: + manifest_raw = _read_bounded_at(directory_descriptor, manifest_path.name, "manifest") manifest = json.loads(manifest_raw) - if manifest.get("profile") != "global-osv" or manifest.get("completeness") != "full-osv-source": - raise ValueError("unexpected global OSV manifest profile") + profile = manifest.get("profile") + completeness = manifest.get("completeness") + if (profile, completeness) not in { + ("global-osv", "full-osv-source"), + ("osv-partial", "partial-osv-prefix"), + }: + raise ValueError("unexpected OSV manifest profile") if manifest.get("production_full_database") is not False or manifest.get("source_record_mode") != "digest-only": raise ValueError("global OSV manifest boundaries are invalid") expected_database_digest = manifest.get("database_sha256") - actual_database_digest = _sha256(database_path) + actual_database_digest = _sha256_descriptor(database_descriptor) if expected_database_digest != actual_database_digest: raise ValueError("database SHA-256 mismatch") - if manifest.get("database_bytes") != database_path.stat().st_size: + if manifest.get("database_bytes") != database_stat.st_size: raise ValueError("database size does not match manifest") snapshot_id = manifest.get("snapshot_id") source = manifest.get("sources", {}).get("osv-global", {}) - if not isinstance(snapshot_id, str) or not snapshot_id: - raise ValueError("snapshot_id is missing") + expected_snapshot_prefix = "global-osv-" if profile == "global-osv" else "osv-partial-" + if not isinstance(snapshot_id, str) or not snapshot_id.startswith(expected_snapshot_prefix) or len(snapshot_id) > 128: + raise ValueError("snapshot_id is missing or outside the profile contract") source_records_expected = source.get("records") + source_status = source.get("status") + available_records = source.get("available_records") + profile_source_valid = source_status == "complete" + if profile == "osv-partial": + profile_source_valid = ( + source_status == "bounded-prefix" + and type(available_records) is int + and type(source_records_expected) is int + and available_records > source_records_expected + ) if ( - source.get("status") != "complete" + not profile_source_valid or type(source_records_expected) is not int or source_records_expected < minimum_records or source.get("members_seen") != source_records_expected @@ -81,12 +134,12 @@ def verify_global_osv_snapshot(database_path: Path, manifest_path: Path, *, mini or manifest.get("quality", {}).get("import_errors") != 0 ): raise ValueError("global OSV source record gate failed") - ready_path = database_path.with_suffix(database_path.suffix + ".ready.json") - ready_raw = _read_bounded_regular(ready_path, "release ready marker") + ready_name = database_path.name + ".ready.json" + ready_raw = _read_bounded_at(directory_descriptor, ready_name, "release ready marker") ready = json.loads(ready_raw) if ready.get("schema") != "coderisktools.vulnerability.release-set-ready.v1" or ready.get("snapshot_id") != snapshot_id: raise ValueError("release ready marker contract mismatch") - if ready.get("database") != {"name": database_path.name, "bytes": database_path.stat().st_size, "sha256": actual_database_digest}: + if ready.get("database") != {"name": database_path.name, "bytes": database_stat.st_size, "sha256": actual_database_digest}: raise ValueError("release ready marker database mismatch") manifest_ready = ready.get("manifest", {}) if manifest_ready.get("name") != manifest_path.name or manifest_ready.get("sha256") != "sha256:" + hashlib.sha256(manifest_raw).hexdigest(): @@ -95,10 +148,13 @@ def verify_global_osv_snapshot(database_path: Path, manifest_path: Path, *, mini checksum_name = checksum_ready.get("name") if not isinstance(checksum_name, str) or Path(checksum_name).name != checksum_name: raise ValueError("release ready marker checksum name is invalid") - checksum_raw = _read_bounded_regular(database_path.parent / checksum_name, "checksum") + checksum_raw = _read_bounded_at(directory_descriptor, checksum_name, "checksum") if checksum_ready.get("sha256") != "sha256:" + hashlib.sha256(checksum_raw).hexdigest(): raise ValueError("release ready marker checksum mismatch") - database = VulnerabilityDatabase.read_only(str(database_path)) + expected_checksum = f"{actual_database_digest.removeprefix('sha256:')} {database_path.name}\n".encode("ascii") + if checksum_raw != expected_checksum: + raise ValueError("checksum sidecar content does not bind the database") + database = VulnerabilityDatabase.from_file_descriptor(database_descriptor, readonly=True) try: integrity = database.connection.execute("PRAGMA integrity_check").fetchone()[0] foreign_key_errors = len(database.connection.execute("PRAGMA foreign_key_check").fetchall()) @@ -137,7 +193,7 @@ def verify_global_osv_snapshot(database_path: Path, manifest_path: Path, *, mini "advisory_count": actual_manifest["advisory_count"], "affected_package_count": actual_manifest["affected_package_count"], "content_digest": actual_manifest["content_digest"], - "database_bytes": database_path.stat().st_size, + "database_bytes": database_stat.st_size, "database_sha256": actual_database_digest, "foreign_key_errors": foreign_key_errors, "integrity_check": integrity, @@ -149,6 +205,39 @@ def verify_global_osv_snapshot(database_path: Path, manifest_path: Path, *, mini database.close() +def verify_global_osv_snapshot(database_path: Path, manifest_path: Path, *, minimum_records: int = 1) -> dict[str, Any]: + if minimum_records <= 0: + raise ValueError("minimum_records must be positive") + if database_path.parent.resolve() != manifest_path.parent.resolve(): + raise ValueError("database and manifest must share one directory") + directory_descriptor = os.open( + database_path.parent, + os.O_RDONLY | getattr(os, "O_DIRECTORY", 0) | getattr(os, "O_CLOEXEC", 0) | getattr(os, "O_NOFOLLOW", 0), + ) + database_descriptor = -1 + try: + database_descriptor = os.open( + database_path.name, + os.O_RDONLY | getattr(os, "O_CLOEXEC", 0) | getattr(os, "O_NOFOLLOW", 0), + dir_fd=directory_descriptor, + ) + database_stat = os.fstat(database_descriptor) + if not stat.S_ISREG(database_stat.st_mode) or database_stat.st_size <= 0: + raise ValueError("database must be a non-empty regular file") + return _verify_global_osv_snapshot_pinned( + database_path, + manifest_path, + directory_descriptor, + database_descriptor, + database_stat, + minimum_records=minimum_records, + ) + finally: + if database_descriptor >= 0: + os.close(database_descriptor) + os.close(directory_descriptor) + + def main() -> int: parser = argparse.ArgumentParser() parser.add_argument("--database", type=Path, required=True) diff --git a/src/__main__.py b/src/__main__.py index 4c18b13..2694880 100644 --- a/src/__main__.py +++ b/src/__main__.py @@ -13,7 +13,7 @@ from .safeio import write_private_atomic from . import __version__ -_DEFAULT_GLOBAL_DATABASE = "~/.local/share/coderisktools/vuln-db/global-osv.sqlite" +_DEFAULT_GLOBAL_DATABASE = "~/.local/share/coderisktools/vuln-db/osv-partial.sqlite" def main(): @@ -153,7 +153,7 @@ def main(): bootstrap_seed_parser.add_argument("--signature-url", required=True, metavar="URL") bootstrap_seed_parser.add_argument("--destination", required=True, metavar="FILE") bootstrap_seed_parser.add_argument("--keyring", required=True, metavar="FILE") - bootstrap_global_parser = vuln_db_actions.add_parser("bootstrap-global", help="Install and activate the pinned signed global OSV SQLite ZIP") + bootstrap_global_parser = vuln_db_actions.add_parser("bootstrap-global", help="Install and activate the pinned signed partial OSV SQLite ZIP") bootstrap_global_parser.add_argument("--destination", default=_DEFAULT_GLOBAL_DATABASE, metavar="FILE") bootstrap_global_parser.add_argument("--no-activate", action="store_true", help="Install as staged without activating it") explain_parser = vuln_db_actions.add_parser("explain", help="Explain one persisted vulnerability match") diff --git a/src/vulnerability/global_bootstrap.py b/src/vulnerability/global_bootstrap.py index b390bbf..8e9fb69 100644 --- a/src/vulnerability/global_bootstrap.py +++ b/src/vulnerability/global_bootstrap.py @@ -23,6 +23,23 @@ _SHA256_RE = re.compile(r"sha256:[0-9a-f]{64}") +def _file_identity(file_stat: os.stat_result) -> tuple[int, int]: + return file_stat.st_dev, file_stat.st_ino + + +def _unlink_if_owned(path: Path, expected: os.stat_result | None) -> bool: + if expected is None: + return False + try: + current = os.stat(path, follow_symlinks=False) + except FileNotFoundError: + return False + if _file_identity(current) != _file_identity(expected): + return False + path.unlink() + return True + + @contextmanager def _installation_lock(path: Path) -> Iterator[None]: descriptor = os.open(path, os.O_RDWR | os.O_CREAT | getattr(os, "O_CLOEXEC", 0) | getattr(os, "O_NOFOLLOW", 0), 0o600) @@ -116,9 +133,12 @@ def validate_signed_global_manifest( } if not required.issubset(verified): raise ValueError("signed global manifest is incomplete") + profile_contract = (verified["profile"], verified["completeness"]) if ( - verified["profile"] != "global-osv" - or verified["completeness"] != "full-osv-source" + profile_contract not in { + ("global-osv", "full-osv-source"), + ("osv-partial", "partial-osv-prefix"), + } or verified["production_full_database"] is not False or verified["manifest_format"] != "compact-v1" ): @@ -133,7 +153,8 @@ def validate_signed_global_manifest( if not isinstance(member, str) or Path(member).name != member or not member.endswith(".sqlite"): raise ValueError("signed global manifest archive member is invalid") snapshot_id = verified["snapshot_id"] - if not isinstance(snapshot_id, str) or not snapshot_id.startswith("global-osv-"): + expected_snapshot_prefix = "global-osv-" if verified["profile"] == "global-osv" else "osv-partial-" + if not isinstance(snapshot_id, str) or not snapshot_id.startswith(expected_snapshot_prefix) or len(snapshot_id) > 128: raise ValueError("signed global manifest snapshot id is invalid") return dict(verified) @@ -200,10 +221,17 @@ def bootstrap_global_osv_asset( allowed_hosts: frozenset[str] = frozenset({"github.com", "objects.githubusercontent.com", "release-assets.githubusercontent.com"}), timeout: float = 120.0, activate: bool = True, + expected_profile: str = "osv-partial", + expected_completeness: str = "partial-osv-prefix", ) -> dict[str, object]: """Stream, verify, extract, and atomically install one pinned SQLite ZIP.""" if timeout <= 0 or timeout > 300: raise ValueError("bootstrap timeout must be in (0, 300]") + if (expected_profile, expected_completeness) not in { + ("global-osv", "full-osv-source"), + ("osv-partial", "partial-osv-prefix"), + }: + raise ValueError("expected release profile contract is invalid") target = Path(destination).expanduser() target.parent.mkdir(parents=True, exist_ok=True) lock_path = target.with_suffix(target.suffix + ".bootstrap.lock") @@ -220,10 +248,17 @@ def bootstrap_global_osv_asset( if not isinstance(manifest, dict) or not isinstance(envelope, dict): raise ValueError("global release metadata must be JSON objects") verified = validate_signed_global_manifest(manifest, envelope, trusted_keys) + if (verified["profile"], verified["completeness"]) != (expected_profile, expected_completeness): + raise ValueError("signed release does not match the expected profile contract") zip_fd, zip_name = tempfile.mkstemp(prefix=f".{target.name}.", suffix=".zip.tmp", dir=str(target.parent)) db_fd, db_name = tempfile.mkstemp(prefix=f".{target.name}.", suffix=".sqlite.tmp", dir=str(target.parent)) zip_temporary = Path(zip_name) database_temporary = Path(db_name) + zip_owned_identity = os.fstat(zip_fd) + database_owned_identity = os.fstat(db_fd) + zip_guard = os.dup(zip_fd) + database_guard = os.dup(db_fd) + target_owned_identity: os.stat_result | None = None installed = False try: with os.fdopen(zip_fd, "w+b") as zip_stream, os.fdopen(db_fd, "w+b") as database_stream: @@ -262,43 +297,49 @@ def bootstrap_global_osv_asset( os.link(database_temporary, target, follow_symlinks=False) target_identity = os.stat(target, follow_symlinks=False) if (final_identity.st_dev, final_identity.st_ino) != (target_identity.st_dev, target_identity.st_ino): - target.unlink() + _unlink_if_owned(target, final_identity) raise OSError("installed database identity mismatch") installed = True + target_owned_identity = final_identity directory_fd = os.open(target.parent, os.O_RDONLY | getattr(os, "O_DIRECTORY", 0)) try: os.fsync(directory_fd) finally: os.close(directory_fd) - zip_temporary.unlink() - database_temporary.unlink() + installed_digest = hashlib.sha256() + database_stream.seek(0) + while chunk := database_stream.read(1024 * 1024): + installed_digest.update(chunk) + database_stream.seek(0) + _unlink_if_owned(zip_temporary, zip_owned_identity) + _unlink_if_owned(database_temporary, database_owned_identity) + if _file_identity(os.stat(target, follow_symlinks=False)) != _file_identity(target_owned_identity): + raise OSError("installed database was replaced before bootstrap completion") return { "state": state, - "profile": "global-osv", - "completeness": "full-osv-source", + "profile": verified["profile"], + "completeness": verified["completeness"], "path": str(target), "snapshot_id": verified["snapshot_id"], - "database_sha256": verified["database_sha256"], + "database_sha256": "sha256:" + installed_digest.hexdigest(), + "release_database_sha256": verified["database_sha256"], "signing_key_id": envelope["key_id"], } except BaseException: if installed: - try: - target.unlink() - except FileNotFoundError: - pass - for temporary in (zip_temporary, database_temporary): - try: - temporary.unlink() - except FileNotFoundError: - pass + _unlink_if_owned(target, target_owned_identity) + _unlink_if_owned(zip_temporary, zip_owned_identity) + _unlink_if_owned(database_temporary, database_owned_identity) raise + finally: + os.close(zip_guard) + os.close(database_guard) DEFAULT_GLOBAL_OSV_RELEASE = { - "asset_url": "https://github.com/9batalion/coderisktools-scanner/releases/download/v3.1.0/coderisktools-vulndb-global-osv-2026-07-23.sqlite.zip", - "manifest_url": "https://github.com/9batalion/coderisktools-scanner/releases/download/v3.1.0/coderisktools-vulndb-global-osv-2026-07-23.manifest.json", - "signature_url": "https://github.com/9batalion/coderisktools-scanner/releases/download/v3.1.0/coderisktools-vulndb-global-osv-2026-07-23.manifest.sig.json", + "asset_url": "https://github.com/9batalion/coderisktools-scanner/releases/download/v3.1.0/coderisktools-vulndb-osv-partial-2026-07-23.sqlite.zip", + "manifest_url": "https://github.com/9batalion/coderisktools-scanner/releases/download/v3.1.0/coderisktools-vulndb-osv-partial-2026-07-23.release.manifest.json", + "signature_url": "https://github.com/9batalion/coderisktools-scanner/releases/download/v3.1.0/coderisktools-vulndb-osv-partial-2026-07-23.release.manifest.sig.json", "key_id": "coderisktools-vulndb-2026", "public_key": bytes.fromhex("5fd70b01c5ef2b0317765fe188f5ef136527d8bceefcff37b0adfc40c4fbf235"), } diff --git a/tests/test_global_bootstrap.py b/tests/test_global_bootstrap.py index 16f0b62..595d839 100644 --- a/tests/test_global_bootstrap.py +++ b/tests/test_global_bootstrap.py @@ -13,6 +13,7 @@ from src.vulnerability.global_bootstrap import ( _installation_lock, _extract_single_database, + _unlink_if_owned, bootstrap_global_osv_asset, validate_signed_global_manifest, ) @@ -27,6 +28,20 @@ def test_installation_lock_rejects_parallel_bootstrap(self): with _installation_lock(lock): self.fail("parallel lock unexpectedly succeeded") + def test_bootstrap_cleanup_never_deletes_replacement_inode(self): + with tempfile.TemporaryDirectory() as directory: + target = Path(directory) / "database.sqlite" + target.write_text("owned") + owned_descriptor = os.open(target, os.O_RDONLY) + try: + owned_identity = os.fstat(owned_descriptor) + target.unlink() + target.write_text("attacker replacement") + self.assertFalse(_unlink_if_owned(target, owned_identity)) + self.assertEqual(target.read_text(), "attacker replacement") + finally: + os.close(owned_descriptor) + def test_database_descriptor_pins_inode_across_path_replacement(self): with tempfile.TemporaryDirectory() as directory: root = Path(directory) @@ -105,6 +120,27 @@ def test_signed_global_manifest_is_profile_bound(self): actual = validate_signed_global_manifest(manifest, envelope, {"global-key": b"k" * 32}) self.assertEqual(actual, manifest) + def test_signed_partial_manifest_is_explicitly_profile_bound(self): + manifest = { + "asset_sha256": "sha256:" + "a" * 64, + "archive_member": "database.sqlite", + "completeness": "partial-osv-prefix", + "database_bytes": 10, + "database_sha256": "sha256:" + "b" * 64, + "manifest_format": "compact-v1", + "production_full_database": False, + "profile": "osv-partial", + "snapshot_id": "osv-partial-2026-07-23", + } + envelope = {"key_id": "global-key"} + with patch("src.vulnerability.global_bootstrap.verify_manifest", return_value=dict(manifest)): + actual = validate_signed_global_manifest(manifest, envelope, {"global-key": b"k" * 32}) + self.assertEqual(actual, manifest) + mismatched = dict(manifest, completeness="full-osv-source") + with patch("src.vulnerability.global_bootstrap.verify_manifest", return_value=dict(mismatched)): + with self.assertRaises(ValueError): + validate_signed_global_manifest(mismatched, envelope, {"global-key": b"k" * 32}) + def test_bootstrap_downloads_zip_verifies_database_and_activates(self): with tempfile.TemporaryDirectory() as directory: root = Path(directory) @@ -180,12 +216,24 @@ def asset_download(_url, destination, _hosts, **_kwargs): patch("src.vulnerability.global_bootstrap._download_to_file", side_effect=asset_download), patch("src.vulnerability.global_bootstrap.verify_manifest", return_value=manifest), ): + rejected_destination = root / "rejected-full.sqlite" + with self.assertRaises(ValueError): + bootstrap_global_osv_asset( + "https://github.com/database.zip", + "https://github.com/manifest", + "https://github.com/sig", + rejected_destination, + trusted_keys={"global-key": public_key}, + ) + self.assertFalse(rejected_destination.exists()) result = bootstrap_global_osv_asset( "https://github.com/database.zip", "https://github.com/manifest", "https://github.com/sig", destination, trusted_keys={"global-key": public_key}, + expected_profile="global-osv", + expected_completeness="full-osv-source", ) self.assertEqual(result["state"], "active") with VulnerabilityDatabase.read_only(str(destination)) as database: diff --git a/tests/test_global_osv_snapshot.py b/tests/test_global_osv_snapshot.py index 0ee3e12..2a538e2 100644 --- a/tests/test_global_osv_snapshot.py +++ b/tests/test_global_osv_snapshot.py @@ -1,5 +1,6 @@ import hashlib import json +import os import struct import tempfile import unittest @@ -8,6 +9,7 @@ from unittest.mock import patch from scripts.build_global_osv_vulndb import build_global_osv_snapshot +from scripts.finalize_partial_osv_vulndb import _unlink_owned_at, finalize_partial_osv_snapshot from scripts.verify_global_osv_vulndb import verify_global_osv_snapshot from src.vulnerability.database import VulnerabilityDatabase from src.vulnerability.full_snapshot import import_osv_zip, validate_zip_central_directory @@ -217,6 +219,154 @@ def test_non_sentinel_zip64_metadata_cannot_bypass_member_limit(self): with self.assertRaises(ValueError): validate_zip_central_directory(stream, max_members=1) + def test_owned_cleanup_never_deletes_replacement_inode(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + owned = root / "published.json" + owned.write_text("owned") + owned_identity = owned.stat() + owned.unlink() + owned.write_text("attacker replacement") + directory_descriptor = os.open(root, os.O_RDONLY | getattr(os, "O_DIRECTORY", 0)) + try: + self.assertFalse(_unlink_owned_at(directory_descriptor, owned.name, owned_identity)) + finally: + os.close(directory_descriptor) + self.assertEqual(owned.read_text(), "attacker replacement") + + def test_partial_finalizer_accepts_only_verified_archive_prefix_and_labels_it_partial(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + archive = root / "osv.zip" + source_manifest = root / "source.json" + partial_database = root / "partial.tmp.sqlite" + output = root / "partial.sqlite" + manifest_output = root / "partial.manifest.json" + sha_output = root / "partial.sha256" + records = [self._record(f"OSV-{index}", f"package-{index}") for index in range(1, 4)] + with zipfile.ZipFile(archive, "w", compression=zipfile.ZIP_DEFLATED) as zipped: + for record in records: + zipped.writestr(f"{record['id']}.json", json.dumps(record)) + with zipfile.ZipFile(archive) as zipped: + uncompressed = sum(info.file_size for info in zipped.infolist()) + source_manifest.write_text(json.dumps({ + "sha256": "sha256:" + hashlib.sha256(archive.read_bytes()).hexdigest(), + "records": 3, + "uncompressed_bytes": uncompressed, + "url": "https://example.invalid/osv.zip", + })) + with VulnerabilityDatabase(str(partial_database)) as database: + database.import_osv_records(records[:2], source_record_mode="digest-only", correlate_aliases=False) + original_partial_digest = hashlib.sha256(partial_database.read_bytes()).hexdigest() + manifest = finalize_partial_osv_snapshot( + archive, + source_manifest, + partial_database, + output, + manifest_output, + sha_output, + snapshot_id="osv-partial-test", + minimum_records=2, + maximum_records=2, + maximum_database_bytes=100_000_000, + reserve_free_bytes=1_000_000, + ) + self.assertEqual(manifest["profile"], "osv-partial") + self.assertEqual(manifest["completeness"], "partial-osv-prefix") + self.assertFalse(manifest["production_full_database"]) + self.assertEqual(manifest["sources"]["osv-global"]["records"], 2) + self.assertEqual(manifest["sources"]["osv-global"]["available_records"], 3) + report = verify_global_osv_snapshot(output, manifest_output, minimum_records=2) + self.assertEqual(report["advisory_count"], 2) + self.assertEqual(report["profile"], "osv-partial") + self.assertEqual(hashlib.sha256(partial_database.read_bytes()).hexdigest(), original_partial_digest) + with VulnerabilityDatabase.read_only(str(partial_database)) as original: + self.assertEqual(original.connection.execute("SELECT COUNT(*) FROM snapshots").fetchone()[0], 0) + bogus_checksum = b"NOT A DATABASE CHECKSUM\n" + sha_output.write_bytes(bogus_checksum) + ready_path = output.with_suffix(output.suffix + ".ready.json") + ready = json.loads(ready_path.read_text()) + ready["checksum"]["sha256"] = "sha256:" + hashlib.sha256(bogus_checksum).hexdigest() + ready_path.write_text(json.dumps(ready)) + with self.assertRaises(ValueError): + verify_global_osv_snapshot(output, manifest_output, minimum_records=2) + + def test_partial_finalizer_rejects_database_that_is_not_archive_prefix(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + archive = root / "osv.zip" + source_manifest = root / "source.json" + partial_database = root / "partial.tmp.sqlite" + output = root / "partial.sqlite" + manifest_output = root / "partial.manifest.json" + sha_output = root / "partial.sha256" + records = [self._record(f"OSV-{index}", f"package-{index}") for index in range(1, 3)] + with zipfile.ZipFile(archive, "w", compression=zipfile.ZIP_DEFLATED) as zipped: + for record in records: + zipped.writestr(f"{record['id']}.json", json.dumps(record)) + with zipfile.ZipFile(archive) as zipped: + uncompressed = sum(info.file_size for info in zipped.infolist()) + source_manifest.write_text(json.dumps({ + "sha256": "sha256:" + hashlib.sha256(archive.read_bytes()).hexdigest(), + "records": 2, + "uncompressed_bytes": uncompressed, + })) + with VulnerabilityDatabase(str(partial_database)) as database: + database.import_osv_records([records[1]], source_record_mode="digest-only", correlate_aliases=False) + with self.assertRaises(ValueError): + finalize_partial_osv_snapshot( + archive, + source_manifest, + partial_database, + output, + manifest_output, + sha_output, + snapshot_id="osv-partial-mismatch", + minimum_records=1, + maximum_records=1, + maximum_database_bytes=100_000_000, + reserve_free_bytes=1_000_000, + ) + self.assertFalse(output.exists()) + with VulnerabilityDatabase.read_only(str(partial_database)) as database: + self.assertEqual(database.connection.execute("SELECT COUNT(*) FROM snapshots").fetchone()[0], 0) + + def test_partial_finalizer_rejects_tampered_normalized_state_without_mutating_input(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + records = [ + {"id": "OSV-1", "summary": "one", "affected": []}, + {"id": "OSV-2", "summary": "two", "affected": []}, + {"id": "OSV-3", "summary": "three", "affected": []}, + ] + archive = root / "all.zip" + with zipfile.ZipFile(archive, "w") as zipped: + for index, record in enumerate(records, 1): + zipped.writestr(f"ecosystem/{index}.json", json.dumps(record)) + source_manifest = root / "source.json" + source_manifest.write_text(json.dumps({ + "records": 3, + "sha256": "sha256:" + hashlib.sha256(archive.read_bytes()).hexdigest(), + "uncompressed_bytes": sum(len(json.dumps(record).encode()) for record in records), + "url": "https://storage.googleapis.com/osv-vulnerabilities/all.zip", + })) + partial_database = root / "partial.sqlite" + with VulnerabilityDatabase(str(partial_database)) as database: + database.import_osv_records(records[:2], source_record_mode="digest-only", correlate_aliases=False) + database.connection.execute("UPDATE advisories SET summary = 'TAMPERED' WHERE id = 'OSV-1'") + database.connection.commit() + original_digest = hashlib.sha256(partial_database.read_bytes()).hexdigest() + output = root / "release.sqlite" + with self.assertRaises(ValueError): + finalize_partial_osv_snapshot( + archive, source_manifest, partial_database, output, + root / "release.manifest.json", root / "release.sqlite.sha256", + snapshot_id="osv-partial-test", minimum_records=2, maximum_records=2, + reserve_free_bytes=1, + ) + self.assertEqual(hashlib.sha256(partial_database.read_bytes()).hexdigest(), original_digest) + self.assertFalse(output.exists()) + def test_builder_rejects_duplicate_advisory_ids_without_publishing_artifacts(self): with tempfile.TemporaryDirectory() as directory: root = Path(directory)