From fdd4759dc07e87e1dff0f60125aa6972ccb50b24 Mon Sep 17 00:00:00 2001 From: CodeRiskTools Date: Thu, 23 Jul 2026 20:37:10 +0000 Subject: [PATCH 1/2] feat(vuln-db): add signed global OSV bootstrap --- CHANGELOG.md | 3 + README.md | 18 +- data/vulnerability-seed/release-keyring.json | 3 +- docs/GLOBAL-OSV-SNAPSHOT.md | 81 +++++ docs/RELEASE-NOTES-3.1.0.md | 8 +- scripts/build_global_osv_vulndb.py | 364 +++++++++++++++++++ scripts/package_global_osv_release.py | 238 ++++++++++++ scripts/verify_global_osv_vulndb.py | 163 +++++++++ src/__main__.py | 35 +- src/vulnerability/bootstrap.py | 46 ++- src/vulnerability/database.py | 229 ++++++++++-- src/vulnerability/full_snapshot.py | 259 +++++++++++++ src/vulnerability/global_bootstrap.py | 304 ++++++++++++++++ tests/test_global_bootstrap.py | 196 ++++++++++ tests/test_global_osv_snapshot.py | 289 +++++++++++++++ 15 files changed, 2176 insertions(+), 60 deletions(-) create mode 100644 docs/GLOBAL-OSV-SNAPSHOT.md create mode 100644 scripts/build_global_osv_vulndb.py create mode 100644 scripts/package_global_osv_release.py create mode 100644 scripts/verify_global_osv_vulndb.py create mode 100644 src/vulnerability/full_snapshot.py create mode 100644 src/vulnerability/global_bootstrap.py create mode 100644 tests/test_global_bootstrap.py create mode 100644 tests/test_global_osv_snapshot.py diff --git a/CHANGELOG.md b/CHANGELOG.md index a6863dd..63855c7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,11 +13,14 @@ All notable changes to `coderisktools-scanner` are documented here. - added bounded public feed adapters and explicit provenance/quality reports without claiming full-feed coverage; - added a real partial `seed` snapshot with 187 advisories, 378 affected-package rows and seven represented OSV ecosystems; - added signed, pinned seed bootstrap and a separate explicit `--profile seed --apply` activation command. +- added a streamed, signed global OSV SQLite ZIP bootstrap that installs and activates the pinned database on first default vulnerability scan; +- added `vuln-db bootstrap-global` and `vuln scan --no-bootstrap` controls. ### Fixed - directory self-scan now skips SQLite database artifacts, preventing the real seed from tripping the scanner byte cap in CI; - bootstrap now verifies the detached Ed25519 manifest envelope, exact database SHA-256, SQLite integrity, foreign keys, snapshot identity and manifest counts before atomic installation. +- global ZIP bootstrap verifies the compressed asset digest, one-member extraction contract, expanded database digest, compact provenance manifest and staged snapshot before local activation. ### Seed boundary diff --git a/README.md b/README.md index f64911e..925da8b 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,8 @@ # CodeRiskTools Secret Scanner Engine -**Local-first, offline-by-default scanner for secret-like values, risky configuration changes and opt-in local vulnerability analysis.** +**Local-first scanner for secret-like values, risky configuration changes and opt-in local vulnerability analysis.** -CodeRiskTools Secret Scanner Engine is MIT licensed and has no runtime dependencies. It scans diffs, staged changes, local directories and bounded Git history without executing target-project code. Vulnerability analysis is a separate, explicitly selected local SQLite/SBOM path. +CodeRiskTools Secret Scanner Engine is MIT licensed and has no runtime dependencies. It scans diffs, staged changes, local directories and bounded Git history without executing target-project code. Vulnerability analysis uses a local SQLite/SBOM path; the pinned signed global database can be installed automatically on its first use. > Evidence, not guarantees. A clean result is not proof that code is secure. Findings can contain false positives and false negatives. This tool is not a security audit, certification, legal opinion or compliance guarantee. @@ -22,7 +22,7 @@ CodeRiskTools Secret Scanner Engine is MIT licensed and has no runtime dependenc - local SQLite vulnerability snapshot scanning and reports; - snapshot reconciliation, verification, status, update, rollback, retention pruning and provenance fetch commands; - OpenVEX/CycloneDX VEX annotations, suppression and vulnerability baselines; -- no telemetry, no target-project execution and no network during ordinary scans. +- no telemetry or target-project execution; secret/config scans and vulnerability scans after database bootstrap perform no network I/O. The stable detector count excludes provisional candidates. See the detector backlog and source records in [`docs/STAGE6_SECRET_DETECTOR_BACKLOG.md`](docs/STAGE6_SECRET_DETECTOR_BACKLOG.md) and [`docs/STAGE8_CI_CD_BATCH1_SOURCES.md`](docs/STAGE8_CI_CD_BATCH1_SOURCES.md). @@ -34,7 +34,7 @@ This repository contains only the public Scanner flagship. It does not contain o - Python 3.10–3.13; - Git is required for `--staged` and `--git-history` modes; -- ordinary scanning does not require network access or third-party runtime packages. +- secret/config scanning does not require network access or third-party runtime packages; first-use default vulnerability scanning downloads a pinned signed database ZIP. From a checkout: @@ -247,12 +247,11 @@ The sidecar schema is `coderisktools.vulnerability.external-evidence-provenance` ## 4. Local vulnerability scanning -The vulnerability path requires an explicitly supplied local SQLite database with an active snapshot: +The vulnerability path uses a local SQLite database with an active snapshot. If `--database` is omitted and the default database is absent, the first run downloads, verifies, extracts and activates the pinned signed global OSV snapshot: ```bash secret-scanner vuln scan \ --root . \ - --database vulnerability.sqlite \ --format json ``` @@ -272,14 +271,15 @@ secret-scanner vuln scan \ Options: - `--root DIR` — local repository root; -- `--database FILE` — local regular SQLite database; +- `--database FILE` — local regular SQLite database; defaults to `~/.local/share/coderisktools/vuln-db/global-osv.sqlite`; +- `--no-bootstrap` — reject a missing default database instead of downloading it; - `--format {json,sarif,markdown,html,csv}`; - `--output FILE` — write the report atomically; - `--baseline FILE` — JSON format emits new/existing/resolved delta; - `--vex FILE` — local OpenVEX or CycloneDX VEX; - `--suppressions FILE` — strict local suppression document. -Matching is offline, active-snapshot-only and read-only. The database path cannot be a symlink, URL or non-regular file. +After first-use bootstrap, matching is offline, active-snapshot-only and read-only. The database path cannot be a symlink, URL or non-regular file. Manual installation is available as `secret-scanner vuln-db bootstrap-global`. ## 5. OSV feed import @@ -396,7 +396,7 @@ secret-scanner vuln-db prune \ ### Explicit allowlisted HTTPS fetch -This is the only `vuln-db` operation that performs network I/O, and it requires an explicit allowlist: +This explicit generic fetch operation requires an explicit allowlist. The separate `bootstrap-global` operation performs only its built-in, pinned, signature-verified GitHub Release download: ```bash secret-scanner vuln-db fetch \ diff --git a/data/vulnerability-seed/release-keyring.json b/data/vulnerability-seed/release-keyring.json index be197ca..41dc2c3 100644 --- a/data/vulnerability-seed/release-keyring.json +++ b/data/vulnerability-seed/release-keyring.json @@ -1,6 +1,7 @@ { "keys": { - "coderisktools-seed-2026": "64d79d903860fc16b2adb99d9a8ebe6a05540b9a9a2437d0062ba1c552a380a1" + "coderisktools-seed-2026": "64d79d903860fc16b2adb99d9a8ebe6a05540b9a9a2437d0062ba1c552a380a1", + "coderisktools-vulndb-2026": "5fd70b01c5ef2b0317765fe188f5ef136527d8bceefcff37b0adfc40c4fbf235" }, "schema": "coderisktools.rule-keyring", "version": 1 diff --git a/docs/GLOBAL-OSV-SNAPSHOT.md b/docs/GLOBAL-OSV-SNAPSHOT.md new file mode 100644 index 0000000..d46e992 --- /dev/null +++ b/docs/GLOBAL-OSV-SNAPSHOT.md @@ -0,0 +1,81 @@ +# Global OSV SQLite snapshot + +The global OSV builder imports the pinned OSV `all.zip` directly into a temporary SQLite database without extracting the archive to disk. + +## Scope and naming + +A successful artifact is labeled: + +- `profile: global-osv` +- `completeness: full-osv-source` +- `production_full_database: false` + +`full-osv-source` means that every accepted JSON member from the pinned global OSV archive was processed. It does **not** mean complete Core coverage, complete vulnerability coverage, or proof that an unmatched component is safe. GHSA, KEV, EPSS, NVD, distro feeds, and other enrichments have separate provenance and completeness requirements. + +## Space-efficient evidence mode + +The large snapshot uses `source_record_mode: digest-only`: + +- normalized advisory, alias, package, range, version, reference, and matching data remain in SQLite; +- valid GIT/CVE records without an OSV `package` object remain as advisories and digest-backed source evidence; their affected entries are counted as `unmapped_affected_entries` and are not package-matchable; +- every source record keeps its source/native ID, SHA-256 content digest, record fingerprint, and advisory mapping; +- the duplicate full source JSON payload is omitted from SQLite; +- the exact pinned ZIP digest and source URL are retained in the manifest. + +This avoids storing the same multi-gigabyte JSON payload twice while preserving verifiable provenance. The pinned ZIP remains the raw source artifact. + +## Build + +```bash +python scripts/build_global_osv_vulndb.py \ + --archive /path/to/all.zip \ + --source-manifest /path/to/manifest.json \ + --output /path/to/coderisktools-vulndb-global-osv.sqlite \ + --manifest-output /path/to/coderisktools-vulndb-global-osv.manifest.json \ + --sha256-output /path/to/coderisktools-vulndb-global-osv.sqlite.sha256 \ + --snapshot-id global-osv-YYYY-MM-DD +``` + +The builder: + +1. rejects symlinks, unsafe paths, duplicate members, encrypted files, non-JSON payloads, oversized members, and oversized expanded archives; +2. verifies the pinned archive SHA-256 before import; +3. imports bounded batches with no archive extraction; +4. rejects the build if the configured import-error threshold is exceeded; +5. checks SQLite integrity and foreign keys; +6. generates a bounded-memory `compact-v1` content digest; +7. publishes manifest and checksum first and the SQLite readiness artifact last with no-overwrite hard links; +8. leaves the embedded snapshot **staged**, never active. + +## GitHub Release ZIP and first-run installation + +The repository never stores the multi-gigabyte database in Git history. Release `v3.1.0` publishes: + +- `coderisktools-vulndb-global-osv-2026-07-23.sqlite.zip`; +- `coderisktools-vulndb-global-osv-2026-07-23.manifest.json`; +- `coderisktools-vulndb-global-osv-2026-07-23.manifest.sig.json`. + +The ZIP contains exactly one SQLite member. On the first `vuln scan` invocation, when the default database path does not exist, the scanner: + +1. downloads the pinned manifest and Ed25519 signature; +2. validates the embedded public key and profile contract; +3. streams the ZIP to disk with a 2 GiB compressed limit; +4. verifies the ZIP SHA-256; +5. validates the single-member ZIP contract and declared expanded size; +6. streams extraction with a 9 GiB database limit and SHA-256 verification; +7. runs SQLite integrity, foreign-key, compact-manifest, and snapshot quality gates; +8. atomically installs and locally activates the verified snapshot. + +Default location: + +```text +~/.local/share/coderisktools/vuln-db/global-osv.sqlite +``` + +Manual bootstrap: + +```bash +secret-scanner vuln-db bootstrap-global +``` + +Automatic network bootstrap can be disabled with `vuln scan --no-bootstrap`. diff --git a/docs/RELEASE-NOTES-3.1.0.md b/docs/RELEASE-NOTES-3.1.0.md index 6a2bcd5..2e6d513 100644 --- a/docs/RELEASE-NOTES-3.1.0.md +++ b/docs/RELEASE-NOTES-3.1.0.md @@ -1,6 +1,6 @@ # CodeRiskTools Scanner 3.1.0 — release notes -CodeRiskTools Scanner 3.1.0 adds a controlled, local-first vulnerability database workflow and publishes a real, verified **partial seed** snapshot for bootstrap and integration testing. +CodeRiskTools Scanner 3.1.0 adds a controlled, local-first vulnerability database workflow, a small verified **partial seed**, and a pinned signed global OSV SQLite ZIP for first-use installation. ## What is included @@ -12,6 +12,9 @@ CodeRiskTools Scanner 3.1.0 adds a controlled, local-first vulnerability databas - signed pinned bootstrap that installs seed as staged only; - explicit `vuln-db activate --profile seed --apply` activation; - real lodash `4.17.15` end-to-end matching evidence with stable fingerprints. +- streamed first-use global database bootstrap with ZIP/database SHA-256, Ed25519, SQLite integrity, foreign-key and compact-manifest verification; +- automatic installation to `~/.local/share/coderisktools/vuln-db/global-osv.sqlite` when the default database is missing; +- `vuln-db bootstrap-global` for an explicit installation and `vuln scan --no-bootstrap` to disable automatic network bootstrap. ## Release assets @@ -21,6 +24,7 @@ CodeRiskTools Scanner 3.1.0 adds a controlled, local-first vulnerability databas - SHA-256 sidecar; - Ed25519 signed manifest envelope (`.sig` JSON); - public release keyring. +- global OSV single-SQLite ZIP, detached manifest and Ed25519 signed manifest envelope. ## Verified seed facts @@ -37,6 +41,6 @@ CodeRiskTools Scanner 3.1.0 adds a controlled, local-first vulnerability databas ## Important limitations -This seed is not Core, Full, Production or Complete. It retains 129 exact-alias conflicts rather than heuristically merging advisories. A zero-finding seed scan is not proof that a project has no vulnerabilities. Updates remain user-triggered; ordinary scanner runs do not download feeds. +The seed is not Core, Full, Production or Complete. The global snapshot is labeled `full-osv-source`, not complete Core coverage or proof that an unmatched component is safe. It retains exact-alias conflicts rather than heuristically merging advisories. A zero-finding scan is not proof that a project has no vulnerabilities. Secret/config scans remain offline; the first default vulnerability scan downloads only the pinned signed release asset, and subsequent matching is local. A clean scanner result is not proof that code is secure. This release is not a security audit, certification, compliance guarantee or legal opinion. diff --git a/scripts/build_global_osv_vulndb.py b/scripts/build_global_osv_vulndb.py new file mode 100644 index 0000000..59b2e91 --- /dev/null +++ b/scripts/build_global_osv_vulndb.py @@ -0,0 +1,364 @@ +#!/usr/bin/env python3 +"""Build a staged, global-OSV SQLite snapshot without expanding the ZIP.""" +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import shutil +import stat +import tempfile +from pathlib import Path +from typing import Any + +from src.vulnerability.database import VulnerabilityDatabase +from src.vulnerability.full_snapshot import import_osv_zip + +_SOURCE_MANIFEST_LIMIT = 1024 * 1024 + + +def _sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as stream: + while True: + chunk = stream.read(1024 * 1024) + if not chunk: + break + digest.update(chunk) + return "sha256:" + digest.hexdigest() + + +def _sha256_descriptor(descriptor: int) -> str: + digest = hashlib.sha256() + os.lseek(descriptor, 0, os.SEEK_SET) + while True: + chunk = os.read(descriptor, 1024 * 1024) + if not chunk: + break + digest.update(chunk) + os.lseek(descriptor, 0, os.SEEK_SET) + return "sha256:" + digest.hexdigest() + + +def _same_file(left: os.stat_result, right: os.stat_result) -> bool: + return (left.st_dev, left.st_ino, left.st_size) == (right.st_dev, right.st_ino, right.st_size) + + +def _link_verified(source: Path, destination: Path) -> None: + descriptor = os.open(source, os.O_RDONLY | getattr(os, "O_CLOEXEC", 0) | getattr(os, "O_NOFOLLOW", 0)) + try: + identity = os.fstat(descriptor) + os.link(source, destination, follow_symlinks=False) + if not _same_file(identity, os.stat(destination, follow_symlinks=False)): + destination.unlink() + raise OSError("published artifact identity changed during no-overwrite link") + finally: + os.close(descriptor) + + +def _read_regular_json(path: Path) -> dict[str, Any]: + flags = os.O_RDONLY | getattr(os, "O_CLOEXEC", 0) | getattr(os, "O_NOFOLLOW", 0) + descriptor = os.open(path, flags) + try: + file_stat = os.fstat(descriptor) + if not stat.S_ISREG(file_stat.st_mode) or file_stat.st_size <= 0 or file_stat.st_size > _SOURCE_MANIFEST_LIMIT: + raise ValueError("source manifest must be a bounded regular file") + with os.fdopen(descriptor, "rb", closefd=False) as stream: + raw = stream.read(_SOURCE_MANIFEST_LIMIT + 1) + payload = json.loads(raw) + if not isinstance(payload, dict): + raise ValueError("source manifest must be a JSON object") + return payload + finally: + os.close(descriptor) + + +def _write_json_atomic(path: Path, payload: dict[str, Any]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + fd, name = tempfile.mkstemp(prefix=f".{path.name}.", suffix=".tmp", dir=str(path.parent)) + temporary = Path(name) + try: + with os.fdopen(fd, "w", encoding="utf-8") as stream: + stream.write(json.dumps(payload, indent=2, sort_keys=True) + "\n") + stream.flush() + os.fsync(stream.fileno()) + os.replace(temporary, path) + except BaseException: + try: + temporary.unlink() + except FileNotFoundError: + pass + raise + + +def _write_private_temp(parent: Path, prefix: str, payload: bytes) -> Path: + fd, name = tempfile.mkstemp(prefix=prefix, suffix=".tmp", dir=str(parent)) + path = Path(name) + try: + with os.fdopen(fd, "wb") as stream: + stream.write(payload) + stream.flush() + os.fsync(stream.fileno()) + return path + except BaseException: + try: + path.unlink() + except FileNotFoundError: + pass + raise + + +def _fsync_directory(path: Path) -> None: + descriptor = os.open(path, os.O_RDONLY | getattr(os, "O_DIRECTORY", 0)) + try: + os.fsync(descriptor) + finally: + os.close(descriptor) + + +def build_global_osv_snapshot( + archive: Path, + source_manifest: Path, + output: Path, + manifest_output: Path, + sha256_output: Path, + *, + snapshot_id: str, + maximum_database_bytes: int = 8_500_000_000, + reserve_free_bytes: int = 3_000_000_000, + max_import_errors: int = 0, +) -> dict[str, Any]: + ready_output = output.with_suffix(output.suffix + ".ready.json") + if len({output.parent.resolve(), manifest_output.parent.resolve(), sha256_output.parent.resolve()}) != 1: + raise ValueError("database, manifest, and checksum must share one publication directory") + if not snapshot_id or len(snapshot_id) > 128: + raise ValueError("snapshot_id is required and must be bounded") + if maximum_database_bytes <= 0 or reserve_free_bytes <= 0 or max_import_errors < 0: + raise ValueError("build limits are invalid") + output.parent.mkdir(parents=True, exist_ok=True) + for path in (output, manifest_output, sha256_output, ready_output): + if path.exists() or path.is_symlink(): + raise FileExistsError(f"refusing to overwrite output: {path}") + source = _read_regular_json(source_manifest) + expected_digest = source.get("sha256") + expected_records = source.get("records") + expected_uncompressed = source.get("uncompressed_bytes") + if ( + not isinstance(expected_digest, str) + or not expected_digest.startswith("sha256:") + or len(expected_digest) != 71 + or type(expected_records) is not int + or expected_records <= 0 + or type(expected_uncompressed) is not int + or expected_uncompressed <= 0 + ): + raise ValueError("source manifest lacks pinned digest/count/size") + required_free = maximum_database_bytes + reserve_free_bytes + free_bytes = shutil.disk_usage(output.parent).free + if free_bytes < required_free: + raise OSError(f"insufficient free disk space: {free_bytes} < {required_free}") + fd, temporary_name = tempfile.mkstemp(prefix=f".{output.name}.", suffix=".tmp", dir=str(output.parent)) + os.close(fd) + temporary = Path(temporary_name) + temporary.unlink() + temporary_manifest: Path | None = None + temporary_sha: Path | None = None + temporary_ready: Path | None = None + published: list[Path] = [] + progress_path = output.with_suffix(output.suffix + ".progress.json") + error_path = output.with_suffix(output.suffix + ".error.json") + try: + def progress(members_seen: int, advisories_imported: int) -> None: + database_bytes = temporary.stat().st_size if temporary.exists() else 0 + current_free = shutil.disk_usage(output.parent).free + if database_bytes > maximum_database_bytes: + raise OSError("SQLite database exceeded the configured byte limit") + if current_free < reserve_free_bytes: + raise OSError("free disk reserve was exhausted during build") + _write_json_atomic(progress_path, { + "advisories_imported": advisories_imported, + "database_bytes": database_bytes, + "free_bytes": current_free, + "members_seen": members_seen, + "snapshot_id": snapshot_id, + "state": "building", + }) + + with VulnerabilityDatabase(str(temporary)) as database: + database.connection.execute("PRAGMA journal_mode=OFF") + database.connection.execute("PRAGMA synchronous=OFF") + database.connection.execute("PRAGMA temp_store=FILE") + database.connection.execute("PRAGMA cache_size=-65536") + report = import_osv_zip( + database, + archive, + expected_archive_sha256=expected_digest, + expected_payload_members=expected_records, + expected_uncompressed_bytes=expected_uncompressed, + max_errors=max_import_errors, + source_record_mode="digest-only", + progress=progress, + ) + if report.error_count > max_import_errors: + raise ValueError(f"OSV import errors exceeded limit: {report.error_count} > {max_import_errors}") + database.correlate_aliases() + advisory_count = database.advisory_count() + source_record_count = int(database.connection.execute( + "SELECT COUNT(*) FROM source_records WHERE source_id = 'osv'" + ).fetchone()[0]) + if not ( + report.members_seen == expected_records + and report.advisories_imported == expected_records + and advisory_count == expected_records + and source_record_count == expected_records + ): + raise ValueError("OSV completeness gate failed for members/advisories/source records") + database.connection.execute( + "INSERT INTO source_snapshots " + "(snapshot_id, source_id, content_digest, observed_at, record_count, status, metadata_json) " + "VALUES (?, 'osv-global', ?, NULL, ?, 'complete', ?)", + ( + snapshot_id, + expected_digest, + report.advisories_imported, + json.dumps({ + "archive_bytes": report.archive_bytes, + "declared_uncompressed_bytes": report.declared_uncompressed_bytes, + "source_record_mode": "digest-only", + "url": source.get("url"), + }, sort_keys=True, separators=(",", ":")), + ), + ) + database.connection.execute( + "INSERT INTO quality_metrics(snapshot_id, metric_name, metric_value, details_json) VALUES (?, 'import_errors', ?, '{}')", + (snapshot_id, float(report.error_count)), + ) + database.connection.execute( + "INSERT INTO quality_metrics(snapshot_id, metric_name, metric_value, details_json) VALUES (?, 'unmapped_affected_entries', ?, ?)", + (snapshot_id, float(report.unmapped_affected_entries), json.dumps({"reason": "OSV affected entry has no package ecosystem/name"}, sort_keys=True)), + ) + database.connection.commit() + integrity = database.connection.execute("PRAGMA integrity_check").fetchone()[0] + foreign_key_errors = len(database.connection.execute("PRAGMA foreign_key_check").fetchall()) + if integrity != "ok" or foreign_key_errors: + raise ValueError("SQLite integrity or foreign-key verification failed") + sources = { + "osv-global": { + "archive_bytes": report.archive_bytes, + "declared_uncompressed_bytes": report.declared_uncompressed_bytes, + "members_seen": report.members_seen, + "records": report.advisories_imported, + "sha256": expected_digest, + "status": "complete", + "unmapped_affected_entries": report.unmapped_affected_entries, + "url": source.get("url"), + } + } + provenance = { + "profile": "global-osv", + "completeness": "full-osv-source", + "production_full_database": False, + "snapshot_id": snapshot_id, + "source_digest": expected_digest, + "source_record_mode": "digest-only", + "sources": sources, + } + manifest = database.build_compact_snapshot_manifest(provenance) + manifest.update(provenance) + manifest["quality"] = { + "foreign_key_errors": foreign_key_errors, + "import_errors": report.error_count, + "integrity_check": integrity, + "unmapped_affected_entries": report.unmapped_affected_entries, + } + database.stage_snapshot(snapshot_id, expected_digest, manifest) + database_descriptor = os.open( + temporary, + os.O_RDONLY | getattr(os, "O_CLOEXEC", 0) | getattr(os, "O_NOFOLLOW", 0), + ) + try: + database_identity = os.fstat(database_descriptor) + database_bytes = database_identity.st_size + current_free = shutil.disk_usage(output.parent).free + if database_bytes > maximum_database_bytes or current_free < reserve_free_bytes: + raise OSError("final SQLite size/free-space gate failed") + os.fsync(database_descriptor) + database_digest = _sha256_descriptor(database_descriptor) + manifest["database_bytes"] = database_bytes + manifest["database_sha256"] = database_digest + manifest_bytes = (json.dumps(manifest, indent=2, sort_keys=True) + "\n").encode("utf-8") + sha_bytes = f"{database_digest.removeprefix('sha256:')} {output.name}\n".encode("ascii") + ready = { + "schema": "coderisktools.vulnerability.release-set-ready.v1", + "snapshot_id": snapshot_id, + "database": {"name": output.name, "bytes": database_bytes, "sha256": database_digest}, + "manifest": {"name": manifest_output.name, "sha256": "sha256:" + hashlib.sha256(manifest_bytes).hexdigest()}, + "checksum": {"name": sha256_output.name, "sha256": "sha256:" + hashlib.sha256(sha_bytes).hexdigest()}, + } + ready_bytes = (json.dumps(ready, indent=2, sort_keys=True) + "\n").encode("utf-8") + temporary_manifest = _write_private_temp(output.parent, f".{manifest_output.name}.", manifest_bytes) + temporary_sha = _write_private_temp(output.parent, f".{sha256_output.name}.", sha_bytes) + temporary_ready = _write_private_temp(output.parent, f".{ready_output.name}.", ready_bytes) + _link_verified(temporary_manifest, manifest_output) + published.append(manifest_output) + _link_verified(temporary_sha, sha256_output) + published.append(sha256_output) + os.link(temporary, output, follow_symlinks=False) + if not _same_file(database_identity, os.stat(output, follow_symlinks=False)): + output.unlink() + raise OSError("database identity changed between verification and publication") + published.append(output) + _link_verified(temporary_ready, ready_output) + published.append(ready_output) + _fsync_directory(output.parent) + finally: + os.close(database_descriptor) + for path in (temporary, temporary_manifest, temporary_sha, temporary_ready): + path.unlink() + if progress_path.exists(): + progress_path.unlink() + if error_path.exists(): + error_path.unlink() + return manifest + except BaseException as exc: + for path in reversed(published): + try: + path.unlink() + except FileNotFoundError: + pass + for path in (temporary, temporary_manifest, temporary_sha, temporary_ready): + if path is not None: + try: + path.unlink() + except FileNotFoundError: + pass + _write_json_atomic(error_path, {"error": f"{type(exc).__name__}: {exc}", "snapshot_id": snapshot_id, "state": "rejected"}) + raise + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--archive", type=Path, required=True) + parser.add_argument("--source-manifest", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--manifest-output", type=Path, required=True) + parser.add_argument("--sha256-output", type=Path, required=True) + parser.add_argument("--snapshot-id", required=True) + parser.add_argument("--max-import-errors", type=int, default=0) + args = parser.parse_args() + manifest = build_global_osv_snapshot( + args.archive, + args.source_manifest, + args.output, + args.manifest_output, + args.sha256_output, + snapshot_id=args.snapshot_id, + max_import_errors=args.max_import_errors, + ) + print(json.dumps(manifest, indent=2, sort_keys=True)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/package_global_osv_release.py b/scripts/package_global_osv_release.py new file mode 100644 index 0000000..cbad1d4 --- /dev/null +++ b/scripts/package_global_osv_release.py @@ -0,0 +1,238 @@ +#!/usr/bin/env python3 +"""Create and sign a deterministic single-SQLite GitHub Release ZIP.""" +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import stat +import tempfile +import zipfile +from pathlib import Path +from typing import Any + +from scripts.verify_global_osv_vulndb import _read_bounded_regular, verify_global_osv_snapshot +from src.vulnerability.manifest_signing import sign_manifest + +_GITHUB_RELEASE_ASSET_LIMIT = 2 * 1024 * 1024 * 1024 + + +def _sha256_stream(stream: Any) -> str: + digest = hashlib.sha256() + stream.seek(0) + while chunk := stream.read(1024 * 1024): + digest.update(chunk) + stream.seek(0) + return "sha256:" + digest.hexdigest() + + +def _same_file(left: os.stat_result, right: os.stat_result) -> bool: + return (left.st_dev, left.st_ino, left.st_size) == (right.st_dev, right.st_ino, right.st_size) + + +def _write_temp(parent: Path, prefix: str, content: bytes) -> Path: + descriptor, name = tempfile.mkstemp(prefix=prefix, suffix=".tmp", dir=str(parent)) + path = Path(name) + try: + with os.fdopen(descriptor, "wb") as output: + output.write(content) + output.flush() + os.fsync(output.fileno()) + return path + except BaseException: + try: + path.unlink() + except FileNotFoundError: + pass + raise + + +def _link_verified(source: Path, destination: Path) -> None: + descriptor = os.open(source, os.O_RDONLY | getattr(os, "O_CLOEXEC", 0) | getattr(os, "O_NOFOLLOW", 0)) + try: + identity = os.fstat(descriptor) + os.link(source, destination, follow_symlinks=False) + if not _same_file(identity, os.stat(destination, follow_symlinks=False)): + destination.unlink() + raise OSError("release artifact identity changed during publication") + finally: + os.close(descriptor) + + +def _fsync_directory(path: Path) -> None: + descriptor = os.open(path, os.O_RDONLY | getattr(os, "O_DIRECTORY", 0)) + try: + os.fsync(descriptor) + finally: + os.close(descriptor) + + +def _read_private_key(path: Path) -> bytes: + descriptor = os.open(path, os.O_RDONLY | getattr(os, "O_CLOEXEC", 0) | getattr(os, "O_NOFOLLOW", 0)) + try: + file_stat = os.fstat(descriptor) + if not stat.S_ISREG(file_stat.st_mode) or file_stat.st_size != 32: + raise ValueError("Ed25519 private key must be one 32-byte regular file") + key = os.read(descriptor, 33) + if len(key) != 32: + raise ValueError("Ed25519 private key changed while being read") + return key + finally: + os.close(descriptor) + + +def package_global_osv_release( + database: Path, + build_manifest_path: Path, + zip_output: Path, + release_manifest_output: Path, + signature_output: Path, + private_key_path: Path, + *, + key_id: str, + minimum_records: int = 800_000, +) -> dict[str, Any]: + output_parent = zip_output.parent + ready_output = zip_output.with_suffix(zip_output.suffix + ".ready.json") + if len({path.parent.resolve() for path in (zip_output, release_manifest_output, signature_output, ready_output)}) != 1: + raise ValueError("release outputs must share one directory") + for candidate in (zip_output, release_manifest_output, signature_output, ready_output): + if candidate.exists() or candidate.is_symlink(): + raise FileExistsError(f"refusing to overwrite release artifact: {candidate}") + output_parent.mkdir(parents=True, exist_ok=True) + database_descriptor = os.open(database, os.O_RDONLY | getattr(os, "O_CLOEXEC", 0) | getattr(os, "O_NOFOLLOW", 0)) + temporary_zip: Path | None = None + temporary_manifest: Path | None = None + temporary_signature: Path | None = None + temporary_ready: Path | None = None + published: list[Path] = [] + try: + database_identity = os.fstat(database_descriptor) + if not stat.S_ISREG(database_identity.st_mode): + raise ValueError("database must be a regular non-symlink file") + manifest_raw = _read_bounded_regular(build_manifest_path, "build manifest") + manifest_identity = os.stat(build_manifest_path, follow_symlinks=False) + private_key = _read_private_key(private_key_path) + verify_global_osv_snapshot(database, build_manifest_path, minimum_records=minimum_records) + if not _same_file(database_identity, os.stat(database, follow_symlinks=False)): + raise OSError("database changed during release verification") + if manifest_raw != _read_bounded_regular(build_manifest_path, "build manifest") or not _same_file(manifest_identity, os.stat(build_manifest_path, follow_symlinks=False)): + raise OSError("build manifest changed during release verification") + manifest = json.loads(manifest_raw) + with os.fdopen(os.dup(database_descriptor), "rb") as pinned_database: + if _sha256_stream(pinned_database) != manifest.get("database_sha256"): + raise ValueError("pinned database does not match the verified build manifest") + member = database.name + zip_descriptor, zip_name = tempfile.mkstemp(prefix=f".{zip_output.name}.", suffix=".tmp", dir=str(output_parent)) + os.close(zip_descriptor) + temporary_zip = Path(zip_name) + temporary_zip.unlink() + with zipfile.ZipFile(temporary_zip, "x", compression=zipfile.ZIP_DEFLATED, compresslevel=9, allowZip64=True) as archive: + info = zipfile.ZipInfo(member, date_time=(2026, 7, 23, 0, 0, 0)) + info.compress_type = zipfile.ZIP_DEFLATED + info.create_system = 3 + info.external_attr = 0o100644 << 16 + info.file_size = database_identity.st_size + with os.fdopen(os.dup(database_descriptor), "rb") as source, archive.open(info, "w", force_zip64=True) as target: + source.seek(0) + while chunk := source.read(1024 * 1024): + target.write(chunk) + if not _same_file(database_identity, os.fstat(database_descriptor)): + raise OSError("database changed while the release ZIP was created") + asset_bytes = temporary_zip.stat().st_size + if asset_bytes > _GITHUB_RELEASE_ASSET_LIMIT: + raise ValueError("SQLite ZIP exceeds the 2 GiB GitHub Release asset limit") + zip_read_descriptor = os.open( + temporary_zip, + os.O_RDONLY | getattr(os, "O_CLOEXEC", 0) | getattr(os, "O_NOFOLLOW", 0), + ) + try: + os.fsync(zip_read_descriptor) + with os.fdopen(os.dup(zip_read_descriptor), "rb") as stream: + asset_sha256 = _sha256_stream(stream) + finally: + os.close(zip_read_descriptor) + manifest.update({"archive_member": member, "asset_bytes": asset_bytes, "asset_sha256": asset_sha256}) + manifest_bytes = (json.dumps(manifest, indent=2, sort_keys=True) + "\n").encode("utf-8") + envelope = sign_manifest(manifest, key_id, private_key) + signature_bytes = (json.dumps(envelope, indent=2, sort_keys=True) + "\n").encode("utf-8") + ready = { + "schema": "coderisktools.vulnerability.github-release-ready.v1", + "snapshot_id": manifest["snapshot_id"], + "asset": {"name": zip_output.name, "bytes": asset_bytes, "sha256": asset_sha256}, + "manifest": {"name": release_manifest_output.name, "sha256": "sha256:" + hashlib.sha256(manifest_bytes).hexdigest()}, + "signature": {"name": signature_output.name, "sha256": "sha256:" + hashlib.sha256(signature_bytes).hexdigest()}, + } + ready_bytes = (json.dumps(ready, indent=2, sort_keys=True) + "\n").encode("utf-8") + temporary_manifest = _write_temp(output_parent, f".{release_manifest_output.name}.", manifest_bytes) + temporary_signature = _write_temp(output_parent, f".{signature_output.name}.", signature_bytes) + temporary_ready = _write_temp(output_parent, f".{ready_output.name}.", ready_bytes) + _link_verified(temporary_signature, signature_output) + published.append(signature_output) + _link_verified(temporary_manifest, release_manifest_output) + published.append(release_manifest_output) + _link_verified(temporary_zip, zip_output) + published.append(zip_output) + _fsync_directory(output_parent) + _link_verified(temporary_ready, ready_output) + published.append(ready_output) + _fsync_directory(output_parent) + for temporary in (temporary_zip, temporary_manifest, temporary_signature, temporary_ready): + temporary.unlink() + _fsync_directory(output_parent) + return { + "asset": str(zip_output), + "asset_bytes": asset_bytes, + "asset_sha256": asset_sha256, + "database_bytes": manifest["database_bytes"], + "database_sha256": manifest["database_sha256"], + "manifest": str(release_manifest_output), + "ready": str(ready_output), + "signature": str(signature_output), + "signing_key_id": key_id, + } + except BaseException: + for candidate in reversed(published): + try: + candidate.unlink() + except FileNotFoundError: + pass + for temporary in (temporary_zip, temporary_manifest, temporary_signature, temporary_ready): + if temporary is not None: + try: + temporary.unlink() + except FileNotFoundError: + pass + raise + finally: + os.close(database_descriptor) + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--database", type=Path, required=True) + parser.add_argument("--build-manifest", type=Path, required=True) + parser.add_argument("--zip-output", type=Path, required=True) + parser.add_argument("--release-manifest-output", type=Path, required=True) + parser.add_argument("--signature-output", type=Path, required=True) + parser.add_argument("--private-key", type=Path, required=True) + parser.add_argument("--key-id", default="coderisktools-vulndb-2026") + parser.add_argument("--minimum-records", type=int, default=800_000) + args = parser.parse_args() + report = package_global_osv_release( + args.database, + args.build_manifest, + args.zip_output, + args.release_manifest_output, + args.signature_output, + args.private_key, + key_id=args.key_id, + minimum_records=args.minimum_records, + ) + print(json.dumps(report, indent=2, sort_keys=True)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/verify_global_osv_vulndb.py b/scripts/verify_global_osv_vulndb.py new file mode 100644 index 0000000..c135330 --- /dev/null +++ b/scripts/verify_global_osv_vulndb.py @@ -0,0 +1,163 @@ +#!/usr/bin/env python3 +"""Independently verify a staged global OSV SQLite snapshot.""" +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import stat +from pathlib import Path +from typing import Any + +from src.vulnerability.database import VulnerabilityDatabase + +_MAX_METADATA_BYTES = 2 * 1024 * 1024 + + +def _read_bounded_regular(path: Path, label: str) -> bytes: + descriptor = os.open(path, os.O_RDONLY | getattr(os, "O_CLOEXEC", 0) | getattr(os, "O_NOFOLLOW", 0)) + try: + file_stat = os.fstat(descriptor) + if not stat.S_ISREG(file_stat.st_mode) or file_stat.st_size <= 0 or file_stat.st_size > _MAX_METADATA_BYTES: + raise ValueError(f"{label} must be a bounded regular file") + chunks: list[bytes] = [] + remaining = file_stat.st_size + while remaining: + chunk = os.read(descriptor, min(1024 * 1024, remaining)) + if not chunk: + break + chunks.append(chunk) + remaining -= len(chunk) + data = b"".join(chunks) + if len(data) != file_stat.st_size or os.read(descriptor, 1): + raise ValueError(f"{label} changed while being read") + return data + finally: + os.close(descriptor) + + +def _sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as stream: + while True: + chunk = stream.read(1024 * 1024) + if not chunk: + break + digest.update(chunk) + return "sha256:" + digest.hexdigest() + + +def verify_global_osv_snapshot(database_path: Path, manifest_path: Path, *, minimum_records: int = 1) -> dict[str, Any]: + for path, label in ((database_path, "database"), (manifest_path, "manifest")): + if path.is_symlink() or not path.is_file(): + raise ValueError(f"{label} must be a regular non-symlink file") + if minimum_records <= 0: + raise ValueError("minimum_records must be positive") + manifest_raw = _read_bounded_regular(manifest_path, "manifest") + manifest = json.loads(manifest_raw) + if manifest.get("profile") != "global-osv" or manifest.get("completeness") != "full-osv-source": + raise ValueError("unexpected global OSV manifest profile") + if manifest.get("production_full_database") is not False or manifest.get("source_record_mode") != "digest-only": + raise ValueError("global OSV manifest boundaries are invalid") + expected_database_digest = manifest.get("database_sha256") + actual_database_digest = _sha256(database_path) + if expected_database_digest != actual_database_digest: + raise ValueError("database SHA-256 mismatch") + if manifest.get("database_bytes") != database_path.stat().st_size: + raise ValueError("database size does not match manifest") + snapshot_id = manifest.get("snapshot_id") + source = manifest.get("sources", {}).get("osv-global", {}) + if not isinstance(snapshot_id, str) or not snapshot_id: + raise ValueError("snapshot_id is missing") + source_records_expected = source.get("records") + if ( + source.get("status") != "complete" + or type(source_records_expected) is not int + or source_records_expected < minimum_records + or source.get("members_seen") != source_records_expected + or manifest.get("advisory_count") != source_records_expected + or manifest.get("source_digest") != source.get("sha256") + or manifest.get("quality", {}).get("import_errors") != 0 + ): + raise ValueError("global OSV source record gate failed") + ready_path = database_path.with_suffix(database_path.suffix + ".ready.json") + ready_raw = _read_bounded_regular(ready_path, "release ready marker") + ready = json.loads(ready_raw) + if ready.get("schema") != "coderisktools.vulnerability.release-set-ready.v1" or ready.get("snapshot_id") != snapshot_id: + raise ValueError("release ready marker contract mismatch") + if ready.get("database") != {"name": database_path.name, "bytes": database_path.stat().st_size, "sha256": actual_database_digest}: + raise ValueError("release ready marker database mismatch") + manifest_ready = ready.get("manifest", {}) + if manifest_ready.get("name") != manifest_path.name or manifest_ready.get("sha256") != "sha256:" + hashlib.sha256(manifest_raw).hexdigest(): + raise ValueError("release ready marker manifest mismatch") + checksum_ready = ready.get("checksum", {}) + checksum_name = checksum_ready.get("name") + if not isinstance(checksum_name, str) or Path(checksum_name).name != checksum_name: + raise ValueError("release ready marker checksum name is invalid") + checksum_raw = _read_bounded_regular(database_path.parent / checksum_name, "checksum") + if checksum_ready.get("sha256") != "sha256:" + hashlib.sha256(checksum_raw).hexdigest(): + raise ValueError("release ready marker checksum mismatch") + database = VulnerabilityDatabase.read_only(str(database_path)) + try: + integrity = database.connection.execute("PRAGMA integrity_check").fetchone()[0] + foreign_key_errors = len(database.connection.execute("PRAGMA foreign_key_check").fetchall()) + if integrity != "ok" or foreign_key_errors: + raise ValueError("SQLite integrity or foreign-key verification failed") + provenance = { + key: manifest[key] + for key in ( + "completeness", "production_full_database", "profile", "snapshot_id", + "source_digest", "source_record_mode", "sources", + ) + if key in manifest + } + actual_manifest = database.build_compact_snapshot_manifest(provenance) + for key in ("content_digest", "advisory_count", "affected_package_count", "table_counts", "provenance_digest"): + if actual_manifest.get(key) != manifest.get(key): + raise ValueError(f"compact manifest mismatch: {key}") + snapshot = database.snapshot_status(snapshot_id) + quality = database.snapshot_quality_gate(snapshot_id) + if not quality["healthy"]: + raise ValueError("stored snapshot quality gate failed") + if snapshot.get("state") != "staged" or database.active_snapshot() is not None: + raise ValueError("global OSV snapshot must remain staged and inactive") + if snapshot.get("source_digest") != source.get("sha256"): + raise ValueError("snapshot source digest does not match global OSV source") + non_digest_only = int(database.connection.execute( + "SELECT COUNT(*) FROM source_records " + "WHERE json_extract(record_json, '$._payload_omitted') IS NOT 1" + ).fetchone()[0]) + if non_digest_only: + raise ValueError("source record evidence mode mismatch") + source_records = int(database.connection.execute("SELECT COUNT(*) FROM source_records WHERE source_id = 'osv'").fetchone()[0]) + if source_records != source_records_expected or database.advisory_count() != source_records_expected: + raise ValueError("source record count does not match manifest") + return { + "advisory_count": actual_manifest["advisory_count"], + "affected_package_count": actual_manifest["affected_package_count"], + "content_digest": actual_manifest["content_digest"], + "database_bytes": database_path.stat().st_size, + "database_sha256": actual_database_digest, + "foreign_key_errors": foreign_key_errors, + "integrity_check": integrity, + "profile": manifest["profile"], + "snapshot_id": snapshot_id, + "state": snapshot["state"], + } + finally: + database.close() + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--database", type=Path, required=True) + parser.add_argument("--manifest", type=Path, required=True) + parser.add_argument("--minimum-records", type=int, default=1) + args = parser.parse_args() + print(json.dumps(verify_global_osv_snapshot(args.database, args.manifest, minimum_records=args.minimum_records), indent=2, sort_keys=True)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/__main__.py b/src/__main__.py index 1e54d03..4c18b13 100644 --- a/src/__main__.py +++ b/src/__main__.py @@ -13,6 +13,8 @@ from .safeio import write_private_atomic from . import __version__ +_DEFAULT_GLOBAL_DATABASE = "~/.local/share/coderisktools/vuln-db/global-osv.sqlite" + def main(): parser = argparse.ArgumentParser( @@ -90,7 +92,8 @@ def main(): inventory_parser.add_argument("--provenance", metavar="FILE", help="Verified provenance sidecar for external evidence") scan_parser = vuln_actions.add_parser("scan", help="Scan a local repository against an active local vulnerability database") scan_parser.add_argument("--root", required=True, metavar="DIR") - scan_parser.add_argument("--database", required=True, metavar="FILE") + scan_parser.add_argument("--database", default=_DEFAULT_GLOBAL_DATABASE, metavar="FILE", help="Local SQLite database; the pinned global snapshot is installed on first use by default") + scan_parser.add_argument("--no-bootstrap", action="store_true", help="Do not download the pinned global database when the default path is missing") scan_parser.add_argument("--format", choices=["json", "sarif", "markdown", "html", "csv"], default="json") scan_parser.add_argument("--output", metavar="FILE") scan_parser.add_argument("--baseline", metavar="FILE", help="Strict local vulnerability baseline; JSON format emits new/existing/resolved delta") @@ -150,6 +153,9 @@ def main(): bootstrap_seed_parser.add_argument("--signature-url", required=True, metavar="URL") bootstrap_seed_parser.add_argument("--destination", required=True, metavar="FILE") bootstrap_seed_parser.add_argument("--keyring", required=True, metavar="FILE") + bootstrap_global_parser = vuln_db_actions.add_parser("bootstrap-global", help="Install and activate the pinned signed global OSV SQLite ZIP") + bootstrap_global_parser.add_argument("--destination", default=_DEFAULT_GLOBAL_DATABASE, metavar="FILE") + bootstrap_global_parser.add_argument("--no-activate", action="store_true", help="Install as staged without activating it") explain_parser = vuln_db_actions.add_parser("explain", help="Explain one persisted vulnerability match") explain_parser.add_argument("--database", required=True, metavar="FILE") explain_parser.add_argument("--fingerprint", required=True, metavar="FINGERPRINT") @@ -245,10 +251,21 @@ def main(): build_markdown_vulnerability_report, build_sarif_vulnerability_report, ) - database_path = Path(args.database) - if database_path.is_symlink() or not database_path.is_file() or "\n" in args.database or "://" in args.database: + database_path = Path(args.database).expanduser() + if not database_path.exists() and args.database == _DEFAULT_GLOBAL_DATABASE and not args.no_bootstrap: + from .vulnerability.global_bootstrap import DEFAULT_GLOBAL_OSV_RELEASE, bootstrap_global_osv_asset + bootstrap_global_osv_asset( + DEFAULT_GLOBAL_OSV_RELEASE["asset_url"], + DEFAULT_GLOBAL_OSV_RELEASE["manifest_url"], + DEFAULT_GLOBAL_OSV_RELEASE["signature_url"], + database_path, + trusted_keys={DEFAULT_GLOBAL_OSV_RELEASE["key_id"]: DEFAULT_GLOBAL_OSV_RELEASE["public_key"]}, + activate=True, + ) + database_argument = str(database_path) + if database_path.is_symlink() or not database_path.is_file() or "\n" in database_argument or "://" in database_argument: raise ValueError("scan requires a local regular SQLite database path") - database = VulnerabilityDatabase.read_only(args.database) + database = VulnerabilityDatabase.read_only(database_argument) try: findings = scan_inventory(args.root, database) if args.vex or args.suppressions: @@ -327,6 +344,16 @@ def main(): args.destination, trusted_keys=load_trusted_keyring(args.keyring), ) + elif args.vuln_db_action == "bootstrap-global": + from .vulnerability.global_bootstrap import DEFAULT_GLOBAL_OSV_RELEASE, bootstrap_global_osv_asset + result = bootstrap_global_osv_asset( + DEFAULT_GLOBAL_OSV_RELEASE["asset_url"], + DEFAULT_GLOBAL_OSV_RELEASE["manifest_url"], + DEFAULT_GLOBAL_OSV_RELEASE["signature_url"], + Path(args.destination).expanduser(), + trusted_keys={DEFAULT_GLOBAL_OSV_RELEASE["key_id"]: DEFAULT_GLOBAL_OSV_RELEASE["public_key"]}, + activate=not args.no_activate, + ) elif args.vuln_db_action == "init-config": from .vulnerability.update_config import default_update_config output = Path(args.output).expanduser() diff --git a/src/vulnerability/bootstrap.py b/src/vulnerability/bootstrap.py index c6c9615..bb93099 100644 --- a/src/vulnerability/bootstrap.py +++ b/src/vulnerability/bootstrap.py @@ -17,6 +17,7 @@ _MAX_METADATA_BYTES = 2 * 1024 * 1024 _MAX_ASSET_BYTES = 64 * 1024 * 1024 + _SHA256_RE = re.compile(r"sha256:[0-9a-f]{64}") @@ -99,17 +100,49 @@ def validate_signed_release_manifest( return dict(verified) -def _verify_database_contract(database_path: Path, manifest: dict[str, Any]) -> None: +def _verify_database_contract( + database_path: Path, + manifest: dict[str, Any], + *, + descriptor: int | None = None, +) -> None: from .database import VulnerabilityDatabase - verify_asset_sha256(database_path, manifest["database_sha256"]) - with VulnerabilityDatabase.read_only(str(database_path)) as database: + if descriptor is None: + verify_asset_sha256(database_path, manifest["database_sha256"]) + database = VulnerabilityDatabase.read_only(str(database_path)) + else: + digest = hashlib.sha256() + offset = 0 + while True: + chunk = os.pread(descriptor, 1024 * 1024, offset) + if not chunk: + break + digest.update(chunk) + offset += len(chunk) + if "sha256:" + digest.hexdigest() != manifest["database_sha256"]: + raise ValueError("database descriptor SHA-256 mismatch") + database = VulnerabilityDatabase.from_file_descriptor(descriptor, readonly=True) + with database: if database.integrity_check() != "ok": raise ValueError("seed SQLite integrity check failed") if database.connection.execute("PRAGMA foreign_key_check").fetchall(): raise ValueError("seed SQLite foreign-key check failed") - actual = database.build_snapshot_manifest() - for key in ("content_digest", "advisory_count", "affected_package_count"): + if manifest.get("manifest_format") == "compact-v1": + provenance = { + key: manifest[key] + for key in ( + "completeness", "production_full_database", "profile", "snapshot_id", + "source_digest", "source_record_mode", "sources", + ) + if key in manifest + } + actual = database.build_compact_snapshot_manifest(provenance) + keys = ("content_digest", "advisory_count", "affected_package_count", "table_counts", "provenance_digest") + else: + actual = database.build_snapshot_manifest() + keys = ("content_digest", "advisory_count", "affected_package_count") + for key in keys: if actual.get(key) != manifest.get(key): raise ValueError(f"seed database manifest mismatch: {key}") row = database.connection.execute( @@ -118,6 +151,9 @@ def _verify_database_contract(database_path: Path, manifest: dict[str, Any]) -> ).fetchone() if row is None or row["state"] not in {"staged", "active"}: raise ValueError("seed database does not contain the declared staged snapshot") + quality = database.snapshot_quality_gate(str(manifest.get("snapshot_id"))) + if not quality["healthy"]: + raise ValueError("database snapshot quality gate failed") def activate_seed_database( diff --git a/src/vulnerability/database.py b/src/vulnerability/database.py index 918888d..a157705 100644 --- a/src/vulnerability/database.py +++ b/src/vulnerability/database.py @@ -4,8 +4,10 @@ import json import hashlib +import os import re import sqlite3 +import stat from dataclasses import dataclass from pathlib import Path from typing import Any, Iterable @@ -59,6 +61,7 @@ class ImportStats: advisories_imported: int affected_packages_imported: int errors: tuple[str, ...] = () + unmapped_affected_entries: int = 0 @dataclass(frozen=True) @@ -99,6 +102,29 @@ def __init__(self, path: str = ":memory:", *, readonly: bool = False) -> None: def read_only(cls, path: str) -> "VulnerabilityDatabase": return cls(path, readonly=True) + @classmethod + def from_file_descriptor( + cls, + descriptor: int, + *, + readonly: bool, + initialize_schema: bool = False, + ) -> "VulnerabilityDatabase": + """Open the exact inode pinned by an existing descriptor on procfs platforms.""" + if os.name != "posix" or not Path(f"/proc/self/fd/{descriptor}").exists(): + raise OSError("descriptor-pinned SQLite access requires procfs") + file_stat = os.fstat(descriptor) + if not stat.S_ISREG(file_stat.st_mode): + raise ValueError("vulnerability database descriptor must reference a regular file") + instance = cls.__new__(cls) + mode = "ro" if readonly else "rw" + instance.connection = sqlite3.connect(f"file:/proc/self/fd/{descriptor}?mode={mode}", uri=True) + instance.connection.execute("PRAGMA foreign_keys = ON") + instance.connection.row_factory = sqlite3.Row + if not readonly and initialize_schema: + instance._create_schema() + return instance + def _create_schema(self) -> None: self.connection.executescript( """ @@ -731,14 +757,31 @@ def _index_alias(self, alias: str, advisory_id: str, source: str = "osv") -> Non ) def correlate_aliases(self) -> dict[str, int]: - """Rebuild exact native-ID/explicit-alias index without heuristic merging.""" + """Rebuild exact native-ID/explicit-alias index in set-oriented SQL.""" + self.connection.create_function("crt_normalize_alias", 1, self._normalize_alias, deterministic=True) with self.connection: self.connection.execute("DELETE FROM advisory_aliases") self.connection.execute("DELETE FROM alias_conflicts") - for row in self.connection.execute("SELECT id, aliases_json, source FROM advisories ORDER BY id").fetchall(): - self._index_alias(row["id"], row["id"], row["source"]) - for alias in json.loads(row["aliases_json"] or "[]"): - self._index_alias(alias, row["id"], row["source"]) + self.connection.execute( + "INSERT OR IGNORE INTO advisory_aliases(alias, advisory_id, source) " + "SELECT normalized, advisory_id, source FROM (" + "SELECT crt_normalize_alias(id) AS normalized, id AS advisory_id, source FROM advisories" + ") WHERE normalized IS NOT NULL" + ) + self.connection.execute( + "INSERT OR IGNORE INTO advisory_aliases(alias, advisory_id, source) " + "SELECT normalized, advisory_id, source FROM (" + "SELECT crt_normalize_alias(CAST(item.value AS TEXT)) AS normalized, " + "advisories.id AS advisory_id, advisories.source AS source " + "FROM advisories JOIN json_each(advisories.aliases_json) AS item" + ") WHERE normalized IS NOT NULL" + ) + self.connection.execute( + "INSERT INTO alias_conflicts(alias, advisory_ids_json) " + "SELECT alias, json_group_array(advisory_id) FROM (" + "SELECT DISTINCT alias, advisory_id FROM advisory_aliases ORDER BY alias, advisory_id" + ") GROUP BY alias HAVING COUNT(*) > 1" + ) return {"aliases": self.alias_count(), "conflicts": self.alias_conflict_count()} def lookup_advisory(self, identifier: str) -> dict[str, Any]: @@ -1315,10 +1358,15 @@ def import_osv_records( *, max_records: int = MAX_OSV_RECORDS, max_record_bytes: int = MAX_OSV_RECORD_BYTES, + source_record_mode: str = "full", + correlate_aliases: bool = True, + record_savepoints: bool = True, ) -> ImportStats: if max_records <= 0 or max_record_bytes <= 0: raise ValueError("OSV import limits must be positive") - seen = imported = packages_imported = 0 + if source_record_mode not in {"full", "digest-only"}: + raise ValueError("source_record_mode must be full or digest-only") + seen = imported = packages_imported = unmapped_affected = 0 errors: list[str] = [] with self.connection: for record in records: @@ -1326,22 +1374,32 @@ def import_osv_records( errors.append(f"max_records limit exceeded: {max_records}") break seen += 1 - self.connection.execute("SAVEPOINT osv_record") + if record_savepoints: + self.connection.execute("SAVEPOINT osv_record") try: if not isinstance(record, dict): raise ValueError("OSV record must be an object") - if len(canonical_json_bytes(record)) > max_record_bytes: + record_canonical = canonical_json_bytes(record) + if len(record_canonical) > max_record_bytes: raise ValueError(f"max_record_bytes limit exceeded: {max_record_bytes}") advisory_id = record["id"] if not isinstance(advisory_id, str) or not advisory_id: raise ValueError("OSV record id is required") source_record = record.get("_source_record", record) - source_canonical = canonical_json_bytes(source_record) + source_canonical = record_canonical if source_record is record else canonical_json_bytes(source_record) + stored_source_record = source_canonical.decode("utf-8") + if source_record_mode == "digest-only": + stored_source_record = json.dumps( + {"_payload_omitted": True, "id": advisory_id}, + ensure_ascii=False, + sort_keys=True, + separators=(",", ":"), + ) self.connection.execute( "INSERT OR IGNORE INTO source_records " "(source_id, native_record_id, content_digest, record_fingerprint, advisory_id, record_json) " "VALUES (?, ?, ?, ?, ?, ?)", - (source, advisory_id, "sha256:" + hashlib.sha256(source_canonical).hexdigest(), source_record_fingerprint(source, advisory_id), advisory_id, source_canonical.decode("utf-8")), + (source, advisory_id, "sha256:" + hashlib.sha256(source_canonical).hexdigest(), source_record_fingerprint(source, advisory_id), advisory_id, stored_source_record), ) aliases = tuple(str(alias) for alias in record.get("aliases", [])) self.connection.execute( @@ -1360,9 +1418,10 @@ def import_osv_records( json.dumps(record.get("severity", []), ensure_ascii=False), json.dumps(record.get("database_specific", {}), ensure_ascii=False), source), ) - self._index_alias(advisory_id, advisory_id, source) - for alias in aliases: - self._index_alias(alias, advisory_id, source) + if correlate_aliases: + self._index_alias(advisory_id, advisory_id, source) + for alias in aliases: + self._index_alias(alias, advisory_id, source) self.connection.execute("DELETE FROM affected_packages WHERE advisory_id = ?", (advisory_id,)) self.connection.execute("DELETE FROM advisory_references WHERE advisory_id = ?", (advisory_id,)) for affected in record.get("affected", []): @@ -1370,38 +1429,44 @@ def import_osv_records( ecosystem = _ecosystem(str(package.get("ecosystem", ""))) name = str(package.get("name", "")).strip().lower() if not ecosystem or not name: - raise ValueError("affected package requires ecosystem and name") + unmapped_affected += 1 + continue cursor = self.connection.execute( "INSERT INTO affected_packages(advisory_id, ecosystem, name, purl) VALUES (?, ?, ?, ?)", (advisory_id, ecosystem, name, package.get("purl")), ) package_id = cursor.lastrowid packages_imported += 1 - for version in affected.get("versions", []): - self.connection.execute( - "INSERT OR IGNORE INTO affected_versions(affected_package_id, version) VALUES (?, ?)", - (package_id, str(version)), - ) - for range_data in affected.get("ranges", []): - events = range_data.get("events", []) - self.connection.execute( - "INSERT INTO affected_ranges(affected_package_id, range_type, events_json) VALUES (?, ?, ?)", - (package_id, str(range_data.get("type", "UNKNOWN")), json.dumps(events)), - ) - for reference in record.get("references", []): - if reference.get("url"): - self.connection.execute( - "INSERT INTO advisory_references(advisory_id, reference_type, url) VALUES (?, ?, ?)", - (advisory_id, reference.get("type"), reference["url"]), - ) + self.connection.executemany( + "INSERT OR IGNORE INTO affected_versions(affected_package_id, version) VALUES (?, ?)", + ((package_id, str(version)) for version in affected.get("versions", [])), + ) + self.connection.executemany( + "INSERT INTO affected_ranges(affected_package_id, range_type, events_json) VALUES (?, ?, ?)", + ( + (package_id, str(range_data.get("type", "UNKNOWN")), json.dumps(range_data.get("events", []))) + for range_data in affected.get("ranges", []) + ), + ) + self.connection.executemany( + "INSERT INTO advisory_references(advisory_id, reference_type, url) VALUES (?, ?, ?)", + ( + (advisory_id, reference.get("type"), reference["url"]) + for reference in record.get("references", []) + if reference.get("url") + ), + ) imported += 1 - self.connection.execute("RELEASE SAVEPOINT osv_record") + if record_savepoints: + self.connection.execute("RELEASE SAVEPOINT osv_record") except (KeyError, TypeError, ValueError, sqlite3.Error) as exc: - self.connection.execute("ROLLBACK TO SAVEPOINT osv_record") - self.connection.execute("RELEASE SAVEPOINT osv_record") + if record_savepoints: + self.connection.execute("ROLLBACK TO SAVEPOINT osv_record") + self.connection.execute("RELEASE SAVEPOINT osv_record") errors.append(f"record {seen}: {exc}") - self.correlate_aliases() - return ImportStats(seen, imported, packages_imported, tuple(errors)) + if correlate_aliases: + self.correlate_aliases() + return ImportStats(seen, imported, packages_imported, tuple(errors), unmapped_affected) def advisory_metadata(self, advisory_id: str) -> dict[str, Any]: row = self.connection.execute("SELECT * FROM advisories WHERE id = ?", (advisory_id,)).fetchone() @@ -1472,12 +1537,85 @@ def build_snapshot_manifest(self) -> dict[str, Any]: manifest["content_digest"] = "sha256:" + hashlib.sha256(canonical).hexdigest() return manifest + def build_compact_snapshot_manifest(self, provenance: dict[str, Any] | None = None) -> dict[str, Any]: + """Build a bounded-memory digest over persisted vulnerability evidence.""" + tables = ( + "advisories", "advisory_aliases", "advisory_relations", "advisory_references", + "affected_packages", "affected_ranges", "affected_versions", "alias_conflicts", + "cpe_mappings", "epss_scores", "import_errors", "kev_records", "merge_decisions", + "nvd_cpe_matches", "nvd_cvss", "nvd_enrichments", "nvd_weaknesses", + "quality_metrics", "schema_migrations", "source_records", "source_snapshots", + "unresolved_enrichments", "vulnrichment_records", + ) + provenance = dict(provenance or {}) + provenance_bytes = json.dumps(provenance, ensure_ascii=False, sort_keys=True, separators=(",", ":")).encode("utf-8") + digest = hashlib.sha256() + digest.update(b'{"manifest_format":"compact-v1","schema_version":"2"}\n') + digest.update(provenance_bytes + b"\n") + table_counts: dict[str, int] = {} + existing = { + str(row[0]) + for row in self.connection.execute("SELECT name FROM sqlite_master WHERE type = 'table'") + } + for table in tables: + if table not in existing: + continue + columns = self.connection.execute(f'PRAGMA table_info("{table}")').fetchall() + names = [str(column[1]) for column in columns] + primary = [str(column[1]) for column in sorted(columns, key=lambda item: int(item[5])) if int(column[5]) > 0] + ordering = primary or names + select_columns = ", ".join(f'"{name}"' for name in names) + order_columns = ", ".join(f'"{name}"' for name in ordering) + digest.update(json.dumps({"columns": names, "table": table}, sort_keys=True, separators=(",", ":")).encode("utf-8") + b"\n") + cursor = self.connection.execute( + f'SELECT {select_columns} FROM "{table}" ORDER BY {order_columns}' + ) + count = 0 + while True: + rows = cursor.fetchmany(1000) + if not rows: + break + for row in rows: + values = [ + {"bytes_hex": value.hex()} if isinstance(value, bytes) else value + for value in tuple(row) + ] + digest.update(json.dumps(values, ensure_ascii=False, separators=(",", ":")).encode("utf-8") + b"\n") + count += 1 + table_counts[table] = count + return { + "schema_version": "2", + "manifest_format": "compact-v1", + "advisory_count": self.advisory_count(), + "affected_package_count": self.affected_package_count(), + "table_counts": table_counts, + "provenance_digest": "sha256:" + hashlib.sha256(provenance_bytes).hexdigest(), + "content_digest": "sha256:" + digest.hexdigest(), + } + + def _manifest_for_verification(self, expected: dict[str, Any]) -> dict[str, Any]: + if expected.get("manifest_format") == "compact-v1": + provenance = { + key: expected[key] + for key in ( + "completeness", "production_full_database", "profile", "snapshot_id", + "source_digest", "source_record_mode", "sources", + ) + if key in expected + } + return self.build_compact_snapshot_manifest(provenance) + return self.build_snapshot_manifest() + def stage_snapshot(self, snapshot_id: str, source_digest: str, manifest: dict[str, Any]) -> None: if not snapshot_id or not source_digest.startswith("sha256:") or len(source_digest) <= len("sha256:"): raise ValueError("snapshot_id and sha256 source_digest are required") required = {"content_digest", "advisory_count", "affected_package_count"} if not required.issubset(manifest): raise ValueError("snapshot manifest is incomplete") + if manifest.get("snapshot_id", snapshot_id) != snapshot_id: + raise ValueError("snapshot manifest id mismatch") + if manifest.get("source_digest", source_digest) != source_digest: + raise ValueError("snapshot manifest source digest mismatch") with self.connection: self.connection.execute( "INSERT OR REPLACE INTO snapshots " @@ -1510,8 +1648,21 @@ def snapshot_quality_gate(self, snapshot_id: str | None = None) -> dict[str, Any issues.append("snapshot_state") else: expected = json.loads(row["manifest_json"]) - actual = self.build_snapshot_manifest() - for key in ("content_digest", "advisory_count", "affected_package_count"): + actual = self._manifest_for_verification(expected) + manifest_keys = ["content_digest", "advisory_count", "affected_package_count"] + if expected.get("manifest_format") == "compact-v1": + manifest_keys.extend(["table_counts", "provenance_digest"]) + if expected.get("snapshot_id") != row["snapshot_id"]: + checks["manifest:snapshot_id"] = "failed" + issues.append("manifest:snapshot_id") + else: + checks["manifest:snapshot_id"] = "ok" + if expected.get("source_digest") != row["source_digest"]: + checks["manifest:source_digest"] = "failed" + issues.append("manifest:source_digest") + else: + checks["manifest:source_digest"] = "ok" + for key in manifest_keys: checks[f"manifest:{key}"] = "ok" if expected.get(key) == actual.get(key) else "failed" if expected.get(key) != actual.get(key): issues.append(f"manifest:{key}") @@ -1532,7 +1683,7 @@ def activate_snapshot(self, snapshot_id: str) -> dict[str, Any]: if not quality["healthy"]: raise SnapshotActivationError("snapshot quality gate failed: " + ", ".join(quality["issues"])) expected = json.loads(row["manifest_json"]) - actual = self.build_snapshot_manifest() + actual = self._manifest_for_verification(expected) mismatches = [] for key in ("content_digest", "advisory_count", "affected_package_count"): if expected.get(key) != actual.get(key): diff --git a/src/vulnerability/full_snapshot.py b/src/vulnerability/full_snapshot.py new file mode 100644 index 0000000..d1eec4c --- /dev/null +++ b/src/vulnerability/full_snapshot.py @@ -0,0 +1,259 @@ +"""Bounded streaming import for a global OSV ZIP snapshot.""" +from __future__ import annotations + +import hashlib +import json +import os +import stat +import struct +import zipfile +from dataclasses import dataclass +from pathlib import Path, PurePosixPath +from typing import BinaryIO, Callable + +from .database import VulnerabilityDatabase + + +@dataclass(frozen=True) +class OsvZipImportReport: + archive_sha256: str + archive_bytes: int + declared_uncompressed_bytes: int + members_seen: int + advisories_imported: int + affected_packages_imported: int + unmapped_affected_entries: int + error_count: int + errors: tuple[str, ...] + + +def _stream_sha256(stream: BinaryIO) -> str: + digest = hashlib.sha256() + stream.seek(0) + while True: + chunk = stream.read(1024 * 1024) + if not chunk: + break + digest.update(chunk) + stream.seek(0) + return "sha256:" + digest.hexdigest() + + +def validate_zip_central_directory( + stream: BinaryIO, + *, + max_members: int, + max_central_directory_bytes: int = 256 * 1024 * 1024, +) -> tuple[int, int]: + """Bound ZIP member count and central-directory bytes before ZipFile allocates them.""" + if max_members <= 0 or max_central_directory_bytes <= 0: + raise ValueError("ZIP central-directory limits must be positive") + stream.seek(0, os.SEEK_END) + archive_size = stream.tell() + tail_size = min(archive_size, 65_557) + stream.seek(archive_size - tail_size) + tail = stream.read(tail_size) + marker = tail.rfind(b"PK\x05\x06") + if marker < 0 or len(tail) - marker < 22: + raise ValueError("ZIP end-of-central-directory record is missing") + eocd = struct.unpack_from("<4s4H2LH", tail, marker) + disk_number, central_disk, disk_entries, total_entries, central_bytes, _central_offset, comment_bytes = eocd[1:] + if marker + 22 + comment_bytes != len(tail) or disk_number != 0 or central_disk != 0: + raise ValueError("multi-disk or malformed ZIP is not supported") + eocd_offset = archive_size - tail_size + marker + locator_present = eocd_offset >= 20 + locator = b"" + if locator_present: + stream.seek(eocd_offset - 20) + locator = stream.read(20) + locator_present = locator[:4] == b"PK\x06\x07" + sentinels = total_entries == 0xFFFF or disk_entries == 0xFFFF or central_bytes == 0xFFFFFFFF + if sentinels and not locator_present: + raise ValueError("ZIP64 locator is missing") + if locator_present: + signature, zip64_disk, zip64_offset, disk_count = struct.unpack("<4sLQL", locator) + if signature != b"PK\x06\x07" or zip64_disk != 0 or disk_count != 1: + raise ValueError("multi-disk or malformed ZIP64 is not supported") + stream.seek(zip64_offset) + record = stream.read(56) + if len(record) != 56: + raise ValueError("ZIP64 end-of-central-directory record is truncated") + values = struct.unpack("<4sQ2H2L4Q", record) + if values[0] != b"PK\x06\x06" or values[4] != 0 or values[5] != 0: + raise ValueError("multi-disk or malformed ZIP64 is not supported") + zip64_disk_entries, zip64_total_entries, zip64_central_bytes, zip64_central_offset = values[6:10] + if not sentinels and ( + disk_entries != zip64_disk_entries + or total_entries != zip64_total_entries + or central_bytes != zip64_central_bytes + or _central_offset != zip64_central_offset + ): + raise ValueError("legacy and ZIP64 central-directory metadata disagree") + disk_entries, total_entries, central_bytes, _central_offset = ( + zip64_disk_entries, + zip64_total_entries, + zip64_central_bytes, + zip64_central_offset, + ) + if _central_offset < 0 or _central_offset + central_bytes > eocd_offset: + raise ValueError("ZIP central-directory bounds are invalid") + if disk_entries != total_entries or total_entries <= 0 or total_entries > max_members: + raise ValueError("ZIP member count is outside the allowed range") + if central_bytes <= 0 or central_bytes > max_central_directory_bytes: + raise ValueError("ZIP central directory exceeds the memory safety limit") + stream.seek(0) + return int(total_entries), int(central_bytes) + + +def _validate_member(info: zipfile.ZipInfo, *, max_member_bytes: int) -> None: + name = info.filename + path = PurePosixPath(name) + if ( + not name + or len(name.encode("utf-8")) > 1024 + or path.is_absolute() + or ".." in path.parts + or "\\" in name + or "\x00" in name + ): + raise ValueError("OSV ZIP contains an unsafe member path") + if info.flag_bits & 0x1: + raise ValueError("OSV ZIP contains an encrypted member") + unix_mode = (info.external_attr >> 16) & 0xFFFF + if unix_mode and stat.S_ISLNK(unix_mode): + raise ValueError("OSV ZIP contains a symbolic link") + if not info.is_dir() and not name.endswith(".json"): + raise ValueError("OSV ZIP contains a non-JSON payload") + if info.file_size > max_member_bytes: + raise ValueError("OSV ZIP member exceeds the byte limit") + + +def import_osv_zip( + database: VulnerabilityDatabase, + archive: str | os.PathLike[str], + *, + expected_archive_sha256: str | None = None, + expected_payload_members: int | None = None, + expected_uncompressed_bytes: int | None = None, + batch_records: int = 200, + batch_bytes: int = 4 * 1024 * 1024, + max_members: int = 1_000_000, + max_archive_bytes: int = 2 * 1024 * 1024 * 1024, + max_member_bytes: int = 24 * 1024 * 1024, + max_uncompressed_bytes: int = 12 * 1024 * 1024 * 1024, + max_errors: int = 1000, + source_record_mode: str = "digest-only", + progress: Callable[[int, int], None] | None = None, +) -> OsvZipImportReport: + """Open one pinned descriptor, validate centrally, then import bounded batches.""" + if batch_records <= 0 or batch_bytes <= 0: + raise ValueError("OSV ZIP batch limits must be positive") + if max_members <= 0 or max_archive_bytes <= 0 or max_member_bytes <= 0 or max_uncompressed_bytes <= 0 or max_errors < 0: + raise ValueError("OSV ZIP safety limits must be positive") + if expected_payload_members is not None and expected_payload_members <= 0: + raise ValueError("expected_payload_members must be positive") + if expected_uncompressed_bytes is not None and expected_uncompressed_bytes <= 0: + raise ValueError("expected_uncompressed_bytes must be positive") + path = Path(archive) + flags = os.O_RDONLY | getattr(os, "O_CLOEXEC", 0) | getattr(os, "O_NOFOLLOW", 0) + descriptor = os.open(path, flags) + try: + file_stat = os.fstat(descriptor) + if not stat.S_ISREG(file_stat.st_mode): + raise ValueError("OSV ZIP must be a regular non-symlink file") + if file_stat.st_size <= 0 or file_stat.st_size > max_archive_bytes: + raise ValueError("OSV ZIP compressed size is outside the allowed range") + with os.fdopen(descriptor, "rb", closefd=False) as archive_file: + archive_digest = _stream_sha256(archive_file) + if expected_archive_sha256 is not None and archive_digest != expected_archive_sha256: + raise ValueError("OSV ZIP SHA-256 does not match pinned provenance") + declared_members, _ = validate_zip_central_directory(archive_file, max_members=max_members) + errors: list[str] = [] + error_count = 0 + imported = packages = unmapped_affected = members_seen = 0 + with zipfile.ZipFile(archive_file) as zipped: + infos = zipped.infolist() + if len(infos) != declared_members: + raise ValueError("ZIP central-directory member count changed during open") + infos.sort(key=lambda item: item.filename) + total_uncompressed = 0 + payload_count = 0 + previous_name: str | None = None + for info in infos: + _validate_member(info, max_member_bytes=max_member_bytes) + if info.filename == previous_name: + raise ValueError("OSV ZIP contains duplicate member paths") + previous_name = info.filename + total_uncompressed += info.file_size + if total_uncompressed > max_uncompressed_bytes: + raise ValueError("OSV ZIP exceeds the total uncompressed byte limit") + if not info.is_dir(): + payload_count += 1 + if payload_count == 0: + raise ValueError("OSV ZIP contains no JSON payload records") + if expected_payload_members is not None and payload_count != expected_payload_members: + raise ValueError("OSV ZIP payload member count does not match the pinned manifest") + if expected_uncompressed_bytes is not None and total_uncompressed != expected_uncompressed_bytes: + raise ValueError("OSV ZIP uncompressed byte count does not match the pinned manifest") + batch: list[dict] = [] + current_bytes = 0 + + def flush() -> None: + nonlocal batch, current_bytes, imported, packages, unmapped_affected, error_count + if not batch: + return + stats = database.import_osv_records( + batch, + max_records=len(batch), + max_record_bytes=max_member_bytes, + source_record_mode=source_record_mode, + correlate_aliases=False, + record_savepoints=False, + ) + imported += stats.advisories_imported + packages += stats.affected_packages_imported + unmapped_affected += stats.unmapped_affected_entries + for error in stats.errors: + error_count += 1 + if len(errors) < 1000: + errors.append(error) + if error_count > max_errors: + raise ValueError("OSV ZIP import error limit exceeded") + batch = [] + current_bytes = 0 + if progress is not None: + progress(members_seen, imported) + + for info in infos: + if info.is_dir(): + continue + members_seen += 1 + try: + raw = zipped.read(info) + record = json.loads(raw) + if not isinstance(record, dict): + raise ValueError("record must be a JSON object") + if batch and (len(batch) >= batch_records or current_bytes + len(raw) > batch_bytes): + flush() + batch.append(record) + current_bytes += len(raw) + except (json.JSONDecodeError, UnicodeDecodeError, ValueError, RuntimeError, zipfile.BadZipFile) as exc: + error_count += 1 + if len(errors) < 1000: + errors.append(f"{info.filename}: {exc}") + if error_count > max_errors: + raise ValueError("OSV ZIP import error limit exceeded") from exc + flush() + return OsvZipImportReport( + archive_sha256=archive_digest, + archive_bytes=file_stat.st_size, + declared_uncompressed_bytes=total_uncompressed, + members_seen=members_seen, + advisories_imported=imported, + affected_packages_imported=packages, + unmapped_affected_entries=unmapped_affected, + error_count=error_count, + errors=tuple(errors), + ) + finally: + os.close(descriptor) diff --git a/src/vulnerability/global_bootstrap.py b/src/vulnerability/global_bootstrap.py new file mode 100644 index 0000000..b390bbf --- /dev/null +++ b/src/vulnerability/global_bootstrap.py @@ -0,0 +1,304 @@ +"""Pinned, signed bootstrap for the global OSV SQLite ZIP release.""" +from __future__ import annotations + +import hashlib +import json +import os +import re +import shutil +import stat +import tempfile +import zipfile +from contextlib import contextmanager +from pathlib import Path +from typing import Any, BinaryIO, Iterator, Mapping +from urllib.request import Request, urlopen + +from .bootstrap import _MAX_METADATA_BYTES, _download, _https_url, _verify_database_contract +from .full_snapshot import validate_zip_central_directory +from .manifest_signing import verify_manifest + +_MAX_GLOBAL_ZIP_BYTES = 2 * 1024 * 1024 * 1024 +_MAX_GLOBAL_DATABASE_BYTES = 9 * 1024 * 1024 * 1024 +_SHA256_RE = re.compile(r"sha256:[0-9a-f]{64}") + + +@contextmanager +def _installation_lock(path: Path) -> Iterator[None]: + descriptor = os.open(path, os.O_RDWR | os.O_CREAT | getattr(os, "O_CLOEXEC", 0) | getattr(os, "O_NOFOLLOW", 0), 0o600) + locked = False + try: + try: + import fcntl + fcntl.flock(descriptor, fcntl.LOCK_EX | fcntl.LOCK_NB) + except ImportError: # pragma: no cover - Windows + import msvcrt + if os.fstat(descriptor).st_size == 0: + os.write(descriptor, b"0") + os.lseek(descriptor, 0, os.SEEK_SET) + getattr(msvcrt, "locking")(descriptor, getattr(msvcrt, "LK_NBLCK"), 1) + except OSError as exc: + raise RuntimeError("another vulnerability database bootstrap is already running") from exc + locked = True + yield + finally: + try: + if locked and os.name == "nt": # pragma: no cover - Windows + import msvcrt + os.lseek(descriptor, 0, os.SEEK_SET) + getattr(msvcrt, "locking")(descriptor, getattr(msvcrt, "LK_UNLCK"), 1) + elif locked: + import fcntl + fcntl.flock(descriptor, fcntl.LOCK_UN) + finally: + os.close(descriptor) + + +def _download_to_file( + url: str, + output: BinaryIO, + allowed_hosts: frozenset[str], + *, + maximum: int, + expected_sha256: str, + timeout: float, +) -> tuple[int, str]: + _https_url(url, allowed_hosts) + request = Request(url, headers={"Accept": "application/octet-stream", "User-Agent": "coderisktools-bootstrap/1"}) + digest = hashlib.sha256() + total = 0 + with urlopen(request, timeout=timeout) as response: + _https_url(response.geturl(), allowed_hosts) + length = response.headers.get("Content-Length") + if length is not None: + try: + declared = int(length) + except (TypeError, ValueError) as exc: + raise ValueError("database ZIP has invalid Content-Length") from exc + if declared < 0 or declared > maximum: + raise ValueError("database ZIP exceeds byte limit") + output.seek(0) + output.truncate(0) + while True: + chunk = response.read(1024 * 1024) + if not chunk: + break + total += len(chunk) + if total > maximum: + raise ValueError("database ZIP exceeds byte limit") + digest.update(chunk) + output.write(chunk) + output.flush() + os.fsync(output.fileno()) + output.seek(0) + actual = "sha256:" + digest.hexdigest() + if actual != expected_sha256: + raise ValueError("database ZIP SHA-256 mismatch") + return total, actual + + +def validate_signed_global_manifest( + manifest: dict[str, Any], + envelope: dict[str, Any], + trusted_keys: Mapping[str, bytes], +) -> dict[str, Any]: + if not isinstance(envelope, dict): + raise ValueError("signed global manifest envelope must be an object") + key_id = envelope.get("key_id") + if not isinstance(key_id, str) or key_id not in trusted_keys: + raise ValueError("signed global manifest uses an untrusted key") + verified = verify_manifest(envelope, trusted_keys[key_id]) + if verified != manifest: + raise ValueError("signed global manifest does not match detached manifest") + required = { + "asset_sha256", "archive_member", "completeness", "database_bytes", "database_sha256", + "manifest_format", "production_full_database", "profile", "snapshot_id", + } + if not required.issubset(verified): + raise ValueError("signed global manifest is incomplete") + if ( + verified["profile"] != "global-osv" + or verified["completeness"] != "full-osv-source" + or verified["production_full_database"] is not False + or verified["manifest_format"] != "compact-v1" + ): + raise ValueError("signed global manifest profile contract is invalid") + for field in ("asset_sha256", "database_sha256"): + if not isinstance(verified[field], str) or _SHA256_RE.fullmatch(verified[field]) is None: + raise ValueError(f"signed global manifest has an invalid {field}") + database_bytes = verified["database_bytes"] + if type(database_bytes) is not int or database_bytes <= 0 or database_bytes > _MAX_GLOBAL_DATABASE_BYTES: + raise ValueError("signed global manifest database size is invalid") + member = verified["archive_member"] + if not isinstance(member, str) or Path(member).name != member or not member.endswith(".sqlite"): + raise ValueError("signed global manifest archive member is invalid") + snapshot_id = verified["snapshot_id"] + if not isinstance(snapshot_id, str) or not snapshot_id.startswith("global-osv-"): + raise ValueError("signed global manifest snapshot id is invalid") + return dict(verified) + + +def _extract_single_database(zip_stream: BinaryIO, output: BinaryIO, output_parent: Path, manifest: dict[str, Any]) -> None: + expected_member = manifest["archive_member"] + expected_size = int(manifest["database_bytes"]) + expected_digest = manifest["database_sha256"] + reserve = 512 * 1024 * 1024 + if shutil.disk_usage(output_parent).free < expected_size + reserve: + raise OSError("insufficient disk space to install vulnerability database") + zip_stream.seek(0) + declared_members, _ = validate_zip_central_directory( + zip_stream, + max_members=1, + max_central_directory_bytes=1024 * 1024, + ) + with zipfile.ZipFile(zip_stream) as archive: + infos = archive.infolist() + if declared_members != 1 or len(infos) != declared_members: + raise ValueError("database ZIP must contain exactly one member") + info = infos[0] + mode = (info.external_attr >> 16) & 0xFFFF + if ( + info.is_dir() + or info.filename != expected_member + or Path(info.filename).name != info.filename + or "\\" in info.filename + or "\x00" in info.filename + or info.flag_bits & 0x1 + or (mode and stat.S_ISLNK(mode)) + or info.file_size != expected_size + or info.file_size > _MAX_GLOBAL_DATABASE_BYTES + ): + raise ValueError("database ZIP member contract is invalid") + digest = hashlib.sha256() + total = 0 + output.seek(0) + output.truncate(0) + with archive.open(info, "r") as source: + while True: + chunk = source.read(1024 * 1024) + if not chunk: + break + total += len(chunk) + if total > expected_size: + raise ValueError("extracted database exceeds declared size") + digest.update(chunk) + output.write(chunk) + output.flush() + os.fsync(output.fileno()) + output.seek(0) + if total != expected_size or "sha256:" + digest.hexdigest() != expected_digest: + raise ValueError("extracted database digest or size mismatch") + + +def bootstrap_global_osv_asset( + asset_url: str, + manifest_url: str, + signature_url: str, + destination: str | os.PathLike[str], + *, + trusted_keys: Mapping[str, bytes], + allowed_hosts: frozenset[str] = frozenset({"github.com", "objects.githubusercontent.com", "release-assets.githubusercontent.com"}), + timeout: float = 120.0, + activate: bool = True, +) -> dict[str, object]: + """Stream, verify, extract, and atomically install one pinned SQLite ZIP.""" + if timeout <= 0 or timeout > 300: + raise ValueError("bootstrap timeout must be in (0, 300]") + target = Path(destination).expanduser() + target.parent.mkdir(parents=True, exist_ok=True) + lock_path = target.with_suffix(target.suffix + ".bootstrap.lock") + with _installation_lock(lock_path): + if target.exists() or target.is_symlink(): + raise FileExistsError(f"vulnerability database destination already exists: {target}") + manifest_raw = _download(manifest_url, allowed_hosts, "application/json", _MAX_METADATA_BYTES, "global manifest", timeout) + signature_raw = _download(signature_url, allowed_hosts, "application/json", _MAX_METADATA_BYTES, "global signature", timeout) + try: + manifest = json.loads(manifest_raw.decode("utf-8")) + envelope = json.loads(signature_raw.decode("utf-8")) + except (UnicodeError, json.JSONDecodeError) as exc: + raise ValueError("global release metadata is invalid JSON") from exc + if not isinstance(manifest, dict) or not isinstance(envelope, dict): + raise ValueError("global release metadata must be JSON objects") + verified = validate_signed_global_manifest(manifest, envelope, trusted_keys) + zip_fd, zip_name = tempfile.mkstemp(prefix=f".{target.name}.", suffix=".zip.tmp", dir=str(target.parent)) + db_fd, db_name = tempfile.mkstemp(prefix=f".{target.name}.", suffix=".sqlite.tmp", dir=str(target.parent)) + zip_temporary = Path(zip_name) + database_temporary = Path(db_name) + installed = False + try: + with os.fdopen(zip_fd, "w+b") as zip_stream, os.fdopen(db_fd, "w+b") as database_stream: + _download_to_file( + asset_url, + zip_stream, + allowed_hosts, + maximum=_MAX_GLOBAL_ZIP_BYTES, + expected_sha256=verified["asset_sha256"], + timeout=timeout, + ) + _extract_single_database(zip_stream, database_stream, target.parent, verified) + database_identity = os.fstat(database_stream.fileno()) + _verify_database_contract( + database_temporary, + verified, + descriptor=database_stream.fileno(), + ) + path_identity = os.stat(database_temporary, follow_symlinks=False) + if (database_identity.st_dev, database_identity.st_ino) != (path_identity.st_dev, path_identity.st_ino): + raise OSError("staged database identity changed during verification") + state = "installed_staged" + if activate: + from .database import VulnerabilityDatabase + with VulnerabilityDatabase.from_file_descriptor( + database_stream.fileno(), + readonly=False, + ) as database: + database.connection.execute("PRAGMA journal_mode=MEMORY") + status = database.activate_snapshot(str(verified["snapshot_id"])) + state = status["state"] + final_identity = os.fstat(database_stream.fileno()) + path_identity = os.stat(database_temporary, follow_symlinks=False) + if (final_identity.st_dev, final_identity.st_ino) != (path_identity.st_dev, path_identity.st_ino): + raise OSError("staged database identity changed before installation") + os.link(database_temporary, target, follow_symlinks=False) + target_identity = os.stat(target, follow_symlinks=False) + if (final_identity.st_dev, final_identity.st_ino) != (target_identity.st_dev, target_identity.st_ino): + target.unlink() + raise OSError("installed database identity mismatch") + installed = True + directory_fd = os.open(target.parent, os.O_RDONLY | getattr(os, "O_DIRECTORY", 0)) + try: + os.fsync(directory_fd) + finally: + os.close(directory_fd) + zip_temporary.unlink() + database_temporary.unlink() + return { + "state": state, + "profile": "global-osv", + "completeness": "full-osv-source", + "path": str(target), + "snapshot_id": verified["snapshot_id"], + "database_sha256": verified["database_sha256"], + "signing_key_id": envelope["key_id"], + } + except BaseException: + if installed: + try: + target.unlink() + except FileNotFoundError: + pass + for temporary in (zip_temporary, database_temporary): + try: + temporary.unlink() + except FileNotFoundError: + pass + raise + + +DEFAULT_GLOBAL_OSV_RELEASE = { + "asset_url": "https://github.com/9batalion/coderisktools-scanner/releases/download/v3.1.0/coderisktools-vulndb-global-osv-2026-07-23.sqlite.zip", + "manifest_url": "https://github.com/9batalion/coderisktools-scanner/releases/download/v3.1.0/coderisktools-vulndb-global-osv-2026-07-23.manifest.json", + "signature_url": "https://github.com/9batalion/coderisktools-scanner/releases/download/v3.1.0/coderisktools-vulndb-global-osv-2026-07-23.manifest.sig.json", + "key_id": "coderisktools-vulndb-2026", + "public_key": bytes.fromhex("5fd70b01c5ef2b0317765fe188f5ef136527d8bceefcff37b0adfc40c4fbf235"), +} diff --git a/tests/test_global_bootstrap.py b/tests/test_global_bootstrap.py new file mode 100644 index 0000000..5aa54c6 --- /dev/null +++ b/tests/test_global_bootstrap.py @@ -0,0 +1,196 @@ +import hashlib +import json +import os +import tempfile +import unittest +import zipfile +from pathlib import Path +from unittest.mock import patch + +from cryptography.hazmat.primitives import serialization +from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PrivateKey +from scripts.build_global_osv_vulndb import build_global_osv_snapshot +from scripts.package_global_osv_release import package_global_osv_release +from src.vulnerability.database import VulnerabilityDatabase +from src.vulnerability.global_bootstrap import ( + _installation_lock, + _extract_single_database, + bootstrap_global_osv_asset, + validate_signed_global_manifest, +) + + +class GlobalBootstrapTests(unittest.TestCase): + def test_installation_lock_rejects_parallel_bootstrap(self): + with tempfile.TemporaryDirectory() as directory: + lock = Path(directory) / "database.bootstrap.lock" + with _installation_lock(lock): + with self.assertRaises(RuntimeError): + with _installation_lock(lock): + self.fail("parallel lock unexpectedly succeeded") + + def test_database_descriptor_pins_inode_across_path_replacement(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + database_path = root / "database.sqlite" + moved_path = root / "moved.sqlite" + with VulnerabilityDatabase(str(database_path)) as database: + database.import_osv_records([self._record()], source_record_mode="digest-only") + descriptor = os.open(database_path, os.O_RDONLY) + try: + database_path.rename(moved_path) + database_path.write_bytes(b"not sqlite") + with VulnerabilityDatabase.from_file_descriptor(descriptor, readonly=True) as pinned: + self.assertEqual(pinned.advisory_count(), 1) + finally: + os.close(descriptor) + + @staticmethod + def _record(): + return { + "id": "OSV-BOOTSTRAP-1", + "aliases": ["CVE-2026-1000"], + "affected": [{ + "package": {"ecosystem": "PyPI", "name": "example"}, + "versions": ["1.0.0"], + }], + } + + def test_extracts_exactly_one_digest_bound_sqlite_member(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + archive = root / "database.zip" + output = root / "database.sqlite" + payload = b"SQLite bytes" + with zipfile.ZipFile(archive, "w", compression=zipfile.ZIP_DEFLATED) as zipped: + zipped.writestr("database.sqlite", payload) + manifest = { + "archive_member": "database.sqlite", + "database_bytes": len(payload), + "database_sha256": "sha256:" + hashlib.sha256(payload).hexdigest(), + } + with archive.open("rb") as source, output.open("w+b") as target: + _extract_single_database(source, target, root, manifest) + self.assertEqual(output.read_bytes(), payload) + + def test_rejects_zip_with_more_than_one_member(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + archive = root / "database.zip" + output = root / "database.sqlite" + with zipfile.ZipFile(archive, "w") as zipped: + zipped.writestr("database.sqlite", b"db") + zipped.writestr("extra.txt", b"not allowed") + manifest = { + "archive_member": "database.sqlite", + "database_bytes": 2, + "database_sha256": "sha256:" + hashlib.sha256(b"db").hexdigest(), + } + with self.assertRaises(ValueError): + with archive.open("rb") as source, output.open("w+b") as target: + _extract_single_database(source, target, root, manifest) + + def test_signed_global_manifest_is_profile_bound(self): + manifest = { + "asset_sha256": "sha256:" + "a" * 64, + "archive_member": "database.sqlite", + "completeness": "full-osv-source", + "database_bytes": 10, + "database_sha256": "sha256:" + "b" * 64, + "manifest_format": "compact-v1", + "production_full_database": False, + "profile": "global-osv", + "snapshot_id": "global-osv-2026-07-23", + } + envelope = {"key_id": "global-key"} + with patch("src.vulnerability.global_bootstrap.verify_manifest", return_value=dict(manifest)): + actual = validate_signed_global_manifest(manifest, envelope, {"global-key": b"k" * 32}) + self.assertEqual(actual, manifest) + + def test_bootstrap_downloads_zip_verifies_database_and_activates(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + source_zip = root / "source.zip" + source_manifest = root / "source.json" + source_database = root / "source.sqlite" + database_manifest = root / "database.manifest.json" + database_sha = root / "database.sha256" + osv_payload = json.dumps(self._record()) + with zipfile.ZipFile(source_zip, "w", compression=zipfile.ZIP_DEFLATED) as zipped: + zipped.writestr("OSV-BOOTSTRAP-1.json", osv_payload) + with zipfile.ZipFile(source_zip) as zipped: + uncompressed = sum(item.file_size for item in zipped.infolist()) + source_manifest.write_text(json.dumps({ + "sha256": "sha256:" + hashlib.sha256(source_zip.read_bytes()).hexdigest(), + "records": 1, + "uncompressed_bytes": uncompressed, + })) + build_global_osv_snapshot( + source_zip, + source_manifest, + source_database, + database_manifest, + database_sha, + snapshot_id="global-osv-bootstrap-test", + maximum_database_bytes=100_000_000, + reserve_free_bytes=1_000_000, + ) + asset = root / "database.zip" + release_manifest_path = root / "database.release.manifest.json" + signature_path = root / "database.sig.json" + private_key_path = root / "private.key" + private_key = Ed25519PrivateKey.generate() + private_key_path.write_bytes(private_key.private_bytes( + serialization.Encoding.Raw, + serialization.PrivateFormat.Raw, + serialization.NoEncryption(), + )) + public_key = private_key.public_key().public_bytes( + serialization.Encoding.Raw, + serialization.PublicFormat.Raw, + ) + package_global_osv_release( + source_database, + database_manifest, + asset, + release_manifest_path, + signature_path, + private_key_path, + key_id="global-key", + minimum_records=1, + ) + manifest = json.loads(release_manifest_path.read_text()) + envelope = json.loads(signature_path.read_text()) + + def metadata_download(url, *_args, **_kwargs): + return json.dumps(envelope if url.endswith("sig") else manifest).encode() + + def asset_download(_url, destination, _hosts, **_kwargs): + destination.seek(0) + destination.truncate(0) + destination.write(asset.read_bytes()) + destination.flush() + destination.seek(0) + return asset.stat().st_size, manifest["asset_sha256"] + + destination = root / "installed.sqlite" + with ( + patch("src.vulnerability.global_bootstrap._download", side_effect=metadata_download), + patch("src.vulnerability.global_bootstrap._download_to_file", side_effect=asset_download), + ): + result = bootstrap_global_osv_asset( + "https://github.com/database.zip", + "https://github.com/manifest", + "https://github.com/sig", + destination, + trusted_keys={"global-key": public_key}, + ) + self.assertEqual(result["state"], "active") + with VulnerabilityDatabase.read_only(str(destination)) as database: + active = database.active_snapshot() + self.assertIsNotNone(active) + self.assertEqual((active or {})["snapshot_id"], "global-osv-bootstrap-test") + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_global_osv_snapshot.py b/tests/test_global_osv_snapshot.py new file mode 100644 index 0000000..0ee3e12 --- /dev/null +++ b/tests/test_global_osv_snapshot.py @@ -0,0 +1,289 @@ +import hashlib +import json +import struct +import tempfile +import unittest +import zipfile +from pathlib import Path +from unittest.mock import patch + +from scripts.build_global_osv_vulndb import build_global_osv_snapshot +from scripts.verify_global_osv_vulndb import verify_global_osv_snapshot +from src.vulnerability.database import VulnerabilityDatabase +from src.vulnerability.full_snapshot import import_osv_zip, validate_zip_central_directory +from src.vulnerability.models import Component + + +class GlobalOsvSnapshotTests(unittest.TestCase): + @staticmethod + def _record(identifier: str, package: str, fixed: str = "2.0.0") -> dict: + return { + "id": identifier, + "aliases": ["CVE-2026-0001"] if identifier.endswith("1") else [], + "summary": "real source summary", + "details": "source-backed details", + "modified": "2026-07-23T00:00:00Z", + "affected": [ + { + "package": {"ecosystem": "PyPI", "name": package}, + "versions": ["1.0.0"], + "ranges": [{"type": "ECOSYSTEM", "events": [{"introduced": "0"}, {"fixed": fixed}]}], + } + ], + } + + def test_digest_only_source_evidence_keeps_hash_and_matching_data(self): + with tempfile.TemporaryDirectory() as directory: + database_path = Path(directory) / "database.sqlite" + record = self._record("OSV-1", "example") + with VulnerabilityDatabase(str(database_path)) as database: + stats = database.import_osv_records([record], source_record_mode="digest-only") + self.assertEqual(stats.advisories_imported, 1) + evidence = database.connection.execute( + "SELECT content_digest, record_json FROM source_records WHERE native_record_id = 'OSV-1'" + ).fetchone() + self.assertTrue(evidence["content_digest"].startswith("sha256:")) + marker = json.loads(evidence["record_json"]) + self.assertEqual(marker, {"_payload_omitted": True, "id": "OSV-1"}) + matches = database.match_component(Component("pypi", "example", "1.0.0")) + self.assertEqual([match.advisory_id for match in matches], ["OSV-1"]) + + def test_large_batch_can_defer_global_alias_correlation(self): + with VulnerabilityDatabase(":memory:") as database: + with patch.object(database, "correlate_aliases", wraps=database.correlate_aliases) as correlate: + stats = database.import_osv_records( + [self._record("OSV-1", "example")], + source_record_mode="digest-only", + correlate_aliases=False, + ) + self.assertEqual(stats.advisories_imported, 1) + correlate.assert_not_called() + self.assertEqual(database.alias_count(), 0) + + def test_package_less_git_advisory_is_retained_as_unmapped_source_evidence(self): + record = { + "id": "CVE-2026-9999", + "summary": "GIT-only advisory", + "affected": [{"ranges": [{"type": "GIT", "repo": "https://example.invalid/repo", "events": [{"introduced": "0"}]}]}], + } + with VulnerabilityDatabase(":memory:") as database: + stats = database.import_osv_records([record], source_record_mode="digest-only") + self.assertEqual(stats.advisories_imported, 1) + self.assertEqual(stats.affected_packages_imported, 0) + self.assertEqual(stats.unmapped_affected_entries, 1) + self.assertEqual(stats.errors, ()) + self.assertEqual(database.advisory_count(), 1) + + def test_compact_manifest_is_bounded_and_detects_content_mutation(self): + with tempfile.TemporaryDirectory() as directory: + database_path = Path(directory) / "database.sqlite" + with VulnerabilityDatabase(str(database_path)) as database: + database.import_osv_records([self._record("OSV-1", "example")], source_record_mode="digest-only") + first = database.build_compact_snapshot_manifest() + self.assertEqual(first["manifest_format"], "compact-v1") + self.assertNotIn("advisories", first) + self.assertEqual(first["advisory_count"], 1) + database.connection.execute("UPDATE advisories SET summary = 'changed' WHERE id = 'OSV-1'") + database.connection.commit() + second = database.build_compact_snapshot_manifest() + self.assertNotEqual(first["content_digest"], second["content_digest"]) + + def test_compact_snapshot_can_be_staged_and_quality_checked(self): + with tempfile.TemporaryDirectory() as directory: + database_path = Path(directory) / "database.sqlite" + with VulnerabilityDatabase(str(database_path)) as database: + database.import_osv_records([self._record("OSV-1", "example")], source_record_mode="digest-only") + provenance = { + "profile": "global-osv", + "completeness": "full-osv-source", + "production_full_database": False, + "snapshot_id": "global-osv-test", + "source_digest": "sha256:" + "a" * 64, + "source_record_mode": "digest-only", + "sources": {"osv-global": {"sha256": "sha256:" + "a" * 64}}, + } + manifest = database.build_compact_snapshot_manifest(provenance) + manifest.update(provenance) + database.stage_snapshot("global-osv-test", "sha256:" + "a" * 64, manifest) + quality = database.snapshot_quality_gate("global-osv-test") + self.assertTrue(quality["healthy"], quality) + database.connection.execute( + "UPDATE snapshots SET source_digest = ? WHERE snapshot_id = 'global-osv-test'", + ("sha256:" + "b" * 64,), + ) + database.connection.commit() + tampered = database.snapshot_quality_gate("global-osv-test") + self.assertFalse(tampered["healthy"]) + self.assertIn("manifest:source_digest", tampered["issues"]) + + def test_zip_import_is_streamed_and_reports_provenance(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + archive_path = root / "osv.zip" + database_path = root / "database.sqlite" + with zipfile.ZipFile(archive_path, "w", compression=zipfile.ZIP_DEFLATED) as archive: + archive.writestr("PyPI/OSV-1.json", json.dumps(self._record("OSV-1", "example"))) + archive.writestr("npm/OSV-2.json", json.dumps(self._record("OSV-2", "example-js"))) + with VulnerabilityDatabase(str(database_path)) as database: + report = import_osv_zip(database, archive_path, batch_records=1, source_record_mode="digest-only") + self.assertEqual(report.members_seen, 2) + self.assertEqual(report.advisories_imported, 2) + self.assertEqual(report.errors, ()) + self.assertTrue(report.archive_sha256.startswith("sha256:")) + self.assertEqual(database.advisory_count(), 2) + + def test_builder_publishes_only_a_staged_verified_snapshot(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + archive_path = root / "osv.zip" + source_manifest = root / "source.json" + output = root / "global.sqlite" + manifest_output = root / "global.manifest.json" + sha_output = root / "global.sha256" + with zipfile.ZipFile(archive_path, "w", compression=zipfile.ZIP_DEFLATED) as archive: + archive.writestr("PyPI/OSV-1.json", json.dumps(self._record("OSV-1", "example"))) + digest = "sha256:" + hashlib.sha256(archive_path.read_bytes()).hexdigest() + with zipfile.ZipFile(archive_path) as archive: + uncompressed = sum(info.file_size for info in archive.infolist()) + source_manifest.write_text(json.dumps({"sha256": digest, "url": "https://example.invalid/osv.zip", "records": 1, "uncompressed_bytes": uncompressed})) + manifest = build_global_osv_snapshot( + archive_path, + source_manifest, + output, + manifest_output, + sha_output, + snapshot_id="global-osv-test", + maximum_database_bytes=100_000_000, + reserve_free_bytes=1_000_000, + ) + self.assertTrue(output.is_file()) + ready_output = output.with_suffix(output.suffix + ".ready.json") + self.assertTrue(ready_output.is_file()) + self.assertEqual(manifest["completeness"], "full-osv-source") + self.assertFalse(manifest["production_full_database"]) + verification = verify_global_osv_snapshot(output, manifest_output, minimum_records=1) + self.assertEqual(verification["state"], "staged") + self.assertEqual(verification["integrity_check"], "ok") + with VulnerabilityDatabase(str(output)) as database: + status = database.snapshot_status("global-osv-test") + self.assertEqual(status["state"], "staged") + self.assertIsNone(database.active_snapshot()) + self.assertTrue(database.snapshot_quality_gate("global-osv-test")["healthy"]) + ready_output.unlink() + with self.assertRaises(ValueError): + verify_global_osv_snapshot(output, manifest_output, minimum_records=1) + + def test_zip_import_rejects_compressed_archive_over_limit_before_import(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + archive_path = root / "osv.zip" + with zipfile.ZipFile(archive_path, "w") as archive: + archive.writestr("OSV-1.json", json.dumps(self._record("OSV-1", "example"))) + with VulnerabilityDatabase(":memory:") as database: + with self.assertRaises(ValueError): + import_osv_zip(database, archive_path, max_archive_bytes=1) + self.assertEqual(database.advisory_count(), 0) + + def test_central_directory_member_limit_is_checked_before_zipfile_open(self): + with tempfile.TemporaryDirectory() as directory: + archive_path = Path(directory) / "osv.zip" + with zipfile.ZipFile(archive_path, "w") as archive: + archive.writestr("OSV-1.json", b"{}") + raw = bytearray(archive_path.read_bytes()) + eocd = raw.rfind(b"PK\x05\x06") + struct.pack_into(" Date: Thu, 23 Jul 2026 20:40:47 +0000 Subject: [PATCH 2/2] test: keep global bootstrap suite dependency-free --- tests/test_global_bootstrap.py | 46 ++++++++++++++++++---------------- 1 file changed, 24 insertions(+), 22 deletions(-) diff --git a/tests/test_global_bootstrap.py b/tests/test_global_bootstrap.py index 5aa54c6..16f0b62 100644 --- a/tests/test_global_bootstrap.py +++ b/tests/test_global_bootstrap.py @@ -7,8 +7,6 @@ from pathlib import Path from unittest.mock import patch -from cryptography.hazmat.primitives import serialization -from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PrivateKey from scripts.build_global_osv_vulndb import build_global_osv_snapshot from scripts.package_global_osv_release import package_global_osv_release from src.vulnerability.database import VulnerabilityDatabase @@ -139,26 +137,29 @@ def test_bootstrap_downloads_zip_verifies_database_and_activates(self): release_manifest_path = root / "database.release.manifest.json" signature_path = root / "database.sig.json" private_key_path = root / "private.key" - private_key = Ed25519PrivateKey.generate() - private_key_path.write_bytes(private_key.private_bytes( - serialization.Encoding.Raw, - serialization.PrivateFormat.Raw, - serialization.NoEncryption(), - )) - public_key = private_key.public_key().public_bytes( - serialization.Encoding.Raw, - serialization.PublicFormat.Raw, - ) - package_global_osv_release( - source_database, - database_manifest, - asset, - release_manifest_path, - signature_path, - private_key_path, - key_id="global-key", - minimum_records=1, - ) + private_key_path.write_bytes(b"k" * 32) + public_key = b"p" * 32 + + def fake_sign(payload, key_id, _private_key): + return { + "schema": "coderisktools.vulnerability.signed-manifest", + "version": 1, + "key_id": key_id, + "manifest": payload, + "signature": "test-only-envelope", + } + + with patch("scripts.package_global_osv_release.sign_manifest", side_effect=fake_sign): + package_global_osv_release( + source_database, + database_manifest, + asset, + release_manifest_path, + signature_path, + private_key_path, + key_id="global-key", + minimum_records=1, + ) manifest = json.loads(release_manifest_path.read_text()) envelope = json.loads(signature_path.read_text()) @@ -177,6 +178,7 @@ def asset_download(_url, destination, _hosts, **_kwargs): with ( patch("src.vulnerability.global_bootstrap._download", side_effect=metadata_download), patch("src.vulnerability.global_bootstrap._download_to_file", side_effect=asset_download), + patch("src.vulnerability.global_bootstrap.verify_manifest", return_value=manifest), ): result = bootstrap_global_osv_asset( "https://github.com/database.zip",