diff --git a/.coderisktools-baseline.json b/.coderisktools-baseline.json index 06e7994..43d9ec3 100644 --- a/.coderisktools-baseline.json +++ b/.coderisktools-baseline.json @@ -18,6 +18,7 @@ "sha256:317334d25e7f63f731739102c0f3324e149163dc9aa89fd062cb56a002c38e90", "sha256:32003d386f4c13bbd8e39554ecf8e790165bf5d8027978c3fd53eedadc18a6a3", "sha256:3e8c5ab6e930af06663435256e3205c0dcc548c366b128138504e64ad0a35340", + "sha256:3fa4350533e8d0b945c36390441343258bcf356aa7f90ffcec205723a48acfbb", "sha256:41a2d2511602221b2e73923b12ec4ed17126a52a1e4eefc457bad128e6cd4582", "sha256:43fbf8cbca5bb69e800d6bad8d2e93e5b0c9782cfd77b1a24e01d7dcaa23cb43", "sha256:44fff6f8123baedbb3805ab5510e8e75ef98479060a1db6814a23c2cadad7f72", @@ -51,7 +52,6 @@ "sha256:95f4fb0457e624269f2e6db5ad54c88942b2d7fe1092c79343b9c2ba0ae0e4f6", "sha256:9912c72fe03e28b66d5b4667c666714d1ba43cc93124187b65a453927d239cbb", "sha256:9f33a910547b5beef79ce1c0aba074f306a966846d1ca608d4a9fb72a2b2c601", - "sha256:a02ba68f0be4d9b78998ae99f3ee9197b26de0762b701a55986d33879d71064c", "sha256:a052f7f69ccc1e3c69286d4b4be5a8b64c951345afdd7a8c3a1568efff22e87a", "sha256:a153a81f7c9c09b188669b04c2e94fb1ab97c6cb257713180938b64bc24292fa", "sha256:a422a63976b0052a6999f2d10c8ad7b270d79c4abcb54b3e87c1b575aec957bc", diff --git a/CHANGELOG.md b/CHANGELOG.md index 115f136..a6863dd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,17 +4,27 @@ All notable changes to `coderisktools-scanner` are documented here. ## [Unreleased] -### Vulnerability enrichment +## [3.1.0] — 2026-07-23 -- versioned the local vulnerability database schema with explicit initialization, migration, current-state reporting and fail-closed future-version handling; -- preserved NVD configuration logic instead of flattening it: node `AND`/`OR`, `negate`, nested children and a legacy flat CPE projection are all available; -- preserved NVD references, source tags and CVE change history through parser, SQLite enrichment and normalized reports; -- added deterministic readback tests and documentation for the V5x–V5z contracts. +### Added -### Scanner evidence boundary +- added the opt-in, read-only vulnerability inventory and matching pipeline with explicit local SQLite selection; +- added controlled, user-triggered staging/update, verification, reconciliation, rollback and retention operations; +- added bounded public feed adapters and explicit provenance/quality reports without claiming full-feed coverage; +- added a real partial `seed` snapshot with 187 advisories, 378 affected-package rows and seven represented OSV ecosystems; +- added signed, pinned seed bootstrap and a separate explicit `--profile seed --apply` activation command. -- documented the required distinction between runtime remediation evidence and publication-safe redaction; -- runtime evidence preservation remains an open formatter/pipeline audit finding; checked-in/public artifacts remain synthetic or explicitly redacted. +### Fixed + +- directory self-scan now skips SQLite database artifacts, preventing the real seed from tripping the scanner byte cap in CI; +- bootstrap now verifies the detached Ed25519 manifest envelope, exact database SHA-256, SQLite integrity, foreign keys, snapshot identity and manifest counts before atomic installation. + +### Seed boundary + +- the seed is `completeness=partial` and `production_full_database=false`; +- 1,688 unresolved KEV/EPSS enrichments and 129 exact-alias conflicts are retained and disclosed rather than heuristically merged; +- an empty seed scan is not evidence that a project has no vulnerabilities; +- Core/Full activation remains separate and is never replaced automatically by seed bootstrap. ## [3.0.1] — 2026-07-20 diff --git a/README.md b/README.md index 7960faa..f64911e 100644 --- a/README.md +++ b/README.md @@ -500,7 +500,7 @@ Valid agents: `generic`, `codex`, `claude-code`. This is an explicit input contr ```yaml repos: - repo: https://github.com/9batalion/coderisktools-scanner - rev: v3.0.1 + rev: v3.1.0 hooks: - id: coderisktools-secret-scan ``` @@ -522,7 +522,7 @@ jobs: - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: fetch-depth: 0 - - uses: 9batalion/coderisktools-scanner@v3.0.1 + - uses: 9batalion/coderisktools-scanner@v3.1.0 with: profile: balanced ``` diff --git a/data/vulnerability-seed/README.md b/data/vulnerability-seed/README.md new file mode 100644 index 0000000..c9ce1c0 --- /dev/null +++ b/data/vulnerability-seed/README.md @@ -0,0 +1,63 @@ +# Verified vulnerability seed + +This directory contains a real, bounded starter snapshot for local development and integration tests. + +## Scope + +- profile: `seed` +- completeness: `partial` +- `production_full_database`: `false` +- advisories: 187 +- affected packages: 378 +- SQLite size: approximately 5.6 MB +- OSV: 87 imported advisories from selected PyPI, npm, Go, crates.io, Maven, NuGet and Packagist records +- CISA KEV: 1,653 source records processed; 2 exact enrichments and 1,651 unresolved records retained +- GHSA: bounded 100-record batch, 100 imported +- EPSS: 54 targeted records; 17 exact enrichments and 37 unresolved records retained +- exact alias index: 129 ambiguous alias conflicts are reported and are not heuristically merged + +This is not a complete Core database and must not be used to interpret an empty scan as proof of no vulnerabilities. The seed intentionally preserves unresolved enrichment and alias conflicts instead of making unsafe automatic merge decisions. + +## Verify + +From the repository root: + +```bash +sha256sum -c data/vulnerability-seed/seed-vulndb.sqlite.sha256 +python3 scripts/verify_seed_vulndb.py \ + --database data/vulnerability-seed/seed-vulndb.sqlite \ + --manifest data/vulnerability-seed/manifest.json +``` + +The verifier checks the SQLite integrity, foreign-key integrity, database digest, and manifest counts. + +## Pinned release bootstrap + +The `v3.1.0` release publishes the exact SQLite, detached manifest, SHA-256 sidecar, signed manifest envelope and public keyring. Bootstrap installs the database as **staged** only: + +```bash +secret-scanner vuln-db bootstrap \ + --asset-url https://github.com/9batalion/coderisktools-scanner/releases/download/v3.1.0/coderisktools-vulndb-seed-2026-07-23.sqlite \ + --manifest-url https://github.com/9batalion/coderisktools-scanner/releases/download/v3.1.0/coderisktools-vulndb-seed-2026-07-23.manifest.json \ + --signature-url https://github.com/9batalion/coderisktools-scanner/releases/download/v3.1.0/coderisktools-vulndb-seed-2026-07-23.sig \ + --keyring data/vulnerability-seed/release-keyring.json \ + --destination "$HOME/.local/share/coderisktools/vuln-db/seed.sqlite" +``` + +Activation is a separate explicit operation and is limited to `--profile seed`: + +```bash +secret-scanner vuln-db activate \ + --database "$HOME/.local/share/coderisktools/vuln-db/seed.sqlite" \ + --manifest data/vulnerability-seed/manifest.json \ + --profile seed \ + --apply +``` + +A seed activation never replaces a Core/Full pointer automatically. + +## Provenance + +The snapshot was built from verified staged CISA KEV and GHSA source artifacts, targeted EPSS data from FIRST, and OSV vulnerability records obtained through the public OSV API using explicit package probes. Raw source feeds are not committed to this repository. + +A larger Core snapshot belongs in a GitHub Release Asset, not in Git history. Any future replacement must update the manifest, SHA-256 sidecar, signed envelope, and independent verification evidence together. diff --git a/data/vulnerability-seed/manifest.json b/data/vulnerability-seed/manifest.json new file mode 100644 index 0000000..f3d6346 --- /dev/null +++ b/data/vulnerability-seed/manifest.json @@ -0,0 +1,68 @@ +{ + "advisory_count": 187, + "affected_package_count": 378, + "alias_conflict_count": 129, + "completeness": "partial", + "content_digest": "sha256:8c122da9ffbbe70f677503068382416ad0d9d31adcdff3ceb1a9adf50de48f04", + "correlation": "exact-alias-index-with-unresolved-conflicts", + "database": { + "advisory_count": 187, + "affected_package_count": 378, + "content_digest": "sha256:8c122da9ffbbe70f677503068382416ad0d9d31adcdff3ceb1a9adf50de48f04", + "path": "seed-vulndb.sqlite", + "sha256": "sha256:2bf63ca969a56a12551c491a956746d504409f13ea1c006abcf19e52ff0e6b04" + }, + "database_sha256": "sha256:2bf63ca969a56a12551c491a956746d504409f13ea1c006abcf19e52ff0e6b04", + "production_full_database": false, + "profile": "seed", + "provenance": { + "built_at": "2026-07-23", + "cache_reuse": "CISA KEV and GHSA verified staging cache; EPSS targeted CVEs from FIRST API", + "osv_transport": "OSV API per-package probes plus OSV vulnerability records" + }, + "release": { + "asset": "coderisktools-vulndb-seed-2026-07-23.sqlite", + "key_id": "coderisktools-seed-2026", + "manifest": "coderisktools-vulndb-seed-2026-07-23.manifest.json", + "sha256": "coderisktools-vulndb-seed-2026-07-23.sha256", + "signature": "coderisktools-vulndb-seed-2026-07-23.sig", + "tag": "v3.1.0" + }, + "schema": "coderisktools.vulnerability.seed-manifest.v1", + "snapshot_id": "seed-8c122da9ffbbe70f677503068382416a", + "source_digest": "sha256:a9499f0d50184de077690431a98027dbb7c52fe916e1dc8879db77f2a2ac4ea9", + "sources": { + "cisa-kev": { + "imported": 2, + "records": 1653, + "status": "complete", + "unresolved": 1651 + }, + "epss": { + "imported": 17, + "records": 54, + "status": "bounded", + "unresolved": 37 + }, + "ghsa": { + "imported": 100, + "records": 100, + "status": "bounded" + }, + "osv": { + "failed_ecosystems": [], + "imported": 87, + "records_per_ecosystem": 500, + "status": "partial", + "successful_ecosystems": [ + "PyPI", + "npm", + "Go", + "crates.io", + "Maven", + "NuGet", + "Packagist" + ] + } + } +} diff --git a/data/vulnerability-seed/release-keyring.json b/data/vulnerability-seed/release-keyring.json new file mode 100644 index 0000000..be197ca --- /dev/null +++ b/data/vulnerability-seed/release-keyring.json @@ -0,0 +1,7 @@ +{ + "keys": { + "coderisktools-seed-2026": "64d79d903860fc16b2adb99d9a8ebe6a05540b9a9a2437d0062ba1c552a380a1" + }, + "schema": "coderisktools.rule-keyring", + "version": 1 +} diff --git a/data/vulnerability-seed/seed-vulndb.sqlite b/data/vulnerability-seed/seed-vulndb.sqlite new file mode 100644 index 0000000..3681849 Binary files /dev/null and b/data/vulnerability-seed/seed-vulndb.sqlite differ diff --git a/data/vulnerability-seed/seed-vulndb.sqlite.sha256 b/data/vulnerability-seed/seed-vulndb.sqlite.sha256 new file mode 100644 index 0000000..664ef02 --- /dev/null +++ b/data/vulnerability-seed/seed-vulndb.sqlite.sha256 @@ -0,0 +1 @@ +2bf63ca969a56a12551c491a956746d504409f13ea1c006abcf19e52ff0e6b04 seed-vulndb.sqlite diff --git a/docs/EXTERNAL-EVIDENCE-COMPARISON.md b/docs/EXTERNAL-EVIDENCE-COMPARISON.md new file mode 100644 index 0000000..ad91684 --- /dev/null +++ b/docs/EXTERNAL-EVIDENCE-COMPARISON.md @@ -0,0 +1,24 @@ +# External evidence comparison + +The benchmark comparison fixture is intentionally offline and synthetic. It +compares identifier sets representing an internal result and explicitly +supplied OSV-Scanner, Trivy and Grype evidence. It does not execute any of the +external tools and does not merge their findings into `Finding`, `ScanResult` +or the local vulnerability database. + +## Difference meanings + +- `aligned`: the identifier is present in both result sets; +- `external_only`: the supplied external evidence contains an identifier not + present in the internal result; +- `internal_only`: the internal result contains an identifier absent from the + supplied external evidence. + +Differences are not proof that one tool is correct. They can result from +package identity normalization, version/range semantics, advisory database +freshness, severity policy, ignored paths, transitive dependency handling or +source-specific matching rules. Investigations must use the original evidence +and provenance rather than copying an external result. + +The fixture is a contract test for comparison/reporting behavior, not an +accuracy ranking of OSV-Scanner, Trivy or Grype. diff --git a/docs/FULL-FEED-COVERAGE-PLAN.md b/docs/FULL-FEED-COVERAGE-PLAN.md new file mode 100644 index 0000000..3d8817a --- /dev/null +++ b/docs/FULL-FEED-COVERAGE-PLAN.md @@ -0,0 +1,87 @@ +# Full feed coverage plan + +## Objective + +Build a reproducible local vulnerability database from declared sources without +claiming universal ecosystem coverage. A feed becomes `ready` only after its +adapter, fixture corpus, provenance, license/terms review, quality gate, +rebuild determinism and rollback path pass. + +## Source waves + +### Wave 1 — core advisory and exploitation sources + +- OSV +- NVD +- CISA KEV +- EPSS +- GitHub Advisory Database + +### Wave 2 — Linux distributions + +- Debian Security +- Ubuntu Security +- Red Hat Security +- SUSE Security +- Alpine Security + +### Wave 3 — CSAF providers + +- Generic CSAF 2.0 +- Provider-specific product trees +- Provider-specific status and remediation semantics + +### Wave 4 — package ecosystems + +- PyPI +- npm +- crates.io +- Maven Central +- NuGet +- remaining ecosystem adapters + +## Per-feed implementation contract + +1. Declare endpoint, source identity, terms/license status and update cadence. +2. Fetch only through the existing HTTPS allowlist, bounded streaming and + conditional request policy. +3. Verify content type, size, digest and source provenance. +4. Parse into the existing OSV-shaped import boundary without changing + `Finding`, `ScanResult` or fingerprint contracts. +5. Preserve source-native identifiers, aliases, references and raw status. +6. Run quality metrics and reject invalid snapshots before activation. +7. Test malformed, oversized, duplicate, stale and semantically ambiguous data. +8. Rebuild twice and compare artifact digests. +9. Verify rollback and air-gap export/import. +10. Mark the feed `ready` only when all evidence is stored. + +The OSV adapter now supports a separate staging operation: an allowlisted HTTPS +URL is streamed to an atomically replaced file and then passed to the existing +bounded OSV importer. Activation remains explicit (`activate=False` by default), +and the adapter reports the downloaded payload digest. Live feed acceptance, +license verification and a verified full snapshot are still required before OSV +can become `ready`. + +NVD now has a bounded API 2.0 file ingestion contract with source digest, +strict CVE parsing, exact advisory correlation, partial-record errors and +explicit activation. Live NVD feed acceptance and terms verification remain +required before `ready`. + +KEV, EPSS and GitHub Advisory now have bounded local feed orchestration with +source digest, delegation to existing database importers, partial-error +reporting and explicit activation. Their live feed, terms and full snapshot +evidence remain required before `ready`. + +When network access is unavailable, `evaluate_feed_artifact()` provides the +acceptance path for an externally acquired artifact from CI, cache or an +air-gapped transfer. It verifies JSON envelope, size/record bounds and SHA-256 +without downloading or activating the artifact; this is not equivalent to live +transport acceptance. + +## Non-goals + +- No assertion that all advisories worldwide are covered. +- No package-manager execution. +- No execution of scanned repository code. +- No automatic activation after download. +- No redistribution of source data before terms/license verification. diff --git a/docs/KNOWN-LIMITATIONS.md b/docs/KNOWN-LIMITATIONS.md new file mode 100644 index 0000000..435ca15 --- /dev/null +++ b/docs/KNOWN-LIMITATIONS.md @@ -0,0 +1,53 @@ +# Known limitations + +This document is part of the production snapshot boundary. It intentionally +lists limitations instead of implying complete vulnerability coverage. + +## Vulnerability data + +- Feed adapters are bounded contracts, not a claim of complete live-feed + ingestion or complete historical coverage. +- OSV-shaped records are supported locally; source-specific fields outside the + bounded adapter contract may be preserved only as metadata or omitted. +- Matching can be `indeterminate` when a package, version, range or ecosystem + cannot be compared safely. +- Enrichment and correlation evidence is additive and provenance-aware; it is + not a replacement for vendor advisory review. + +## Ecosystem coverage + +- Python, npm, Cargo/ crates.io and Go have the strongest local inventory path. +- Linux distribution, Maven, NuGet, RubyGems, Swift, Dart, Elixir, Haskell, + R, Conan and vcpkg adapters are bounded and fixture-driven. +- CSAF support is bounded to the documented product/PURL, advisory, provider, + remediation, vendor-status and quality-gate contract. +- The project must not be described as providing full support for all package + managers, distributions, feeds or CSAF provider extensions. + +## Benchmarking + +- Public fixtures are deterministic but intentionally small; they are not a + statistically complete vulnerability corpus. +- Precision/recall results are fixture results, not a security certification. +- Latency depends on hardware, Python version and database state. Absolute + cross-platform baselines require separate measurement. +- Comparisons with OSV-Scanner, Trivy and Grype are external-evidence work and + must not be represented as copied or authoritative results. + +## Production and recovery + +- Manifest signing verifies supplied keys and bytes; key custody and rotation + remain deployment responsibilities. +- Air-gap import restores a verified database but does not activate it. +- Rollback planning is non-destructive until an explicit apply operation. +- Release readiness reports validate supplied metadata; they cannot prove the + integrity of an upstream build system or hosting provider. + +## Scanner boundary + +- A clean result is not proof that code is secure. +- False positives and false negatives remain possible. +- The scanner is not a security audit, certification, legal opinion or + compliance guarantee. +- The scanner does not execute target-project code and ordinary scans do not + require network access. diff --git a/docs/P2-SNAPSHOT-SCHEMA-FOUNDATIONS.md b/docs/P2-SNAPSHOT-SCHEMA-FOUNDATIONS.md index 9939a90..48b3013 100644 --- a/docs/P2-SNAPSHOT-SCHEMA-FOUNDATIONS.md +++ b/docs/P2-SNAPSHOT-SCHEMA-FOUNDATIONS.md @@ -64,3 +64,189 @@ stored manifest against the current deterministic manifest for content digest, advisory count and affected-package count. `activate_snapshot()` now invokes this gate and raises `SnapshotActivationError` on any failed check; no active snapshot pointer or metadata is changed on failure. + +## Signed snapshot manifests + +`vulnerability.manifest_signing` defines the canonical Ed25519 envelope +`coderisktools.vulnerability.signed-manifest` version `1`. The signed payload +is deterministic JSON (`sort_keys`, compact separators, UTF-8). `sign_manifest` +uses the optional `cryptography` backend; installations without that optional +backend fail closed rather than silently falling back to HMAC or an unsigned +manifest. `verify_manifest` uses the repository's existing Ed25519 verifier, +and `stage_signed_snapshot()` verifies the envelope before staging. Tampering +with any manifest field is rejected. + +## Air-gap bundles + +`vulnerability.airgap` provides `export_air_gap_bundle()` and +`import_air_gap_bundle()`. A bundle contains `manifest.json` and +`snapshot.sqlite` in a bounded `tar.gz`. Import uses the safe archive extractor, +opens the staged SQLite database read-only, runs the database health gate and +compares the deterministic manifest before an fsync plus atomic replacement. +It never activates a snapshot; activation remains a separate explicit action. + +## Explicit retention and prune + +`VulnerabilityDatabase.prune_snapshots(keep_snapshot_ids, apply=False)` first +returns a deterministic dry-run plan. Only an explicit `apply=True` deletes +snapshot metadata outside the allowlist; active snapshots are never candidates. +Related `source_snapshots`, `import_errors` and `quality_metrics` rows are +removed together. This API does not infer retention from timestamps because +the current snapshot schema has no trusted acquisition timestamp. + +## V12a Debian adapter boundary + +`vulnerability.sources.debian.ingest_file()` is a bounded, local-only parser +for the versioned Debian feed fixture format. It preserves release, +source-package, binary-package, urgency, fixed-version and backport metadata. +It does not fetch Debian infrastructure, claim complete Security Tracker/OVAL +coverage, or activate/import records into the vulnerability database yet. + +V12a now also provides `ingest_file_to_database()`: it maps Debian binary +packages and fixed versions into the existing OSV-shaped bounded importer, +stages a snapshot without implicit activation, preserves Debian backport data +in `database_specific`, and writes `source_snapshots` plus `quality_metrics`. +This remains fixture-format support, not a claim of complete Debian feed +coverage. + +Debian fixed-only advisories are normalized as an explicit OSV range with +`introduced: "0"` followed by the Debian fixed revision. This makes the +backport boundary testable: a revision below `3.0.11-1~deb12u2` is affected, +while that revision and later revisions are not affected under the Debian +comparator. This is bounded matching behavior, not complete Debian archive +coverage. + +The V12b Ubuntu adapter currently provides the same bounded fixture/provenance +boundary for Ubuntu releases. It is parser-only and does not claim complete +USN/Ubuntu archive coverage or database staging until its dedicated integration +batch is completed. + +V12c adds the Ubuntu staging bridge using the same lifecycle contract as +Debian: normalized OSV-shaped records, staged snapshots without implicit +activation, source provenance, `source_snapshots`, and quality metrics. This +does not claim complete USN or Ubuntu archive coverage. + +Ubuntu fixed-version boundaries are covered by an integration test through +`evaluate_component()`. A version below the normalized fixed revision is +classified as affected, while the fixed revision is classified as not affected. + +V12e applies the same bounded staging contract to Red Hat fixtures, using the +RPM ecosystem and preserving RHSA severity, release, backport, and source +provenance metadata. This is not a claim of complete Red Hat advisory feed +coverage. + +V12f applies the bounded staging contract to SUSE/SLES fixtures through the RPM +ecosystem, preserving release, severity, backport, and provenance metadata. +Full SUSE advisory-feed coverage is intentionally not claimed yet. + +V12g extends the same bounded staging contract to Alpine APK fixtures and +verifies APK version revision ordering through `compare_alpine_version()`. + +V12h adds a cross-distro bounded backport contract test covering Debian, +Ubuntu, Red Hat, SUSE, and Alpine. It verifies preservation of backport flags, +fixed versions, and provenance digests without claiming complete real-feed +backport coverage. + +V13a adds a bounded Maven fixture/staging boundary with Maven coordinate +validation, fixed-version range matching, source provenance, and lifecycle +quality metadata. It does not claim complete Maven advisory coverage. + +V13b adds the same bounded fixture/staging boundary for NuGet package IDs, +including fixed-version matching and lifecycle provenance. Complete NuGet +advisory coverage is not claimed. + +V13c adds the same bounded fixture/staging boundary for RubyGems names, +including fixed-version matching and lifecycle provenance. Complete RubyGems +advisory coverage is not claimed. + +V13d adds the same bounded fixture/staging boundary for Swift package +identities, including fixed-version matching and lifecycle provenance. +Complete Swift advisory coverage is not claimed. + +V13e adds the same bounded fixture/staging boundary for Dart/pub package names, +including fixed-version matching and lifecycle provenance. Complete Dart +advisory coverage is not claimed. + +V13f adds the same bounded fixture/staging boundary for Elixir/Hex package +names, including fixed-version matching and lifecycle provenance. Complete +Elixir advisory coverage is not claimed. + +V13g adds the same bounded fixture/staging boundary for Haskell/Hackage package +names, including fixed-version matching and lifecycle provenance. Complete +Haskell advisory coverage is not claimed. + +V13h adds the same bounded fixture/staging boundary for R/CRAN package names, +including fixed-version matching and lifecycle provenance. Complete R advisory +coverage is not claimed. + +V13i adds the same bounded fixture/staging boundary for Conan package +references, including fixed-version matching and lifecycle provenance. +Complete Conan advisory coverage is not claimed. + +V13j adds the same bounded fixture/staging boundary for vcpkg port names, +including fixed-version matching and lifecycle provenance. Complete vcpkg +advisory coverage is not claimed. + +V14a adds a bounded generic CSAF 2.0 staging boundary with product-tree/PURL +mapping and `known_affected` preservation. Provider registry, remediations, +health and complete CSAF coverage are not claimed. + +V14b adds a deterministic CSAF provider registry and health-state contract. +Registry operations are local and do not perform implicit network calls. + +V14c preserves bounded CSAF remediations and vendor-specific product status in +source records and `database_specific`. Full remediation semantics remain +outside the verified subset. + +V14d adds `csaf_quality_gate()` for bounded validation of identifiers, product +references, provenance, remediation shape, and provider extensions. + +V14e integrates this gate into CSAF staging before import or snapshot write. +Invalid documents return `rejected` and are not activated. + +V15a adds offline benchmark foundations with deterministic precision, recall, +F1 and latency percentile metrics. Public fixture corpus remains open. + +V15b adds a versioned public benchmark fixture with golden results and a +bounded schema-validating loader. + +V15c adds an offline benchmark runner with precision, recall, F1, and p95 +latency acceptance thresholds. Failed gates are reported without raising. + +V15d adds `run_database_benchmark()` against the real +`VulnerabilityDatabase.evaluate_component()` API using local cases only. + +V15e adds a larger multi-ecosystem fixture covering PyPI, npm, crates.io, +Maven, and NuGet with golden precision/recall/F1 thresholds. + +V15f adds repeated performance benchmarking with bounded repetitions and p95 +latency acceptance. Platform-specific absolute baselines remain open. + +V15 quality suites add deterministic precision/recall partitions, explicit +false-positive/false-negative case IDs, and regression gates for newly +introduced FP/FN cases. + +V15 memory adds a bounded `tracemalloc` peak-memory benchmark with an explicit +acceptance limit; it is not a cross-platform absolute memory guarantee. + +External evidence comparison uses offline supplied-evidence fixtures for +OSV-Scanner, Trivy and Grype and reports differences without merging results. + +V15g adds a final benchmark report combining ecosystem status, confusion +matrix quality metrics, and aggregated performance. + +V16a adds a production snapshot readiness gate covering build identity, +digests, reproducibility, licenses, attributions, signature, and air-gap +verification. + +V16b adds database health and source coverage reports with deterministic source +shares, totals, quality errors, and active snapshot status. + +V16c adds non-destructive rollback planning and a real air-gap disaster +recovery test; imported recovery databases remain inactive. + +V16e adds a fail-closed final production snapshot report combining readiness, +health, source coverage, rollback, disaster recovery, and optional benchmark. + +V16 reproducibility evidence compares bounded regular-file artifact sets using +SHA-256 and reports missing, unexpected, or changed artifacts. diff --git a/docs/RELEASE-NOTES-3.0.1.md b/docs/RELEASE-NOTES-3.0.1.md new file mode 100644 index 0000000..800b576 --- /dev/null +++ b/docs/RELEASE-NOTES-3.0.1.md @@ -0,0 +1,43 @@ +# CodeRiskTools Scanner 3.0.1 — release notes + +## Scope + +This release documents the local-first vulnerability snapshot foundations and +production-readiness checks added after the scanner's existing secret/config +scanning features. The release does not claim complete coverage of every +package ecosystem or advisory feed. + +## Included and verified + +- bounded OSV-shaped local ingestion and SQLite vulnerability matching; +- PURL normalization and explicit affected/not-affected/indeterminate status; +- bounded snapshot staging, quality checks, signing and air-gap export/import; +- bounded Linux distribution and package-ecosystem adapters; +- CSAF import with provider metadata, remediation preservation and a quality gate; +- offline benchmark fixtures, database integration, repeated performance runs + and final quality/performance reports; +- production snapshot readiness, database health, source coverage, rollback + planning and disaster-recovery checks; +- existing secret/config scan contracts, SARIF/JSON output and offline policy + boundaries remain unchanged. + +## Operational guarantees + +Ordinary scans remain local-first and do not require network access. The tool +does not execute code from the target repository. Vulnerability evidence and +secret findings remain separate result domains. Secret contents may be +redacted by secret-output formatters; vulnerability evidence is not silently +redacted. + +## Validation + +The release gate was run locally with the project's test suite, optimized +Python test execution, bytecode compilation, whitespace validation and the +secret self-scan. Exact results are recorded in the project session/CI output; +packagers should rerun the same gates for their target platform. + +## Upgrade note + +Existing CLI, `Finding`, `ScanResult`, baseline, suppression, VEX and SARIF +contracts are preserved. Vulnerability database features are opt-in and use +explicit local inputs or already-staged snapshots. diff --git a/docs/RELEASE-NOTES-3.1.0.md b/docs/RELEASE-NOTES-3.1.0.md new file mode 100644 index 0000000..6a2bcd5 --- /dev/null +++ b/docs/RELEASE-NOTES-3.1.0.md @@ -0,0 +1,42 @@ +# CodeRiskTools Scanner 3.1.0 — release notes + +CodeRiskTools Scanner 3.1.0 adds a controlled, local-first vulnerability database workflow and publishes a real, verified **partial seed** snapshot for bootstrap and integration testing. + +## What is included + +- opt-in local dependency inventory and vulnerability matching; +- user-triggered staging, verification, reconciliation, rollback and retention commands; +- bounded source adapters and explicit provenance/quality evidence; +- a 5.9 MB SQLite seed with 187 advisories and 378 affected-package rows; +- represented OSV ecosystems: PyPI, npm, Go, crates.io, Maven, NuGet and Packagist; +- signed pinned bootstrap that installs seed as staged only; +- explicit `vuln-db activate --profile seed --apply` activation; +- real lodash `4.17.15` end-to-end matching evidence with stable fingerprints. + +## Release assets + +- Python wheel and source distribution; +- `coderisktools-vulndb-seed-2026-07-23.sqlite`; +- detached seed manifest; +- SHA-256 sidecar; +- Ed25519 signed manifest envelope (`.sig` JSON); +- public release keyring. + +## Verified seed facts + +- `profile=seed`; +- `completeness=partial`; +- `production_full_database=false`; +- SQLite `PRAGMA integrity_check=ok`; +- SQLite `PRAGMA foreign_key_check=0`; +- database SHA-256 `2bf63ca969a56a12551c491a956746d504409f13ea1c006abcf19e52ff0e6b04`; +- CISA KEV: 1,653 processed, 2 exact enrichments, 1,651 unresolved retained; +- EPSS: 54 processed, 17 exact enrichments, 37 unresolved retained; +- GHSA: bounded 100 imported; +- OSV: 87 imported. + +## Important limitations + +This seed is not Core, Full, Production or Complete. It retains 129 exact-alias conflicts rather than heuristically merging advisories. A zero-finding seed scan is not proof that a project has no vulnerabilities. Updates remain user-triggered; ordinary scanner runs do not download feeds. + +A clean scanner result is not proof that code is secure. This release is not a security audit, certification, compliance guarantee or legal opinion. diff --git a/docs/V17-CONTROLLED-UPDATE-PIPELINE.md b/docs/V17-CONTROLLED-UPDATE-PIPELINE.md new file mode 100644 index 0000000..d15f85c --- /dev/null +++ b/docs/V17-CONTROLLED-UPDATE-PIPELINE.md @@ -0,0 +1,101 @@ +# Controlled vulnerability database update pipeline + +The vulnerability scanner does not download advisory data. Scanning is read-only +against an already active local snapshot: + +```text +python -m src vuln scan --root ./repo --database DATA/snapshots/.../snapshot.sqlite3 +``` + +The current supported build profile is `core`: + +```text +python -m src vuln-db update --full --profile core +``` + +Core includes OSV JSONL streaming, GitHub Advisory, CISA KEV and EPSS. The +pipeline requires at least 8 GiB free space before network transfer starts; +this accounts for downloads, staging and the temporary SQLite snapshot. The +larger extended profile is not enabled by this command yet. + +At the start of this explicit update operation the updater loads its source +configuration and fetches the declared feeds. The scanner itself never performs +this fetch. + +To create the starter configuration explicitly: +``` + +The default configuration path is: + +```text +~/.config/coderisktools/vuln-db.json +``` + +The built-in starter configuration contains bounded entries for NVD, CISA KEV, +EPSS and GitHub Advisory. It also lists OSV, CVE v5, Debian, Ubuntu and RustSec +as `enabled: false` until their dedicated format/archive/pagination adapters are +connected to the multi-source importer. It is not a claim of complete global +coverage. + +The default data root is: + +```text +~/.local/share/coderisktools/vuln-db/ + staging/ + snapshots/ + active +``` + +## Configuration + +The bounded full-update configuration declares one adapter-shaped source per +entry: + +```json +{ + "sources": [ + { + "source_id": "osv", + "format": "osv", + "url": "https://example.invalid/osv.json", + "allowed_hosts": ["example.invalid"] + } + ] +} +``` + +Supported importer formats currently include `osv`, `nvd`, `cve-v5`, `kev`, +`epss`, and `github-advisories`. Each source is fetched with HTTPS, host +allowlisting, timeout and a byte limit. Its bytes are retained in the staging +run, hashed, parsed, and imported through the existing database importer. + +## Activation boundary + +Without `--apply`, the command only builds and verifies a staged snapshot: + +```text +python -m src vuln-db update --full \ + --config ./vuln-db.json \ + --root ./coderisktools-data +``` + +Activation is explicit: + +```text +python -m src vuln-db update --full \ + --config ./vuln-db.json \ + --root ./coderisktools-data \ + --active ./coderisktools-data/active \ + --apply +``` + +The database is built under a temporary staging directory, integrity-checked, +manifested, moved into `snapshots/`, verified again, and only then +can the active pointer be atomically replaced. A failed import preserves a +small `update-error.json` diagnostic in staging and cannot replace the active +snapshot. + +This implementation is deliberately conservative: a source is not considered +full-coverage `ready` merely because its adapter can fetch and import a file. +Live acceptance, terms/license evidence, quality metrics, reproducibility and +rollback evidence remain separate gates. diff --git a/docs/VULNERABILITY-DATABASE-MASTER-TODO-HERMES.md b/docs/VULNERABILITY-DATABASE-MASTER-TODO-HERMES.md index 43a9539..607c57d 100644 --- a/docs/VULNERABILITY-DATABASE-MASTER-TODO-HERMES.md +++ b/docs/VULNERABILITY-DATABASE-MASTER-TODO-HERMES.md @@ -2594,10 +2594,15 @@ src/ - [x] Quality metrics. - [x] Source snapshots. - [ ] Pełna baza lokalna — nieukończone. -- [ ] Benchmark — pełny benchmark jakości/wydajności pending. +- [x] Benchmark — bounded quality, latency, repeated performance and memory + benchmark suites; full global-feed benchmark remains open. - [x] Rebuild determinism — w zakresie obecnego fixture/snapshot contract. - [x] Ograniczenia pamięci — bounded `max_records` i `max_record_bytes`; pełny low-memory contract dla globalnych feedów pending. +V4 bounded scope is complete: supplied-record import, limits, metrics, source +snapshots, deterministic fixtures, and benchmark suites are verified. Full +all-ecosystem/global-feed local database remains intentionally open. + ## Stage V5 — CVE i GHSA correlation - [x] CVE List V5. @@ -2682,69 +2687,176 @@ src/ ## Stage V12 — dystrybucje Linux -- [ ] Debian. -- [ ] Ubuntu. -- [ ] Red Hat. -- [ ] SUSE. -- [ ] Alpine. -- [ ] RPM comparator. -- [ ] Debian comparator. -- [ ] APK comparator. -- [ ] Backport awareness. +V12a: dodano bounded lokalny parser Debian Security feed fixture (`debian.py`) +z zachowaniem release/source/binary/backport metadata. Pełny Debian adapter, +provenance, staging bridge i bounded backport-aware matching są częściowo +zaimplementowane; real feed mapping i pełna coverage pozostają otwarte. +V12b: dodano bounded lokalny Ubuntu fixture parser (`ubuntu.py`) z release, +provenance i source digest; staging bridge oraz pełny Ubuntu feed mapping są +jeszcze otwarte. +V12c: Ubuntu fixture advisories są mapowane do OSV-shaped importera i staged +z `source_snapshots` oraz `quality_metrics`; pełne USN mapping i coverage nadal +pozostają otwarte. +V12d: zweryfikowano Ubuntu fixed-version boundary przez istniejący comparator +(`affected` przed `3.0.11-1ubuntu2.1`, `not_affected` od wersji fixed). +V12e: dodano bounded Red Hat fixture parser i staging bridge z RPM ecosystem, +provenance, source snapshots, quality metrics oraz zweryfikowanym fixed-version +boundary; pełne RHSA feed mapping pozostaje otwarte. +V12f: dodano bounded SUSE/SLES fixture parser i staging bridge z RPM ecosystem, +provenance, source snapshots, quality metrics oraz zweryfikowanym fixed-version +boundary; pełne SUSE advisory mapping pozostaje otwarte. +V12g: Alpine fixture parser i staging bridge używają istniejącego +`compare_alpine_version()`; APK fixed-version boundary jest zweryfikowany, +ale pełne Alpine feed mapping pozostaje otwarte. +V12h: dodano przekrojowy bounded backport contract test dla Debian, Ubuntu, +Red Hat, SUSE i Alpine; wszystkie adaptery zachowują `backport`, fixed version +i provenance digest. Pełna backport awareness dla realnych feedów pozostaje +otwarta. + +- [x] Debian — bounded fixture parser, provenance, staging and matching. +- [x] Ubuntu — bounded fixture parser, provenance, staging and matching. +- [x] Red Hat — bounded fixture parser, provenance, staging and matching. +- [x] SUSE — bounded fixture parser, provenance, staging and matching. +- [x] Alpine — bounded fixture parser, provenance, staging and matching. +- [x] RPM comparator — implemented and tested in P1; V12 source integration remains open. +- [x] Debian comparator — implemented and tested in P1; V12 source integration remains bounded. +- [x] APK comparator — istniejący `compare_alpine_version()` zweryfikowany testem V12g. +- [x] Backport awareness — bounded cross-distro contract; full live-feed awareness remains open. ## Stage V13 — pozostałe ekosystemy -- [ ] Maven. -- [ ] NuGet. -- [ ] Ruby. -- [ ] Swift. -- [ ] Dart. -- [ ] Elixir. -- [ ] Haskell. -- [ ] R. -- [ ] Conan. -- [ ] vcpkg. +V13a: dodano bounded Maven fixture parser i staging bridge z coordinate +validation, provenance, source snapshots, quality metrics oraz fixed-version +boundary; pełne Maven advisory/feed mapping pozostaje otwarte. +V13b: dodano bounded NuGet fixture parser i staging bridge z package-ID +validation, provenance, source snapshots, quality metrics oraz fixed-version +boundary; pełne NuGet advisory/feed mapping pozostaje otwarte. +V13c: dodano bounded RubyGems fixture parser i staging bridge z gem-name +validation, provenance, source snapshots, quality metrics oraz fixed-version +boundary; pełne RubyGems advisory/feed mapping pozostaje otwarte. +V13d: dodano bounded Swift fixture parser i staging bridge z package-identity +validation, provenance, source snapshots, quality metrics oraz fixed-version +boundary; pełne Swift advisory/feed mapping pozostaje otwarte. +V13e: dodano bounded Dart/pub fixture parser i staging bridge z package-name +validation, provenance, source snapshots, quality metrics oraz fixed-version +boundary; pełne Dart advisory/feed mapping pozostaje otwarte. +V13f: dodano bounded Elixir/Hex fixture parser i staging bridge z package-name +validation, provenance, source snapshots, quality metrics oraz fixed-version +boundary; pełne Elixir advisory/feed mapping pozostaje otwarte. +V13g: dodano bounded Haskell/Hackage fixture parser i staging bridge z +package-name validation, provenance, source snapshots, quality metrics oraz +fixed-version boundary; pełne Haskell advisory/feed mapping pozostaje otwarte. +V13h: dodano bounded R/CRAN fixture parser i staging bridge z package-name +validation, provenance, source snapshots, quality metrics oraz fixed-version +boundary; pełne R advisory/feed mapping pozostaje otwarte. +V13i: dodano bounded Conan fixture parser i staging bridge z package-reference +validation, provenance, source snapshots, quality metrics oraz fixed-version +boundary; pełne Conan advisory/feed mapping pozostaje otwarte. +V13j: dodano bounded vcpkg fixture parser i staging bridge z port-name +validation, provenance, source snapshots, quality metrics oraz fixed-version +boundary; pełne vcpkg advisory/feed mapping pozostaje otwarte. + +- [x] Maven — bounded fixture/staging contract. +- [x] NuGet — bounded fixture/staging contract. +- [x] Ruby — bounded fixture/staging contract. +- [x] Swift — bounded fixture/staging contract. +- [x] Dart — bounded fixture/staging contract. +- [x] Elixir — bounded fixture/staging contract. +- [x] Haskell — bounded fixture/staging contract. +- [x] R — bounded fixture/staging contract. +- [x] Conan — bounded fixture/staging contract. +- [x] vcpkg — bounded fixture/staging contract. ## Stage V14 — CSAF vendor feeds -- [ ] Generic CSAF importer. -- [ ] Provider registry. -- [ ] Provider health. -- [ ] Product tree. -- [ ] Remediations. -- [ ] Vendor-specific status. +V14a: dodano bounded generic CSAF 2.0 importer z product-tree/PURL mapping, +`known_affected` status, provenance, source snapshots i quality metrics. +Provider registry, health, remediations, vendor-specific status i pełne CSAF +coverage pozostają otwarte. +V14b: dodano additive CSAF provider registry z walidacją endpointów, +deterministycznym sortowaniem i rejestrowaniem wyników bez implicit network +calls. Raport health rozróżnia `healthy`, `unhealthy`, `disabled` i `unknown`. +V14c: importer CSAF zachowuje bounded `remediations` oraz vendor-specific +product status w rekordzie źródłowym i `database_specific`; pełne semantics +remediation/status pozostają ograniczone do zweryfikowanego podzbioru. +V14d: dodano additive `csaf_quality_gate()` walidujący identyfikatory, +referencje produktów, provenance, remediation shape i provider extensions oraz +raportujący metryki jakości. +V14e: CSAF staging wywołuje quality gate przed importem i zapisem snapshotu; +niepoprawne dokumenty zwracają jawny `rejected` bez aktywacji. + +- [x] Generic CSAF importer — bounded CSAF 2.0 contract. +- [x] Provider registry — no implicit network calls. +- [x] Provider health — deterministic registry health states. +- [x] Product tree — bounded product/PURL mapping. +- [x] Remediations — bounded preservation contract. +- [x] Vendor-specific status — bounded preservation contract. - [ ] Licencje. -- [ ] Quality gates. - -## Stage V15 — pełny benchmark - -- [ ] Public fixtures. -- [ ] Precision. -- [ ] Recall. -- [ ] Performance. -- [ ] Memory. -- [ ] False positive suite. -- [ ] False negative suite. -- [ ] Regression gates. -- [ ] Comparison with OSV-Scanner. -- [ ] Comparison with Trivy. -- [ ] Comparison with Grype. -- [ ] Wyjaśnienie różnic bez kopiowania ich wyników. +- [x] Quality gates — enforced before CSAF staging. + +## Stage V15 — benchmark and acceptance gates + +V15a: dodano offline benchmark core z fixture cases, confusion-matrix +metrics (`precision`, `recall`, `F1`) oraz latency summary (`min`, `mean`, +`p50`, `p95`, `max`). Benchmark nie wykonuje kodu skanowanego repozytorium. +V15b: dodano wersjonowany public fixture `tests/fixtures/v15_public_benchmark.json` +z golden results oraz bounded loader z walidacją schema, limits i metryk. +V15c: dodano offline `run_fixture_benchmark()` z progami precision/recall/F1 +i max p95 latency; niespełnione gate’y są raportowane bez wyjątku. +V15d: dodano `run_database_benchmark()` wywołujący rzeczywiste +`VulnerabilityDatabase.evaluate_component()` na lokalnych cases i raportujący +precision/recall/F1 oraz latency. +V15e: dodano większy fixture wieloekosystemowy (PyPI, npm, crates.io, Maven, +NuGet) z golden thresholds precision/recall/F1 i integracyjną walidacją przez +rzeczywistą bazę podatności. +V15f: dodano repeated performance runner z kontrolowaną liczbą powtórzeń, +agregacją latency (`min`, `mean`, `p95`, `max`) i limitem `max_p95_ms`. +V15g: dodano `build_final_benchmark_report()` łączący statusy ekosystemów, +confusion matrix, precision/recall/F1 oraz zagregowaną wydajność. + +- [x] Public benchmark fixtures — versioned local fixtures and multi-ecosystem fixture. +- [x] Precision — benchmark metrics and database integration. +- [x] Recall — benchmark metrics and database integration. +- [x] Performance — repeated benchmark with p95 acceptance. +- [x] Memory — bounded `tracemalloc` peak-memory benchmark with acceptance limit. +- [x] False positive suite — deterministic FP partition and case IDs. +- [x] False negative suite — deterministic FN partition and case IDs. +- [x] Regression gates — rejects newly introduced FP/FN cases. +- [x] Comparison with OSV-Scanner — offline supplied-evidence fixture. +- [x] Comparison with Trivy — offline supplied-evidence fixture. +- [x] Comparison with Grype — offline supplied-evidence fixture. +- [x] Wyjaśnienie różnic bez kopiowania ich wyników — documented difference semantics. ## Stage V16 — produkcyjny snapshot +V16a: dodano `production_snapshot_gate()` sprawdzający build ID, digesty, +reproducibility metadata, licenses, attributions, manifest signature i +zweryfikowany air-gap bundle. +V16b: dodano `database_health_report()` oraz `source_coverage_report()` z +deterministycznym raportowaniem statusu snapshotu, błędów jakości, udziałów +źródeł i sumarycznych rekordów/advisories. +V16c: dodano nieinwazyjny `rollback_readiness_report()` oparty o planowanie +`prune_snapshots(..., apply=False)` oraz test disaster recovery przez realny +air-gap export/import, z zachowaniem `activated: false` po imporcie. +V16d: dodano `docs/RELEASE-NOTES-3.0.1.md` i `docs/KNOWN-LIMITATIONS.md` +opisujące zweryfikowany zakres, bounded adapters, benchmark boundaries, +recovery semantics i brak deklaracji pełnego pokrycia ekosystemów. +V16e: dodano `build_production_snapshot_report()` łączący readiness, +database health, source coverage, rollback, disaster recovery i opcjonalny +benchmark w jeden fail-closed raport `ready`/`failed`. + - [ ] Full build. -- [ ] Reproducibility. -- [ ] Licenses. -- [ ] Attributions. -- [ ] Sign manifest. -- [ ] Air-gap bundle. -- [ ] Release notes. -- [ ] Database health report. -- [ ] Source coverage report. -- [ ] Known limitations. -- [ ] Rollback test. -- [ ] Disaster recovery test. +- [x] Reproducibility — bounded regular-file SHA-256 comparison report. +- [x] Licenses. +- [x] Attributions. +- [x] Sign manifest. +- [x] Air-gap bundle. +- [x] Release notes. +- [x] Database health report. +- [x] Source coverage report. +- [x] Known limitations. +- [x] Rollback test. +- [x] Disaster recovery test. --- diff --git a/pyproject.toml b/pyproject.toml index 272b342..d212a20 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "coderisktools-scanner" -version = "3.0.1" +version = "3.1.0" description = "Local-first open-source scanner for secret-like values and risky configuration changes in code diffs." readme = "README.md" license = "MIT" diff --git a/scripts/build_seed_vulndb.py b/scripts/build_seed_vulndb.py new file mode 100644 index 0000000..13397cb --- /dev/null +++ b/scripts/build_seed_vulndb.py @@ -0,0 +1,149 @@ +#!/usr/bin/env python3 +"""Build a bounded real seed database from verified cache plus OSV API queries.""" +from __future__ import annotations + +import argparse +import json +import shutil +import sys +import tempfile +import urllib.parse +import urllib.request +from pathlib import Path +from urllib.error import HTTPError + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +from src.vulnerability.seed import SEED_ECOSYSTEMS, build_seed_database + + +PACKAGE_PROBES = ( + ("PyPI", "django", "2.2.0"), + ("npm", "lodash", "4.17.15"), + ("Go", "github.com/golang-jwt/jwt/v4", "4.0.0"), + ("crates.io", "time", "0.1.44"), + ("Maven", "org.apache.logging.log4j:log4j-core", "2.14.1"), + ("NuGet", "System.Text.Encodings.Web", "4.7.0"), + ("Packagist", "symfony/http-foundation", "5.4.0"), +) + + +def _query_package(ecosystem: str, package: str, version: str) -> list[str]: + payload = json.dumps({"package": {"ecosystem": ecosystem, "name": package}, "version": version}).encode("utf-8") + request = urllib.request.Request("https://api.osv.dev/v1/query", data=payload, headers={"Accept": "application/json", "Content-Type": "application/json", "User-Agent": "coderisktools-seed/1"}, method="POST") + with urllib.request.urlopen(request, timeout=30) as response: + value = json.loads(response.read().decode("utf-8")) + return [item["id"] for item in value.get("vulns", []) if isinstance(item, dict) and isinstance(item.get("id"), str)] + + +def _fetch_epss(cves: list[str]) -> dict: + if not cves: + return {"data": []} + query = ",".join(cves[:1000]) + request = urllib.request.Request( + "https://api.first.org/data/v1/epss?cve=" + urllib.parse.quote(query), + headers={"Accept": "application/json", "User-Agent": "coderisktools-seed/1"}, + ) + with urllib.request.urlopen(request, timeout=30) as response: + value = json.loads(response.read().decode("utf-8")) + return value if isinstance(value, dict) else {"data": []} + + +def _cache_with_targeted_epss(cache: Path, records_by_ecosystem: dict[str, list[dict]]) -> Path: + temporary = Path(tempfile.mkdtemp(prefix="coderisktools-seed-cache-")) + for filename in ("cisa-kev.json", "github-advisories.json"): + shutil.copy2(cache / filename, temporary / filename) + cves = sorted({alias for records in records_by_ecosystem.values() for record in records for alias in record.get("aliases", []) if isinstance(alias, str) and alias.startswith("CVE-")}) + (temporary / "epss.json").write_text(json.dumps(_fetch_epss(cves), sort_keys=True), encoding="utf-8") + return temporary + + +def _fetch_vulnerability(cve: str) -> dict | None: + request = urllib.request.Request( + f"https://api.osv.dev/v1/vulns/{cve}", + headers={"Accept": "application/json", "User-Agent": "coderisktools-seed/1"}, + ) + try: + with urllib.request.urlopen(request, timeout=30) as response: + value = json.loads(response.read().decode("utf-8")) + except HTTPError as exc: + if exc.code == 404: + return None + raise + return value if isinstance(value, dict) else None + + +def _post_querybatch(cves: list[str]) -> list[dict]: + payload = json.dumps({"queries": [{"query": cve} for cve in cves]}).encode("utf-8") + request = urllib.request.Request( + "https://api.osv.dev/v1/querybatch", + data=payload, + headers={"Accept": "application/json", "Content-Type": "application/json", "User-Agent": "coderisktools-seed/1"}, + method="POST", + ) + with urllib.request.urlopen(request, timeout=30) as response: + body = json.loads(response.read().decode("utf-8")) + records: list[dict] = [] + for result in body.get("results", []): + for vulnerability in result.get("vulns", []): + if isinstance(vulnerability, dict) and isinstance(vulnerability.get("id"), str): + records.append(vulnerability) + return records + + +def _cache_cves(cache: Path, limit: int) -> list[str]: + cves: list[str] = [] + for filename in ("cisa-kev.json", "github-advisories.json"): + payload = json.loads((cache / filename).read_text(encoding="utf-8")) + records = payload.get("vulnerabilities", []) if isinstance(payload, dict) else payload + for record in records: + if filename.startswith("cisa"): + value = record.get("cveID") + else: + value = record.get("cve_id") + if isinstance(value, str) and value.startswith("CVE-") and value not in cves: + cves.append(value) + if len(cves) >= limit: + return cves + return cves + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--cache", required=True) + parser.add_argument("--output", required=True) + parser.add_argument("--max-records-per-ecosystem", type=int, default=500) + parser.add_argument("--max-cves", type=int, default=500) + args = parser.parse_args() + cache = Path(args.cache) + cves = _cache_cves(cache, args.max_cves) + if not cves: + raise SystemExit("verified cache contains no CVE identifiers") + records_by_ecosystem = {ecosystem: [] for ecosystem in SEED_ECOSYSTEMS} + seen_ids: set[str] = set() + for ecosystem, package, version in PACKAGE_PROBES: + for identifier in _query_package(ecosystem, package, version): + if identifier in seen_ids: + continue + record = _fetch_vulnerability(identifier) + if record: + records_by_ecosystem[ecosystem].append(record) + seen_ids.add(identifier) + for cve in cves: + record = _fetch_vulnerability(cve) + if not record: + continue + ecosystems = {affected.get("package", {}).get("ecosystem") for affected in record.get("affected", []) if isinstance(affected, dict)} + for ecosystem in ecosystems & set(SEED_ECOSYSTEMS): + if len(records_by_ecosystem[ecosystem]) < args.max_records_per_ecosystem: + records_by_ecosystem[ecosystem].append(record) + targeted_cache = _cache_with_targeted_epss(cache, records_by_ecosystem) + try: + manifest = build_seed_database(targeted_cache, args.output, records_by_ecosystem, max_records_per_ecosystem=args.max_records_per_ecosystem) + finally: + shutil.rmtree(targeted_cache, ignore_errors=True) + print(json.dumps(manifest, ensure_ascii=False, sort_keys=True, indent=2)) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/normalize_sdist.py b/scripts/normalize_sdist.py new file mode 100644 index 0000000..0a9e673 --- /dev/null +++ b/scripts/normalize_sdist.py @@ -0,0 +1,93 @@ +#!/usr/bin/env python3 +"""Normalize a Python sdist tar.gz to deterministic release bytes.""" +from __future__ import annotations + +import argparse +import copy +import gzip +import os +import tarfile +import tempfile +from pathlib import Path, PurePosixPath + +MAX_MEMBERS = 10_000 +MAX_TOTAL_BYTES = 256 * 1024 * 1024 + + +def _safe_name(name: str) -> None: + path = PurePosixPath(name) + if not name or path.is_absolute() or ".." in path.parts or "\\" in name or "\x00" in name: + raise ValueError("sdist contains an unsafe member path") + + +def normalize_sdist(source: str | os.PathLike[str], destination: str | os.PathLike[str], *, epoch: int) -> None: + """Rewrite a regular-file/directory sdist with fixed metadata and gzip header.""" + if type(epoch) is not int or epoch < 0: + raise ValueError("epoch must be a non-negative integer") + source_path = Path(source) + destination_path = Path(destination) + if source_path.is_symlink() or not source_path.is_file(): + raise ValueError("source sdist must be a regular non-symlink file") + if destination_path.is_symlink() or source_path.resolve() == destination_path.resolve(strict=False): + raise ValueError("destination must be a distinct non-symlink path") + destination_path.parent.mkdir(parents=True, exist_ok=True) + fd, temp_name = tempfile.mkstemp(prefix=f".{destination_path.name}.", dir=str(destination_path.parent)) + os.close(fd) + temporary = Path(temp_name) + seen: set[str] = set() + total = 0 + try: + with tarfile.open(source_path, "r:gz") as input_archive: + members = input_archive.getmembers() + if not members or len(members) > MAX_MEMBERS: + raise ValueError("sdist member count is outside the allowed range") + with temporary.open("wb") as raw_output: + with gzip.GzipFile(filename="", mode="wb", fileobj=raw_output, compresslevel=9, mtime=epoch) as gzip_output: + with tarfile.open(fileobj=gzip_output, mode="w", format=tarfile.PAX_FORMAT) as output_archive: + for original in members: + _safe_name(original.name) + if original.name in seen: + raise ValueError("sdist contains duplicate member paths") + seen.add(original.name) + if not (original.isfile() or original.isdir()): + raise ValueError("sdist contains an unsupported member type") + total += original.size + if total > MAX_TOTAL_BYTES: + raise ValueError("sdist payload exceeds the byte limit") + member = copy.copy(original) + member.mtime = epoch + member.uid = 0 + member.gid = 0 + member.uname = "" + member.gname = "" + member.pax_headers = {} + if member.isfile(): + stream = input_archive.extractfile(original) + if stream is None: + raise ValueError("sdist regular member cannot be read") + output_archive.addfile(member, stream) + else: + output_archive.addfile(member) + raw_output.flush() + os.fsync(raw_output.fileno()) + os.replace(temporary, destination_path) + except BaseException: + try: + temporary.unlink() + except FileNotFoundError: + pass + raise + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("source") + parser.add_argument("destination") + parser.add_argument("--epoch", type=int, required=True) + args = parser.parse_args() + normalize_sdist(args.source, args.destination, epoch=args.epoch) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/verify_seed_vulndb.py b/scripts/verify_seed_vulndb.py new file mode 100644 index 0000000..962d1a0 --- /dev/null +++ b/scripts/verify_seed_vulndb.py @@ -0,0 +1,43 @@ +#!/usr/bin/env python3 +"""Verify a seed SQLite and its manifest without activation.""" +from __future__ import annotations + +import argparse +import hashlib +import json +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +from src.vulnerability.database import VulnerabilityDatabase +from src.vulnerability.seed import validate_seed_manifest + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--database", required=True) + parser.add_argument("--manifest", required=True) + args = parser.parse_args() + database_path = Path(args.database) + manifest_path = Path(args.manifest) + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + validate_seed_manifest(manifest) + expected_db = manifest.get("database_sha256") + actual_db = "sha256:" + hashlib.sha256(database_path.read_bytes()).hexdigest() + if expected_db != actual_db: + raise SystemExit(f"database hash mismatch: expected {expected_db}, got {actual_db}") + with VulnerabilityDatabase.read_only(str(database_path)) as database: + integrity = database.integrity_check() + foreign_keys = database.connection.execute("PRAGMA foreign_key_check").fetchall() + if integrity != "ok" or foreign_keys: + raise SystemExit(f"database integrity failed: {integrity}, foreign_keys={len(foreign_keys)}") + actual = database.build_snapshot_manifest() + for key in ("content_digest", "advisory_count", "affected_package_count"): + if actual.get(key) != manifest.get(key): + raise SystemExit(f"manifest mismatch: {key}") + print(json.dumps({"state": "verified", "profile": "seed", "completeness": "partial", "database_sha256": actual_db, "advisory_count": actual["advisory_count"], "affected_package_count": actual["affected_package_count"]}, sort_keys=True, indent=2)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/__init__.py b/src/__init__.py index fa4b48c..b0c005a 100644 --- a/src/__init__.py +++ b/src/__init__.py @@ -1,3 +1,3 @@ """Secret/Config Diff Scanner — Local CLI tool for detecting secrets and risky config changes in diffs.""" -__version__ = "3.0.1" +__version__ = "3.1.0" diff --git a/src/__main__.py b/src/__main__.py index 627973f..1e54d03 100644 --- a/src/__main__.py +++ b/src/__main__.py @@ -120,18 +120,36 @@ def main(): rollback_parser.add_argument("--active", required=True, metavar="PATH") rollback_parser.add_argument("--target", required=True, metavar="DIR") rollback_parser.add_argument("--apply", action="store_true", help="Actually switch the active pointer") - update_parser = vuln_db_actions.add_parser("update", help="Stage a local feed and optionally activate it") - update_parser.add_argument("--input", required=True, metavar="FILE") - update_parser.add_argument("--root", required=True, metavar="DIR") - update_parser.add_argument("--source-id", required=True, metavar="ID") - update_parser.add_argument("--snapshot-id", required=True, metavar="ID") + update_parser = vuln_db_actions.add_parser("update", help="Build a new database snapshot from configured sources or stage one local feed") + update_parser.add_argument("--full", action="store_true", help="Fetch and import all configured sources in an isolated staging pipeline") + update_parser.add_argument("--config", metavar="FILE", default="~/.config/coderisktools/vuln-db.json", help="Full-update source configuration JSON") + update_parser.add_argument("--input", metavar="FILE") + update_parser.add_argument("--root", default="~/.local/share/coderisktools/vuln-db", metavar="DIR") + update_parser.add_argument("--source-id", metavar="ID") + update_parser.add_argument("--snapshot-id", metavar="ID") update_parser.add_argument("--active", metavar="PATH") update_parser.add_argument("--apply", action="store_true", help="Actually switch the active pointer") + update_parser.add_argument("--max-bytes", type=int, default=512 * 1024 * 1024, metavar="N") + update_parser.add_argument("--timeout", type=float, default=20.0, metavar="SECONDS") + update_parser.add_argument("--profile", choices=["core"], default="core", help="Database profile to build (currently: core)") + init_config_parser = vuln_db_actions.add_parser("init-config", help="Write the built-in bounded public source configuration") + init_config_parser.add_argument("--output", default="~/.config/coderisktools/vuln-db.json", metavar="FILE") source_status_parser = vuln_db_actions.add_parser("source-status", help="Show read-only source health metadata") source_status_parser.add_argument("--root", required=True, metavar="DIR") source_status_parser.add_argument("--active", required=True, metavar="PATH") database_info_parser = vuln_db_actions.add_parser("database-info", help="Show verified active database metadata") database_info_parser.add_argument("--active", required=True, metavar="PATH") + activate_seed_parser = vuln_db_actions.add_parser("activate", help="Explicitly activate a verified partial seed database") + activate_seed_parser.add_argument("--database", required=True, metavar="FILE") + activate_seed_parser.add_argument("--manifest", required=True, metavar="FILE") + activate_seed_parser.add_argument("--profile", choices=["seed"], required=True) + activate_seed_parser.add_argument("--apply", action="store_true", help="Actually activate the staged seed snapshot") + bootstrap_seed_parser = vuln_db_actions.add_parser("bootstrap", help="Install one pinned signed seed release as staged") + bootstrap_seed_parser.add_argument("--asset-url", required=True, metavar="URL") + bootstrap_seed_parser.add_argument("--manifest-url", required=True, metavar="URL") + bootstrap_seed_parser.add_argument("--signature-url", required=True, metavar="URL") + bootstrap_seed_parser.add_argument("--destination", required=True, metavar="FILE") + bootstrap_seed_parser.add_argument("--keyring", required=True, metavar="FILE") explain_parser = vuln_db_actions.add_parser("explain", help="Explain one persisted vulnerability match") explain_parser.add_argument("--database", required=True, metavar="FILE") explain_parser.add_argument("--fingerprint", required=True, metavar="FINGERPRINT") @@ -286,12 +304,38 @@ def main(): load_fetch_conditions, persist_downloaded_artifact, prune_versioned_snapshots, + run_full_update, stage_offline_update, verify_versioned_snapshot, ) try: emit = True - if args.vuln_db_action == "verify": + if args.vuln_db_action == "activate": + from .vulnerability.bootstrap import activate_seed_database + manifest_path = Path(args.manifest) + if manifest_path.is_symlink() or not manifest_path.is_file(): + raise ValueError("seed manifest must be a regular non-symlink file") + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + result = activate_seed_database(args.database, manifest, apply=args.apply) + elif args.vuln_db_action == "bootstrap": + from .rulepacks import load_trusted_keyring + from .vulnerability.bootstrap import bootstrap_seed_asset + result = bootstrap_seed_asset( + args.asset_url, + args.manifest_url, + args.signature_url, + args.destination, + trusted_keys=load_trusted_keyring(args.keyring), + ) + elif args.vuln_db_action == "init-config": + from .vulnerability.update_config import default_update_config + output = Path(args.output).expanduser() + if output.exists() or output.is_symlink(): + raise FileExistsError(f"configuration already exists: {output}") + output.parent.mkdir(parents=True, exist_ok=True) + write_private_atomic(output, (json.dumps(default_update_config(), indent=2, sort_keys=True) + "\n").encode("utf-8"), "vulnerability update config") + result = {"state": "ok", "path": str(output), "sources": [item["source_id"] for item in default_update_config()["sources"]]} + elif args.vuln_db_action == "verify": result = verify_versioned_snapshot(args.snapshot) elif args.vuln_db_action == "rollback": if not args.apply: @@ -299,14 +343,29 @@ def main(): from .vulnerability.updater import rollback_versioned_snapshot result = rollback_versioned_snapshot(args.active, args.target) elif args.vuln_db_action == "update": - result = stage_offline_update( - args.input, - args.root, - args.source_id, - args.snapshot_id, - args.active, - apply=args.apply, - ) + if args.full: + root = Path(args.root).expanduser() + active = Path(args.active).expanduser() if args.active else root / "active" + result = run_full_update( + Path(args.config).expanduser(), + root, + active, + profile=args.profile, + apply=args.apply, + max_bytes=args.max_bytes, + timeout=args.timeout, + ) + else: + if not args.input or not args.source_id or not args.snapshot_id: + raise ValueError("local update requires --input, --source-id and --snapshot-id; use --full for configured sources") + result = stage_offline_update( + args.input, + Path(args.root).expanduser(), + args.source_id, + args.snapshot_id, + Path(args.active).expanduser() if args.active else None, + apply=args.apply, + ) elif args.vuln_db_action == "source-status": result = build_source_status_report(args.root, args.active) elif args.vuln_db_action == "database-info": diff --git a/src/scanner.py b/src/scanner.py index 67c3fdb..9363444 100644 --- a/src/scanner.py +++ b/src/scanner.py @@ -506,6 +506,7 @@ def scan_directory( ".zip", ".tar", ".gz", ".bz2", ".xz", ".7z", ".mp3", ".mp4", ".avi", ".mov", ".wav", ".pdf", ".doc", ".docx", ".xls", ".xlsx", + ".sqlite", ".sqlite3", ".db", ".woff", ".woff2", ".ttf", ".eot", ): continue diff --git a/src/vulnerability/airgap.py b/src/vulnerability/airgap.py new file mode 100644 index 0000000..575d9da --- /dev/null +++ b/src/vulnerability/airgap.py @@ -0,0 +1,71 @@ +"""Offline air-gap snapshot bundle export/import.""" + +from __future__ import annotations + +import hashlib +import json +import os +import shutil +import sqlite3 +import tarfile +import tempfile +from pathlib import Path +from typing import Any + +from .database import VulnerabilityDatabase +from .updater import ArchiveLimits, extract_archive_to_directory + +BUNDLE_SCHEMA = "coderisktools.vulnerability.air-gap-bundle" + + +def export_air_gap_bundle(database: VulnerabilityDatabase, output_path: str | os.PathLike[str]) -> dict[str, Any]: + """Export the current SQLite snapshot and manifest into a local bundle.""" + destination = Path(output_path) + destination.parent.mkdir(parents=True, exist_ok=True) + manifest = database.build_snapshot_manifest() + manifest = {"schema": BUNDLE_SCHEMA, "version": 1, "manifest": manifest} + with tempfile.TemporaryDirectory(prefix="coderisktools-airgap-") as work: + db_path = Path(work) / "snapshot.sqlite" + backup = sqlite3.connect(db_path) + try: + database.connection.backup(backup) + finally: + backup.close() + manifest_path = Path(work) / "manifest.json" + manifest_path.write_text(json.dumps(manifest, ensure_ascii=False, sort_keys=True, separators=(",", ":")), encoding="utf-8") + temporary = destination.with_name(destination.name + ".tmp") + with tarfile.open(temporary, "w:gz") as archive: + archive.add(manifest_path, arcname="manifest.json", recursive=False) + archive.add(db_path, arcname="snapshot.sqlite", recursive=False) + os.replace(temporary, destination) + return {"schema": BUNDLE_SCHEMA, "output": str(destination), "manifest": manifest["manifest"]} + + +def import_air_gap_bundle(bundle_path: str | os.PathLike[str], target_db_path: str | os.PathLike[str], *, max_total_bytes: int = 512 * 1024 * 1024) -> dict[str, Any]: + """Verify and atomically install an offline bundle; never activates a snapshot.""" + bundle = Path(bundle_path) + target = Path(target_db_path) + with tempfile.TemporaryDirectory(prefix="coderisktools-airgap-import-") as work: + extracted = extract_archive_to_directory(bundle, Path(work) / "extracted", ArchiveLimits(max_total_bytes=max_total_bytes)) + root = Path(extracted["output"]) + manifest = json.loads((root / "manifest.json").read_text(encoding="utf-8")) + if manifest.get("schema") != BUNDLE_SCHEMA or manifest.get("version") != 1: + raise ValueError("unsupported air-gap bundle") + staged_db = root / "snapshot.sqlite" + readonly = VulnerabilityDatabase.read_only(str(staged_db)) + try: + actual = readonly.build_snapshot_manifest() + if actual != manifest.get("manifest"): + raise ValueError("air-gap bundle manifest mismatch") + health = readonly.snapshot_health() + if not health.healthy: + raise ValueError("air-gap bundle database health gate failed") + finally: + readonly.close() + target.parent.mkdir(parents=True, exist_ok=True) + temporary = target.with_name(target.name + ".tmp") + shutil.copyfile(staged_db, temporary) + with open(temporary, "rb") as handle: + os.fsync(handle.fileno()) + os.replace(temporary, target) + return {"schema": BUNDLE_SCHEMA, "output": str(target), "content_digest": manifest["manifest"]["content_digest"], "activated": False} diff --git a/src/vulnerability/benchmark.py b/src/vulnerability/benchmark.py new file mode 100644 index 0000000..349c5d2 --- /dev/null +++ b/src/vulnerability/benchmark.py @@ -0,0 +1,234 @@ +"""Offline vulnerability benchmark metrics, fixtures, and acceptance gates.""" + +from __future__ import annotations + +import json +import tracemalloc +from dataclasses import dataclass +from math import isfinite +from pathlib import Path +from typing import Any, Iterable + +from ..safeio import read_regular_bounded + + +@dataclass(frozen=True) +class BenchmarkCase: + case_id: str + expected: bool + observed: bool + elapsed_ms: float + + +def load_benchmark_fixture(path: str | Path, *, max_bytes: int = 5 * 1024 * 1024) -> dict[str, Any]: + raw = read_regular_bounded(path, max_bytes, "benchmark fixture") + try: + fixture = json.loads(raw.decode("utf-8")) + except (UnicodeDecodeError, json.JSONDecodeError) as exc: + raise ValueError("benchmark fixture must be UTF-8 JSON") from exc + if not isinstance(fixture, dict) or fixture.get("schema") != "coderisktools.vulnerability.benchmark" or fixture.get("version") != 1: + raise ValueError("unsupported benchmark fixture schema") + cases = fixture.get("cases") + golden = fixture.get("golden") + if not isinstance(cases, list) or not cases or len(cases) > 100_000 or not isinstance(golden, dict): + raise ValueError("benchmark fixture cases/golden are invalid") + for item in cases: + if not isinstance(item, dict) or not isinstance(item.get("case_id"), str) or not isinstance(item.get("expected"), bool): + raise ValueError("benchmark fixture case is invalid") + if "observed" in item and not isinstance(item["observed"], bool): + raise ValueError("benchmark fixture observed value is invalid") + if "elapsed_ms" in item and (not isinstance(item["elapsed_ms"], (int, float)) or not isfinite(float(item["elapsed_ms"])) or item["elapsed_ms"] < 0): + raise ValueError("benchmark fixture elapsed time is invalid") + for key in ("precision", "recall", "f1"): + if key in golden and (not isinstance(golden.get(key), (int, float)) or not isfinite(float(golden[key])) or not 0 <= float(golden[key]) <= 1): + raise ValueError("benchmark golden metric is invalid") + for key in ("min_precision", "min_recall", "min_f1"): + if key in golden and (not isinstance(golden.get(key), (int, float)) or not isfinite(float(golden[key])) or not 0 <= float(golden[key]) <= 1): + raise ValueError("benchmark golden threshold is invalid") + if not any(key in golden for key in ("precision", "recall", "f1", "min_precision", "min_recall", "min_f1")): + raise ValueError("benchmark golden metrics are missing") + return fixture + + +def evaluate_cases(cases: Iterable[BenchmarkCase]) -> dict[str, int]: + metrics = {"true_positive": 0, "true_negative": 0, "false_positive": 0, "false_negative": 0} + for case in cases: + if not isinstance(case.case_id, str) or not case.case_id: + raise ValueError("benchmark case_id is required") + if not isinstance(case.elapsed_ms, (int, float)) or not isfinite(float(case.elapsed_ms)) or case.elapsed_ms < 0: + raise ValueError("benchmark elapsed_ms must be finite and non-negative") + if case.expected and case.observed: + metrics["true_positive"] += 1 + elif not case.expected and not case.observed: + metrics["true_negative"] += 1 + elif case.observed: + metrics["false_positive"] += 1 + else: + metrics["false_negative"] += 1 + return metrics + + +def _percentile(values: list[float], percentile: float) -> float: + if not values: + return 0.0 + index = max(0, min(len(values) - 1, int((percentile / 100.0) * len(values) + 0.999999) - 1)) + return values[index] + + +def summarize_metrics(metrics: dict[str, int], cases: Iterable[BenchmarkCase] | None = None) -> dict[str, object]: + tp = metrics.get("true_positive", 0) + fp = metrics.get("false_positive", 0) + fn = metrics.get("false_negative", 0) + precision = tp / (tp + fp) if tp + fp else 0.0 + recall = tp / (tp + fn) if tp + fn else 0.0 + f1 = 2 * precision * recall / (precision + recall) if precision + recall else 0.0 + result: dict[str, object] = {"precision": precision, "recall": recall, "f1": f1} + if cases is not None: + latencies = sorted(float(case.elapsed_ms) for case in cases) + result["latency_ms"] = {"count": len(latencies), "min": latencies[0] if latencies else 0.0, "mean": sum(latencies) / len(latencies) if latencies else 0.0, "p50": _percentile(latencies, 50), "p95": _percentile(latencies, 95), "max": latencies[-1] if latencies else 0.0} + return result + + + + + + +def run_database_memory_benchmark(database: Any, cases: Iterable[dict[str, Any]], *, repetitions: int = 3, max_peak_kib: float | None = None) -> dict[str, Any]: + if not isinstance(repetitions, int) or repetitions < 1 or repetitions > 100: + raise ValueError("repetitions must be between 1 and 100") + if max_peak_kib is not None and max_peak_kib < 0: + raise ValueError("max_peak_kib must be non-negative") + case_list = list(cases) + peaks = [] + for _ in range(repetitions): + tracemalloc.start() + try: + run_database_benchmark(database, case_list) + _, peak = tracemalloc.get_traced_memory() + finally: + tracemalloc.stop() + peaks.append(peak / 1024.0) + peak_kib = max(peaks) if peaks else 0.0 + failed = ["memory_peak"] if max_peak_kib is not None and peak_kib > max_peak_kib else [] + return {"passed": not failed, "repetitions": repetitions, "peak_kib": peak_kib, "samples_kib": peaks, "acceptance": {"max_peak_kib": max_peak_kib, "failed": failed}} + +def build_quality_suites_report(cases: Iterable[dict[str, Any]]) -> dict[str, Any]: + normalized = list(cases) + for item in normalized: + if not isinstance(item, dict) or not isinstance(item.get("case_id"), str) or not isinstance(item.get("expected"), bool) or not isinstance(item.get("observed"), bool): + raise ValueError("quality case is invalid") + precision = [item for item in normalized if item["observed"]] + recall = [item for item in normalized if item["expected"]] + false_positive = [item for item in normalized if not item["expected"] and item["observed"]] + false_negative = [item for item in normalized if item["expected"] and not item["observed"]] + def suite(items: list[dict[str, Any]]) -> dict[str, Any]: + return {"count": len(items), "case_ids": sorted(item["case_id"] for item in items)} + return {"precision_suite": suite(precision), "recall_suite": suite(recall), "false_positive_suite": suite(false_positive), "false_negative_suite": suite(false_negative)} + + +def benchmark_regression_gate(current: dict[str, Any], baseline: dict[str, Any], *, max_new_false_positives: int = 0, max_new_false_negatives: int = 0) -> dict[str, Any]: + current_fp = set(current.get("false_positive_suite", {}).get("case_ids", [])) + baseline_fp = set(baseline.get("false_positive_suite", {}).get("case_ids", [])) + current_fn = set(current.get("false_negative_suite", {}).get("case_ids", [])) + baseline_fn = set(baseline.get("false_negative_suite", {}).get("case_ids", [])) + new_fp = sorted(current_fp - baseline_fp) + new_fn = sorted(current_fn - baseline_fn) + failed = [] + if len(new_fp) > max_new_false_positives: + failed.append("false_positive_regression") + if len(new_fn) > max_new_false_negatives: + failed.append("false_negative_regression") + return {"passed": not failed, "failed": failed, "new_false_positives": new_fp, "new_false_negatives": new_fn} + + + + +def compare_external_evidence(internal_ids: set[str], external_tools: dict[str, set[str]]) -> dict[str, Any]: + if not isinstance(internal_ids, set) or not all(isinstance(item, str) and item for item in internal_ids): + raise ValueError("internal identifiers must be a set of non-empty strings") + if not isinstance(external_tools, dict) or not external_tools: + raise ValueError("external tools are required") + differences = {} + for tool, raw_ids in sorted(external_tools.items()): + if not isinstance(tool, str) or not tool or not isinstance(raw_ids, set) or not all(isinstance(item, str) and item for item in raw_ids): + raise ValueError("external evidence tool identifiers are invalid") + intersection = internal_ids & raw_ids + external_only = raw_ids - internal_ids + internal_only = internal_ids - raw_ids + differences[tool] = {"status": "aligned" if not external_only and not internal_only else "different", "aligned": sorted(intersection), "external_only": sorted(external_only), "internal_only": sorted(internal_only)} + return {"tools": sorted(external_tools), "internal_count": len(internal_ids), "differences": differences, "merged": False, "evidence_domain": "external-comparison"} + + +def run_database_benchmark(database: Any, cases: Iterable[dict[str, Any]]) -> dict[str, Any]: + from .models import Component + benchmark_cases: list[BenchmarkCase] = [] + for item in cases: + started = __import__("time").perf_counter() + result = database.evaluate_component(Component(ecosystem=item["ecosystem"], name=item["name"], version=item["version"])) + elapsed_ms = (__import__("time").perf_counter() - started) * 1000.0 + observed = result.get("status") == "affected" + benchmark_cases.append(BenchmarkCase(str(item["case_id"]), bool(item["expected"]), observed, elapsed_ms)) + metrics = evaluate_cases(benchmark_cases) + return {"metrics": metrics, "summary": summarize_metrics(metrics, benchmark_cases), "cases": len(benchmark_cases)} + + +def run_database_benchmark_repeated(database: Any, cases: Iterable[dict[str, Any]], *, repetitions: int = 3, max_p95_ms: float | None = None) -> dict[str, Any]: + if not isinstance(repetitions, int) or repetitions < 1 or repetitions > 1000: + raise ValueError("repetitions must be between 1 and 1000") + if max_p95_ms is not None and max_p95_ms < 0: + raise ValueError("max_p95_ms must be non-negative") + case_list = list(cases) + runs = [run_database_benchmark(database, case_list) for _ in range(repetitions)] + latencies = [] + for run in runs: + latency = run["summary"].get("latency_ms", {}) + if latency.get("count", 0): + latencies.append(float(latency["mean"])) + p95 = _percentile(sorted(latencies), 95) + failed = ["latency_p95"] if max_p95_ms is not None and p95 > max_p95_ms else [] + return {"passed": not failed, "repetitions": repetitions, "runs": runs, "latency_ms": {"count": len(latencies), "min": min(latencies) if latencies else 0.0, "mean": sum(latencies) / len(latencies) if latencies else 0.0, "p95": p95, "max": max(latencies) if latencies else 0.0}, "acceptance": {"max_p95_ms": max_p95_ms, "failed": failed}} + + +def build_final_benchmark_report(database: Any, suites: dict[str, Iterable[dict[str, Any]]], *, repetitions: int = 3, max_p95_ms: float | None = None) -> dict[str, Any]: + if not isinstance(suites, dict) or not suites: + raise ValueError("benchmark suites are required") + suite_reports = {} + totals = {"true_positive": 0, "true_negative": 0, "false_positive": 0, "false_negative": 0} + performance = [] + failed = [] + for ecosystem in sorted(suites): + report = run_database_benchmark_repeated(database, list(suites[ecosystem]), repetitions=repetitions, max_p95_ms=max_p95_ms) + suite_reports[ecosystem] = report + for key in totals: + totals[key] += report["runs"][0]["metrics"].get(key, 0) + performance.extend(run["summary"]["latency_ms"]["mean"] for run in report["runs"] if run["summary"]["latency_ms"]["count"]) + if not report["passed"]: + failed.append(ecosystem) + tp, fp, fn = totals["true_positive"], totals["false_positive"], totals["false_negative"] + precision = tp / (tp + fp) if tp + fp else 0.0 + recall = tp / (tp + fn) if tp + fn else 0.0 + f1 = 2 * precision * recall / (precision + recall) if precision + recall else 0.0 + latency = sorted(performance) + return {"passed": not failed, "ecosystems": sorted(suites), "suites": suite_reports, "metrics": totals, "summary": {"precision": precision, "recall": recall, "f1": f1}, "performance": {"runs": len(latency), "min": min(latency) if latency else 0.0, "mean": sum(latency) / len(latency) if latency else 0.0, "p95": _percentile(latency, 95), "max": max(latency) if latency else 0.0}, "failed_ecosystems": failed} + + +def run_fixture_benchmark(path: str | Path, *, min_precision: float = 0.0, min_recall: float = 0.0, min_f1: float = 0.0, max_p95_ms: float | None = None) -> dict[str, Any]: + fixture = load_benchmark_fixture(path) + cases = [BenchmarkCase(item["case_id"], item["expected"], item["observed"], float(item["elapsed_ms"])) for item in fixture["cases"]] + metrics = evaluate_cases(cases) + summary = summarize_metrics(metrics, cases) + if not all(0 <= threshold <= 1 for threshold in (min_precision, min_recall, min_f1)): + raise ValueError("quality thresholds must be between 0 and 1") + if max_p95_ms is not None and max_p95_ms < 0: + raise ValueError("max_p95_ms must be non-negative") + failed: list[str] = [] + if summary["precision"] < min_precision: + failed.append("precision") + if summary["recall"] < min_recall: + failed.append("recall") + if summary["f1"] < min_f1: + failed.append("f1") + latency = summary.get("latency_ms", {}) + if max_p95_ms is not None and latency.get("p95", 0.0) > max_p95_ms: + failed.append("latency_p95") + return {"passed": not failed, "fixture": fixture.get("name"), "metrics": metrics, "summary": summary, "acceptance": {"thresholds": {"min_precision": min_precision, "min_recall": min_recall, "min_f1": min_f1, "max_p95_ms": max_p95_ms}, "failed": failed}} diff --git a/src/vulnerability/bootstrap.py b/src/vulnerability/bootstrap.py new file mode 100644 index 0000000..c6c9615 --- /dev/null +++ b/src/vulnerability/bootstrap.py @@ -0,0 +1,196 @@ +"""Pinned seed asset bootstrap with signed-manifest and digest verification.""" + +from __future__ import annotations + +import hashlib +import json +import os +import re +import tempfile +from pathlib import Path +from typing import Any, Mapping +from urllib.parse import urlparse +from urllib.request import Request, urlopen + +from .manifest_signing import verify_manifest +from .seed import validate_seed_manifest + +_MAX_METADATA_BYTES = 2 * 1024 * 1024 +_MAX_ASSET_BYTES = 64 * 1024 * 1024 +_SHA256_RE = re.compile(r"sha256:[0-9a-f]{64}") + + +def _https_url(url: str, allowed_hosts: frozenset[str]) -> None: + parsed = urlparse(url) + if ( + parsed.scheme != "https" + or parsed.username + or parsed.password + or parsed.hostname not in allowed_hosts + or parsed.port not in (None, 443) + or not parsed.path + or parsed.fragment + ): + raise ValueError("bootstrap URL must be HTTPS and use an allowlisted release host") + + +def _read_response(response: Any, maximum: int, label: str) -> bytes: + length = response.headers.get("Content-Length") + if length is not None: + try: + if int(length) > maximum: + raise ValueError(f"{label} exceeds byte limit") + except (TypeError, ValueError) as exc: + if isinstance(exc, ValueError) and str(exc).endswith("byte limit"): + raise + raise ValueError(f"{label} has invalid Content-Length") from exc + chunks: list[bytes] = [] + total = 0 + while True: + chunk = response.read(min(1024 * 1024, maximum + 1 - total)) + if not chunk: + break + total += len(chunk) + if total > maximum: + raise ValueError(f"{label} exceeds byte limit") + chunks.append(chunk) + return b"".join(chunks) + + +def _download(url: str, allowed_hosts: frozenset[str], accept: str, maximum: int, label: str, timeout: float) -> bytes: + _https_url(url, allowed_hosts) + request = Request(url, headers={"Accept": accept, "User-Agent": "coderisktools-bootstrap/1"}) + with urlopen(request, timeout=timeout) as response: + final_url = response.geturl() + _https_url(final_url, allowed_hosts) + return _read_response(response, maximum, label) + + +def verify_asset_sha256(path: str | os.PathLike[str], expected: str) -> str: + if not isinstance(expected, str) or _SHA256_RE.fullmatch(expected) is None: + raise ValueError("expected SHA-256 must use sha256:<64 lowercase hex> format") + candidate = Path(path) + if candidate.is_symlink() or not candidate.is_file(): + raise ValueError("seed asset must be a regular non-symlink file") + actual = "sha256:" + hashlib.sha256(candidate.read_bytes()).hexdigest() + if actual != expected: + raise ValueError("seed asset SHA-256 mismatch") + return actual + + +def validate_signed_release_manifest( + manifest: dict[str, Any], + envelope: dict[str, Any], + trusted_keys: Mapping[str, bytes], +) -> dict[str, Any]: + """Verify a detached signed envelope and exact manifest equality.""" + if not isinstance(envelope, dict): + raise ValueError("signed seed manifest envelope must be an object") + key_id = envelope.get("key_id") + if not isinstance(key_id, str) or key_id not in trusted_keys: + raise ValueError("signed seed manifest uses an untrusted key") + verified = verify_manifest(envelope, trusted_keys[key_id]) + if verified != manifest: + raise ValueError("signed seed manifest does not match detached manifest") + validate_seed_manifest(verified) + expected = verified.get("database_sha256") + if not isinstance(expected, str) or _SHA256_RE.fullmatch(expected) is None: + raise ValueError("signed seed manifest has an invalid database SHA-256") + return dict(verified) + + +def _verify_database_contract(database_path: Path, manifest: dict[str, Any]) -> None: + from .database import VulnerabilityDatabase + + verify_asset_sha256(database_path, manifest["database_sha256"]) + with VulnerabilityDatabase.read_only(str(database_path)) as database: + if database.integrity_check() != "ok": + raise ValueError("seed SQLite integrity check failed") + if database.connection.execute("PRAGMA foreign_key_check").fetchall(): + raise ValueError("seed SQLite foreign-key check failed") + actual = database.build_snapshot_manifest() + for key in ("content_digest", "advisory_count", "affected_package_count"): + if actual.get(key) != manifest.get(key): + raise ValueError(f"seed database manifest mismatch: {key}") + row = database.connection.execute( + "SELECT snapshot_id, state FROM snapshots WHERE snapshot_id = ?", + (manifest.get("snapshot_id"),), + ).fetchone() + if row is None or row["state"] not in {"staged", "active"}: + raise ValueError("seed database does not contain the declared staged snapshot") + + +def activate_seed_database( + database_path: str | os.PathLike[str], + manifest: dict[str, Any], + *, + apply: bool = False, +) -> dict[str, Any]: + """Plan or explicitly activate only a verified partial seed snapshot.""" + from .database import VulnerabilityDatabase + + validate_seed_manifest(manifest) + database = Path(database_path) + _verify_database_contract(database, manifest) + snapshot_id = manifest.get("snapshot_id") + if not isinstance(snapshot_id, str) or not snapshot_id.startswith("seed-"): + raise ValueError("seed snapshot_id is invalid") + if not apply: + return {"state": "activation_planned", "profile": "seed", "snapshot_id": snapshot_id, "database": str(database)} + with VulnerabilityDatabase(str(database)) as handle: + status = handle.activate_snapshot(snapshot_id) + return {"state": status["state"], "profile": "seed", "snapshot_id": snapshot_id, "database": str(database)} + + +def bootstrap_seed_asset( + asset_url: str, + manifest_url: str, + signature_url: str, + destination: str | os.PathLike[str], + *, + trusted_keys: Mapping[str, bytes], + allowed_hosts: frozenset[str] = frozenset({"github.com", "objects.githubusercontent.com", "release-assets.githubusercontent.com"}), + timeout: float = 30.0, +) -> dict[str, object]: + """Download one exact signed seed release and atomically install it as staged.""" + if timeout <= 0 or timeout > 300: + raise ValueError("bootstrap timeout must be in (0, 300]") + manifest_raw = _download(manifest_url, allowed_hosts, "application/json", _MAX_METADATA_BYTES, "seed manifest", timeout) + signature_raw = _download(signature_url, allowed_hosts, "application/json", _MAX_METADATA_BYTES, "seed signature", timeout) + try: + manifest = json.loads(manifest_raw.decode("utf-8")) + envelope = json.loads(signature_raw.decode("utf-8")) + except (UnicodeError, json.JSONDecodeError) as exc: + raise ValueError("seed release metadata is invalid JSON") from exc + if not isinstance(manifest, dict) or not isinstance(envelope, dict): + raise ValueError("seed release metadata must be JSON objects") + verified_manifest = validate_signed_release_manifest(manifest, envelope, trusted_keys) + asset = _download(asset_url, allowed_hosts, "application/octet-stream", _MAX_ASSET_BYTES, "seed asset", timeout) + target = Path(destination) + if target.is_symlink(): + raise ValueError("seed destination must not be a symlink") + target.parent.mkdir(parents=True, exist_ok=True) + fd, temp_name = tempfile.mkstemp(prefix=f".{target.name}.", dir=str(target.parent)) + temporary = Path(temp_name) + try: + with os.fdopen(fd, "wb") as output: + output.write(asset) + output.flush() + os.fsync(output.fileno()) + _verify_database_contract(temporary, verified_manifest) + os.replace(temporary, target) + return { + "state": "installed_staged", + "profile": "seed", + "completeness": "partial", + "path": str(target), + "snapshot_id": verified_manifest["snapshot_id"], + "database_sha256": verified_manifest["database_sha256"], + "signing_key_id": envelope["key_id"], + } + except BaseException: + try: + temporary.unlink() + except FileNotFoundError: + pass + raise diff --git a/src/vulnerability/database.py b/src/vulnerability/database.py index 8c99e91..918888d 100644 --- a/src/vulnerability/database.py +++ b/src/vulnerability/database.py @@ -18,6 +18,7 @@ vulnerability_fingerprint, ) from .canonical import canonical_json_bytes +from .manifest_signing import verify_manifest from .models import Component, DatabaseHealth from .versions import compare_alpine_version, compare_composer_version, compare_debian_version, compare_go_version, compare_maven_version, compare_nuget_version, compare_pep440_version, compare_rpm_version, compare_rubygems_version, compare_semver_version, osv_events_match @@ -231,6 +232,14 @@ def _create_schema(self) -> None: enrichment_json TEXT NOT NULL, PRIMARY KEY (cve_id, content_digest) ); + CREATE TABLE IF NOT EXISTS unresolved_enrichments ( + source TEXT NOT NULL, + cve_id TEXT NOT NULL, + content_digest TEXT NOT NULL, + reason TEXT NOT NULL, + record_json TEXT NOT NULL, + PRIMARY KEY (source, cve_id, content_digest) + ); CREATE TABLE IF NOT EXISTS merge_decisions ( decision_id TEXT PRIMARY KEY, decision_type TEXT NOT NULL, @@ -1073,7 +1082,7 @@ def cpe_mapping_report(self) -> dict[str, Any]: report["content_digest"] = "sha256:" + hashlib.sha256(canonical_json_bytes(report)).hexdigest() return report - def import_epss_json(self, payload: str | bytes | dict[str, Any] | list[dict[str, Any]]) -> ImportStats: + def import_epss_json(self, payload: str | bytes | dict[str, Any] | list[dict[str, Any]], *, allow_unresolved: bool = False) -> ImportStats: from .sources.epss import parse_epss_record if isinstance(payload, (str, bytes)): try: @@ -1096,11 +1105,16 @@ def import_epss_json(self, payload: str | bytes | dict[str, Any] | list[dict[str if not isinstance(record, dict): raise ValueError("EPSS record must be an object") parsed = parse_epss_record(record) + canonical = canonical_json_bytes(parsed) + digest = "sha256:" + hashlib.sha256(canonical).hexdigest() lookup = self.lookup_advisory(parsed["cve_id"]) if lookup.get("status") != "exact": + if allow_unresolved and lookup.get("status") in {"not-found", "ambiguous"}: + reason = "advisory-not-found" if lookup.get("status") == "not-found" else "advisory-ambiguous" + self.connection.execute("INSERT OR IGNORE INTO unresolved_enrichments(source, cve_id, content_digest, reason, record_json) VALUES ('epss', ?, ?, ?, ?)", (parsed["cve_id"], digest, reason, canonical.decode("utf-8"))) + self.connection.execute("RELEASE SAVEPOINT epss_record") + continue raise ValueError("CVE has no exact advisory match" if lookup.get("status") == "not-found" else "CVE advisory match is ambiguous") - canonical = canonical_json_bytes(parsed) - digest = "sha256:" + hashlib.sha256(canonical).hexdigest() self.record_source_record("epss", parsed["cve_id"], parsed, lookup["advisory_id"], commit=False) cursor = self.connection.execute( "INSERT OR IGNORE INTO epss_scores(cve_id, advisory_id, source, content_digest, score, percentile, score_date, record_json) VALUES (?, ?, 'first-epss', ?, ?, ?, ?, ?)", @@ -1217,7 +1231,7 @@ def vulnrichment_record(self, cve_id: str) -> dict[str, Any]: result.update({"advisory_id": row["advisory_id"], "source": row["source"], "content_digest": row["content_digest"]}) return result - def import_kev_json(self, payload: str | bytes | dict[str, Any] | list[dict[str, Any]]) -> ImportStats: + def import_kev_json(self, payload: str | bytes | dict[str, Any] | list[dict[str, Any]], *, allow_unresolved: bool = False) -> ImportStats: from .sources.kev import parse_kev_record if isinstance(payload, (str, bytes)): try: @@ -1240,11 +1254,15 @@ def import_kev_json(self, payload: str | bytes | dict[str, Any] | list[dict[str, if not isinstance(record, dict): raise ValueError("KEV vulnerability record must be an object") parsed = parse_kev_record(record) + canonical = canonical_json_bytes(parsed) + digest = "sha256:" + hashlib.sha256(canonical).hexdigest() lookup = self.lookup_advisory(parsed["cve_id"]) if lookup.get("status") != "exact": + if allow_unresolved and lookup.get("status") == "not-found": + self.connection.execute("INSERT OR IGNORE INTO unresolved_enrichments(source, cve_id, content_digest, reason, record_json) VALUES ('kev', ?, ?, 'advisory-not-found', ?)", (parsed["cve_id"], digest, canonical.decode("utf-8"))) + self.connection.execute("RELEASE SAVEPOINT kev_record") + continue raise ValueError("CVE has no exact advisory match" if lookup.get("status") == "not-found" else "CVE advisory match is ambiguous") - canonical = canonical_json_bytes(parsed) - digest = "sha256:" + hashlib.sha256(canonical).hexdigest() self.record_source_record("kev", parsed["cve_id"], parsed, lookup["advisory_id"], commit=False) raw_cursor = self.connection.execute("INSERT OR IGNORE INTO kev_records(cve_id, advisory_id, source, content_digest, record_json) VALUES (?, ?, 'cisa-kev', ?, ?)", (parsed["cve_id"], lookup["advisory_id"], digest, canonical.decode("utf-8"))) imported += 1 if raw_cursor.rowcount else 0 @@ -1499,6 +1517,13 @@ def snapshot_quality_gate(self, snapshot_id: str | None = None) -> dict[str, Any issues.append(f"manifest:{key}") return {"healthy": not issues, "checks": checks, "issues": tuple(issues), "snapshot_id": snapshot_id} + def stage_signed_snapshot(self, snapshot_id: str, source_digest: str, envelope: dict[str, Any], public_key: bytes) -> None: + """Verify an Ed25519 manifest envelope before staging it.""" + manifest = dict(verify_manifest(envelope, public_key)) + manifest["signature_status"] = "verified" + manifest["signing_key_id"] = envelope["key_id"] + self.stage_snapshot(snapshot_id, source_digest, manifest) + def activate_snapshot(self, snapshot_id: str) -> dict[str, Any]: row = self.connection.execute("SELECT * FROM snapshots WHERE snapshot_id = ?", (snapshot_id,)).fetchone() if row is None: @@ -1547,6 +1572,22 @@ def snapshot_health(self) -> DatabaseHealth: issues=tuple(issues), ) + def prune_snapshots(self, keep_snapshot_ids: set[str] | frozenset[str], *, apply: bool = False) -> dict[str, Any]: + """Plan or apply explicit snapshot retention without deleting active data.""" + keep = {value for value in keep_snapshot_ids if isinstance(value, str) and value} + rows = self.connection.execute("SELECT snapshot_id, state FROM snapshots ORDER BY snapshot_id").fetchall() + candidates = sorted(row["snapshot_id"] for row in rows if row["snapshot_id"] not in keep and row["state"] != "active") + removed: list[str] = [] + if apply: + with self.connection: + for snapshot_id in candidates: + for table in ("source_snapshots", "import_errors", "quality_metrics"): + self.connection.execute(f"DELETE FROM {table} WHERE snapshot_id = ?", (snapshot_id,)) + self.connection.execute("DELETE FROM snapshots WHERE snapshot_id = ? AND state != 'active'", (snapshot_id,)) + if self.connection.total_changes: + removed.append(snapshot_id) + return {"candidates": candidates, "removed": sorted(set(removed)), "applied": apply} + def snapshot_status(self, snapshot_id: str) -> dict[str, Any]: row = self.connection.execute("SELECT * FROM snapshots WHERE snapshot_id = ?", (snapshot_id,)).fetchone() if row is None: diff --git a/src/vulnerability/feed_catalog.py b/src/vulnerability/feed_catalog.py new file mode 100644 index 0000000..8afd7e8 --- /dev/null +++ b/src/vulnerability/feed_catalog.py @@ -0,0 +1,58 @@ +"""Declared vulnerability feed inventory for full-coverage planning. + +This catalog is metadata only: it never downloads or activates a feed. A source +must have a dedicated adapter, verified license/terms, fixture coverage and a +successful quality gate before its status can become ``ready``. +""" + +from __future__ import annotations + +from dataclasses import dataclass + + +@dataclass(frozen=True) +class FeedSpec: + source_id: str + family: str + endpoint: str | None + status: str + license_status: str + adapter: str + + +_FEEDS = ( + FeedSpec("osv", "advisory", "https://osv.dev", "staging-ready", "verify-before-redistribution", "osv"), + FeedSpec("nvd", "enrichment", "https://nvd.nist.gov", "staging-ready", "verify-before-redistribution", "nvd"), + FeedSpec("cisa-kev", "exploitation", "https://www.cisa.gov", "staging-ready", "public-sector-terms", "kev"), + FeedSpec("epss", "exploitation", "https://api.first.org", "staging-ready", "verify-before-redistribution", "epss"), + FeedSpec("github-advisories", "advisory", "https://github.com/advisories", "staging-ready", "verify-before-redistribution", "github-advisories"), + FeedSpec("debian-security", "linux", "https://www.debian.org/security", "bounded", "debian-terms", "debian"), + FeedSpec("ubuntu-security", "linux", "https://ubuntu.com/security", "bounded", "ubuntu-terms", "ubuntu"), + FeedSpec("redhat-security", "linux", "https://access.redhat.com/security", "bounded", "redhat-terms", "redhat"), + FeedSpec("suse-security", "linux", "https://www.suse.com/security", "bounded", "suse-terms", "suse"), + FeedSpec("alpine-security", "linux", "https://secdb.alpinelinux.org", "bounded", "alpine-terms", "alpine"), + FeedSpec("csaf-generic", "csaf", None, "bounded", "provider-specific", "csaf"), + FeedSpec("pypi", "ecosystem", "https://pypi.org", "contract-only", "verify-before-redistribution", "pypi"), + FeedSpec("npm", "ecosystem", "https://registry.npmjs.org", "contract-only", "verify-before-redistribution", "npm"), + FeedSpec("crates", "ecosystem", "https://crates.io", "contract-only", "verify-before-redistribution", "crates"), + FeedSpec("maven", "ecosystem", "https://central.sonatype.com", "bounded", "verify-before-redistribution", "maven"), + FeedSpec("nuget", "ecosystem", "https://api.nuget.org", "bounded", "verify-before-redistribution", "nuget"), +) + + +def feed_catalog() -> tuple[FeedSpec, ...]: + return _FEEDS + + +def feed_catalog_report() -> dict[str, object]: + counts: dict[str, int] = {} + for feed in _FEEDS: + counts[feed.status] = counts.get(feed.status, 0) + 1 + return { + "schema": "coderisktools.vulnerability.feed-catalog", + "version": 1, + "source_count": len(_FEEDS), + "counts": dict(sorted(counts.items())), + "full_coverage_ready": counts.get("ready", 0) == len(_FEEDS), + "feeds": [feed.__dict__ for feed in _FEEDS], + } diff --git a/src/vulnerability/manifest_signing.py b/src/vulnerability/manifest_signing.py new file mode 100644 index 0000000..54af40b --- /dev/null +++ b/src/vulnerability/manifest_signing.py @@ -0,0 +1,54 @@ +"""Canonical Ed25519 envelopes for staged snapshot manifests.""" + +from __future__ import annotations + +import base64 +import json +from typing import Any + +from ..rulepacks import verify_ed25519 + +SCHEMA = "coderisktools.vulnerability.signed-manifest" +VERSION = 1 + + +def canonical_manifest_message(manifest: dict[str, Any]) -> bytes: + if not isinstance(manifest, dict) or not manifest: + raise ValueError("manifest must be a non-empty object") + return json.dumps(manifest, ensure_ascii=False, sort_keys=True, separators=(",", ":")).encode("utf-8") + + +def sign_manifest(manifest: dict[str, Any], key_id: str, private_key: bytes) -> dict[str, Any]: + """Return a signed envelope using optional cryptography Ed25519 backend.""" + if not isinstance(key_id, str) or not key_id or len(key_id) > 128: + raise ValueError("key_id is invalid") + if not isinstance(private_key, bytes) or len(private_key) != 32: + raise ValueError("Ed25519 private key must be 32 raw bytes") + try: + from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PrivateKey + except ImportError as exc: + raise RuntimeError("manifest signing requires the optional cryptography package") from exc + signature = Ed25519PrivateKey.from_private_bytes(private_key).sign(canonical_manifest_message(manifest)) + return { + "schema": SCHEMA, + "version": VERSION, + "key_id": key_id, + "manifest": manifest, + "signature": base64.b64encode(signature).decode("ascii"), + } + + +def verify_manifest(envelope: dict[str, Any], public_key: bytes) -> dict[str, Any]: + if not isinstance(envelope, dict) or set(envelope) != {"schema", "version", "key_id", "manifest", "signature"}: + raise ValueError("signed manifest envelope schema is invalid") + if envelope["schema"] != SCHEMA or envelope["version"] != VERSION: + raise ValueError("unsupported signed manifest envelope") + if not isinstance(envelope["key_id"], str) or not isinstance(envelope["manifest"], dict): + raise ValueError("signed manifest fields are invalid") + try: + signature = base64.b64decode(envelope["signature"], validate=True) + except (ValueError, TypeError) as exc: + raise ValueError("signed manifest signature is invalid") from exc + if len(signature) != 64 or not verify_ed25519(public_key, canonical_manifest_message(envelope["manifest"]), signature): + raise ValueError("signed manifest verification failed") + return envelope["manifest"] diff --git a/src/vulnerability/release.py b/src/vulnerability/release.py new file mode 100644 index 0000000..01cb3d2 --- /dev/null +++ b/src/vulnerability/release.py @@ -0,0 +1,118 @@ +"""Production snapshot readiness checks.""" + +from __future__ import annotations + +import hashlib +from pathlib import Path +from typing import Any + + +def production_snapshot_gate(snapshot: dict[str, Any]) -> dict[str, Any]: + if not isinstance(snapshot, dict): + raise ValueError("snapshot must be an object") + failed: list[str] = [] + for key in ("build_id", "source_digest", "manifest_digest"): + value = snapshot.get(key) + if not isinstance(value, str) or not value: + failed.append(key) + for key in ("source_digest", "manifest_digest"): + value = snapshot.get(key) + if isinstance(value, str) and not value.startswith("sha256:"): + failed.append(key) + reproducibility = snapshot.get("reproducibility") + if not isinstance(reproducibility, dict) or not reproducibility.get("deterministic") or not reproducibility.get("python") or not reproducibility.get("platform"): + failed.append("reproducibility") + for key in ("licenses", "attributions"): + if not isinstance(snapshot.get(key), list) or not snapshot[key]: + failed.append(key) + for key in ("manifest_signature", "airgap_bundle"): + value = snapshot.get(key) + if not isinstance(value, dict) or value.get("verified") is not True: + failed.append(key) + return {"passed": not failed, "failed": sorted(set(failed)), "requirements": {"build_id": True, "source_digest": True, "manifest_digest": True, "reproducibility": True, "licenses": True, "attributions": True, "manifest_signature": True, "airgap_bundle": True}} + + + + +def reproducibility_report(first_artifacts: list[str | Path], second_artifacts: list[str | Path], *, max_artifacts: int = 10000, max_bytes_per_artifact: int = 512 * 1024 * 1024) -> dict[str, Any]: + if len(first_artifacts) > max_artifacts or len(second_artifacts) > max_artifacts: + raise ValueError("too many reproducibility artifacts") + def digest(paths: list[str | Path]) -> dict[str, str]: + result = {} + for raw_path in paths: + path = Path(raw_path) + if not path.is_file() or path.is_symlink(): + raise ValueError("reproducibility artifacts must be regular files") + if path.stat().st_size > max_bytes_per_artifact: + raise ValueError("reproducibility artifact exceeds size limit") + result[path.name] = hashlib.sha256(path.read_bytes()).hexdigest() + return result + first = digest(first_artifacts) + second = digest(second_artifacts) + failed = [] + for name in sorted(set(first) - set(second)): + failed.append(f"missing_artifact:{name}") + for name in sorted(set(second) - set(first)): + failed.append(f"unexpected_artifact:{name}") + if any(first.get(name) != second.get(name) for name in set(first) & set(second)): + failed.append("digest_mismatch") + return {"reproducible": not failed, "failed": failed, "first": first, "second": second} + +def database_health_report(snapshot: dict[str, Any]) -> dict[str, Any]: + if not isinstance(snapshot, dict): + raise ValueError("database snapshot must be an object") + errors = snapshot.get("errors", []) + quality = snapshot.get("quality", {}) + invalid = quality.get("invalid", 0) if isinstance(quality, dict) else 0 + healthy = snapshot.get("status") == "active" and isinstance(snapshot.get("records"), int) and snapshot["records"] >= 0 and not errors and invalid == 0 + return {"healthy": healthy, "snapshot_id": snapshot.get("snapshot_id"), "status": snapshot.get("status"), "records": snapshot.get("records", 0), "errors": list(errors) if isinstance(errors, list) else ["errors must be a list"], "invalid_records": invalid} + + + + +def rollback_readiness_report(database: Any, keep_snapshot_ids: set[str] | frozenset[str]) -> dict[str, Any]: + if not hasattr(database, "prune_snapshots"): + raise ValueError("database does not support snapshot rollback") + plan = database.prune_snapshots(keep_snapshot_ids, apply=False) + return {"ready": isinstance(plan, dict) and plan.get("applied") is False, "applied": False, "plan": plan} + + + + +def build_production_snapshot_report(snapshot: dict[str, Any], health_snapshot: dict[str, Any], sources: dict[str, dict[str, Any]], *, rollback: dict[str, Any] | None = None, disaster_recovery: dict[str, Any] | None = None, benchmark: dict[str, Any] | None = None) -> dict[str, Any]: + readiness = production_snapshot_gate(snapshot) + health = database_health_report(health_snapshot) + coverage = source_coverage_report(sources) + failed = list(readiness["failed"]) + if not health["healthy"]: + failed.append("database_health") + if not coverage["source_count"] or not coverage["total_records"]: + failed.append("source_coverage") + if rollback is not None and rollback.get("ready") is not True: + failed.append("rollback") + if disaster_recovery is not None and (disaster_recovery.get("verified") is not True or disaster_recovery.get("activated") is not False): + failed.append("disaster_recovery") + if benchmark is not None and benchmark.get("passed") is not True: + failed.append("benchmark") + return {"ready": not failed, "failed": sorted(set(failed)), "readiness": readiness, "health": health, "coverage": coverage, "rollback": rollback, "disaster_recovery": disaster_recovery, "benchmark": benchmark} + + +def source_coverage_report(sources: dict[str, dict[str, Any]]) -> dict[str, Any]: + if not isinstance(sources, dict): + raise ValueError("sources must be an object") + normalized = [] + for source_id, values in sources.items(): + if not isinstance(source_id, str) or not source_id or not isinstance(values, dict): + raise ValueError("invalid source coverage entry") + records = values.get("records", 0) + advisories = values.get("advisories", 0) + if not isinstance(records, int) or records < 0 or not isinstance(advisories, int) or advisories < 0: + raise ValueError("source coverage counts must be non-negative integers") + normalized.append({"source_id": source_id, "records": records, "advisories": advisories}) + normalized.sort(key=lambda item: item["source_id"]) + total_records = sum(item["records"] for item in normalized) + total_advisories = sum(item["advisories"] for item in normalized) + for item in normalized: + item["record_share"] = item["records"] / total_records if total_records else 0.0 + item["advisory_share"] = item["advisories"] / total_advisories if total_advisories else 0.0 + return {"source_count": len(normalized), "total_records": total_records, "total_advisories": total_advisories, "sources": normalized} diff --git a/src/vulnerability/seed.py b/src/vulnerability/seed.py new file mode 100644 index 0000000..ee81265 --- /dev/null +++ b/src/vulnerability/seed.py @@ -0,0 +1,165 @@ +"""Bounded, verified seed snapshot helpers.""" + +from __future__ import annotations + +import hashlib +import json +import os +import tempfile +from pathlib import Path +from typing import Any, Iterable + +SEED_ECOSYSTEMS = ("PyPI", "npm", "Go", "crates.io", "Maven", "NuGet", "Packagist") +_REQUIRED_SOURCES = ("cisa-kev", "epss", "ghsa", "osv") + + +def build_seed_manifest(source_reports: dict[str, Any], database_manifest: dict[str, Any]) -> dict[str, Any]: + manifest = { + "profile": "seed", + "completeness": "partial", + "production_full_database": False, + "sources": source_reports, + "database": database_manifest, + "advisory_count": int(database_manifest.get("advisory_count", 0)), + "affected_package_count": int(database_manifest.get("affected_package_count", 0)), + "content_digest": database_manifest.get("content_digest"), + } + validate_seed_manifest(manifest) + return manifest + + +def validate_seed_manifest(manifest: dict[str, Any]) -> None: + if not isinstance(manifest, dict): + raise ValueError("seed manifest must be an object") + if manifest.get("profile") != "seed" or manifest.get("completeness") != "partial": + raise ValueError("seed manifest must declare profile=seed and completeness=partial") + if manifest.get("production_full_database") is not False: + raise ValueError("seed manifest must declare production_full_database=false") + sources = manifest.get("sources") + if not isinstance(sources, dict) or any(source not in sources for source in _REQUIRED_SOURCES): + raise ValueError("seed manifest is missing a required source") + for source in _REQUIRED_SOURCES: + if not isinstance(sources[source], dict): + raise ValueError(f"seed source report is invalid: {source}") + expected_status = {"cisa-kev": "complete", "epss": "bounded", "ghsa": "bounded", "osv": "partial"} + for source, status in expected_status.items(): + report = sources[source] + if report.get("status") != status: + raise ValueError(f"seed source status is invalid: {source}") + if type(report.get("imported")) is not int or report["imported"] <= 0: + raise ValueError(f"seed source imported count must be positive: {source}") + if source != "osv" and (type(report.get("records")) is not int or report["records"] < report["imported"]): + raise ValueError(f"seed source record count is invalid: {source}") + osv = sources["osv"] + successful = osv.get("successful_ecosystems", []) + failed = osv.get("failed_ecosystems", []) + if ( + not isinstance(successful, list) + or len(successful) < 4 + or len(successful) != len(set(successful)) + or any(ecosystem not in SEED_ECOSYSTEMS for ecosystem in successful) + or not isinstance(failed, list) + or set(successful) & set(failed) + ): + raise ValueError("seed requires at least four unique successful OSV ecosystems") + if int(manifest.get("advisory_count", 0)) <= 0: + raise ValueError("seed must contain at least one advisory") + digest = manifest.get("content_digest") + if not isinstance(digest, str) or not digest.startswith("sha256:"): + raise ValueError("seed content_digest is required") + + +def manifest_digest(manifest: dict[str, Any]) -> str: + unsigned = dict(manifest) + unsigned.pop("manifest_sha256", None) + canonical = json.dumps(unsigned, ensure_ascii=False, sort_keys=True, separators=(",", ":")).encode("utf-8") + return "sha256:" + hashlib.sha256(canonical).hexdigest() + + +def _load_json(path: Path) -> Any: + if path.is_symlink() or not path.is_file(): + raise ValueError(f"seed cache file is not a regular file: {path}") + return json.loads(path.read_text(encoding="utf-8")) + + +def _osv_ecosystems(records: Iterable[dict[str, Any]]) -> set[str]: + found: set[str] = set() + for record in records: + for affected in record.get("affected", []): + package = affected.get("package", {}) if isinstance(affected, dict) else {} + ecosystem = package.get("ecosystem") if isinstance(package, dict) else None + if ecosystem in SEED_ECOSYSTEMS: + found.add(ecosystem) + return found + + +def build_seed_database( + cache_dir: str | Path, + output_path: str | Path, + osv_records_by_ecosystem: dict[str, list[dict[str, Any]]], + *, + max_records_per_ecosystem: int = 500, + minimum_successful_ecosystems: int = 4, +) -> dict[str, Any]: + """Build a fresh seed SQLite from real cached JSON and bounded OSV records.""" + from .database import VulnerabilityDatabase + + cache = Path(cache_dir) + output = Path(output_path) + if cache.is_symlink() or not cache.is_dir() or output.is_symlink(): + raise ValueError("seed cache/output path is invalid") + if max_records_per_ecosystem < 1 or minimum_successful_ecosystems < 1: + raise ValueError("seed limits must be positive") + output.parent.mkdir(parents=True, exist_ok=True) + temporary = Path(tempfile.mkstemp(prefix=f".{output.name}.", dir=str(output.parent))[1]) + reports: dict[str, Any] = {"cisa-kev": {}, "epss": {}, "ghsa": {}} + successful: list[str] = [] + failed: list[str] = [] + osv_imported = 0 + try: + with VulnerabilityDatabase(str(temporary)) as database: + for ecosystem in SEED_ECOSYSTEMS: + records = list(osv_records_by_ecosystem.get(ecosystem, []))[:max_records_per_ecosystem] + if not records: + failed.append(ecosystem) + continue + stats = database.import_osv_json(records) + if stats.errors or stats.advisories_imported == 0: + failed.append(ecosystem) + continue + successful.append(ecosystem) + osv_imported += stats.advisories_imported + database.correlate_aliases() + source_files = {"cisa-kev": "cisa-kev.json", "epss": "epss.json", "ghsa": "github-advisories.json"} + importers = {"cisa-kev": ("import_kev_json", {"allow_unresolved": True}), "epss": ("import_epss_json", {"allow_unresolved": True}), "ghsa": ("import_ghsa_json", {})} + for source, filename in source_files.items(): + payload = _load_json(cache / filename) + importer, kwargs = importers[source] + stats = getattr(database, importer)(payload, **kwargs) + if stats.errors: + raise ValueError(f"cached {source} import failed: {stats.errors[0]}") + reports[source] = {"status": "complete" if source == "cisa-kev" else "bounded", "records": stats.records_seen, "imported": stats.advisories_imported} + database.correlate_aliases() + if len(successful) < minimum_successful_ecosystems: + raise ValueError(f"seed requires {minimum_successful_ecosystems} successful OSV ecosystems; got {len(successful)}") + if database.integrity_check() != "ok": + raise ValueError("seed SQLite integrity check failed") + database_manifest = database.build_snapshot_manifest() + os.replace(temporary, output) + reports["osv"] = {"status": "partial", "imported": osv_imported, "successful_ecosystems": successful, "failed_ecosystems": failed, "records_per_ecosystem": max_records_per_ecosystem} + manifest = build_seed_manifest(reports, database_manifest) + snapshot_id = "seed-" + database_manifest["content_digest"].split(":", 1)[1][:32] + source_digest = "sha256:" + hashlib.sha256(json.dumps(reports, sort_keys=True, separators=(",", ":")).encode("utf-8")).hexdigest() + manifest["snapshot_id"] = snapshot_id + manifest["source_digest"] = source_digest + with VulnerabilityDatabase(str(output)) as database: + database.stage_snapshot(snapshot_id, source_digest, manifest) + manifest["database_sha256"] = "sha256:" + hashlib.sha256(output.read_bytes()).hexdigest() + manifest["manifest_sha256"] = manifest_digest(manifest) + return manifest + except Exception: + try: + temporary.unlink() + except FileNotFoundError: + pass + raise diff --git a/src/vulnerability/sources/alpine.py b/src/vulnerability/sources/alpine.py new file mode 100644 index 0000000..90a567c --- /dev/null +++ b/src/vulnerability/sources/alpine.py @@ -0,0 +1,74 @@ +"""Bounded local Alpine APK advisory feed adapter.""" + +from __future__ import annotations + +import hashlib +import json +from datetime import datetime, timezone +from pathlib import Path +from typing import Any + +from ...safeio import read_regular_bounded +from ..database import VulnerabilityDatabase + +ALPINE_SOURCE_ID = "alpine-security" +SCHEMA = "coderisktools.vulnerability.alpine-feed" +MAX_FEED_BYTES = 50 * 1024 * 1024 +MAX_ADVISORIES = 100_000 +_MAX_TEXT = 256 + + +def _text(value: Any, field: str, default: str | None = None) -> str: + if value is None and default is not None: + value = default + if not isinstance(value, str) or not value or len(value) > _MAX_TEXT: + raise ValueError(f"Alpine {field} is invalid") + return value + + +def ingest_file(path: str | Path, *, max_bytes: int = MAX_FEED_BYTES) -> dict[str, Any]: + raw = read_regular_bounded(path, max_bytes, "Alpine feed") + try: + document = json.loads(raw.decode("utf-8")) + except (UnicodeDecodeError, json.JSONDecodeError) as exc: + raise ValueError("Alpine feed must be bounded UTF-8 JSON") from exc + if not isinstance(document, dict) or document.get("schema") != SCHEMA or document.get("version") != 1: + raise ValueError("unsupported Alpine feed schema") + release = _text(document.get("release"), "release") + advisories = document.get("advisories") + if not isinstance(advisories, list) or len(advisories) > MAX_ADVISORIES: + raise ValueError("Alpine advisory list exceeds bounds") + normalized = [] + for item in advisories: + if not isinstance(item, dict): + raise ValueError("Alpine advisory must be an object") + binaries = item.get("binary_packages") + if not isinstance(binaries, list) or not binaries or len(binaries) > 1000: + raise ValueError("Alpine binary package list is invalid") + normalized.append({"id": _text(item.get("id"), "advisory id"), "package": _text(item.get("package"), "source package"), "source_package": _text(item.get("source_package"), "source package", item.get("package")), "binary_packages": sorted({_text(name, "binary package") for name in binaries}), "fixed": _text(item.get("fixed"), "fixed APK version"), "severity": _text(item.get("severity", "Unknown"), "severity"), "backport": item.get("backport", False) is True}) + digest = "sha256:" + hashlib.sha256(raw).hexdigest() + provenance = {"source_id": ALPINE_SOURCE_ID, "source_path": str(Path(path)), "source_digest": digest, "schema": SCHEMA, "version": 1, "release": release} + return {"source_id": ALPINE_SOURCE_ID, "source_path": str(Path(path)), "source_digest": digest, "release": release, "advisory_count": len(normalized), "advisories": normalized, "provenance": provenance} + + +def ingest_file_to_database(path: str | Path, database: VulnerabilityDatabase, snapshot_id: str, *, activate: bool = False, max_bytes: int = MAX_FEED_BYTES) -> dict[str, Any]: + document = ingest_file(path, max_bytes=max_bytes) + source = f"{ALPINE_SOURCE_ID}:{document['release']}" + records = [] + for advisory in document["advisories"]: + affected = [{"package": {"ecosystem": "Alpine", "name": name}, "ranges": [{"type": "ECOSYSTEM", "events": [{"introduced": "0"}, {"fixed": advisory["fixed"]}]}]} for name in advisory["binary_packages"]] + records.append({"id": advisory["id"], "summary": f"Alpine {document['release']} advisory {advisory['id']}", "affected": affected, "database_specific": {"alpine_release": document["release"], "source_package": advisory["source_package"], "severity": advisory["severity"], "backport": advisory["backport"]}, "_source_record": advisory}) + stats = database.import_osv_records(records, source=source) + if stats.errors: + return {"source_id": source, "snapshot_id": snapshot_id, "state": "rejected", "activated": False, "errors": list(stats.errors)} + manifest = database.build_snapshot_manifest() + manifest.update({"source_id": source, "source_digest": document["source_digest"], "source_path": document["source_path"], "signature_status": "unsigned", "ingestion_schema": "coderisktools.vulnerability.alpine-staging.v1"}) + database.stage_snapshot(snapshot_id, document["source_digest"], manifest) + now = datetime.now(timezone.utc).isoformat() + database.connection.execute("INSERT OR REPLACE INTO source_snapshots(snapshot_id, source_id, content_digest, observed_at, record_count, status, metadata_json) VALUES (?, ?, ?, ?, ?, ?, ?)", (snapshot_id, source, document["source_digest"], now, stats.records_seen, "staged", json.dumps(document["provenance"], sort_keys=True))) + for metric_name, metric_value in (("records_seen", stats.records_seen), ("advisories_imported", stats.advisories_imported), ("affected_packages_imported", stats.affected_packages_imported)): + database.connection.execute("INSERT OR REPLACE INTO quality_metrics(snapshot_id, metric_name, metric_value, details_json) VALUES (?, ?, ?, ?)", (snapshot_id, metric_name, float(metric_value), "{}")) + database.connection.commit() + if activate: + database.activate_snapshot(snapshot_id) + return {"source_id": source, "snapshot_id": snapshot_id, "state": "active" if activate else "staged", "activated": activate, "records_seen": stats.records_seen, "advisories_imported": stats.advisories_imported, "affected_packages_imported": stats.affected_packages_imported, "errors": []} diff --git a/src/vulnerability/sources/conan.py b/src/vulnerability/sources/conan.py new file mode 100644 index 0000000..97f43c2 --- /dev/null +++ b/src/vulnerability/sources/conan.py @@ -0,0 +1,65 @@ +"""Bounded local Conan advisory source adapter.""" + +from __future__ import annotations + +import hashlib +import json +from datetime import datetime, timezone +from pathlib import Path +from typing import Any + +from ...safeio import read_regular_bounded +from ..database import VulnerabilityDatabase + +CONAN_SOURCE_ID = "conan-security" +SCHEMA = "coderisktools.vulnerability.conan-feed" +MAX_FEED_BYTES = 50 * 1024 * 1024 +MAX_ADVISORIES = 100_000 +_MAX_TEXT = 256 + + +def _text(value: Any, field: str) -> str: + if not isinstance(value, str) or not value or len(value) > _MAX_TEXT: + raise ValueError(f"Conan {field} is invalid") + return value + + +def ingest_file(path: str | Path, *, max_bytes: int = MAX_FEED_BYTES) -> dict[str, Any]: + raw = read_regular_bounded(path, max_bytes, "Conan feed") + try: + document = json.loads(raw.decode("utf-8")) + except (UnicodeDecodeError, json.JSONDecodeError) as exc: + raise ValueError("Conan feed must be bounded UTF-8 JSON") from exc + if not isinstance(document, dict) or document.get("schema") != SCHEMA or document.get("version") != 1: + raise ValueError("unsupported Conan feed schema") + advisories = document.get("advisories") + if not isinstance(advisories, list) or len(advisories) > MAX_ADVISORIES: + raise ValueError("Conan advisory list exceeds bounds") + normalized = [] + for item in advisories: + if not isinstance(item, dict): + raise ValueError("Conan advisory must be an object") + package = _text(item.get("package"), "reference") + normalized.append({"id": _text(item.get("id"), "advisory id"), "package": package, "fixed": _text(item.get("fixed"), "fixed version"), "severity": _text(item.get("severity", "Unknown"), "severity")}) + digest = "sha256:" + hashlib.sha256(raw).hexdigest() + provenance = {"source_id": CONAN_SOURCE_ID, "source_path": str(Path(path)), "source_digest": digest, "schema": SCHEMA, "version": 1} + return {"source_id": CONAN_SOURCE_ID, "source_path": str(Path(path)), "source_digest": digest, "advisory_count": len(normalized), "advisories": normalized, "provenance": provenance} + + +def ingest_file_to_database(path: str | Path, database: VulnerabilityDatabase, snapshot_id: str, *, activate: bool = False, max_bytes: int = MAX_FEED_BYTES) -> dict[str, Any]: + document = ingest_file(path, max_bytes=max_bytes) + records = [{"id": item["id"], "summary": f"Conan advisory {item['id']}", "affected": [{"package": {"ecosystem": "Conan", "name": item["package"]}, "ranges": [{"type": "ECOSYSTEM", "events": [{"introduced": "0"}, {"fixed": item["fixed"]}]}]}], "database_specific": {"severity": item["severity"]}, "_source_record": item} for item in document["advisories"]] + stats = database.import_osv_records(records, source=CONAN_SOURCE_ID) + if stats.errors: + return {"source_id": CONAN_SOURCE_ID, "snapshot_id": snapshot_id, "state": "rejected", "activated": False, "errors": list(stats.errors)} + manifest = database.build_snapshot_manifest() + manifest.update({"source_id": CONAN_SOURCE_ID, "source_digest": document["source_digest"], "source_path": document["source_path"], "signature_status": "unsigned", "ingestion_schema": "coderisktools.vulnerability.conan-staging.v1"}) + database.stage_snapshot(snapshot_id, document["source_digest"], manifest) + now = datetime.now(timezone.utc).isoformat() + database.connection.execute("INSERT OR REPLACE INTO source_snapshots(snapshot_id, source_id, content_digest, observed_at, record_count, status, metadata_json) VALUES (?, ?, ?, ?, ?, ?, ?)", (snapshot_id, CONAN_SOURCE_ID, document["source_digest"], now, stats.records_seen, "staged", json.dumps(document["provenance"], sort_keys=True))) + for name, value in (("records_seen", stats.records_seen), ("advisories_imported", stats.advisories_imported), ("affected_packages_imported", stats.affected_packages_imported)): + database.connection.execute("INSERT OR REPLACE INTO quality_metrics(snapshot_id, metric_name, metric_value, details_json) VALUES (?, ?, ?, ?)", (snapshot_id, name, float(value), "{}")) + database.connection.commit() + if activate: + database.activate_snapshot(snapshot_id) + return {"source_id": CONAN_SOURCE_ID, "snapshot_id": snapshot_id, "state": "active" if activate else "staged", "activated": activate, "records_seen": stats.records_seen, "advisories_imported": stats.advisories_imported, "affected_packages_imported": stats.affected_packages_imported, "errors": []} diff --git a/src/vulnerability/sources/csaf.py b/src/vulnerability/sources/csaf.py new file mode 100644 index 0000000..ce74fb3 --- /dev/null +++ b/src/vulnerability/sources/csaf.py @@ -0,0 +1,98 @@ +"""Bounded generic CSAF 2.0 security-advisory importer.""" + +from __future__ import annotations + +import hashlib +import json +from datetime import datetime, timezone +from pathlib import Path +from typing import Any + +from ...safeio import read_regular_bounded +from ..database import VulnerabilityDatabase +from .csaf_quality import csaf_quality_gate + +CSAF_SOURCE_ID = "csaf-security" +MAX_FEED_BYTES = 50 * 1024 * 1024 +MAX_VULNERABILITIES = 100_000 +MAX_PRODUCTS = 200_000 + +def _purl_component(purl: Any) -> tuple[str, str, str]: + if not isinstance(purl, str) or not purl.startswith("pkg:") or len(purl) > 2048: + raise ValueError("CSAF product PURL is invalid") + value = purl[4:] + kind, _, remainder = value.partition("/") + name_version = remainder.split("?", 1)[0].split("#", 1)[0] + parts = name_version.rsplit("@", 1) + if len(parts) != 2 or not all(parts): + raise ValueError("CSAF product PURL must include name and version") + ecosystem = {"pypi": "PyPI", "npm": "npm", "cargo": "crates.io", "golang": "Go", "maven": "Maven", "nuget": "NuGet"}.get(kind.lower(), kind) + return ecosystem, parts[0], parts[1] + +def _products(node: Any, output: dict[str, tuple[str, str, str]]) -> None: + if isinstance(node, dict): + product = node.get("product") + if isinstance(product, dict) and isinstance(product.get("product_id"), str): + helper = product.get("product_identification_helper", {}) + if isinstance(helper, dict) and "purl" in helper: + output[product["product_id"]] = _purl_component(helper["purl"]) + for value in node.values(): + _products(value, output) + elif isinstance(node, list): + for value in node: + _products(value, output) + +def ingest_file(path: str | Path, *, max_bytes: int = MAX_FEED_BYTES) -> dict[str, Any]: + raw = read_regular_bounded(path, max_bytes, "CSAF feed") + try: + document = json.loads(raw.decode("utf-8")) + except (UnicodeDecodeError, json.JSONDecodeError) as exc: + raise ValueError("CSAF feed must be bounded UTF-8 JSON") from exc + if not isinstance(document, dict) or document.get("csaf_version") != "2.0": + raise ValueError("unsupported CSAF schema") + vulnerabilities = document.get("vulnerabilities") + if not isinstance(vulnerabilities, list) or len(vulnerabilities) > MAX_VULNERABILITIES: + raise ValueError("CSAF vulnerability list exceeds bounds") + products: dict[str, tuple[str, str, str]] = {} + _products(document.get("product_tree", {}), products) + if len(products) > MAX_PRODUCTS: + raise ValueError("CSAF product tree exceeds bounds") + normalized = [] + for item in vulnerabilities: + if not isinstance(item, dict) or not isinstance(item.get("cve"), str): + raise ValueError("CSAF vulnerability requires cve") + status = item.get("product_status", {}) + if not isinstance(status, dict): + raise ValueError("CSAF product_status must be an object") + for key in ("known_affected", "known_not_affected"): + if not isinstance(status.get(key, []), list): + raise ValueError("CSAF product status list is invalid") + normalized.append({"id": item["cve"], "affected_product_ids": list(status.get("known_affected", [])), "not_affected_product_ids": list(status.get("known_not_affected", [])), "vendor_status": dict(status.get("vendor_specific", {})) if isinstance(status.get("vendor_specific", {}), dict) else {}, "remediations": list(item.get("remediations", [])) if isinstance(item.get("remediations", []), list) else [], "products": products, "raw": item}) + digest = "sha256:" + hashlib.sha256(raw).hexdigest() + provenance = {"source_id": CSAF_SOURCE_ID, "source_path": str(Path(path)), "source_digest": digest, "schema": "csaf_security_advisory", "version": "2.0"} + return {"source_id": CSAF_SOURCE_ID, "source_path": str(Path(path)), "source_digest": digest, "advisory_count": len(normalized), "product_count": len(products), "advisories": normalized, "provenance": provenance} + +def ingest_file_to_database(path: str | Path, database: VulnerabilityDatabase, snapshot_id: str, *, activate: bool = False, max_bytes: int = MAX_FEED_BYTES) -> dict[str, Any]: + document = ingest_file(path, max_bytes=max_bytes) + quality = csaf_quality_gate(document) + if not quality["passed"]: + return {"source_id": CSAF_SOURCE_ID, "snapshot_id": snapshot_id, "state": "rejected", "activated": False, "records_seen": 0, "advisories_imported": 0, "affected_packages_imported": 0, "errors": list(quality["errors"])} + records = [] + for advisory in document["advisories"]: + for product_id in advisory["affected_product_ids"]: + if product_id not in advisory["products"]: + raise ValueError(f"CSAF references unknown product {product_id}") + ecosystem, name, version = advisory["products"][product_id] + records.append({"id": advisory["id"], "summary": f"CSAF advisory {advisory['id']}", "affected": [{"package": {"ecosystem": ecosystem, "name": name}, "ranges": [{"type": "ECOSYSTEM", "events": [{"introduced": "0"}]}]}], "database_specific": {"csaf_product_id": product_id, "csaf_product_status": "known_affected", "csaf_vendor_status": advisory["vendor_status"].get(product_id), "csaf_remediations": advisory["remediations"]}, "_source_record": advisory["raw"]}) + stats = database.import_osv_records(records, source=CSAF_SOURCE_ID) + manifest = database.build_snapshot_manifest() + manifest.update({"source_id": CSAF_SOURCE_ID, "source_digest": document["source_digest"], "source_path": document["source_path"], "signature_status": "unsigned", "ingestion_schema": "coderisktools.vulnerability.csaf-staging.v1"}) + database.stage_snapshot(snapshot_id, document["source_digest"], manifest) + now = datetime.now(timezone.utc).isoformat() + database.connection.execute("INSERT OR REPLACE INTO source_snapshots(snapshot_id, source_id, content_digest, observed_at, record_count, status, metadata_json) VALUES (?, ?, ?, ?, ?, ?, ?)", (snapshot_id, CSAF_SOURCE_ID, document["source_digest"], now, stats.records_seen, "staged", json.dumps(document["provenance"], sort_keys=True))) + for name, value in (("records_seen", stats.records_seen), ("advisories_imported", stats.advisories_imported), ("affected_packages_imported", stats.affected_packages_imported)): + database.connection.execute("INSERT OR REPLACE INTO quality_metrics(snapshot_id, metric_name, metric_value, details_json) VALUES (?, ?, ?, ?)", (snapshot_id, name, float(value), "{}")) + database.connection.commit() + if activate: + database.activate_snapshot(snapshot_id) + return {"source_id": CSAF_SOURCE_ID, "snapshot_id": snapshot_id, "state": "active" if activate else "staged", "activated": activate, "records_seen": stats.records_seen, "advisories_imported": stats.advisories_imported, "affected_packages_imported": stats.affected_packages_imported, "errors": list(stats.errors)} diff --git a/src/vulnerability/sources/csaf_quality.py b/src/vulnerability/sources/csaf_quality.py new file mode 100644 index 0000000..0e2df16 --- /dev/null +++ b/src/vulnerability/sources/csaf_quality.py @@ -0,0 +1,57 @@ +"""Quality gate for the bounded CSAF staging subset.""" + +from __future__ import annotations + +import re +from typing import Any + +_CVE_RE = re.compile(r"^(CVE|GHSA)-[A-Za-z0-9._-]+$") + + +def csaf_quality_gate(document: dict[str, Any]) -> dict[str, Any]: + errors: list[str] = [] + if document.get("source_id") != "csaf-security": + errors.append("invalid source_id") + digest = document.get("source_digest") + if not isinstance(digest, str) or not digest.startswith("sha256:"): + errors.append("missing source digest") + provenance = document.get("provenance") + if not isinstance(provenance, dict) or provenance.get("schema") != "csaf_security_advisory" or provenance.get("version") != "2.0": + errors.append("incomplete CSAF provenance") + advisories = document.get("advisories") + if not isinstance(advisories, list): + errors.append("advisories must be a list") + advisories = [] + product_extension_count = 0 + remediation_count = 0 + for index, advisory in enumerate(advisories): + if not isinstance(advisory, dict) or not isinstance(advisory.get("id"), str) or not _CVE_RE.fullmatch(advisory.get("id", "")): + errors.append(f"advisory[{index}] has invalid identifier") + continue + products = advisory.get("products", {}) + if not isinstance(products, dict): + errors.append(f"advisory[{index}] products must be an object") + products = {} + for product_id in advisory.get("affected_product_ids", []) + advisory.get("not_affected_product_ids", []): + if product_id not in products: + errors.append(f"advisory[{index}] references unknown product {product_id}") + statuses = advisory.get("vendor_status", {}) + if isinstance(statuses, dict): + for status in statuses.values(): + if isinstance(status, dict) and status.get("provider_extension") is not None: + product_extension_count += 1 + else: + errors.append(f"advisory[{index}] vendor_status must be an object") + remediations = advisory.get("remediations", []) + if not isinstance(remediations, list): + errors.append(f"advisory[{index}] remediations must be a list") + continue + for remediation in remediations: + if not isinstance(remediation, dict) or not isinstance(remediation.get("category"), str) or not isinstance(remediation.get("details"), str): + errors.append(f"advisory[{index}] remediation is malformed") + continue + remediation_count += 1 + for product_id in remediation.get("product_ids", []): + if product_id not in products: + errors.append(f"advisory[{index}] remediation references unknown product {product_id}") + return {"passed": not errors, "errors": errors, "advisory_count": len(advisories), "provider_extension_count": product_extension_count, "remediation_count": remediation_count} diff --git a/src/vulnerability/sources/csaf_registry.py b/src/vulnerability/sources/csaf_registry.py new file mode 100644 index 0000000..6cd2fa5 --- /dev/null +++ b/src/vulnerability/sources/csaf_registry.py @@ -0,0 +1,59 @@ +"""Deterministic CSAF provider registry and health state.""" + +from __future__ import annotations + +import re +from datetime import datetime, timezone +from typing import Any +from urllib.parse import urlparse + +_PROVIDER_RE = re.compile(r"^[a-z0-9][a-z0-9_.-]{0,63}$") + + +class CsafProviderRegistry: + def __init__(self) -> None: + self._providers: dict[str, dict[str, Any]] = {} + + def register(self, provider_id: str, endpoint: str, *, enabled: bool = True) -> None: + if not isinstance(provider_id, str) or not _PROVIDER_RE.fullmatch(provider_id): + raise ValueError("invalid CSAF provider_id") + parsed = urlparse(endpoint) + if parsed.scheme not in {"http", "https"} or not parsed.netloc: + raise ValueError("CSAF provider endpoint must be an absolute HTTP(S) URL") + self._providers[provider_id] = {"provider_id": provider_id, "endpoint": endpoint, "enabled": bool(enabled), "successes": 0, "failures": 0, "last_success": None, "last_failure": None, "last_records": 0, "last_digest": None} + + def record_result(self, provider_id: str, *, success: bool, records: int = 0, digest: str | None = None, observed_at: str | None = None) -> None: + provider = self._providers.get(provider_id) + if provider is None: + raise KeyError(provider_id) + if not isinstance(records, int) or records < 0: + raise ValueError("records must be a non-negative integer") + timestamp = observed_at or datetime.now(timezone.utc).isoformat() + if success: + provider["successes"] += 1 + provider["last_success"] = timestamp + provider["last_records"] = records + provider["last_digest"] = digest + else: + provider["failures"] += 1 + provider["last_failure"] = timestamp + + def list_providers(self) -> list[dict[str, Any]]: + return [{"provider_id": p["provider_id"], "endpoint": p["endpoint"], "enabled": p["enabled"]} for p in sorted(self._providers.values(), key=lambda item: item["provider_id"])] + + def health_report(self, provider_id: str) -> dict[str, Any]: + provider = self._providers.get(provider_id) + if provider is None: + raise KeyError(provider_id) + if not provider["enabled"]: + status = "disabled" + elif provider["last_success"] is not None and (provider["last_failure"] is None or provider["last_success"] >= provider["last_failure"]): + status = "healthy" + elif provider["failures"]: + status = "unhealthy" + else: + status = "unknown" + return {"provider_id": provider_id, "endpoint": provider["endpoint"], "enabled": provider["enabled"], "status": status, "successes": provider["successes"], "failures": provider["failures"], "last_success": provider["last_success"], "last_failure": provider["last_failure"], "last_records": provider["last_records"], "last_digest": provider["last_digest"]} + + def health_reports(self) -> list[dict[str, Any]]: + return [self.health_report(provider_id) for provider_id in sorted(self._providers)] diff --git a/src/vulnerability/sources/dart.py b/src/vulnerability/sources/dart.py new file mode 100644 index 0000000..382d30b --- /dev/null +++ b/src/vulnerability/sources/dart.py @@ -0,0 +1,64 @@ +"""Bounded local Dart/pub advisory source adapter.""" + +from __future__ import annotations + +import hashlib +import json +from datetime import datetime, timezone +from pathlib import Path +from typing import Any + +from ...safeio import read_regular_bounded +from ..database import VulnerabilityDatabase + +DART_SOURCE_ID = "dart-security" +SCHEMA = "coderisktools.vulnerability.dart-feed" +MAX_FEED_BYTES = 50 * 1024 * 1024 +MAX_ADVISORIES = 100_000 +_MAX_TEXT = 256 + + +def _text(value: Any, field: str) -> str: + if not isinstance(value, str) or not value or len(value) > _MAX_TEXT: + raise ValueError(f"Dart {field} is invalid") + return value + + +def ingest_file(path: str | Path, *, max_bytes: int = MAX_FEED_BYTES) -> dict[str, Any]: + raw = read_regular_bounded(path, max_bytes, "Dart feed") + try: + document = json.loads(raw.decode("utf-8")) + except (UnicodeDecodeError, json.JSONDecodeError) as exc: + raise ValueError("Dart feed must be bounded UTF-8 JSON") from exc + if not isinstance(document, dict) or document.get("schema") != SCHEMA or document.get("version") != 1: + raise ValueError("unsupported Dart feed schema") + advisories = document.get("advisories") + if not isinstance(advisories, list) or len(advisories) > MAX_ADVISORIES: + raise ValueError("Dart advisory list exceeds bounds") + normalized = [] + for item in advisories: + if not isinstance(item, dict): + raise ValueError("Dart advisory must be an object") + normalized.append({"id": _text(item.get("id"), "advisory id"), "package": _text(item.get("package"), "package name"), "fixed": _text(item.get("fixed"), "fixed version"), "severity": _text(item.get("severity", "Unknown"), "severity")}) + digest = "sha256:" + hashlib.sha256(raw).hexdigest() + provenance = {"source_id": DART_SOURCE_ID, "source_path": str(Path(path)), "source_digest": digest, "schema": SCHEMA, "version": 1} + return {"source_id": DART_SOURCE_ID, "source_path": str(Path(path)), "source_digest": digest, "advisory_count": len(normalized), "advisories": normalized, "provenance": provenance} + + +def ingest_file_to_database(path: str | Path, database: VulnerabilityDatabase, snapshot_id: str, *, activate: bool = False, max_bytes: int = MAX_FEED_BYTES) -> dict[str, Any]: + document = ingest_file(path, max_bytes=max_bytes) + records = [{"id": item["id"], "summary": f"Dart advisory {item['id']}", "affected": [{"package": {"ecosystem": "Dart", "name": item["package"]}, "ranges": [{"type": "ECOSYSTEM", "events": [{"introduced": "0"}, {"fixed": item["fixed"]}]}]}], "database_specific": {"severity": item["severity"]}, "_source_record": item} for item in document["advisories"]] + stats = database.import_osv_records(records, source=DART_SOURCE_ID) + if stats.errors: + return {"source_id": DART_SOURCE_ID, "snapshot_id": snapshot_id, "state": "rejected", "activated": False, "errors": list(stats.errors)} + manifest = database.build_snapshot_manifest() + manifest.update({"source_id": DART_SOURCE_ID, "source_digest": document["source_digest"], "source_path": document["source_path"], "signature_status": "unsigned", "ingestion_schema": "coderisktools.vulnerability.dart-staging.v1"}) + database.stage_snapshot(snapshot_id, document["source_digest"], manifest) + now = datetime.now(timezone.utc).isoformat() + database.connection.execute("INSERT OR REPLACE INTO source_snapshots(snapshot_id, source_id, content_digest, observed_at, record_count, status, metadata_json) VALUES (?, ?, ?, ?, ?, ?, ?)", (snapshot_id, DART_SOURCE_ID, document["source_digest"], now, stats.records_seen, "staged", json.dumps(document["provenance"], sort_keys=True))) + for name, value in (("records_seen", stats.records_seen), ("advisories_imported", stats.advisories_imported), ("affected_packages_imported", stats.affected_packages_imported)): + database.connection.execute("INSERT OR REPLACE INTO quality_metrics(snapshot_id, metric_name, metric_value, details_json) VALUES (?, ?, ?, ?)", (snapshot_id, name, float(value), "{}")) + database.connection.commit() + if activate: + database.activate_snapshot(snapshot_id) + return {"source_id": DART_SOURCE_ID, "snapshot_id": snapshot_id, "state": "active" if activate else "staged", "activated": activate, "records_seen": stats.records_seen, "advisories_imported": stats.advisories_imported, "affected_packages_imported": stats.affected_packages_imported, "errors": []} diff --git a/src/vulnerability/sources/debian.py b/src/vulnerability/sources/debian.py new file mode 100644 index 0000000..0e9c304 --- /dev/null +++ b/src/vulnerability/sources/debian.py @@ -0,0 +1,105 @@ +"""Bounded local Debian advisory feed adapter.""" + +from __future__ import annotations + +import hashlib +import json +from datetime import datetime, timezone +from pathlib import Path +from typing import Any + +from ...safeio import read_regular_bounded +from ..database import VulnerabilityDatabase + +DEBIAN_SOURCE_ID = "debian-security" +SCHEMA = "coderisktools.vulnerability.debian-feed" +MAX_FEED_BYTES = 50 * 1024 * 1024 +MAX_ADVISORIES = 100_000 + + +def _validate_advisory(value: Any) -> dict[str, Any]: + if not isinstance(value, dict): + raise ValueError("Debian advisory must be an object") + advisory_id = value.get("id") + package = value.get("package") + fixed = value.get("fixed") + if not all(isinstance(item, str) and 1 <= len(item) <= 512 for item in (advisory_id, package, fixed)): + raise ValueError("Debian advisory id, package and fixed are required bounded strings") + binary = value.get("binary_packages", [package]) + if not isinstance(binary, list) or len(binary) > 256 or not all(isinstance(item, str) and item for item in binary): + raise ValueError("Debian binary_packages is invalid") + return { + "id": advisory_id, + "package": package, + "source_package": value.get("source_package", package), + "fixed": fixed, + "urgency": value.get("urgency", "unknown"), + "backport": value.get("backport", False) is True, + "binary_packages": sorted(set(binary)), + } + + +def ingest_file(path: str | Path, *, max_bytes: int = MAX_FEED_BYTES) -> dict[str, Any]: + """Parse a local Debian feed; no network access or activation is performed.""" + raw = read_regular_bounded(path, max_bytes, "Debian feed") + try: + document = json.loads(raw.decode("utf-8")) + except (UnicodeDecodeError, json.JSONDecodeError) as exc: + raise ValueError("Debian feed must be bounded UTF-8 JSON") from exc + if not isinstance(document, dict) or document.get("schema") != SCHEMA or document.get("version") != 1: + raise ValueError("unsupported Debian feed schema") + release = document.get("release") + advisories = document.get("advisories") + if not isinstance(release, str) or not 1 <= len(release) <= 64: + raise ValueError("Debian release is invalid") + if not isinstance(advisories, list) or len(advisories) > MAX_ADVISORIES: + raise ValueError("Debian advisory list exceeds bounds") + normalized = [_validate_advisory(item) for item in advisories] + digest = "sha256:" + hashlib.sha256(raw).hexdigest() + return { + "source_id": DEBIAN_SOURCE_ID, + "source_path": str(Path(path)), + "source_digest": digest, + "release": release, + "advisory_count": len(normalized), + "advisories": normalized, + "provenance": { + "source_id": DEBIAN_SOURCE_ID, + "source_path": str(Path(path)), + "source_digest": digest, + "schema": SCHEMA, + "version": 1, + "release": release, + }, + } + + +def ingest_file_to_database( + path: str | Path, + database: VulnerabilityDatabase, + snapshot_id: str, + *, + activate: bool = False, + max_bytes: int = MAX_FEED_BYTES, +) -> dict[str, Any]: + """Normalize a Debian fixture into the existing bounded OSV import/staging path.""" + document = ingest_file(path, max_bytes=max_bytes) + source = f"{DEBIAN_SOURCE_ID}:{document['release']}" + records = [] + for advisory in document["advisories"]: + affected = [{"package": {"ecosystem": "Debian", "name": package_name}, "ranges": [{"type": "ECOSYSTEM", "events": [{"introduced": "0"}, {"fixed": advisory["fixed"]}]}]} for package_name in advisory["binary_packages"]] + records.append({"id": advisory["id"], "summary": f"Debian {document['release']} advisory {advisory['id']}", "affected": affected, "database_specific": {"debian_release": document["release"], "source_package": advisory["source_package"], "urgency": advisory["urgency"], "backport": advisory["backport"]}, "_source_record": advisory}) + stats = database.import_osv_records(records, source=source) + if stats.errors: + return {"source_id": source, "snapshot_id": snapshot_id, "state": "rejected", "activated": False, "errors": list(stats.errors)} + manifest = database.build_snapshot_manifest() + manifest.update({"source_id": source, "source_digest": document["source_digest"], "source_path": document["source_path"], "signature_status": "unsigned", "ingestion_schema": "coderisktools.vulnerability.debian-staging.v1"}) + database.stage_snapshot(snapshot_id, document["source_digest"], manifest) + now = datetime.now(timezone.utc).isoformat() + database.connection.execute("INSERT OR REPLACE INTO source_snapshots(snapshot_id, source_id, content_digest, observed_at, record_count, status, metadata_json) VALUES (?, ?, ?, ?, ?, ?, ?)", (snapshot_id, source, document["source_digest"], now, stats.records_seen, "staged", json.dumps(document["provenance"], sort_keys=True))) + for metric_name, metric_value in (("records_seen", stats.records_seen), ("advisories_imported", stats.advisories_imported), ("affected_packages_imported", stats.affected_packages_imported)): + database.connection.execute("INSERT OR REPLACE INTO quality_metrics(snapshot_id, metric_name, metric_value, details_json) VALUES (?, ?, ?, ?)", (snapshot_id, metric_name, float(metric_value), "{}")) + database.connection.commit() + if activate: + database.activate_snapshot(snapshot_id) + return {"source_id": source, "snapshot_id": snapshot_id, "state": "active" if activate else "staged", "activated": activate, "records_seen": stats.records_seen, "advisories_imported": stats.advisories_imported, "affected_packages_imported": stats.affected_packages_imported, "errors": []} diff --git a/src/vulnerability/sources/elixir.py b/src/vulnerability/sources/elixir.py new file mode 100644 index 0000000..3e6e87f --- /dev/null +++ b/src/vulnerability/sources/elixir.py @@ -0,0 +1,64 @@ +"""Bounded local Elixir/Hex advisory source adapter.""" + +from __future__ import annotations + +import hashlib +import json +from datetime import datetime, timezone +from pathlib import Path +from typing import Any + +from ...safeio import read_regular_bounded +from ..database import VulnerabilityDatabase + +ELIXIR_SOURCE_ID = "elixir-security" +SCHEMA = "coderisktools.vulnerability.elixir-feed" +MAX_FEED_BYTES = 50 * 1024 * 1024 +MAX_ADVISORIES = 100_000 +_MAX_TEXT = 256 + + +def _text(value: Any, field: str) -> str: + if not isinstance(value, str) or not value or len(value) > _MAX_TEXT: + raise ValueError(f"Elixir {field} is invalid") + return value + + +def ingest_file(path: str | Path, *, max_bytes: int = MAX_FEED_BYTES) -> dict[str, Any]: + raw = read_regular_bounded(path, max_bytes, "Elixir feed") + try: + document = json.loads(raw.decode("utf-8")) + except (UnicodeDecodeError, json.JSONDecodeError) as exc: + raise ValueError("Elixir feed must be bounded UTF-8 JSON") from exc + if not isinstance(document, dict) or document.get("schema") != SCHEMA or document.get("version") != 1: + raise ValueError("unsupported Elixir feed schema") + advisories = document.get("advisories") + if not isinstance(advisories, list) or len(advisories) > MAX_ADVISORIES: + raise ValueError("Elixir advisory list exceeds bounds") + normalized = [] + for item in advisories: + if not isinstance(item, dict): + raise ValueError("Elixir advisory must be an object") + normalized.append({"id": _text(item.get("id"), "advisory id"), "package": _text(item.get("package"), "Hex package name"), "fixed": _text(item.get("fixed"), "fixed version"), "severity": _text(item.get("severity", "Unknown"), "severity")}) + digest = "sha256:" + hashlib.sha256(raw).hexdigest() + provenance = {"source_id": ELIXIR_SOURCE_ID, "source_path": str(Path(path)), "source_digest": digest, "schema": SCHEMA, "version": 1} + return {"source_id": ELIXIR_SOURCE_ID, "source_path": str(Path(path)), "source_digest": digest, "advisory_count": len(normalized), "advisories": normalized, "provenance": provenance} + + +def ingest_file_to_database(path: str | Path, database: VulnerabilityDatabase, snapshot_id: str, *, activate: bool = False, max_bytes: int = MAX_FEED_BYTES) -> dict[str, Any]: + document = ingest_file(path, max_bytes=max_bytes) + records = [{"id": item["id"], "summary": f"Elixir advisory {item['id']}", "affected": [{"package": {"ecosystem": "Hex", "name": item["package"]}, "ranges": [{"type": "ECOSYSTEM", "events": [{"introduced": "0"}, {"fixed": item["fixed"]}]}]}], "database_specific": {"severity": item["severity"]}, "_source_record": item} for item in document["advisories"]] + stats = database.import_osv_records(records, source=ELIXIR_SOURCE_ID) + if stats.errors: + return {"source_id": ELIXIR_SOURCE_ID, "snapshot_id": snapshot_id, "state": "rejected", "activated": False, "errors": list(stats.errors)} + manifest = database.build_snapshot_manifest() + manifest.update({"source_id": ELIXIR_SOURCE_ID, "source_digest": document["source_digest"], "source_path": document["source_path"], "signature_status": "unsigned", "ingestion_schema": "coderisktools.vulnerability.elixir-staging.v1"}) + database.stage_snapshot(snapshot_id, document["source_digest"], manifest) + now = datetime.now(timezone.utc).isoformat() + database.connection.execute("INSERT OR REPLACE INTO source_snapshots(snapshot_id, source_id, content_digest, observed_at, record_count, status, metadata_json) VALUES (?, ?, ?, ?, ?, ?, ?)", (snapshot_id, ELIXIR_SOURCE_ID, document["source_digest"], now, stats.records_seen, "staged", json.dumps(document["provenance"], sort_keys=True))) + for name, value in (("records_seen", stats.records_seen), ("advisories_imported", stats.advisories_imported), ("affected_packages_imported", stats.affected_packages_imported)): + database.connection.execute("INSERT OR REPLACE INTO quality_metrics(snapshot_id, metric_name, metric_value, details_json) VALUES (?, ?, ?, ?)", (snapshot_id, name, float(value), "{}")) + database.connection.commit() + if activate: + database.activate_snapshot(snapshot_id) + return {"source_id": ELIXIR_SOURCE_ID, "snapshot_id": snapshot_id, "state": "active" if activate else "staged", "activated": activate, "records_seen": stats.records_seen, "advisories_imported": stats.advisories_imported, "affected_packages_imported": stats.affected_packages_imported, "errors": []} diff --git a/src/vulnerability/sources/epss.py b/src/vulnerability/sources/epss.py index 50d063d..cc250ba 100644 --- a/src/vulnerability/sources/epss.py +++ b/src/vulnerability/sources/epss.py @@ -6,6 +6,10 @@ from datetime import date from typing import Any +from ..database import VulnerabilityDatabase +from ..updater import FetchConditions, FetchPolicy +from .feed_file import fetch_json_to_file, read_json_feed + _CVE_ID = re.compile(r"^CVE-[0-9]{4}-[0-9]{4,}$") _DATE = re.compile(r"^[0-9]{4}-[0-9]{2}-[0-9]{2}$") @@ -46,3 +50,21 @@ def parse_epss_record(record: dict[str, Any]) -> dict[str, Any]: "date": score_date, "_source_record": record, } + + +def ingest_file(path: str, database: VulnerabilityDatabase, snapshot_id: str, *, max_records: int = 1_000_000, activate: bool = False) -> dict[str, Any]: + records, digest, source_name = read_json_feed(path, list_key="data", max_records=max_records) + stats = database.import_epss_json(records) + return {"source_id": "epss", "source_path": source_name, "source_digest": digest, "snapshot_id": snapshot_id, "state": "staged" if not stats.errors else "partial", "activated": bool(activate and not stats.errors), "records_seen": stats.records_seen, "records_imported": stats.advisories_imported, "errors": list(stats.errors)} + + + + +def fetch_and_ingest(url: str, output_path: str, database: VulnerabilityDatabase, snapshot_id: str, policy: FetchPolicy, *, conditions: FetchConditions | None = None, activate: bool = False) -> dict[str, Any]: + download = fetch_json_to_file(url, output_path, policy, conditions=conditions) + if download.get("state") == "not_modified": + return {**download, "source_id": "epss", "snapshot_id": snapshot_id, "activated": False} + return {**ingest_file(output_path, database, snapshot_id, activate=activate), "download": download} + + +__all__ = ["parse_epss_record", "ingest_file", "fetch_and_ingest"] diff --git a/src/vulnerability/sources/feed_acceptance.py b/src/vulnerability/sources/feed_acceptance.py new file mode 100644 index 0000000..4a43672 --- /dev/null +++ b/src/vulnerability/sources/feed_acceptance.py @@ -0,0 +1,38 @@ +"""Offline acceptance evidence for externally acquired feed artifacts.""" + +from __future__ import annotations + +import hashlib +import json +from pathlib import Path +from typing import Any + +from ...safeio import read_regular_bounded + + +def evaluate_feed_artifact( + path: str | Path, + *, + source_id: str, + expected_list_key: str, + max_bytes: int = 256 * 1024 * 1024, + max_records: int = 1_000_000, +) -> dict[str, Any]: + """Validate an acquired feed artifact without downloading or activating it.""" + if not isinstance(source_id, str) or not source_id: + raise ValueError("source_id is required") + if not isinstance(expected_list_key, str) or not expected_list_key: + raise ValueError("expected_list_key is required") + source_path = Path(path) + raw = read_regular_bounded(source_path, max_bytes, "feed acceptance artifact") + digest = "sha256:" + hashlib.sha256(raw).hexdigest() + try: + payload = json.loads(raw.decode("utf-8")) + except (UnicodeDecodeError, json.JSONDecodeError) as exc: + return {"source_id": source_id, "source_path": source_path.name, "source_digest": digest, "state": "rejected", "activated": False, "records": 0, "errors": [f"invalid JSON: {exc}"]} + records = payload.get(expected_list_key) if isinstance(payload, dict) else None + if not isinstance(records, list): + return {"source_id": source_id, "source_path": source_path.name, "source_digest": digest, "state": "rejected", "activated": False, "records": 0, "errors": [f"missing list: {expected_list_key}"]} + if len(records) > max_records: + return {"source_id": source_id, "source_path": source_path.name, "source_digest": digest, "state": "rejected", "activated": False, "records": len(records), "errors": ["record limit exceeded"]} + return {"source_id": source_id, "source_path": source_path.name, "source_digest": digest, "state": "accepted", "activated": False, "records": len(records), "errors": [], "network_used": False} diff --git a/src/vulnerability/sources/feed_file.py b/src/vulnerability/sources/feed_file.py new file mode 100644 index 0000000..6a01304 --- /dev/null +++ b/src/vulnerability/sources/feed_file.py @@ -0,0 +1,30 @@ +"""Bounded local feed file helpers shared by source adapters.""" + +from __future__ import annotations + +import hashlib +import json +from pathlib import Path +from typing import Any + +from ...safeio import read_regular_bounded +from ..updater import FetchConditions, FetchPolicy, stream_json_artifact_to_file + + +def read_json_feed(path: str | Path, *, list_key: str, max_bytes: int = 256 * 1024 * 1024, max_records: int = 1_000_000) -> tuple[Any, str, str]: + source = Path(path) + raw = read_regular_bounded(source, max_bytes, "source feed") + digest = "sha256:" + hashlib.sha256(raw).hexdigest() + try: + payload = json.loads(raw.decode("utf-8")) + except (UnicodeDecodeError, json.JSONDecodeError) as exc: + raise ValueError("source feed must be valid UTF-8 JSON") from exc + records = payload.get(list_key) if isinstance(payload, dict) else payload + if not isinstance(records, list) or len(records) > max_records: + raise ValueError(f"source feed requires bounded {list_key} array") + return records, digest, source.name + + +def fetch_json_to_file(url: str, output_path: str | Path, policy: FetchPolicy, *, conditions: FetchConditions | None = None) -> dict[str, Any]: + """Fetch a source artifact without importing or activating it.""" + return stream_json_artifact_to_file(url, output_path, policy, conditions=conditions) diff --git a/src/vulnerability/sources/ghsa.py b/src/vulnerability/sources/ghsa.py index bd04032..db83547 100644 --- a/src/vulnerability/sources/ghsa.py +++ b/src/vulnerability/sources/ghsa.py @@ -5,6 +5,10 @@ import re from typing import Any +from ..database import VulnerabilityDatabase +from ..updater import FetchConditions, FetchPolicy +from .feed_file import fetch_json_to_file, read_json_feed + _GHSA_ID = re.compile(r"^GHSA-[a-z0-9]{4}-[a-z0-9]{4}-[a-z0-9]{4}$", re.IGNORECASE) _CVE_ID = re.compile(r"^CVE-[0-9]{4}-[0-9]{4,}$") @@ -86,3 +90,21 @@ def parse_ghsa_advisory(record: dict[str, Any]) -> dict[str, Any]: "affected": affected, "_source_record": record, } + + +def ingest_file(path: str, database: VulnerabilityDatabase, snapshot_id: str, *, max_records: int = 1_000_000, activate: bool = False) -> dict[str, Any]: + records, digest, source_name = read_json_feed(path, list_key="advisories", max_records=max_records) + stats = database.import_ghsa_json(records) + return {"source_id": "github-advisory", "source_path": source_name, "source_digest": digest, "snapshot_id": snapshot_id, "state": "staged" if not stats.errors else "partial", "activated": bool(activate and not stats.errors), "records_seen": stats.records_seen, "records_imported": stats.advisories_imported, "errors": list(stats.errors)} + + + + +def fetch_and_ingest(url: str, output_path: str, database: VulnerabilityDatabase, snapshot_id: str, policy: FetchPolicy, *, conditions: FetchConditions | None = None, activate: bool = False) -> dict[str, Any]: + download = fetch_json_to_file(url, output_path, policy, conditions=conditions) + if download.get("state") == "not_modified": + return {**download, "source_id": "github-advisory", "snapshot_id": snapshot_id, "activated": False} + return {**ingest_file(output_path, database, snapshot_id, activate=activate), "download": download} + + +__all__ = ["parse_ghsa_advisory", "ingest_file", "fetch_and_ingest"] diff --git a/src/vulnerability/sources/haskell.py b/src/vulnerability/sources/haskell.py new file mode 100644 index 0000000..5675e6c --- /dev/null +++ b/src/vulnerability/sources/haskell.py @@ -0,0 +1,64 @@ +"""Bounded local Haskell/Hackage advisory source adapter.""" + +from __future__ import annotations + +import hashlib +import json +from datetime import datetime, timezone +from pathlib import Path +from typing import Any + +from ...safeio import read_regular_bounded +from ..database import VulnerabilityDatabase + +HASKELL_SOURCE_ID = "haskell-security" +SCHEMA = "coderisktools.vulnerability.haskell-feed" +MAX_FEED_BYTES = 50 * 1024 * 1024 +MAX_ADVISORIES = 100_000 +_MAX_TEXT = 256 + + +def _text(value: Any, field: str) -> str: + if not isinstance(value, str) or not value or len(value) > _MAX_TEXT: + raise ValueError(f"Haskell {field} is invalid") + return value + + +def ingest_file(path: str | Path, *, max_bytes: int = MAX_FEED_BYTES) -> dict[str, Any]: + raw = read_regular_bounded(path, max_bytes, "Haskell feed") + try: + document = json.loads(raw.decode("utf-8")) + except (UnicodeDecodeError, json.JSONDecodeError) as exc: + raise ValueError("Haskell feed must be bounded UTF-8 JSON") from exc + if not isinstance(document, dict) or document.get("schema") != SCHEMA or document.get("version") != 1: + raise ValueError("unsupported Haskell feed schema") + advisories = document.get("advisories") + if not isinstance(advisories, list) or len(advisories) > MAX_ADVISORIES: + raise ValueError("Haskell advisory list exceeds bounds") + normalized = [] + for item in advisories: + if not isinstance(item, dict): + raise ValueError("Haskell advisory must be an object") + normalized.append({"id": _text(item.get("id"), "advisory id"), "package": _text(item.get("package"), "Hackage package name"), "fixed": _text(item.get("fixed"), "fixed version"), "severity": _text(item.get("severity", "Unknown"), "severity")}) + digest = "sha256:" + hashlib.sha256(raw).hexdigest() + provenance = {"source_id": HASKELL_SOURCE_ID, "source_path": str(Path(path)), "source_digest": digest, "schema": SCHEMA, "version": 1} + return {"source_id": HASKELL_SOURCE_ID, "source_path": str(Path(path)), "source_digest": digest, "advisory_count": len(normalized), "advisories": normalized, "provenance": provenance} + + +def ingest_file_to_database(path: str | Path, database: VulnerabilityDatabase, snapshot_id: str, *, activate: bool = False, max_bytes: int = MAX_FEED_BYTES) -> dict[str, Any]: + document = ingest_file(path, max_bytes=max_bytes) + records = [{"id": item["id"], "summary": f"Haskell advisory {item['id']}", "affected": [{"package": {"ecosystem": "Hackage", "name": item["package"]}, "ranges": [{"type": "ECOSYSTEM", "events": [{"introduced": "0"}, {"fixed": item["fixed"]}]}]}], "database_specific": {"severity": item["severity"]}, "_source_record": item} for item in document["advisories"]] + stats = database.import_osv_records(records, source=HASKELL_SOURCE_ID) + if stats.errors: + return {"source_id": HASKELL_SOURCE_ID, "snapshot_id": snapshot_id, "state": "rejected", "activated": False, "errors": list(stats.errors)} + manifest = database.build_snapshot_manifest() + manifest.update({"source_id": HASKELL_SOURCE_ID, "source_digest": document["source_digest"], "source_path": document["source_path"], "signature_status": "unsigned", "ingestion_schema": "coderisktools.vulnerability.haskell-staging.v1"}) + database.stage_snapshot(snapshot_id, document["source_digest"], manifest) + now = datetime.now(timezone.utc).isoformat() + database.connection.execute("INSERT OR REPLACE INTO source_snapshots(snapshot_id, source_id, content_digest, observed_at, record_count, status, metadata_json) VALUES (?, ?, ?, ?, ?, ?, ?)", (snapshot_id, HASKELL_SOURCE_ID, document["source_digest"], now, stats.records_seen, "staged", json.dumps(document["provenance"], sort_keys=True))) + for name, value in (("records_seen", stats.records_seen), ("advisories_imported", stats.advisories_imported), ("affected_packages_imported", stats.affected_packages_imported)): + database.connection.execute("INSERT OR REPLACE INTO quality_metrics(snapshot_id, metric_name, metric_value, details_json) VALUES (?, ?, ?, ?)", (snapshot_id, name, float(value), "{}")) + database.connection.commit() + if activate: + database.activate_snapshot(snapshot_id) + return {"source_id": HASKELL_SOURCE_ID, "snapshot_id": snapshot_id, "state": "active" if activate else "staged", "activated": activate, "records_seen": stats.records_seen, "advisories_imported": stats.advisories_imported, "affected_packages_imported": stats.affected_packages_imported, "errors": []} diff --git a/src/vulnerability/sources/kev.py b/src/vulnerability/sources/kev.py index b98769b..eb81622 100644 --- a/src/vulnerability/sources/kev.py +++ b/src/vulnerability/sources/kev.py @@ -6,6 +6,10 @@ from datetime import date from typing import Any +from ..database import VulnerabilityDatabase +from ..updater import FetchConditions, FetchPolicy +from .feed_file import fetch_json_to_file, read_json_feed + _CVE_ID = re.compile(r"^CVE-[0-9]{4}-[0-9]{4,}$") _DATE = re.compile(r"^[0-9]{4}-[0-9]{2}-[0-9]{2}$") _REQUIRED = ("cveID", "vendorProject", "product", "vulnerabilityName", "dateAdded", "shortDescription", "requiredAction", "dueDate") @@ -54,3 +58,16 @@ def parse_kev_record(record: dict[str, Any]) -> dict[str, Any]: "notes": notes, "_source_record": record, } + + +def ingest_file(path: str, database: VulnerabilityDatabase, snapshot_id: str, *, max_records: int = 1_000_000, activate: bool = False) -> dict[str, Any]: + records, digest, source_name = read_json_feed(path, list_key="vulnerabilities", max_records=max_records) + stats = database.import_kev_json(records) + return {"source_id": "kev", "source_path": source_name, "source_digest": digest, "snapshot_id": snapshot_id, "state": "staged" if not stats.errors else "partial", "activated": bool(activate and not stats.errors), "records_seen": stats.records_seen, "records_imported": stats.advisories_imported, "errors": list(stats.errors)} + + +def fetch_and_ingest(url: str, output_path: str, database: VulnerabilityDatabase, snapshot_id: str, policy: FetchPolicy, *, conditions: FetchConditions | None = None, activate: bool = False) -> dict[str, Any]: + download = fetch_json_to_file(url, output_path, policy, conditions=conditions) + if download.get("state") == "not_modified": + return {**download, "source_id": "kev", "snapshot_id": snapshot_id, "activated": False} + return {**ingest_file(output_path, database, snapshot_id, activate=activate), "download": download} diff --git a/src/vulnerability/sources/maven.py b/src/vulnerability/sources/maven.py new file mode 100644 index 0000000..f3dad4a --- /dev/null +++ b/src/vulnerability/sources/maven.py @@ -0,0 +1,69 @@ +"""Bounded local Maven advisory source adapter.""" + +from __future__ import annotations + +import hashlib +import json +from datetime import datetime, timezone +from pathlib import Path +from typing import Any + +from ...safeio import read_regular_bounded +from ..database import VulnerabilityDatabase + +MAVEN_SOURCE_ID = "maven-security" +SCHEMA = "coderisktools.vulnerability.maven-feed" +MAX_FEED_BYTES = 50 * 1024 * 1024 +MAX_ADVISORIES = 100_000 +_MAX_TEXT = 512 + + +def _text(value: Any, field: str) -> str: + if not isinstance(value, str) or not value or len(value) > _MAX_TEXT: + raise ValueError(f"Maven {field} is invalid") + return value + + +def ingest_file(path: str | Path, *, max_bytes: int = MAX_FEED_BYTES) -> dict[str, Any]: + raw = read_regular_bounded(path, max_bytes, "Maven feed") + try: + document = json.loads(raw.decode("utf-8")) + except (UnicodeDecodeError, json.JSONDecodeError) as exc: + raise ValueError("Maven feed must be bounded UTF-8 JSON") from exc + if not isinstance(document, dict) or document.get("schema") != SCHEMA or document.get("version") != 1: + raise ValueError("unsupported Maven feed schema") + advisories = document.get("advisories") + if not isinstance(advisories, list) or len(advisories) > MAX_ADVISORIES: + raise ValueError("Maven advisory list exceeds bounds") + normalized = [] + for item in advisories: + if not isinstance(item, dict): + raise ValueError("Maven advisory must be an object") + package = _text(item.get("package"), "package coordinate") + if package.count(":") < 1 or package.count(":") > 2: + raise ValueError("Maven package must be group:artifact[:classifier]") + normalized.append({"id": _text(item.get("id"), "advisory id"), "package": package, "fixed": _text(item.get("fixed"), "fixed version"), "severity": _text(item.get("severity", "Unknown"), "severity")}) + digest = "sha256:" + hashlib.sha256(raw).hexdigest() + provenance = {"source_id": MAVEN_SOURCE_ID, "source_path": str(Path(path)), "source_digest": digest, "schema": SCHEMA, "version": 1} + return {"source_id": MAVEN_SOURCE_ID, "source_path": str(Path(path)), "source_digest": digest, "advisory_count": len(normalized), "advisories": normalized, "provenance": provenance} + + +def ingest_file_to_database(path: str | Path, database: VulnerabilityDatabase, snapshot_id: str, *, activate: bool = False, max_bytes: int = MAX_FEED_BYTES) -> dict[str, Any]: + document = ingest_file(path, max_bytes=max_bytes) + records = [] + for advisory in document["advisories"]: + records.append({"id": advisory["id"], "summary": f"Maven advisory {advisory['id']}", "affected": [{"package": {"ecosystem": "Maven", "name": advisory["package"]}, "ranges": [{"type": "ECOSYSTEM", "events": [{"introduced": "0"}, {"fixed": advisory["fixed"]}]}]}], "database_specific": {"severity": advisory["severity"]}, "_source_record": advisory}) + stats = database.import_osv_records(records, source=MAVEN_SOURCE_ID) + if stats.errors: + return {"source_id": MAVEN_SOURCE_ID, "snapshot_id": snapshot_id, "state": "rejected", "activated": False, "errors": list(stats.errors)} + manifest = database.build_snapshot_manifest() + manifest.update({"source_id": MAVEN_SOURCE_ID, "source_digest": document["source_digest"], "source_path": document["source_path"], "signature_status": "unsigned", "ingestion_schema": "coderisktools.vulnerability.maven-staging.v1"}) + database.stage_snapshot(snapshot_id, document["source_digest"], manifest) + now = datetime.now(timezone.utc).isoformat() + database.connection.execute("INSERT OR REPLACE INTO source_snapshots(snapshot_id, source_id, content_digest, observed_at, record_count, status, metadata_json) VALUES (?, ?, ?, ?, ?, ?, ?)", (snapshot_id, MAVEN_SOURCE_ID, document["source_digest"], now, stats.records_seen, "staged", json.dumps(document["provenance"], sort_keys=True))) + for metric_name, metric_value in (("records_seen", stats.records_seen), ("advisories_imported", stats.advisories_imported), ("affected_packages_imported", stats.affected_packages_imported)): + database.connection.execute("INSERT OR REPLACE INTO quality_metrics(snapshot_id, metric_name, metric_value, details_json) VALUES (?, ?, ?, ?)", (snapshot_id, metric_name, float(metric_value), "{}")) + database.connection.commit() + if activate: + database.activate_snapshot(snapshot_id) + return {"source_id": MAVEN_SOURCE_ID, "snapshot_id": snapshot_id, "state": "active" if activate else "staged", "activated": activate, "records_seen": stats.records_seen, "advisories_imported": stats.advisories_imported, "affected_packages_imported": stats.affected_packages_imported, "errors": []} diff --git a/src/vulnerability/sources/nuget.py b/src/vulnerability/sources/nuget.py new file mode 100644 index 0000000..043c730 --- /dev/null +++ b/src/vulnerability/sources/nuget.py @@ -0,0 +1,65 @@ +"""Bounded local NuGet advisory source adapter.""" + +from __future__ import annotations + +import hashlib +import json +from datetime import datetime, timezone +from pathlib import Path +from typing import Any + +from ...safeio import read_regular_bounded +from ..database import VulnerabilityDatabase + +NUGET_SOURCE_ID = "nuget-security" +SCHEMA = "coderisktools.vulnerability.nuget-feed" +MAX_FEED_BYTES = 50 * 1024 * 1024 +MAX_ADVISORIES = 100_000 +_MAX_TEXT = 256 + + +def _text(value: Any, field: str) -> str: + if not isinstance(value, str) or not value or len(value) > _MAX_TEXT: + raise ValueError(f"NuGet {field} is invalid") + return value + + +def ingest_file(path: str | Path, *, max_bytes: int = MAX_FEED_BYTES) -> dict[str, Any]: + raw = read_regular_bounded(path, max_bytes, "NuGet feed") + try: + document = json.loads(raw.decode("utf-8")) + except (UnicodeDecodeError, json.JSONDecodeError) as exc: + raise ValueError("NuGet feed must be bounded UTF-8 JSON") from exc + if not isinstance(document, dict) or document.get("schema") != SCHEMA or document.get("version") != 1: + raise ValueError("unsupported NuGet feed schema") + advisories = document.get("advisories") + if not isinstance(advisories, list) or len(advisories) > MAX_ADVISORIES: + raise ValueError("NuGet advisory list exceeds bounds") + normalized = [] + for item in advisories: + if not isinstance(item, dict): + raise ValueError("NuGet advisory must be an object") + package = _text(item.get("package"), "package id") + normalized.append({"id": _text(item.get("id"), "advisory id"), "package": package, "fixed": _text(item.get("fixed"), "fixed version"), "severity": _text(item.get("severity", "Unknown"), "severity")}) + digest = "sha256:" + hashlib.sha256(raw).hexdigest() + provenance = {"source_id": NUGET_SOURCE_ID, "source_path": str(Path(path)), "source_digest": digest, "schema": SCHEMA, "version": 1} + return {"source_id": NUGET_SOURCE_ID, "source_path": str(Path(path)), "source_digest": digest, "advisory_count": len(normalized), "advisories": normalized, "provenance": provenance} + + +def ingest_file_to_database(path: str | Path, database: VulnerabilityDatabase, snapshot_id: str, *, activate: bool = False, max_bytes: int = MAX_FEED_BYTES) -> dict[str, Any]: + document = ingest_file(path, max_bytes=max_bytes) + records = [{"id": item["id"], "summary": f"NuGet advisory {item['id']}", "affected": [{"package": {"ecosystem": "NuGet", "name": item["package"]}, "ranges": [{"type": "ECOSYSTEM", "events": [{"introduced": "0"}, {"fixed": item["fixed"]}]}]}], "database_specific": {"severity": item["severity"]}, "_source_record": item} for item in document["advisories"]] + stats = database.import_osv_records(records, source=NUGET_SOURCE_ID) + if stats.errors: + return {"source_id": NUGET_SOURCE_ID, "snapshot_id": snapshot_id, "state": "rejected", "activated": False, "errors": list(stats.errors)} + manifest = database.build_snapshot_manifest() + manifest.update({"source_id": NUGET_SOURCE_ID, "source_digest": document["source_digest"], "source_path": document["source_path"], "signature_status": "unsigned", "ingestion_schema": "coderisktools.vulnerability.nuget-staging.v1"}) + database.stage_snapshot(snapshot_id, document["source_digest"], manifest) + now = datetime.now(timezone.utc).isoformat() + database.connection.execute("INSERT OR REPLACE INTO source_snapshots(snapshot_id, source_id, content_digest, observed_at, record_count, status, metadata_json) VALUES (?, ?, ?, ?, ?, ?, ?)", (snapshot_id, NUGET_SOURCE_ID, document["source_digest"], now, stats.records_seen, "staged", json.dumps(document["provenance"], sort_keys=True))) + for name, value in (("records_seen", stats.records_seen), ("advisories_imported", stats.advisories_imported), ("affected_packages_imported", stats.affected_packages_imported)): + database.connection.execute("INSERT OR REPLACE INTO quality_metrics(snapshot_id, metric_name, metric_value, details_json) VALUES (?, ?, ?, ?)", (snapshot_id, name, float(value), "{}")) + database.connection.commit() + if activate: + database.activate_snapshot(snapshot_id) + return {"source_id": NUGET_SOURCE_ID, "snapshot_id": snapshot_id, "state": "active" if activate else "staged", "activated": activate, "records_seen": stats.records_seen, "advisories_imported": stats.advisories_imported, "affected_packages_imported": stats.affected_packages_imported, "errors": []} diff --git a/src/vulnerability/sources/nvd.py b/src/vulnerability/sources/nvd.py index c9d12d9..e16c867 100644 --- a/src/vulnerability/sources/nvd.py +++ b/src/vulnerability/sources/nvd.py @@ -2,9 +2,15 @@ from __future__ import annotations +import hashlib +import json import re +from pathlib import Path from typing import Any +from ..database import VulnerabilityDatabase +from ...safeio import read_regular_bounded + _CVE_ID = re.compile(r"^CVE-[0-9]{4}-[0-9]{4,}$") _NVD_OPERATORS = frozenset({"AND", "OR"}) @@ -140,3 +146,39 @@ def parse_node(node: Any) -> dict[str, Any]: "configurations": configurations, "_source_record": record, } + + +def ingest_file(path: str | Path, database: VulnerabilityDatabase, snapshot_id: str, *, max_records: int = 1_000_000, max_bytes: int = 512 * 1024 * 1024, activate: bool = False) -> dict[str, Any]: + """Parse a bounded NVD API 2.0 JSON file and preserve source provenance.""" + source_path = Path(path) + raw = read_regular_bounded(source_path, max_bytes, "NVD feed") + source_digest = "sha256:" + hashlib.sha256(raw).hexdigest() + try: + payload = json.loads(raw.decode("utf-8")) + except (UnicodeDecodeError, json.JSONDecodeError) as exc: + raise ValueError("NVD feed must be valid UTF-8 JSON") from exc + if isinstance(payload, dict) and isinstance(payload.get("vulnerabilities"), list): + records = payload["vulnerabilities"] + elif isinstance(payload, list): + records = payload + else: + raise ValueError("NVD feed requires vulnerabilities array") + if len(records) > max_records: + raise ValueError("NVD feed exceeds max_records") + imported = 0 + errors: list[str] = [] + for index, record in enumerate(records, 1): + try: + parsed = parse_nvd_cve(record) + cve_id = parsed["id"] + lookup = database.lookup_advisory(cve_id) + if lookup.get("status") != "exact": + raise ValueError("CVE has no exact advisory match") + database.record_source_record("nvd", cve_id, parsed, lookup["advisory_id"]) + imported += 1 + except (KeyError, TypeError, ValueError) as exc: + errors.append(f"record {index}: {exc}") + return {"source_id": "nvd", "source_path": source_path.name, "source_digest": source_digest, "snapshot_id": snapshot_id, "state": "staged" if not errors else "partial", "activated": bool(activate and not errors), "records_seen": len(records), "records_imported": imported, "errors": errors} + + +__all__ = ["parse_nvd_cve", "ingest_file"] diff --git a/src/vulnerability/sources/osv.py b/src/vulnerability/sources/osv.py index 192473d..2e6dd53 100644 --- a/src/vulnerability/sources/osv.py +++ b/src/vulnerability/sources/osv.py @@ -11,7 +11,7 @@ from ..database import VulnerabilityDatabase from ..ingestion import IngestionReport, ingest_osv_file, ingest_osv_streaming_file -from ..updater import JsonRecordLimits +from ..updater import FetchConditions, FetchPolicy, JsonRecordLimits, stream_json_artifact_to_file OSV_SOURCE_ID = "osv" @@ -60,4 +60,32 @@ def ingest_streaming_file( return ingest_osv_streaming_file(source_path, database, snapshot_id, source_id, activate=activate, limits=limits) -__all__ = ["OSV_SOURCE_ID", "IngestionReport", "JsonRecordLimits", "ingest_file", "ingest_streaming_file"] + + +def fetch_and_ingest_stream( + url: str, + output_path: str | PathLike[str], + database: VulnerabilityDatabase, + snapshot_id: str, + policy: FetchPolicy, + *, + source_id: str = OSV_SOURCE_ID, + conditions: FetchConditions | None = None, + limits: JsonRecordLimits | None = None, + activate: bool = False, +) -> dict: + """Fetch an allowlisted OSV JSON/JSONL feed, then explicitly ingest it. + + Download and import remain separate stages. The downloaded artifact is + atomically replaced by the updater, and activation is never implicit. + """ + metadata = stream_json_artifact_to_file(url, output_path, policy, conditions=conditions) + if metadata.get("state") == "not_modified": + return {**metadata, "source_id": source_id, "snapshot_id": snapshot_id, "state": "not_modified", "activated": False} + report = ingest_osv_streaming_file(str(output_path), database, snapshot_id, source_id, activate=activate, limits=limits) + result = report.to_dict() + result.update({"download": metadata, "source_digest": metadata.get("payload_sha256"), "activated": bool(result.get("activated", False))}) + return result + + +__all__ = ["OSV_SOURCE_ID", "IngestionReport", "FetchConditions", "FetchPolicy", "JsonRecordLimits", "ingest_file", "ingest_streaming_file", "fetch_and_ingest_stream"] diff --git a/src/vulnerability/sources/r.py b/src/vulnerability/sources/r.py new file mode 100644 index 0000000..72dd224 --- /dev/null +++ b/src/vulnerability/sources/r.py @@ -0,0 +1,64 @@ +"""Bounded local R/CRAN advisory source adapter.""" + +from __future__ import annotations + +import hashlib +import json +from datetime import datetime, timezone +from pathlib import Path +from typing import Any + +from ...safeio import read_regular_bounded +from ..database import VulnerabilityDatabase + +R_SOURCE_ID = "r-security" +SCHEMA = "coderisktools.vulnerability.r-feed" +MAX_FEED_BYTES = 50 * 1024 * 1024 +MAX_ADVISORIES = 100_000 +_MAX_TEXT = 256 + + +def _text(value: Any, field: str) -> str: + if not isinstance(value, str) or not value or len(value) > _MAX_TEXT: + raise ValueError(f"R {field} is invalid") + return value + + +def ingest_file(path: str | Path, *, max_bytes: int = MAX_FEED_BYTES) -> dict[str, Any]: + raw = read_regular_bounded(path, max_bytes, "R feed") + try: + document = json.loads(raw.decode("utf-8")) + except (UnicodeDecodeError, json.JSONDecodeError) as exc: + raise ValueError("R feed must be bounded UTF-8 JSON") from exc + if not isinstance(document, dict) or document.get("schema") != SCHEMA or document.get("version") != 1: + raise ValueError("unsupported R feed schema") + advisories = document.get("advisories") + if not isinstance(advisories, list) or len(advisories) > MAX_ADVISORIES: + raise ValueError("R advisory list exceeds bounds") + normalized = [] + for item in advisories: + if not isinstance(item, dict): + raise ValueError("R advisory must be an object") + normalized.append({"id": _text(item.get("id"), "advisory id"), "package": _text(item.get("package"), "CRAN package name"), "fixed": _text(item.get("fixed"), "fixed version"), "severity": _text(item.get("severity", "Unknown"), "severity")}) + digest = "sha256:" + hashlib.sha256(raw).hexdigest() + provenance = {"source_id": R_SOURCE_ID, "source_path": str(Path(path)), "source_digest": digest, "schema": SCHEMA, "version": 1} + return {"source_id": R_SOURCE_ID, "source_path": str(Path(path)), "source_digest": digest, "advisory_count": len(normalized), "advisories": normalized, "provenance": provenance} + + +def ingest_file_to_database(path: str | Path, database: VulnerabilityDatabase, snapshot_id: str, *, activate: bool = False, max_bytes: int = MAX_FEED_BYTES) -> dict[str, Any]: + document = ingest_file(path, max_bytes=max_bytes) + records = [{"id": item["id"], "summary": f"R advisory {item['id']}", "affected": [{"package": {"ecosystem": "CRAN", "name": item["package"]}, "ranges": [{"type": "ECOSYSTEM", "events": [{"introduced": "0"}, {"fixed": item["fixed"]}]}]}], "database_specific": {"severity": item["severity"]}, "_source_record": item} for item in document["advisories"]] + stats = database.import_osv_records(records, source=R_SOURCE_ID) + if stats.errors: + return {"source_id": R_SOURCE_ID, "snapshot_id": snapshot_id, "state": "rejected", "activated": False, "errors": list(stats.errors)} + manifest = database.build_snapshot_manifest() + manifest.update({"source_id": R_SOURCE_ID, "source_digest": document["source_digest"], "source_path": document["source_path"], "signature_status": "unsigned", "ingestion_schema": "coderisktools.vulnerability.r-staging.v1"}) + database.stage_snapshot(snapshot_id, document["source_digest"], manifest) + now = datetime.now(timezone.utc).isoformat() + database.connection.execute("INSERT OR REPLACE INTO source_snapshots(snapshot_id, source_id, content_digest, observed_at, record_count, status, metadata_json) VALUES (?, ?, ?, ?, ?, ?, ?)", (snapshot_id, R_SOURCE_ID, document["source_digest"], now, stats.records_seen, "staged", json.dumps(document["provenance"], sort_keys=True))) + for name, value in (("records_seen", stats.records_seen), ("advisories_imported", stats.advisories_imported), ("affected_packages_imported", stats.affected_packages_imported)): + database.connection.execute("INSERT OR REPLACE INTO quality_metrics(snapshot_id, metric_name, metric_value, details_json) VALUES (?, ?, ?, ?)", (snapshot_id, name, float(value), "{}")) + database.connection.commit() + if activate: + database.activate_snapshot(snapshot_id) + return {"source_id": R_SOURCE_ID, "snapshot_id": snapshot_id, "state": "active" if activate else "staged", "activated": activate, "records_seen": stats.records_seen, "advisories_imported": stats.advisories_imported, "affected_packages_imported": stats.affected_packages_imported, "errors": []} diff --git a/src/vulnerability/sources/redhat.py b/src/vulnerability/sources/redhat.py new file mode 100644 index 0000000..e7a5872 --- /dev/null +++ b/src/vulnerability/sources/redhat.py @@ -0,0 +1,82 @@ +"""Bounded local Red Hat security advisory feed adapter.""" + +from __future__ import annotations + +import hashlib +import json +from datetime import datetime, timezone +from pathlib import Path +from typing import Any + +from ...safeio import read_regular_bounded +from ..database import VulnerabilityDatabase + +REDHAT_SOURCE_ID = "redhat-security" +SCHEMA = "coderisktools.vulnerability.redhat-feed" +MAX_FEED_BYTES = 50 * 1024 * 1024 +MAX_ADVISORIES = 100_000 +_MAX_TEXT = 256 + + +def _text(value: Any, field: str, *, required: bool = True) -> str: + if not isinstance(value, str) or (required and not value) or len(value) > _MAX_TEXT: + raise ValueError(f"Red Hat {field} is invalid") + return value + + +def ingest_file(path: str | Path, *, max_bytes: int = MAX_FEED_BYTES) -> dict[str, Any]: + """Parse a local Red Hat fixture without network access or activation.""" + raw = read_regular_bounded(path, max_bytes, "Red Hat feed") + try: + document = json.loads(raw.decode("utf-8")) + except (UnicodeDecodeError, json.JSONDecodeError) as exc: + raise ValueError("Red Hat feed must be bounded UTF-8 JSON") from exc + if not isinstance(document, dict) or document.get("schema") != SCHEMA or document.get("version") != 1: + raise ValueError("unsupported Red Hat feed schema") + release = _text(document.get("release"), "release") + advisories = document.get("advisories") + if not isinstance(advisories, list) or len(advisories) > MAX_ADVISORIES: + raise ValueError("Red Hat advisory list exceeds bounds") + normalized = [] + for item in advisories: + if not isinstance(item, dict): + raise ValueError("Red Hat advisory must be an object") + binaries = item.get("binary_packages") + if not isinstance(binaries, list) or not binaries or len(binaries) > 1000: + raise ValueError("Red Hat binary package list is invalid") + normalized.append({ + "id": _text(item.get("id"), "advisory id"), + "package": _text(item.get("package"), "source package"), + "source_package": _text(item.get("source_package", item.get("package")), "source package"), + "binary_packages": sorted({_text(name, "binary package") for name in binaries}), + "fixed": _text(item.get("fixed"), "fixed RPM version"), + "severity": _text(item.get("severity", "Unknown"), "severity"), + "backport": item.get("backport", False) is True, + }) + digest = "sha256:" + hashlib.sha256(raw).hexdigest() + provenance = {"source_id": REDHAT_SOURCE_ID, "source_path": str(Path(path)), "source_digest": digest, "schema": SCHEMA, "version": 1, "release": release} + return {"source_id": REDHAT_SOURCE_ID, "source_path": str(Path(path)), "source_digest": digest, "release": release, "advisory_count": len(normalized), "advisories": normalized, "provenance": provenance} + + +def ingest_file_to_database(path: str | Path, database: VulnerabilityDatabase, snapshot_id: str, *, activate: bool = False, max_bytes: int = MAX_FEED_BYTES) -> dict[str, Any]: + """Normalize a bounded Red Hat fixture into the existing staging pipeline.""" + document = ingest_file(path, max_bytes=max_bytes) + source = f"{REDHAT_SOURCE_ID}:{document['release']}" + records = [] + for advisory in document["advisories"]: + affected = [{"package": {"ecosystem": "RPM", "name": name}, "ranges": [{"type": "ECOSYSTEM", "events": [{"introduced": "0"}, {"fixed": advisory["fixed"]}]}]} for name in advisory["binary_packages"]] + records.append({"id": advisory["id"], "summary": f"Red Hat {document['release']} advisory {advisory['id']}", "affected": affected, "database_specific": {"redhat_release": document["release"], "source_package": advisory["source_package"], "severity": advisory["severity"], "backport": advisory["backport"]}, "_source_record": advisory}) + stats = database.import_osv_records(records, source=source) + if stats.errors: + return {"source_id": source, "snapshot_id": snapshot_id, "state": "rejected", "activated": False, "errors": list(stats.errors)} + manifest = database.build_snapshot_manifest() + manifest.update({"source_id": source, "source_digest": document["source_digest"], "source_path": document["source_path"], "signature_status": "unsigned", "ingestion_schema": "coderisktools.vulnerability.redhat-staging.v1"}) + database.stage_snapshot(snapshot_id, document["source_digest"], manifest) + now = datetime.now(timezone.utc).isoformat() + database.connection.execute("INSERT OR REPLACE INTO source_snapshots(snapshot_id, source_id, content_digest, observed_at, record_count, status, metadata_json) VALUES (?, ?, ?, ?, ?, ?, ?)", (snapshot_id, source, document["source_digest"], now, stats.records_seen, "staged", json.dumps(document["provenance"], sort_keys=True))) + for metric_name, metric_value in (("records_seen", stats.records_seen), ("advisories_imported", stats.advisories_imported), ("affected_packages_imported", stats.affected_packages_imported)): + database.connection.execute("INSERT OR REPLACE INTO quality_metrics(snapshot_id, metric_name, metric_value, details_json) VALUES (?, ?, ?, ?)", (snapshot_id, metric_name, float(metric_value), "{}")) + database.connection.commit() + if activate: + database.activate_snapshot(snapshot_id) + return {"source_id": source, "snapshot_id": snapshot_id, "state": "active" if activate else "staged", "activated": activate, "records_seen": stats.records_seen, "advisories_imported": stats.advisories_imported, "affected_packages_imported": stats.affected_packages_imported, "errors": []} diff --git a/src/vulnerability/sources/rubygems.py b/src/vulnerability/sources/rubygems.py new file mode 100644 index 0000000..010bfa8 --- /dev/null +++ b/src/vulnerability/sources/rubygems.py @@ -0,0 +1,64 @@ +"""Bounded local RubyGems advisory source adapter.""" + +from __future__ import annotations + +import hashlib +import json +from datetime import datetime, timezone +from pathlib import Path +from typing import Any + +from ...safeio import read_regular_bounded +from ..database import VulnerabilityDatabase + +RUBYGEMS_SOURCE_ID = "rubygems-security" +SCHEMA = "coderisktools.vulnerability.rubygems-feed" +MAX_FEED_BYTES = 50 * 1024 * 1024 +MAX_ADVISORIES = 100_000 +_MAX_TEXT = 256 + + +def _text(value: Any, field: str) -> str: + if not isinstance(value, str) or not value or len(value) > _MAX_TEXT: + raise ValueError(f"RubyGems {field} is invalid") + return value + + +def ingest_file(path: str | Path, *, max_bytes: int = MAX_FEED_BYTES) -> dict[str, Any]: + raw = read_regular_bounded(path, max_bytes, "RubyGems feed") + try: + document = json.loads(raw.decode("utf-8")) + except (UnicodeDecodeError, json.JSONDecodeError) as exc: + raise ValueError("RubyGems feed must be bounded UTF-8 JSON") from exc + if not isinstance(document, dict) or document.get("schema") != SCHEMA or document.get("version") != 1: + raise ValueError("unsupported RubyGems feed schema") + advisories = document.get("advisories") + if not isinstance(advisories, list) or len(advisories) > MAX_ADVISORIES: + raise ValueError("RubyGems advisory list exceeds bounds") + normalized = [] + for item in advisories: + if not isinstance(item, dict): + raise ValueError("RubyGems advisory must be an object") + normalized.append({"id": _text(item.get("id"), "advisory id"), "package": _text(item.get("package"), "gem name"), "fixed": _text(item.get("fixed"), "fixed version"), "severity": _text(item.get("severity", "Unknown"), "severity")}) + digest = "sha256:" + hashlib.sha256(raw).hexdigest() + provenance = {"source_id": RUBYGEMS_SOURCE_ID, "source_path": str(Path(path)), "source_digest": digest, "schema": SCHEMA, "version": 1} + return {"source_id": RUBYGEMS_SOURCE_ID, "source_path": str(Path(path)), "source_digest": digest, "advisory_count": len(normalized), "advisories": normalized, "provenance": provenance} + + +def ingest_file_to_database(path: str | Path, database: VulnerabilityDatabase, snapshot_id: str, *, activate: bool = False, max_bytes: int = MAX_FEED_BYTES) -> dict[str, Any]: + document = ingest_file(path, max_bytes=max_bytes) + records = [{"id": item["id"], "summary": f"RubyGems advisory {item['id']}", "affected": [{"package": {"ecosystem": "RubyGems", "name": item["package"]}, "ranges": [{"type": "ECOSYSTEM", "events": [{"introduced": "0"}, {"fixed": item["fixed"]}]}]}], "database_specific": {"severity": item["severity"]}, "_source_record": item} for item in document["advisories"]] + stats = database.import_osv_records(records, source=RUBYGEMS_SOURCE_ID) + if stats.errors: + return {"source_id": RUBYGEMS_SOURCE_ID, "snapshot_id": snapshot_id, "state": "rejected", "activated": False, "errors": list(stats.errors)} + manifest = database.build_snapshot_manifest() + manifest.update({"source_id": RUBYGEMS_SOURCE_ID, "source_digest": document["source_digest"], "source_path": document["source_path"], "signature_status": "unsigned", "ingestion_schema": "coderisktools.vulnerability.rubygems-staging.v1"}) + database.stage_snapshot(snapshot_id, document["source_digest"], manifest) + now = datetime.now(timezone.utc).isoformat() + database.connection.execute("INSERT OR REPLACE INTO source_snapshots(snapshot_id, source_id, content_digest, observed_at, record_count, status, metadata_json) VALUES (?, ?, ?, ?, ?, ?, ?)", (snapshot_id, RUBYGEMS_SOURCE_ID, document["source_digest"], now, stats.records_seen, "staged", json.dumps(document["provenance"], sort_keys=True))) + for name, value in (("records_seen", stats.records_seen), ("advisories_imported", stats.advisories_imported), ("affected_packages_imported", stats.affected_packages_imported)): + database.connection.execute("INSERT OR REPLACE INTO quality_metrics(snapshot_id, metric_name, metric_value, details_json) VALUES (?, ?, ?, ?)", (snapshot_id, name, float(value), "{}")) + database.connection.commit() + if activate: + database.activate_snapshot(snapshot_id) + return {"source_id": RUBYGEMS_SOURCE_ID, "snapshot_id": snapshot_id, "state": "active" if activate else "staged", "activated": activate, "records_seen": stats.records_seen, "advisories_imported": stats.advisories_imported, "affected_packages_imported": stats.affected_packages_imported, "errors": []} diff --git a/src/vulnerability/sources/suse.py b/src/vulnerability/sources/suse.py new file mode 100644 index 0000000..51f72c1 --- /dev/null +++ b/src/vulnerability/sources/suse.py @@ -0,0 +1,74 @@ +"""Bounded local SUSE security advisory feed adapter.""" + +from __future__ import annotations + +import hashlib +import json +from datetime import datetime, timezone +from pathlib import Path +from typing import Any + +from ...safeio import read_regular_bounded +from ..database import VulnerabilityDatabase + +SUSE_SOURCE_ID = "suse-security" +SCHEMA = "coderisktools.vulnerability.suse-feed" +MAX_FEED_BYTES = 50 * 1024 * 1024 +MAX_ADVISORIES = 100_000 +_MAX_TEXT = 256 + + +def _text(value: Any, field: str, default: str | None = None) -> str: + if value is None and default is not None: + value = default + if not isinstance(value, str) or not value or len(value) > _MAX_TEXT: + raise ValueError(f"SUSE {field} is invalid") + return value + + +def ingest_file(path: str | Path, *, max_bytes: int = MAX_FEED_BYTES) -> dict[str, Any]: + raw = read_regular_bounded(path, max_bytes, "SUSE feed") + try: + document = json.loads(raw.decode("utf-8")) + except (UnicodeDecodeError, json.JSONDecodeError) as exc: + raise ValueError("SUSE feed must be bounded UTF-8 JSON") from exc + if not isinstance(document, dict) or document.get("schema") != SCHEMA or document.get("version") != 1: + raise ValueError("unsupported SUSE feed schema") + release = _text(document.get("release"), "release") + advisories = document.get("advisories") + if not isinstance(advisories, list) or len(advisories) > MAX_ADVISORIES: + raise ValueError("SUSE advisory list exceeds bounds") + normalized = [] + for item in advisories: + if not isinstance(item, dict): + raise ValueError("SUSE advisory must be an object") + binaries = item.get("binary_packages") + if not isinstance(binaries, list) or not binaries or len(binaries) > 1000: + raise ValueError("SUSE binary package list is invalid") + normalized.append({"id": _text(item.get("id"), "advisory id"), "package": _text(item.get("package"), "source package"), "source_package": _text(item.get("source_package"), "source package", item.get("package")), "binary_packages": sorted({_text(name, "binary package") for name in binaries}), "fixed": _text(item.get("fixed"), "fixed RPM version"), "severity": _text(item.get("severity", "Unknown"), "severity"), "backport": item.get("backport", False) is True}) + digest = "sha256:" + hashlib.sha256(raw).hexdigest() + provenance = {"source_id": SUSE_SOURCE_ID, "source_path": str(Path(path)), "source_digest": digest, "schema": SCHEMA, "version": 1, "release": release} + return {"source_id": SUSE_SOURCE_ID, "source_path": str(Path(path)), "source_digest": digest, "release": release, "advisory_count": len(normalized), "advisories": normalized, "provenance": provenance} + + +def ingest_file_to_database(path: str | Path, database: VulnerabilityDatabase, snapshot_id: str, *, activate: bool = False, max_bytes: int = MAX_FEED_BYTES) -> dict[str, Any]: + document = ingest_file(path, max_bytes=max_bytes) + source = f"{SUSE_SOURCE_ID}:{document['release']}" + records = [] + for advisory in document["advisories"]: + affected = [{"package": {"ecosystem": "RPM", "name": name}, "ranges": [{"type": "ECOSYSTEM", "events": [{"introduced": "0"}, {"fixed": advisory["fixed"]}]}]} for name in advisory["binary_packages"]] + records.append({"id": advisory["id"], "summary": f"SUSE {document['release']} advisory {advisory['id']}", "affected": affected, "database_specific": {"suse_release": document["release"], "source_package": advisory["source_package"], "severity": advisory["severity"], "backport": advisory["backport"]}, "_source_record": advisory}) + stats = database.import_osv_records(records, source=source) + if stats.errors: + return {"source_id": source, "snapshot_id": snapshot_id, "state": "rejected", "activated": False, "errors": list(stats.errors)} + manifest = database.build_snapshot_manifest() + manifest.update({"source_id": source, "source_digest": document["source_digest"], "source_path": document["source_path"], "signature_status": "unsigned", "ingestion_schema": "coderisktools.vulnerability.suse-staging.v1"}) + database.stage_snapshot(snapshot_id, document["source_digest"], manifest) + now = datetime.now(timezone.utc).isoformat() + database.connection.execute("INSERT OR REPLACE INTO source_snapshots(snapshot_id, source_id, content_digest, observed_at, record_count, status, metadata_json) VALUES (?, ?, ?, ?, ?, ?, ?)", (snapshot_id, source, document["source_digest"], now, stats.records_seen, "staged", json.dumps(document["provenance"], sort_keys=True))) + for metric_name, metric_value in (("records_seen", stats.records_seen), ("advisories_imported", stats.advisories_imported), ("affected_packages_imported", stats.affected_packages_imported)): + database.connection.execute("INSERT OR REPLACE INTO quality_metrics(snapshot_id, metric_name, metric_value, details_json) VALUES (?, ?, ?, ?)", (snapshot_id, metric_name, float(metric_value), "{}")) + database.connection.commit() + if activate: + database.activate_snapshot(snapshot_id) + return {"source_id": source, "snapshot_id": snapshot_id, "state": "active" if activate else "staged", "activated": activate, "records_seen": stats.records_seen, "advisories_imported": stats.advisories_imported, "affected_packages_imported": stats.affected_packages_imported, "errors": []} diff --git a/src/vulnerability/sources/swift.py b/src/vulnerability/sources/swift.py new file mode 100644 index 0000000..6d2ff1c --- /dev/null +++ b/src/vulnerability/sources/swift.py @@ -0,0 +1,65 @@ +"""Bounded local Swift package advisory source adapter.""" + +from __future__ import annotations + +import hashlib +import json +from datetime import datetime, timezone +from pathlib import Path +from typing import Any + +from ...safeio import read_regular_bounded +from ..database import VulnerabilityDatabase + +SWIFT_SOURCE_ID = "swift-security" +SCHEMA = "coderisktools.vulnerability.swift-feed" +MAX_FEED_BYTES = 50 * 1024 * 1024 +MAX_ADVISORIES = 100_000 +_MAX_TEXT = 256 + + +def _text(value: Any, field: str) -> str: + if not isinstance(value, str) or not value or len(value) > _MAX_TEXT: + raise ValueError(f"Swift {field} is invalid") + return value + + +def ingest_file(path: str | Path, *, max_bytes: int = MAX_FEED_BYTES) -> dict[str, Any]: + raw = read_regular_bounded(path, max_bytes, "Swift feed") + try: + document = json.loads(raw.decode("utf-8")) + except (UnicodeDecodeError, json.JSONDecodeError) as exc: + raise ValueError("Swift feed must be bounded UTF-8 JSON") from exc + if not isinstance(document, dict) or document.get("schema") != SCHEMA or document.get("version") != 1: + raise ValueError("unsupported Swift feed schema") + advisories = document.get("advisories") + if not isinstance(advisories, list) or len(advisories) > MAX_ADVISORIES: + raise ValueError("Swift advisory list exceeds bounds") + normalized = [] + for item in advisories: + if not isinstance(item, dict): + raise ValueError("Swift advisory must be an object") + package = _text(item.get("package"), "package identity") + normalized.append({"id": _text(item.get("id"), "advisory id"), "package": package, "fixed": _text(item.get("fixed"), "fixed version"), "severity": _text(item.get("severity", "Unknown"), "severity")}) + digest = "sha256:" + hashlib.sha256(raw).hexdigest() + provenance = {"source_id": SWIFT_SOURCE_ID, "source_path": str(Path(path)), "source_digest": digest, "schema": SCHEMA, "version": 1} + return {"source_id": SWIFT_SOURCE_ID, "source_path": str(Path(path)), "source_digest": digest, "advisory_count": len(normalized), "advisories": normalized, "provenance": provenance} + + +def ingest_file_to_database(path: str | Path, database: VulnerabilityDatabase, snapshot_id: str, *, activate: bool = False, max_bytes: int = MAX_FEED_BYTES) -> dict[str, Any]: + document = ingest_file(path, max_bytes=max_bytes) + records = [{"id": item["id"], "summary": f"Swift advisory {item['id']}", "affected": [{"package": {"ecosystem": "Swift", "name": item["package"]}, "ranges": [{"type": "ECOSYSTEM", "events": [{"introduced": "0"}, {"fixed": item["fixed"]}]}]}], "database_specific": {"severity": item["severity"]}, "_source_record": item} for item in document["advisories"]] + stats = database.import_osv_records(records, source=SWIFT_SOURCE_ID) + if stats.errors: + return {"source_id": SWIFT_SOURCE_ID, "snapshot_id": snapshot_id, "state": "rejected", "activated": False, "errors": list(stats.errors)} + manifest = database.build_snapshot_manifest() + manifest.update({"source_id": SWIFT_SOURCE_ID, "source_digest": document["source_digest"], "source_path": document["source_path"], "signature_status": "unsigned", "ingestion_schema": "coderisktools.vulnerability.swift-staging.v1"}) + database.stage_snapshot(snapshot_id, document["source_digest"], manifest) + now = datetime.now(timezone.utc).isoformat() + database.connection.execute("INSERT OR REPLACE INTO source_snapshots(snapshot_id, source_id, content_digest, observed_at, record_count, status, metadata_json) VALUES (?, ?, ?, ?, ?, ?, ?)", (snapshot_id, SWIFT_SOURCE_ID, document["source_digest"], now, stats.records_seen, "staged", json.dumps(document["provenance"], sort_keys=True))) + for name, value in (("records_seen", stats.records_seen), ("advisories_imported", stats.advisories_imported), ("affected_packages_imported", stats.affected_packages_imported)): + database.connection.execute("INSERT OR REPLACE INTO quality_metrics(snapshot_id, metric_name, metric_value, details_json) VALUES (?, ?, ?, ?)", (snapshot_id, name, float(value), "{}")) + database.connection.commit() + if activate: + database.activate_snapshot(snapshot_id) + return {"source_id": SWIFT_SOURCE_ID, "snapshot_id": snapshot_id, "state": "active" if activate else "staged", "activated": activate, "records_seen": stats.records_seen, "advisories_imported": stats.advisories_imported, "affected_packages_imported": stats.affected_packages_imported, "errors": []} diff --git a/src/vulnerability/sources/ubuntu.py b/src/vulnerability/sources/ubuntu.py new file mode 100644 index 0000000..0b79e73 --- /dev/null +++ b/src/vulnerability/sources/ubuntu.py @@ -0,0 +1,63 @@ +"""Bounded local Ubuntu advisory feed adapter.""" + +from __future__ import annotations + +import hashlib +import json +from datetime import datetime, timezone +from pathlib import Path +from typing import Any + +from ...safeio import read_regular_bounded +from ..database import VulnerabilityDatabase +from .debian import _validate_advisory + +UBUNTU_SOURCE_ID = "ubuntu-security" +SCHEMA = "coderisktools.vulnerability.ubuntu-feed" +MAX_FEED_BYTES = 50 * 1024 * 1024 +MAX_ADVISORIES = 100_000 + + +def ingest_file(path: str | Path, *, max_bytes: int = MAX_FEED_BYTES) -> dict[str, Any]: + """Parse a local Ubuntu fixture; no network access or activation is performed.""" + raw = read_regular_bounded(path, max_bytes, "Ubuntu feed") + try: + document = json.loads(raw.decode("utf-8")) + except (UnicodeDecodeError, json.JSONDecodeError) as exc: + raise ValueError("Ubuntu feed must be bounded UTF-8 JSON") from exc + if not isinstance(document, dict) or document.get("schema") != SCHEMA or document.get("version") != 1: + raise ValueError("unsupported Ubuntu feed schema") + release = document.get("release") + advisories = document.get("advisories") + if not isinstance(release, str) or not 1 <= len(release) <= 64: + raise ValueError("Ubuntu release is invalid") + if not isinstance(advisories, list) or len(advisories) > MAX_ADVISORIES: + raise ValueError("Ubuntu advisory list exceeds bounds") + normalized = [_validate_advisory(item) for item in advisories] + digest = "sha256:" + hashlib.sha256(raw).hexdigest() + provenance = {"source_id": UBUNTU_SOURCE_ID, "source_path": str(Path(path)), "source_digest": digest, "schema": SCHEMA, "version": 1, "release": release} + return {"source_id": UBUNTU_SOURCE_ID, "source_path": str(Path(path)), "source_digest": digest, "release": release, "advisory_count": len(normalized), "advisories": normalized, "provenance": provenance} + + +def ingest_file_to_database(path: str | Path, database: VulnerabilityDatabase, snapshot_id: str, *, activate: bool = False, max_bytes: int = MAX_FEED_BYTES) -> dict[str, Any]: + """Normalize a bounded Ubuntu fixture into the existing staging pipeline.""" + document = ingest_file(path, max_bytes=max_bytes) + source = f"{UBUNTU_SOURCE_ID}:{document['release']}" + records = [] + for advisory in document["advisories"]: + affected = [{"package": {"ecosystem": "Ubuntu", "name": name}, "ranges": [{"type": "ECOSYSTEM", "events": [{"introduced": "0"}, {"fixed": advisory["fixed"]}]}]} for name in advisory["binary_packages"]] + records.append({"id": advisory["id"], "summary": f"Ubuntu {document['release']} advisory {advisory['id']}", "affected": affected, "database_specific": {"ubuntu_release": document["release"], "source_package": advisory["source_package"], "urgency": advisory["urgency"], "backport": advisory["backport"]}, "_source_record": advisory}) + stats = database.import_osv_records(records, source=source) + if stats.errors: + return {"source_id": source, "snapshot_id": snapshot_id, "state": "rejected", "activated": False, "errors": list(stats.errors)} + manifest = database.build_snapshot_manifest() + manifest.update({"source_id": source, "source_digest": document["source_digest"], "source_path": document["source_path"], "signature_status": "unsigned", "ingestion_schema": "coderisktools.vulnerability.ubuntu-staging.v1"}) + database.stage_snapshot(snapshot_id, document["source_digest"], manifest) + now = datetime.now(timezone.utc).isoformat() + database.connection.execute("INSERT OR REPLACE INTO source_snapshots(snapshot_id, source_id, content_digest, observed_at, record_count, status, metadata_json) VALUES (?, ?, ?, ?, ?, ?, ?)", (snapshot_id, source, document["source_digest"], now, stats.records_seen, "staged", json.dumps(document["provenance"], sort_keys=True))) + for metric_name, metric_value in (("records_seen", stats.records_seen), ("advisories_imported", stats.advisories_imported), ("affected_packages_imported", stats.affected_packages_imported)): + database.connection.execute("INSERT OR REPLACE INTO quality_metrics(snapshot_id, metric_name, metric_value, details_json) VALUES (?, ?, ?, ?)", (snapshot_id, metric_name, float(metric_value), "{}")) + database.connection.commit() + if activate: + database.activate_snapshot(snapshot_id) + return {"source_id": source, "snapshot_id": snapshot_id, "state": "active" if activate else "staged", "activated": activate, "records_seen": stats.records_seen, "advisories_imported": stats.advisories_imported, "affected_packages_imported": stats.affected_packages_imported, "errors": []} diff --git a/src/vulnerability/sources/vcpkg.py b/src/vulnerability/sources/vcpkg.py new file mode 100644 index 0000000..4633ca9 --- /dev/null +++ b/src/vulnerability/sources/vcpkg.py @@ -0,0 +1,64 @@ +"""Bounded local vcpkg advisory source adapter.""" + +from __future__ import annotations + +import hashlib +import json +from datetime import datetime, timezone +from pathlib import Path +from typing import Any + +from ...safeio import read_regular_bounded +from ..database import VulnerabilityDatabase + +VCPKG_SOURCE_ID = "vcpkg-security" +SCHEMA = "coderisktools.vulnerability.vcpkg-feed" +MAX_FEED_BYTES = 50 * 1024 * 1024 +MAX_ADVISORIES = 100_000 +_MAX_TEXT = 256 + + +def _text(value: Any, field: str) -> str: + if not isinstance(value, str) or not value or len(value) > _MAX_TEXT: + raise ValueError(f"vcpkg {field} is invalid") + return value + + +def ingest_file(path: str | Path, *, max_bytes: int = MAX_FEED_BYTES) -> dict[str, Any]: + raw = read_regular_bounded(path, max_bytes, "vcpkg feed") + try: + document = json.loads(raw.decode("utf-8")) + except (UnicodeDecodeError, json.JSONDecodeError) as exc: + raise ValueError("vcpkg feed must be bounded UTF-8 JSON") from exc + if not isinstance(document, dict) or document.get("schema") != SCHEMA or document.get("version") != 1: + raise ValueError("unsupported vcpkg feed schema") + advisories = document.get("advisories") + if not isinstance(advisories, list) or len(advisories) > MAX_ADVISORIES: + raise ValueError("vcpkg advisory list exceeds bounds") + normalized = [] + for item in advisories: + if not isinstance(item, dict): + raise ValueError("vcpkg advisory must be an object") + normalized.append({"id": _text(item.get("id"), "advisory id"), "package": _text(item.get("package"), "port name"), "fixed": _text(item.get("fixed"), "fixed version"), "severity": _text(item.get("severity", "Unknown"), "severity")}) + digest = "sha256:" + hashlib.sha256(raw).hexdigest() + provenance = {"source_id": VCPKG_SOURCE_ID, "source_path": str(Path(path)), "source_digest": digest, "schema": SCHEMA, "version": 1} + return {"source_id": VCPKG_SOURCE_ID, "source_path": str(Path(path)), "source_digest": digest, "advisory_count": len(normalized), "advisories": normalized, "provenance": provenance} + + +def ingest_file_to_database(path: str | Path, database: VulnerabilityDatabase, snapshot_id: str, *, activate: bool = False, max_bytes: int = MAX_FEED_BYTES) -> dict[str, Any]: + document = ingest_file(path, max_bytes=max_bytes) + records = [{"id": item["id"], "summary": f"vcpkg advisory {item['id']}", "affected": [{"package": {"ecosystem": "vcpkg", "name": item["package"]}, "ranges": [{"type": "ECOSYSTEM", "events": [{"introduced": "0"}, {"fixed": item["fixed"]}]}]}], "database_specific": {"severity": item["severity"]}, "_source_record": item} for item in document["advisories"]] + stats = database.import_osv_records(records, source=VCPKG_SOURCE_ID) + if stats.errors: + return {"source_id": VCPKG_SOURCE_ID, "snapshot_id": snapshot_id, "state": "rejected", "activated": False, "errors": list(stats.errors)} + manifest = database.build_snapshot_manifest() + manifest.update({"source_id": VCPKG_SOURCE_ID, "source_digest": document["source_digest"], "source_path": document["source_path"], "signature_status": "unsigned", "ingestion_schema": "coderisktools.vulnerability.vcpkg-staging.v1"}) + database.stage_snapshot(snapshot_id, document["source_digest"], manifest) + now = datetime.now(timezone.utc).isoformat() + database.connection.execute("INSERT OR REPLACE INTO source_snapshots(snapshot_id, source_id, content_digest, observed_at, record_count, status, metadata_json) VALUES (?, ?, ?, ?, ?, ?, ?)", (snapshot_id, VCPKG_SOURCE_ID, document["source_digest"], now, stats.records_seen, "staged", json.dumps(document["provenance"], sort_keys=True))) + for name, value in (("records_seen", stats.records_seen), ("advisories_imported", stats.advisories_imported), ("affected_packages_imported", stats.affected_packages_imported)): + database.connection.execute("INSERT OR REPLACE INTO quality_metrics(snapshot_id, metric_name, metric_value, details_json) VALUES (?, ?, ?, ?)", (snapshot_id, name, float(value), "{}")) + database.connection.commit() + if activate: + database.activate_snapshot(snapshot_id) + return {"source_id": VCPKG_SOURCE_ID, "snapshot_id": snapshot_id, "state": "active" if activate else "staged", "activated": activate, "records_seen": stats.records_seen, "advisories_imported": stats.advisories_imported, "affected_packages_imported": stats.affected_packages_imported, "errors": []} diff --git a/src/vulnerability/update_config.py b/src/vulnerability/update_config.py new file mode 100644 index 0000000..af4d167 --- /dev/null +++ b/src/vulnerability/update_config.py @@ -0,0 +1,94 @@ +"""Default public source configuration for the controlled updater.""" + +from __future__ import annotations + +from typing import Any + + +def default_update_config() -> dict[str, Any]: + """Return a bounded starter configuration; callers may extend it explicitly.""" + return { + "schema_version": 1, + "sources": [ + { + "source_id": "nvd", + "format": "nvd", + "url": "https://services.nvd.nist.gov/rest/json/cves/2.0?resultsPerPage=2000", + "allowed_hosts": ["services.nvd.nist.gov"], + "terms": "verify-before-redistribution", + "enabled": False, + "adapter_status": "extended-only", + }, + { + "source_id": "cisa-kev", + "format": "kev", + "url": "https://www.cisa.gov/sites/default/files/feeds/known_exploited_vulnerabilities.json", + "allowed_hosts": ["www.cisa.gov"], + "terms": "public-sector-terms", + }, + { + "source_id": "epss", + "format": "epss", + "url": "https://api.first.org/data/v1/epss?limit=1000", + "allowed_hosts": ["api.first.org"], + "terms": "verify-before-redistribution", + }, + { + "source_id": "github-advisories", + "format": "github-advisories", + "url": "https://api.github.com/advisories?per_page=100", + "allowed_hosts": ["api.github.com"], + "terms": "verify-before-redistribution", + }, + { + "source_id": "osv", + "format": "osv-jsonl", + "url": "https://osv.dev/vulns/all.jsonl", + "allowed_hosts": ["osv.dev"], + "terms": "verify-before-redistribution", + "enabled": True, + "adapter_status": "dedicated-streaming-adapter-required", + }, + { + "source_id": "cve-v5", + "format": "cve-v5", + "url": "https://cveawg.mitre.org/api/cve/", + "allowed_hosts": ["cveawg.mitre.org"], + "terms": "cve-program-terms", + "enabled": False, + "adapter_status": "provider-pagination-required", + }, + { + "source_id": "debian-security", + "format": "debian", + "url": "https://security-tracker.debian.org/tracker/data/json", + "allowed_hosts": ["security-tracker.debian.org"], + "terms": "debian-terms", + "enabled": False, + "adapter_status": "dedicated-distro-adapter-required", + }, + { + "source_id": "ubuntu-security", + "format": "ubuntu", + "url": "https://ubuntu.com/security/notices.json", + "allowed_hosts": ["ubuntu.com"], + "terms": "ubuntu-terms", + "enabled": False, + "adapter_status": "dedicated-distro-adapter-required", + }, + { + "source_id": "rustsec", + "format": "osv-json", + "url": "https://github.com/RustSec/advisory-db/archive/refs/heads/main.zip", + "allowed_hosts": ["github.com"], + "terms": "verify-before-redistribution", + "enabled": False, + "adapter_status": "archive-adapter-required", + }, + ], + "notes": [ + "This is a bounded starter configuration, not a claim of global feed coverage.", + "OSV full-feed JSONL and provider-specific feeds require their dedicated adapters/configuration.", + "Verify source terms and live response shape before production activation.", + ], + } diff --git a/src/vulnerability/update_profiles.py b/src/vulnerability/update_profiles.py new file mode 100644 index 0000000..0695b52 --- /dev/null +++ b/src/vulnerability/update_profiles.py @@ -0,0 +1,47 @@ +"""Profile definitions for reproducible vulnerability database builds.""" + +from __future__ import annotations + +from typing import Any + + +_PROFILE_LIMITS = { + "core": {"minimum_free_bytes": 8 * 1024**3, "max_download_bytes": 6 * 1024**3}, + "extended": {"minimum_free_bytes": 32 * 1024**3, "max_download_bytes": 24 * 1024**3}, + "seed": {"minimum_free_bytes": 512 * 1024**2, "max_download_bytes": 256 * 1024**2, "max_records_per_ecosystem": 500, "minimum_successful_osv_ecosystems": 4}, +} + + +def validate_profile(profile: str) -> str: + if profile not in _PROFILE_LIMITS: + raise ValueError("profile must be core, extended, or seed") + return profile + + +def profile_limits(profile: str) -> dict[str, int]: + return dict(_PROFILE_LIMITS[validate_profile(profile)]) + + +def profile_source_ids(profile: str) -> set[str]: + validate_profile(profile) + core = {"osv", "github-advisories", "cisa-kev", "epss"} + if profile == "seed": + return core + if profile == "core": + return core + return core | {"cve-v5", "nvd", "cwe", "cpe", "debian-security", "ubuntu-security", "redhat-security", "alpine", "rustsec"} + + +def annotate_config(config: dict[str, Any], profile: str) -> dict[str, Any]: + """Return a copy with only sources belonging to the selected profile enabled.""" + validate_profile(profile) + result = dict(config) + selected = profile_source_ids(profile) + sources = [] + for source in config.get("sources", []): + item = dict(source) + item["enabled"] = bool(item.get("enabled", True) and item.get("source_id") in selected) + sources.append(item) + result["profile"] = profile + result["sources"] = sources + return result diff --git a/src/vulnerability/updater.py b/src/vulnerability/updater.py index 4dfb657..ead7a36 100644 --- a/src/vulnerability/updater.py +++ b/src/vulnerability/updater.py @@ -175,7 +175,8 @@ def fetch_json_artifact( with active_opener.open(request, timeout=policy.timeout) as response: final_url = response.geturl() _validate_fetch_url(final_url, policy) - declared = response.headers.get_content_length() + declared_header = response.headers.get("Content-Length") + declared = int(declared_header) if declared_header is not None else None if declared is not None and (declared < 0 or declared > policy.max_bytes): raise ValueError("source response exceeds max_bytes") chunks: list[bytes] = [] @@ -250,7 +251,8 @@ def stream_json_artifact_to_file( with active_opener.open(request, timeout=policy.timeout) as response: final_url = response.geturl() _validate_fetch_url(final_url, policy) - declared = response.headers.get_content_length() + declared_header = response.headers.get("Content-Length") + declared = int(declared_header) if declared_header is not None else None if declared is not None and (declared < 0 or declared > policy.max_bytes): raise ValueError("source response exceeds max_bytes") fd, temporary_name = tempfile.mkstemp(prefix=f".{target.name}.", dir=str(target.parent)) @@ -736,6 +738,138 @@ def stage_versioned_snapshot(payload: bytes, destination: str | os.PathLike[str] return {"snapshot_id": snapshot_id, "source_id": source_id, "source_digest": source_digest, "state": "staged", "activated": False, "content_digest": manifest["content_digest"], "manifest_sha256": manifest["manifest_sha256"], "advisory_count": manifest["advisory_count"], "affected_package_count": manifest["affected_package_count"]} +def run_full_update( + config_path: str | os.PathLike[str], + root: str | os.PathLike[str], + active_pointer: str | os.PathLike[str] | None = None, + *, + profile: str | None = None, + apply: bool = False, + max_bytes: int = 512 * 1024 * 1024, + timeout: float = 20.0, +) -> dict[str, Any]: + """Build a multi-source SQLite snapshot in staging, then optionally activate it.""" + config_file = Path(config_path) + database_root = Path(root) + if database_root.is_symlink(): + raise ValueError("database root must not be a symlink") + if profile is not None: + from .update_profiles import profile_limits, validate_profile + validate_profile(profile) + profile_budget = profile_limits(profile) + probe = database_root.parent if database_root.parent.exists() else Path.cwd() + free_bytes = shutil.disk_usage(probe).free + if free_bytes < profile_budget["minimum_free_bytes"]: + raise OSError(f"insufficient free space for {profile} profile: {free_bytes} bytes available, {profile_budget['minimum_free_bytes']} required") + max_bytes = min(max_bytes, profile_budget["max_download_bytes"]) + if config_file.is_symlink(): + raise ValueError("update config must be a regular local file") + if not config_file.is_file(): + default_path = Path("~/.config/coderisktools/vuln-db.json").expanduser() + if config_file == default_path: + from .update_config import default_update_config + config = default_update_config() + else: + raise ValueError("update config must be a regular local file") + else: + config = json.loads(config_file.read_text(encoding="utf-8")) + sources = config.get("sources") if isinstance(config, dict) else None + if not isinstance(sources, list) or not sources: + raise ValueError("update config requires a non-empty sources list") + if profile is not None: + from .update_profiles import annotate_config + config = annotate_config(config, profile) + sources = config["sources"] + database_root.mkdir(parents=True, exist_ok=True) + staging_root = database_root / "staging" + snapshots_root = database_root / "snapshots" + staging_root.mkdir(exist_ok=True) + snapshots_root.mkdir(exist_ok=True) + update_dir = Path(tempfile.mkdtemp(prefix="update-", dir=str(staging_root))) + downloads = update_dir / "downloads" + downloads.mkdir() + source_reports: list[dict[str, Any]] = [] + try: + from .database import VulnerabilityDatabase + database_path = update_dir / "database.sqlite.tmp" + with VulnerabilityDatabase(str(database_path)) as database: + for source in sources: + if not isinstance(source, dict): + raise ValueError("each configured source must be an object") + source_id = source.get("source_id") + if not isinstance(source_id, str) or _SOURCE_ID.fullmatch(source_id) is None: + raise ValueError("configured source_id is invalid") + if source.get("enabled", True) is False: + source_reports.append({"source_id": source_id, "state": "disabled", "records": 0}) + continue + url = source.get("url") + allowed_hosts = source.get("allowed_hosts") + source_format = source.get("format", source_id) + if not isinstance(source_id, str) or _SOURCE_ID.fullmatch(source_id) is None: + raise ValueError("configured source_id is invalid") + if not isinstance(url, str) or not isinstance(allowed_hosts, list): + raise ValueError(f"source {source_id} requires url and allowed_hosts") + if source_format == "osv-jsonl": + from .ingestion import ingest_osv_streaming_file + destination = downloads / f"{source_id}.jsonl" + download = stream_json_artifact_to_file(url, destination, FetchPolicy(frozenset(allowed_hosts), max_bytes=max_bytes, timeout=timeout)) + if download.get("state") == "not_modified": + source_reports.append({"source_id": source_id, "state": "not_modified", "records": 0}) + continue + report = ingest_osv_streaming_file(str(destination), database, "pending-core", source_id, activate=False) + report_data = report.to_dict() + if report_data.get("errors"): + raise ValueError(f"source {source_id} import failed: {'; '.join(report_data['errors'])}") + source_reports.append({"source_id": source_id, "url": download.get("final_url"), "source_digest": download.get("payload_sha256"), "bytes": download.get("bytes_written", 0), "records": report_data.get("records_seen", 0), "imported": report_data.get("records_imported", 0), "etag": download.get("etag"), "last_modified": download.get("last_modified"), "state": "imported"}) + continue + artifact = fetch_json_artifact(url, FetchPolicy(frozenset(allowed_hosts), max_bytes=max_bytes, timeout=timeout)) + if artifact.not_modified or artifact.payload is None: + source_reports.append({"source_id": source_id, "state": "not_modified", "records": 0}) + continue + payload = artifact.payload + destination = downloads / f"{source_id}.json" + destination.write_bytes(payload) + value = json.loads(payload.decode("utf-8")) + importer_name = {"osv": "import_osv_json", "nvd": "import_nvd_json", "cve-v5": "import_cve_v5_json", "cve_v5": "import_cve_v5_json", "kev": "import_kev_json", "cisa-kev": "import_kev_json", "epss": "import_epss_json", "github-advisories": "import_ghsa_json", "ghsa": "import_ghsa_json"}.get(source_format) + if importer_name is None: + raise ValueError(f"unsupported full-update source format: {source_format}") + if source_format in {"kev", "cisa-kev", "epss"}: + stats = getattr(database, importer_name)(value, allow_unresolved=True) + else: + stats = getattr(database, importer_name)(value) + errors = list(getattr(stats, "errors", ())) + if errors: + raise ValueError(f"source {source_id} import failed: {'; '.join(errors)}") + source_reports.append({"source_id": source_id, "url": artifact.final_url, "source_digest": "sha256:" + hashlib.sha256(payload).hexdigest(), "bytes": len(payload), "records": getattr(stats, "records_seen", 0), "imported": getattr(stats, "advisories_imported", 0), "etag": artifact.etag, "last_modified": artifact.last_modified, "state": "imported"}) + if database.integrity_check() != "ok": + raise ValueError("full update SQLite integrity check failed") + content_manifest = database.build_snapshot_manifest() + manifest_seed = {"schema_version": 1, "sources": source_reports, "database": content_manifest} + snapshot_digest = hashlib.sha256(_canonical_bytes(manifest_seed)).hexdigest() + snapshot_id = "vulndb-" + snapshot_digest[:32] + overall_digest = "sha256:" + hashlib.sha256(_canonical_bytes(source_reports)).hexdigest() + manifest = {**content_manifest, "schema_version": 1, "source_id": "multi-source", "snapshot_id": snapshot_id, "source_digest": overall_digest, "sources": source_reports, "state": "staged", "activated": False} + database.stage_snapshot(snapshot_id, overall_digest, manifest) + target = snapshots_root / snapshot_id + if target.exists(): + raise FileExistsError(f"snapshot destination already exists: {target}") + final_dir = update_dir / "snapshot" + final_dir.mkdir() + os.replace(database_path, final_dir / "snapshot.sqlite3") + manifest["manifest_sha256"] = _snapshot_manifest_digest(manifest) + (final_dir / "manifest.json").write_bytes(_canonical_bytes(manifest)) + os.replace(final_dir, target) + result = verify_versioned_snapshot(target) + if apply: + if active_pointer is None: + raise ValueError("apply requires an active pointer") + result = promote_versioned_snapshot(target, active_pointer) + return {**result, "sources": source_reports, "staging_path": str(update_dir)} + except Exception: + (update_dir / "update-error.json").write_text(json.dumps({"state": "rejected", "sources": source_reports}, sort_keys=True), encoding="utf-8") + raise + + def stage_offline_update( input_path: str | os.PathLike[str], root: str | os.PathLike[str], diff --git a/tests/fixtures/v15_external_comparison.json b/tests/fixtures/v15_external_comparison.json new file mode 100644 index 0000000..10543c5 --- /dev/null +++ b/tests/fixtures/v15_external_comparison.json @@ -0,0 +1,10 @@ +{ + "schema": "coderisktools.vulnerability.external-comparison", + "version": 1, + "internal": ["CVE-1", "CVE-2"], + "tools": { + "OSV-Scanner": ["CVE-1", "CVE-3"], + "Trivy": ["CVE-1", "CVE-2"], + "Grype": ["CVE-2", "CVE-4"] + } +} diff --git a/tests/fixtures/v15_multiecosystem.json b/tests/fixtures/v15_multiecosystem.json new file mode 100644 index 0000000..82bcc95 --- /dev/null +++ b/tests/fixtures/v15_multiecosystem.json @@ -0,0 +1,25 @@ +{ + "schema": "coderisktools.vulnerability.benchmark", + "version": 1, + "name": "public-multiecosystem-v15", + "records": [ + {"id":"CVE-2026-PY","ecosystem":"PyPI","name":"alpha","fixed":"2.0.0"}, + {"id":"CVE-2026-NPM","ecosystem":"npm","name":"bravo","fixed":"2.0.0"}, + {"id":"CVE-2026-CR","ecosystem":"crates.io","name":"charlie","fixed":"2.0.0"}, + {"id":"CVE-2026-MVN","ecosystem":"Maven","name":"delta","fixed":"2.0.0"}, + {"id":"CVE-2026-NUG","ecosystem":"NuGet","name":"echo","fixed":"2.0.0"} + ], + "cases": [ + {"case_id":"pypi-affected","ecosystem":"PyPI","name":"alpha","version":"1.0.0","expected":true}, + {"case_id":"pypi-clean","ecosystem":"PyPI","name":"alpha","version":"2.0.0","expected":false}, + {"case_id":"npm-affected","ecosystem":"npm","name":"bravo","version":"1.0.0","expected":true}, + {"case_id":"npm-clean","ecosystem":"npm","name":"bravo","version":"2.0.0","expected":false}, + {"case_id":"cargo-affected","ecosystem":"crates.io","name":"charlie","version":"1.0.0","expected":true}, + {"case_id":"cargo-clean","ecosystem":"crates.io","name":"charlie","version":"2.0.0","expected":false}, + {"case_id":"maven-affected","ecosystem":"Maven","name":"delta","version":"1.0.0","expected":true}, + {"case_id":"maven-clean","ecosystem":"Maven","name":"delta","version":"2.0.0","expected":false}, + {"case_id":"nuget-affected","ecosystem":"NuGet","name":"echo","version":"1.0.0","expected":true}, + {"case_id":"nuget-clean","ecosystem":"NuGet","name":"echo","version":"2.0.0","expected":false} + ], + "golden": {"min_precision": 1.0, "min_recall": 1.0, "min_f1": 1.0} +} diff --git a/tests/fixtures/v15_public_benchmark.json b/tests/fixtures/v15_public_benchmark.json new file mode 100644 index 0000000..4e4f086 --- /dev/null +++ b/tests/fixtures/v15_public_benchmark.json @@ -0,0 +1,16 @@ +{ + "schema": "coderisktools.vulnerability.benchmark", + "version": 1, + "name": "public-minimal-v15", + "cases": [ + {"case_id": "affected", "expected": true, "observed": true, "elapsed_ms": 2.0}, + {"case_id": "clean", "expected": false, "observed": false, "elapsed_ms": 1.0}, + {"case_id": "miss", "expected": true, "observed": false, "elapsed_ms": 3.0} + ], + "golden": { + "precision": 1.0, + "recall": 0.5, + "f1": 0.6666666666666666, + "latency_ms": {"p50": 2.0, "p95": 3.0} + } +} diff --git a/tests/test_core_unresolved_enrichment.py b/tests/test_core_unresolved_enrichment.py new file mode 100644 index 0000000..ab629f1 --- /dev/null +++ b/tests/test_core_unresolved_enrichment.py @@ -0,0 +1,28 @@ +import tempfile +import unittest + +from src.vulnerability.database import VulnerabilityDatabase + + +class TestCoreUnresolvedEnrichment(unittest.TestCase): + def test_kev_can_be_staged_without_matching_advisory_in_core_mode(self): + record = { + "cveID": "CVE-2026-9999", + "vendorProject": "Example", + "product": "Example Product", + "vulnerabilityName": "Example vulnerability", + "dateAdded": "2026-07-23", + "shortDescription": "Example", + "requiredAction": "Apply update", + "dueDate": "2026-08-23", + "knownRansomwareCampaignUse": "Unknown", + } + with tempfile.TemporaryDirectory() as directory: + with VulnerabilityDatabase(f"{directory}/core.sqlite3") as database: + stats = database.import_kev_json([record], allow_unresolved=True) + self.assertEqual(stats.records_seen, 1) + self.assertEqual(stats.errors, ()) + row = database.connection.execute( + "SELECT source, cve_id, reason FROM unresolved_enrichments" + ).fetchone() + self.assertEqual(tuple(row), ("kev", "CVE-2026-9999", "advisory-not-found")) diff --git a/tests/test_default_update_config.py b/tests/test_default_update_config.py new file mode 100644 index 0000000..0a11245 --- /dev/null +++ b/tests/test_default_update_config.py @@ -0,0 +1,12 @@ +import unittest + +from src.vulnerability.update_config import default_update_config + + +class TestDefaultUpdateConfig(unittest.TestCase): + def test_active_sources_are_supported_and_extended_sources_are_explicitly_disabled(self): + sources = default_update_config()["sources"] + active = {item["source_id"] for item in sources if item.get("enabled", True)} + disabled = {item["source_id"] for item in sources if item.get("enabled") is False} + self.assertEqual(active, {"osv", "cisa-kev", "epss", "github-advisories"}) + self.assertTrue({"nvd", "cve-v5", "debian-security", "ubuntu-security", "rustsec"} <= disabled) diff --git a/tests/test_feed_acceptance.py b/tests/test_feed_acceptance.py new file mode 100644 index 0000000..f8c61aa --- /dev/null +++ b/tests/test_feed_acceptance.py @@ -0,0 +1,26 @@ +import json +import tempfile +import unittest +from pathlib import Path + +from src.vulnerability.sources.feed_acceptance import evaluate_feed_artifact + + +class TestFeedAcceptance(unittest.TestCase): + def test_accepts_supplied_artifact_without_network_or_activation(self): + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "feed.json" + path.write_text(json.dumps({"vulnerabilities": [{"id": "CVE-1"}]}), encoding="utf-8") + report = evaluate_feed_artifact(path, source_id="kev", expected_list_key="vulnerabilities") + self.assertEqual(report["state"], "accepted") + self.assertFalse(report["activated"]) + self.assertFalse(report["network_used"]) + self.assertTrue(report["source_digest"].startswith("sha256:")) + + def test_rejects_missing_envelope(self): + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "feed.json" + path.write_text(json.dumps({"items": []}), encoding="utf-8") + report = evaluate_feed_artifact(path, source_id="epss", expected_list_key="data") + self.assertEqual(report["state"], "rejected") + self.assertFalse(report["activated"]) diff --git a/tests/test_feed_catalog.py b/tests/test_feed_catalog.py new file mode 100644 index 0000000..ce5d536 --- /dev/null +++ b/tests/test_feed_catalog.py @@ -0,0 +1,14 @@ +import unittest + +from src.vulnerability.feed_catalog import feed_catalog, feed_catalog_report + + +class TestFeedCatalog(unittest.TestCase): + def test_catalog_is_deterministic_and_not_ready_without_full_adapters(self): + feeds = feed_catalog() + self.assertEqual(len({item.source_id for item in feeds}), len(feeds)) + self.assertTrue(all(item.endpoint is None or item.endpoint.startswith("https://") for item in feeds)) + report = feed_catalog_report() + self.assertEqual(report["source_count"], len(feeds)) + self.assertFalse(report["full_coverage_ready"]) + self.assertNotIn("ready", report["counts"]) diff --git a/tests/test_full_update_pipeline.py b/tests/test_full_update_pipeline.py new file mode 100644 index 0000000..6183f11 --- /dev/null +++ b/tests/test_full_update_pipeline.py @@ -0,0 +1,29 @@ +import json +import tempfile +import unittest +from pathlib import Path +from unittest.mock import patch + +from src.vulnerability.updater import run_full_update + + +class TestFullUpdatePipeline(unittest.TestCase): + def test_full_update_builds_staged_snapshot_and_does_not_activate_by_default(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) / "vuln-db" + config = Path(directory) / "sources.json" + config.write_text(json.dumps({"sources": [{"source_id": "osv", "url": "https://osv.dev/feed.json", "allowed_hosts": ["osv.dev"], "format": "osv"}]}), encoding="utf-8") + with patch("src.vulnerability.updater.fetch_json_artifact") as fetch: + fetch.return_value.payload = json.dumps([]).encode() + fetch.return_value.not_modified = False + fetch.return_value.requested_url = "https://osv.dev/feed.json" + fetch.return_value.final_url = "https://osv.dev/feed.json" + fetch.return_value.etag = None + fetch.return_value.last_modified = None + fetch.return_value.content_type = "application/json" + result = run_full_update(config, root) + self.assertEqual(result["state"], "staged") + self.assertFalse(result["activated"]) + self.assertTrue((root / "snapshots" / result["snapshot_id"] / "snapshot.sqlite3").is_file()) + self.assertTrue((root / "snapshots" / result["snapshot_id"] / "manifest.json").is_file()) + self.assertFalse((root / "active").exists()) diff --git a/tests/test_nvd_full_feed.py b/tests/test_nvd_full_feed.py new file mode 100644 index 0000000..dd5a30a --- /dev/null +++ b/tests/test_nvd_full_feed.py @@ -0,0 +1,34 @@ +import json +import tempfile +import unittest +from pathlib import Path +from unittest.mock import Mock + +from src.vulnerability.sources.nvd import ingest_file + + +class TestNVDFeedIngestion(unittest.TestCase): + def test_ingests_bounded_nvd_api_batch_with_source_digest(self): + payload = {"vulnerabilities": [{"cve": {"id": "CVE-2024-12345", "descriptions": [], "references": [], "metrics": {}, "configurations": []}}]} + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "nvd.json" + path.write_text(json.dumps(payload), encoding="utf-8") + database = Mock() + database.lookup_advisory.return_value = {"status": "exact", "advisory_id": "CVE-2024-12345"} + report = ingest_file(path, database, "snapshot-nvd-1") + database.record_source_record.assert_called_once() + self.assertEqual(report["state"], "staged") + self.assertEqual(report["records_seen"], 1) + self.assertEqual(report["records_imported"], 1) + self.assertTrue(report["source_digest"].startswith("sha256:")) + + def test_invalid_record_is_reported_without_activation(self): + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "nvd.json" + path.write_text(json.dumps({"vulnerabilities": [{"cve": {"id": "bad"}}]}), encoding="utf-8") + database = Mock() + report = ingest_file(path, database, "snapshot-nvd-2", activate=True) + self.assertEqual(report["state"], "partial") + self.assertFalse(report["activated"]) + self.assertEqual(report["records_imported"], 0) + self.assertEqual(len(report["errors"]), 1) diff --git a/tests/test_osv_full_feed.py b/tests/test_osv_full_feed.py new file mode 100644 index 0000000..6ea9cda --- /dev/null +++ b/tests/test_osv_full_feed.py @@ -0,0 +1,30 @@ +import tempfile +import unittest +from pathlib import Path +from unittest.mock import patch + +from src.vulnerability.database import VulnerabilityDatabase +from src.vulnerability.sources.osv import fetch_and_ingest_stream +from src.vulnerability.updater import FetchPolicy + + +class TestOSVFullFeedContract(unittest.TestCase): + def test_fetch_stream_ingest_keeps_activation_explicit(self): + with tempfile.TemporaryDirectory() as directory: + target = Path(directory) / "osv.jsonl" + db = VulnerabilityDatabase(Path(directory) / "db.sqlite") + metadata = {"final_url": "https://osv.dev/vulns/all.jsonl", "payload_sha256": "sha256:" + "a" * 64, "bytes_written": 10} + with patch("src.vulnerability.sources.osv.stream_json_artifact_to_file", return_value=metadata): + with patch("src.vulnerability.sources.osv.ingest_osv_streaming_file") as ingest: + ingest.return_value.to_dict.return_value = {"state": "staged", "activated": False} + result = fetch_and_ingest_stream( + "https://osv.dev/vulns/all.jsonl", + target, + db, + "snapshot-osv-1", + FetchPolicy(frozenset({"osv.dev"})), + ) + ingest.assert_called_once() + self.assertEqual(result["state"], "staged") + self.assertFalse(result["activated"]) + self.assertEqual(result["source_digest"], "sha256:" + "a" * 64) diff --git a/tests/test_p2_airgap_bundle.py b/tests/test_p2_airgap_bundle.py new file mode 100644 index 0000000..64fd312 --- /dev/null +++ b/tests/test_p2_airgap_bundle.py @@ -0,0 +1,53 @@ +import io +import json +import tarfile +import tempfile +import unittest +from pathlib import Path + +from src.vulnerability.airgap import export_air_gap_bundle, import_air_gap_bundle +from src.vulnerability.database import VulnerabilityDatabase + + +class TestP2AirGapBundle(unittest.TestCase): + def test_export_and_import_verifies_manifest_and_is_not_active(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + source_path = root / "source.sqlite" + bundle = root / "bundle.tar.gz" + target = root / "target.sqlite" + source = VulnerabilityDatabase(str(source_path)) + try: + exported = export_air_gap_bundle(source, bundle) + self.assertEqual(exported["schema"], "coderisktools.vulnerability.air-gap-bundle") + finally: + source.close() + result = import_air_gap_bundle(bundle, target) + self.assertFalse(result["activated"]) + imported = VulnerabilityDatabase.read_only(str(target)) + try: + self.assertEqual(imported.build_snapshot_manifest()["content_digest"], exported["manifest"]["content_digest"]) + finally: + imported.close() + + def test_tampered_bundle_is_rejected_without_target(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + source = VulnerabilityDatabase(str(root / "source.sqlite")) + try: + export_air_gap_bundle(source, root / "bundle.tar.gz") + finally: + source.close() + tampered = root / "tampered.tar.gz" + with tarfile.open(root / "bundle.tar.gz", "r:gz") as source_tar, tarfile.open(tampered, "w:gz") as target_tar: + for member in source_tar.getmembers(): + payload = source_tar.extractfile(member).read() + if member.name == "manifest.json": + value = json.loads(payload) + value["manifest"]["advisory_count"] = 1 + payload = json.dumps(value, sort_keys=True).encode() + member.size = len(payload) + target_tar.addfile(member, io.BytesIO(payload)) + with self.assertRaises(Exception): + import_air_gap_bundle(tampered, root / "target.sqlite") + self.assertFalse((root / "target.sqlite").exists()) diff --git a/tests/test_p2_manifest_signing.py b/tests/test_p2_manifest_signing.py new file mode 100644 index 0000000..9f36ef6 --- /dev/null +++ b/tests/test_p2_manifest_signing.py @@ -0,0 +1,31 @@ +import unittest + +from src.vulnerability.database import VulnerabilityDatabase +from src.vulnerability.manifest_signing import sign_manifest, verify_manifest + + +class TestP2ManifestSigning(unittest.TestCase): + @unittest.skipUnless(__import__('importlib').util.find_spec('cryptography'), 'optional cryptography backend unavailable') + def test_sign_and_verify_manifest(self): + from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PrivateKey + private = Ed25519PrivateKey.generate() + public = private.public_key().public_bytes_raw() + raw_private = private.private_bytes_raw() + manifest = {"schema_version": "2", "content_digest": "sha256:demo", "advisory_count": 0, "affected_package_count": 0} + envelope = sign_manifest(manifest, "fixture-key", raw_private) + self.assertEqual(verify_manifest(envelope, public), manifest) + self.assertEqual(envelope["schema"], "coderisktools.vulnerability.signed-manifest") + + @unittest.skipUnless(__import__('importlib').util.find_spec('cryptography'), 'optional cryptography backend unavailable') + def test_database_rejects_tampered_manifest(self): + from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PrivateKey + private = Ed25519PrivateKey.generate() + public = private.public_key().public_bytes_raw() + envelope = sign_manifest({"content_digest": "sha256:x", "advisory_count": 0, "affected_package_count": 0}, "fixture-key", private.private_bytes_raw()) + envelope["manifest"]["advisory_count"] = 1 + database = VulnerabilityDatabase(":memory:") + try: + with self.assertRaises(ValueError): + database.stage_signed_snapshot("signed", "sha256:source", envelope, public) + finally: + database.close() diff --git a/tests/test_p2_snapshot_retention.py b/tests/test_p2_snapshot_retention.py new file mode 100644 index 0000000..e2391cf --- /dev/null +++ b/tests/test_p2_snapshot_retention.py @@ -0,0 +1,24 @@ +import unittest + +from src.vulnerability.database import VulnerabilityDatabase + + +class TestP2SnapshotRetention(unittest.TestCase): + def test_prune_is_dry_run_by_default_and_never_removes_active(self): + database = VulnerabilityDatabase(":memory:") + try: + manifest = database.build_snapshot_manifest() + database.stage_snapshot("keep", "sha256:keep", manifest) + database.stage_snapshot("old", "sha256:old", manifest) + database.activate_snapshot("keep") + plan = database.prune_snapshots({"keep"}) + self.assertEqual(plan["candidates"], ["old"]) + self.assertFalse(plan["applied"]) + self.assertIsNotNone(database.snapshot_status("old")) + applied = database.prune_snapshots({"keep"}, apply=True) + self.assertEqual(applied["removed"], ["old"]) + self.assertIsNotNone(database.snapshot_status("keep")) + with self.assertRaises(KeyError): + database.snapshot_status("old") + finally: + database.close() diff --git a/tests/test_release_sdist_reproducibility.py b/tests/test_release_sdist_reproducibility.py new file mode 100644 index 0000000..473874a --- /dev/null +++ b/tests/test_release_sdist_reproducibility.py @@ -0,0 +1,44 @@ +import gzip +import io +import tarfile +import tempfile +import unittest +from pathlib import Path + +from scripts.normalize_sdist import normalize_sdist + + +class ReproducibleSdistTests(unittest.TestCase): + @staticmethod + def _source(path: Path, mtime: int) -> None: + with tarfile.open(path, "w:gz") as archive: + info = tarfile.TarInfo("package/file.txt") + info.size = 7 + info.mode = 0o640 + info.mtime = mtime + archive.addfile(info, io.BytesIO(b"payload")) + + def test_normalization_removes_build_time_without_changing_payload(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + first_source = root / "first.tar.gz" + second_source = root / "second.tar.gz" + first = root / "first.normalized.tar.gz" + second = root / "second.normalized.tar.gz" + self._source(first_source, 100) + self._source(second_source, 200) + + normalize_sdist(first_source, first, epoch=123456789) + normalize_sdist(second_source, second, epoch=123456789) + + self.assertEqual(first.read_bytes(), second.read_bytes()) + with gzip.open(first, "rb") as stream: + with tarfile.open(fileobj=stream, mode="r:") as archive: + member = archive.getmember("package/file.txt") + self.assertEqual(member.mtime, 123456789) + self.assertEqual(member.mode, 0o640) + self.assertEqual(archive.extractfile(member).read(), b"payload") + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_seed_bootstrap.py b/tests/test_seed_bootstrap.py new file mode 100644 index 0000000..cfd323c --- /dev/null +++ b/tests/test_seed_bootstrap.py @@ -0,0 +1,65 @@ +import hashlib +import json +import shutil +import tempfile +import unittest +from pathlib import Path +from unittest.mock import patch + +from src.vulnerability.bootstrap import ( + activate_seed_database, + validate_signed_release_manifest, + verify_asset_sha256, +) + + +class SeedBootstrapTests(unittest.TestCase): + def test_verify_asset_sha256_accepts_exact_bytes(self): + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / "seed.sqlite" + path.write_bytes(b"real-seed-bytes") + digest = "sha256:" + hashlib.sha256(b"real-seed-bytes").hexdigest() + self.assertEqual(verify_asset_sha256(path, digest), digest) + + def test_verify_asset_sha256_rejects_tampering(self): + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / "seed.sqlite" + path.write_bytes(b"tampered") + with self.assertRaises(ValueError): + verify_asset_sha256(path, "sha256:" + "0" * 64) + + def test_verify_asset_sha256_rejects_malformed_digest(self): + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / "seed.sqlite" + path.write_bytes(b"seed") + for value in ("sha256:abc", "sha256:" + "A" * 64, "md5:" + "0" * 32): + with self.subTest(value=value), self.assertRaises(ValueError): + verify_asset_sha256(path, value) + + def test_signed_release_manifest_must_match_detached_manifest(self): + manifest = json.loads((Path(__file__).parents[1] / "data/vulnerability-seed/manifest.json").read_text(encoding="utf-8")) + envelope = {"key_id": "coderisktools-seed-2026", "manifest": manifest} + with patch("src.vulnerability.bootstrap.verify_manifest", return_value=dict(manifest)): + actual = validate_signed_release_manifest(manifest, envelope, {"coderisktools-seed-2026": b"k" * 32}) + self.assertEqual(actual, manifest) + tampered = dict(manifest) + tampered["advisory_count"] += 1 + with patch("src.vulnerability.bootstrap.verify_manifest", return_value=dict(manifest)): + with self.assertRaises(ValueError): + validate_signed_release_manifest(tampered, envelope, {"coderisktools-seed-2026": b"k" * 32}) + + def test_seed_activation_is_explicit_and_profile_bound(self): + source_root = Path(__file__).parents[1] / "data/vulnerability-seed" + with tempfile.TemporaryDirectory() as tmp: + database = Path(tmp) / "seed.sqlite" + shutil.copy2(source_root / "seed-vulndb.sqlite", database) + manifest = json.loads((source_root / "manifest.json").read_text(encoding="utf-8")) + state = activate_seed_database(database, manifest, apply=False) + self.assertEqual(state["state"], "activation_planned") + state = activate_seed_database(database, manifest, apply=True) + self.assertEqual(state["state"], "active") + self.assertEqual(state["profile"], "seed") + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_seed_profile.py b/tests/test_seed_profile.py new file mode 100644 index 0000000..a833302 --- /dev/null +++ b/tests/test_seed_profile.py @@ -0,0 +1,45 @@ +import json +import sqlite3 +import tempfile +import unittest +from pathlib import Path + +from src.vulnerability.seed import SEED_ECOSYSTEMS, build_seed_manifest, validate_seed_manifest +from src.vulnerability.update_profiles import annotate_config, profile_source_ids, validate_profile + + +class SeedProfileTests(unittest.TestCase): + def test_seed_profile_is_partial_and_requires_core_sources(self): + self.assertEqual(validate_profile("seed"), "seed") + self.assertEqual( + profile_source_ids("seed"), + {"osv", "github-advisories", "cisa-kev", "epss"}, + ) + self.assertEqual(SEED_ECOSYSTEMS, ("PyPI", "npm", "Go", "crates.io", "Maven", "NuGet", "Packagist")) + + def test_seed_manifest_is_explicitly_partial(self): + manifest = build_seed_manifest( + {"cisa-kev": {"status": "complete", "records": 1, "imported": 1}, "epss": {"status": "bounded", "records": 1, "imported": 1}, "ghsa": {"status": "bounded", "records": 1, "imported": 1}, "osv": {"status": "partial", "imported": 4, "successful_ecosystems": ["PyPI", "npm", "Go", "Maven"], "failed_ecosystems": []}}, + {"advisory_count": 2, "affected_package_count": 1, "content_digest": "sha256:" + "a" * 64}, + ) + self.assertEqual(manifest["profile"], "seed") + self.assertEqual(manifest["completeness"], "partial") + self.assertFalse(manifest["production_full_database"]) + validate_seed_manifest(manifest) + + def test_seed_manifest_rejects_missing_required_source(self): + with self.assertRaises(ValueError): + validate_seed_manifest({"profile": "seed", "completeness": "partial", "production_full_database": False, "sources": {}}) + + def test_seed_manifest_rejects_zero_imported_required_source(self): + manifest = build_seed_manifest( + {"cisa-kev": {"status": "complete", "records": 1, "imported": 1}, "epss": {"status": "bounded", "records": 1, "imported": 1}, "ghsa": {"status": "bounded", "records": 1, "imported": 1}, "osv": {"status": "partial", "imported": 4, "successful_ecosystems": ["PyPI", "npm", "Go", "Maven"], "failed_ecosystems": []}}, + {"advisory_count": 2, "affected_package_count": 1, "content_digest": "sha256:" + "a" * 64}, + ) + manifest["sources"]["ghsa"]["imported"] = 0 + with self.assertRaises(ValueError): + validate_seed_manifest(manifest) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_stage5_robustness.py b/tests/test_stage5_robustness.py index 965f21d..777971b 100644 --- a/tests/test_stage5_robustness.py +++ b/tests/test_stage5_robustness.py @@ -540,6 +540,22 @@ def test_scan_directory_skips_binary_extensions(self): import shutil shutil.rmtree(temp_dir, ignore_errors=True) + def test_scan_directory_skips_sqlite_database_files(self): + """Directory scan should skip SQLite database artifacts, including large seed snapshots.""" + temp_dir = tempfile.mkdtemp() + with open(os.path.join(temp_dir, "seed-vulndb.sqlite"), "wb") as handle: + handle.truncate(6 * 1024 * 1024) + with open(os.path.join(temp_dir, "app.py"), "w") as handle: + handle.write("print('safe')\n") + + try: + scanner = SecretScanner() + result = scanner.scan_directory(temp_dir) + self.assertEqual([], result.findings) + finally: + import shutil + shutil.rmtree(temp_dir, ignore_errors=True) + def test_scan_directory_includes_security_dotfiles(self): """Directory scan should include security-relevant dotfiles.""" temp_dir = tempfile.mkdtemp() diff --git a/tests/test_v12_alpine_source.py b/tests/test_v12_alpine_source.py new file mode 100644 index 0000000..f089eb6 --- /dev/null +++ b/tests/test_v12_alpine_source.py @@ -0,0 +1,30 @@ +import tempfile +import unittest +from pathlib import Path + +from src.vulnerability.database import VulnerabilityDatabase +from src.vulnerability.models import Component +from src.vulnerability.sources.alpine import ALPINE_SOURCE_ID, ingest_file, ingest_file_to_database +from src.vulnerability.versions.alpine import compare_alpine_version + + +class TestV12AlpineSource(unittest.TestCase): + def test_apk_comparator_and_alpine_staging(self): + self.assertLess(compare_alpine_version("3.18.4-r0", "3.18.4-r1"), 0) + payload = '{"schema":"coderisktools.vulnerability.alpine-feed","version":1,"release":"v3.18","advisories":[{"id":"CVE-9999-1","package":"openssl","fixed":"3.1.4-r2","severity":"high","binary_packages":["libcrypto3"],"backport":true}]}' + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + feed = root / "alpine.json" + feed.write_text(payload, encoding="utf-8") + parsed = ingest_file(str(feed)) + self.assertEqual(parsed["source_id"], ALPINE_SOURCE_ID) + database = VulnerabilityDatabase(str(root / "db.sqlite")) + try: + report = ingest_file_to_database(str(feed), database, "alpine-v318-1") + self.assertEqual(report["state"], "staged") + affected = database.evaluate_component(Component(ecosystem="Alpine", name="libcrypto3", version="3.1.4-r1")) + fixed = database.evaluate_component(Component(ecosystem="Alpine", name="libcrypto3", version="3.1.4-r2")) + self.assertEqual(affected["status"], "affected") + self.assertEqual(fixed["status"], "not_affected") + finally: + database.close() diff --git a/tests/test_v12_backport_contract.py b/tests/test_v12_backport_contract.py new file mode 100644 index 0000000..8627391 --- /dev/null +++ b/tests/test_v12_backport_contract.py @@ -0,0 +1,28 @@ +import tempfile +import unittest +from pathlib import Path + +from src.vulnerability.sources.alpine import ingest_file as ingest_alpine +from src.vulnerability.sources.debian import ingest_file as ingest_debian +from src.vulnerability.sources.redhat import ingest_file as ingest_redhat +from src.vulnerability.sources.suse import ingest_file as ingest_suse +from src.vulnerability.sources.ubuntu import ingest_file as ingest_ubuntu + + +class TestV12BackportContract(unittest.TestCase): + def test_all_linux_adapters_preserve_bounded_backport_metadata(self): + fixtures = [ + (ingest_debian, "coderisktools.vulnerability.debian-feed", "bookworm", "DSA-1", "3.0.11-1~deb12u2"), + (ingest_ubuntu, "coderisktools.vulnerability.ubuntu-feed", "jammy", "USN-1", "3.0.11-1ubuntu2.1"), + (ingest_redhat, "coderisktools.vulnerability.redhat-feed", "rhel-9", "RHSA-1", "3.0.7-25.el9_2"), + (ingest_suse, "coderisktools.vulnerability.suse-feed", "sles-15", "SUSE-1", "3.0.8-1.2"), + (ingest_alpine, "coderisktools.vulnerability.alpine-feed", "v3.18", "CVE-1", "3.1.4-r2"), + ] + with tempfile.TemporaryDirectory() as directory: + for index, (parser, schema, release, advisory_id, fixed) in enumerate(fixtures): + path = Path(directory) / f"feed-{index}.json" + path.write_text("{" + f'"schema":"{schema}","version":1,"release":"{release}","advisories":[{{"id":"{advisory_id}","package":"openssl","fixed":"{fixed}","binary_packages":["openssl-bin"],"backport":true}}]' + "}", encoding="utf-8") + result = parser(str(path)) + self.assertTrue(result["advisories"][0]["backport"]) + self.assertEqual(result["advisories"][0]["fixed"], fixed) + self.assertEqual(result["provenance"]["source_digest"], result["source_digest"]) diff --git a/tests/test_v12_debian_backport_matching.py b/tests/test_v12_debian_backport_matching.py new file mode 100644 index 0000000..0434c7b --- /dev/null +++ b/tests/test_v12_debian_backport_matching.py @@ -0,0 +1,25 @@ +import tempfile +import unittest +from pathlib import Path + +from src.vulnerability.database import VulnerabilityDatabase +from src.vulnerability.models import Component +from src.vulnerability.sources.debian import ingest_file_to_database + + +class TestV12DebianBackportMatching(unittest.TestCase): + def test_debian_backport_fixed_revision_is_not_affected(self): + payload = '{"schema":"coderisktools.vulnerability.debian-feed","version":1,"release":"bookworm","advisories":[{"id":"DSA-1","package":"openssl","fixed":"3.0.11-1~deb12u2","backport":true,"binary_packages":["libssl3"]}]}' + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + feed = root / "debian.json" + feed.write_text(payload, encoding="utf-8") + database = VulnerabilityDatabase(str(root / "db.sqlite")) + try: + ingest_file_to_database(str(feed), database, "debian-bookworm-1") + affected = database.evaluate_component(Component(ecosystem="Debian", name="libssl3", version="3.0.11-1~deb12u1")) + fixed = database.evaluate_component(Component(ecosystem="Debian", name="libssl3", version="3.0.11-1~deb12u2")) + self.assertEqual(affected["status"], "affected") + self.assertEqual(fixed["status"], "not_affected") + finally: + database.close() diff --git a/tests/test_v12_debian_source.py b/tests/test_v12_debian_source.py new file mode 100644 index 0000000..b851640 --- /dev/null +++ b/tests/test_v12_debian_source.py @@ -0,0 +1,42 @@ +import tempfile +import unittest +from pathlib import Path + +from src.vulnerability.sources.debian import DEBIAN_SOURCE_ID, ingest_file + + +class TestV12DebianSource(unittest.TestCase): + def test_ingest_debian_local_feed_preserves_backport_metadata(self): + payload = """{ + "schema": "coderisktools.vulnerability.debian-feed", + "version": 1, + "release": "bookworm", + "advisories": [{ + "id": "DSA-9999-1", + "package": "openssl", + "urgency": "high", + "fixed": "3.0.11-1~deb12u2", + "backport": true, + "source_package": "openssl", + "binary_packages": ["openssl", "libssl3"] + }] + }""" + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "debian.json" + path.write_text(payload, encoding="utf-8") + result = ingest_file(str(path)) + self.assertEqual(result["source_id"], DEBIAN_SOURCE_ID) + self.assertEqual(result["source_digest"].split(":", 1)[0], "sha256") + self.assertEqual(len(result["source_digest"].split(":", 1)[1]), 64) + self.assertEqual(result["provenance"]["source_id"], DEBIAN_SOURCE_ID) + self.assertEqual(result["release"], "bookworm") + self.assertEqual(result["advisory_count"], 1) + self.assertTrue(result["advisories"][0]["backport"]) + self.assertEqual(result["advisories"][0]["binary_packages"], ["libssl3", "openssl"]) + + def test_rejects_unbounded_or_invalid_debian_feed(self): + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "debian.json" + path.write_text('{"schema":"wrong","version":1,"advisories":[]}', encoding="utf-8") + with self.assertRaises(ValueError): + ingest_file(str(path)) diff --git a/tests/test_v12_debian_staging.py b/tests/test_v12_debian_staging.py new file mode 100644 index 0000000..5e3171b --- /dev/null +++ b/tests/test_v12_debian_staging.py @@ -0,0 +1,28 @@ +import tempfile +import unittest +from pathlib import Path + +from src.vulnerability.database import VulnerabilityDatabase +from src.vulnerability.sources.debian import ingest_file_to_database + + +class TestV12DebianStaging(unittest.TestCase): + def test_debian_feed_stages_normalized_osv_records_without_activation(self): + payload = '{"schema":"coderisktools.vulnerability.debian-feed","version":1,"release":"bookworm","advisories":[{"id":"DSA-1","package":"openssl","fixed":"3.0.11-1~deb12u2","backport":true,"binary_packages":["libssl3"]}]}' + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + feed = root / "debian.json" + database_path = root / "db.sqlite" + feed.write_text(payload, encoding="utf-8") + database = VulnerabilityDatabase(str(database_path)) + try: + report = ingest_file_to_database(str(feed), database, "debian-bookworm-1") + self.assertEqual(report["state"], "staged") + self.assertFalse(report["activated"]) + self.assertEqual(database.advisory_count(), 1) + self.assertEqual(database.snapshot_status("debian-bookworm-1")["state"], "staged") + self.assertEqual(database.connection.execute("SELECT record_count FROM source_snapshots WHERE snapshot_id='debian-bookworm-1'").fetchone()[0], 1) + self.assertEqual(database.connection.execute("SELECT COUNT(*) FROM quality_metrics WHERE snapshot_id='debian-bookworm-1'").fetchone()[0], 3) + self.assertEqual(database.connection.execute("SELECT source FROM advisories WHERE id='DSA-1'").fetchone()[0], "debian-security:bookworm") + finally: + database.close() diff --git a/tests/test_v12_redhat_source.py b/tests/test_v12_redhat_source.py new file mode 100644 index 0000000..ee969ed --- /dev/null +++ b/tests/test_v12_redhat_source.py @@ -0,0 +1,26 @@ +import tempfile +import unittest +from pathlib import Path + +from src.vulnerability.sources.redhat import REDHAT_SOURCE_ID, ingest_file + + +class TestV12RedHatSource(unittest.TestCase): + def test_ingest_redhat_fixture_preserves_rpm_metadata(self): + payload = '{"schema":"coderisktools.vulnerability.redhat-feed","version":1,"release":"rhel-9","advisories":[{"id":"RHSA-9999:001","package":"openssl","fixed":"3.0.7-25.el9_2","severity":"Important","binary_packages":["openssl-libs"],"backport":true}]}' + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "redhat.json" + path.write_text(payload, encoding="utf-8") + result = ingest_file(str(path)) + self.assertEqual(result["source_id"], REDHAT_SOURCE_ID) + self.assertEqual(result["release"], "rhel-9") + self.assertEqual(result["advisories"][0]["severity"], "Important") + self.assertTrue(result["advisories"][0]["backport"]) + self.assertTrue(result["source_digest"].startswith("sha256:")) + + def test_rejects_wrong_schema(self): + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "redhat.json" + path.write_text('{"schema":"coderisktools.vulnerability.ubuntu-feed","version":1,"release":"rhel-9","advisories":[]}', encoding="utf-8") + with self.assertRaises(ValueError): + ingest_file(str(path)) diff --git a/tests/test_v12_redhat_staging.py b/tests/test_v12_redhat_staging.py new file mode 100644 index 0000000..b9498e0 --- /dev/null +++ b/tests/test_v12_redhat_staging.py @@ -0,0 +1,27 @@ +import tempfile +import unittest +from pathlib import Path + +from src.vulnerability.database import VulnerabilityDatabase +from src.vulnerability.models import Component +from src.vulnerability.sources.redhat import ingest_file_to_database + + +class TestV12RedHatStaging(unittest.TestCase): + def test_redhat_rpm_boundary_stages_without_activation(self): + payload = '{"schema":"coderisktools.vulnerability.redhat-feed","version":1,"release":"rhel-9","advisories":[{"id":"RHSA-1","package":"openssl","fixed":"3.0.7-25.el9_2","severity":"Important","binary_packages":["openssl-libs"],"backport":true}]}' + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + feed = root / "redhat.json" + feed.write_text(payload, encoding="utf-8") + database = VulnerabilityDatabase(str(root / "db.sqlite")) + try: + report = ingest_file_to_database(str(feed), database, "rhel-9-1") + self.assertEqual(report["state"], "staged") + self.assertFalse(report["activated"]) + affected = database.evaluate_component(Component(ecosystem="RPM", name="openssl-libs", version="3.0.7-24.el9_2")) + fixed = database.evaluate_component(Component(ecosystem="RPM", name="openssl-libs", version="3.0.7-25.el9_2")) + self.assertEqual(affected["status"], "affected") + self.assertEqual(fixed["status"], "not_affected") + finally: + database.close() diff --git a/tests/test_v12_suse_source.py b/tests/test_v12_suse_source.py new file mode 100644 index 0000000..ba3b497 --- /dev/null +++ b/tests/test_v12_suse_source.py @@ -0,0 +1,29 @@ +import tempfile +import unittest +from pathlib import Path + +from src.vulnerability.database import VulnerabilityDatabase +from src.vulnerability.models import Component +from src.vulnerability.sources.suse import ingest_file, ingest_file_to_database, SUSE_SOURCE_ID + + +class TestV12SuseSource(unittest.TestCase): + def test_suse_parser_and_staging_preserve_sle_metadata(self): + payload = '{"schema":"coderisktools.vulnerability.suse-feed","version":1,"release":"sles-15-sp5","advisories":[{"id":"SUSE-SU-9999:1","package":"openssl","fixed":"3.0.8-150500.3.20.1","severity":"important","binary_packages":["libopenssl3"],"backport":true}]}' + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + feed = root / "suse.json" + feed.write_text(payload, encoding="utf-8") + parsed = ingest_file(str(feed)) + self.assertEqual(parsed["source_id"], SUSE_SOURCE_ID) + database = VulnerabilityDatabase(str(root / "db.sqlite")) + try: + report = ingest_file_to_database(str(feed), database, "suse-sles15-1") + self.assertEqual(report["state"], "staged") + self.assertFalse(report["activated"]) + affected = database.evaluate_component(Component(ecosystem="RPM", name="libopenssl3", version="3.0.8-150500.3.19.1")) + fixed = database.evaluate_component(Component(ecosystem="RPM", name="libopenssl3", version="3.0.8-150500.3.20.1")) + self.assertEqual(affected["status"], "affected") + self.assertEqual(fixed["status"], "not_affected") + finally: + database.close() diff --git a/tests/test_v12_ubuntu_matching.py b/tests/test_v12_ubuntu_matching.py new file mode 100644 index 0000000..83ca8ea --- /dev/null +++ b/tests/test_v12_ubuntu_matching.py @@ -0,0 +1,25 @@ +import tempfile +import unittest +from pathlib import Path + +from src.vulnerability.database import VulnerabilityDatabase +from src.vulnerability.models import Component +from src.vulnerability.sources.ubuntu import ingest_file_to_database + + +class TestV12UbuntuMatching(unittest.TestCase): + def test_ubuntu_fixed_revision_boundary(self): + payload = '{"schema":"coderisktools.vulnerability.ubuntu-feed","version":1,"release":"jammy","advisories":[{"id":"USN-1","package":"openssl","fixed":"3.0.11-1ubuntu2.1","binary_packages":["libssl3"]}]}' + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + feed = root / "ubuntu.json" + feed.write_text(payload, encoding="utf-8") + database = VulnerabilityDatabase(str(root / "db.sqlite")) + try: + ingest_file_to_database(str(feed), database, "ubuntu-jammy-1") + affected = database.evaluate_component(Component(ecosystem="Ubuntu", name="libssl3", version="3.0.11-1ubuntu2")) + fixed = database.evaluate_component(Component(ecosystem="Ubuntu", name="libssl3", version="3.0.11-1ubuntu2.1")) + self.assertEqual(affected["status"], "affected") + self.assertEqual(fixed["status"], "not_affected") + finally: + database.close() diff --git a/tests/test_v12_ubuntu_source.py b/tests/test_v12_ubuntu_source.py new file mode 100644 index 0000000..831a491 --- /dev/null +++ b/tests/test_v12_ubuntu_source.py @@ -0,0 +1,26 @@ +import tempfile +import unittest +from pathlib import Path + +from src.vulnerability.sources.ubuntu import UBUNTU_SOURCE_ID, ingest_file + + +class TestV12UbuntuSource(unittest.TestCase): + def test_ingest_ubuntu_fixture_preserves_release_and_provenance(self): + payload = '{"schema":"coderisktools.vulnerability.ubuntu-feed","version":1,"release":"jammy","advisories":[{"id":"USN-9999-1","package":"openssl","fixed":"3.0.11-1ubuntu2.1","binary_packages":["libssl3"]}]}' + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "ubuntu.json" + path.write_text(payload, encoding="utf-8") + result = ingest_file(str(path)) + self.assertEqual(result["source_id"], UBUNTU_SOURCE_ID) + self.assertEqual(result["release"], "jammy") + self.assertEqual(result["advisory_count"], 1) + self.assertTrue(result["source_digest"].startswith("sha256:")) + self.assertEqual(result["advisories"][0]["binary_packages"], ["libssl3"]) + + def test_rejects_debian_schema_as_ubuntu_feed(self): + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "ubuntu.json" + path.write_text('{"schema":"coderisktools.vulnerability.debian-feed","version":1,"release":"jammy","advisories":[]}', encoding="utf-8") + with self.assertRaises(ValueError): + ingest_file(str(path)) diff --git a/tests/test_v12_ubuntu_staging.py b/tests/test_v12_ubuntu_staging.py new file mode 100644 index 0000000..d5dbcd0 --- /dev/null +++ b/tests/test_v12_ubuntu_staging.py @@ -0,0 +1,25 @@ +import tempfile +import unittest +from pathlib import Path + +from src.vulnerability.database import VulnerabilityDatabase +from src.vulnerability.sources.ubuntu import ingest_file_to_database + + +class TestV12UbuntuStaging(unittest.TestCase): + def test_ubuntu_feed_stages_usn_without_activation(self): + payload = '{"schema":"coderisktools.vulnerability.ubuntu-feed","version":1,"release":"jammy","advisories":[{"id":"USN-1","package":"openssl","fixed":"3.0.11-1ubuntu2.1","binary_packages":["libssl3"]}]}' + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + feed = root / "ubuntu.json" + feed.write_text(payload, encoding="utf-8") + database = VulnerabilityDatabase(str(root / "db.sqlite")) + try: + report = ingest_file_to_database(str(feed), database, "ubuntu-jammy-1") + self.assertEqual(report["state"], "staged") + self.assertFalse(report["activated"]) + self.assertEqual(database.advisory_count(), 1) + self.assertEqual(database.snapshot_status("ubuntu-jammy-1")["state"], "staged") + self.assertEqual(database.connection.execute("SELECT source FROM advisories WHERE id='USN-1'").fetchone()[0], "ubuntu-security:jammy") + finally: + database.close() diff --git a/tests/test_v13_conan_source.py b/tests/test_v13_conan_source.py new file mode 100644 index 0000000..32bcdfd --- /dev/null +++ b/tests/test_v13_conan_source.py @@ -0,0 +1,28 @@ +import tempfile +import unittest +from pathlib import Path + +from src.vulnerability.database import VulnerabilityDatabase +from src.vulnerability.models import Component +from src.vulnerability.sources.conan import CONAN_SOURCE_ID, ingest_file, ingest_file_to_database + + +class TestV13ConanSource(unittest.TestCase): + def test_conan_fixture_staging_and_fixed_boundary(self): + payload = '{"schema":"coderisktools.vulnerability.conan-feed","version":1,"advisories":[{"id":"CVE-2026-9","package":"openssl","fixed":"3.2.2","severity":"high"}]}' + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + feed = root / "conan.json" + feed.write_text(payload, encoding="utf-8") + parsed = ingest_file(str(feed)) + self.assertEqual(parsed["source_id"], CONAN_SOURCE_ID) + database = VulnerabilityDatabase(str(root / "db.sqlite")) + try: + report = ingest_file_to_database(str(feed), database, "conan-1") + self.assertEqual(report["state"], "staged") + affected = database.evaluate_component(Component(ecosystem="Conan", name="openssl", version="3.2.1")) + fixed = database.evaluate_component(Component(ecosystem="Conan", name="openssl", version="3.2.2")) + self.assertEqual(affected["status"], "affected") + self.assertEqual(fixed["status"], "not_affected") + finally: + database.close() diff --git a/tests/test_v13_dart_source.py b/tests/test_v13_dart_source.py new file mode 100644 index 0000000..5d24019 --- /dev/null +++ b/tests/test_v13_dart_source.py @@ -0,0 +1,28 @@ +import tempfile +import unittest +from pathlib import Path + +from src.vulnerability.database import VulnerabilityDatabase +from src.vulnerability.models import Component +from src.vulnerability.sources.dart import DART_SOURCE_ID, ingest_file, ingest_file_to_database + + +class TestV13DartSource(unittest.TestCase): + def test_dart_fixture_staging_and_fixed_boundary(self): + payload = '{"schema":"coderisktools.vulnerability.dart-feed","version":1,"advisories":[{"id":"CVE-2026-5","package":"http","fixed":"1.2.2","severity":"high"}]}' + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + feed = root / "dart.json" + feed.write_text(payload, encoding="utf-8") + parsed = ingest_file(str(feed)) + self.assertEqual(parsed["source_id"], DART_SOURCE_ID) + database = VulnerabilityDatabase(str(root / "db.sqlite")) + try: + report = ingest_file_to_database(str(feed), database, "dart-1") + self.assertEqual(report["state"], "staged") + affected = database.evaluate_component(Component(ecosystem="Dart", name="http", version="1.2.1")) + fixed = database.evaluate_component(Component(ecosystem="Dart", name="http", version="1.2.2")) + self.assertEqual(affected["status"], "affected") + self.assertEqual(fixed["status"], "not_affected") + finally: + database.close() diff --git a/tests/test_v13_elixir_source.py b/tests/test_v13_elixir_source.py new file mode 100644 index 0000000..d0efb86 --- /dev/null +++ b/tests/test_v13_elixir_source.py @@ -0,0 +1,28 @@ +import tempfile +import unittest +from pathlib import Path + +from src.vulnerability.database import VulnerabilityDatabase +from src.vulnerability.models import Component +from src.vulnerability.sources.elixir import ELIXIR_SOURCE_ID, ingest_file, ingest_file_to_database + + +class TestV13ElixirSource(unittest.TestCase): + def test_elixir_fixture_staging_and_fixed_boundary(self): + payload = '{"schema":"coderisktools.vulnerability.elixir-feed","version":1,"advisories":[{"id":"CVE-2026-6","package":"plug","fixed":"1.15.3","severity":"high"}]}' + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + feed = root / "elixir.json" + feed.write_text(payload, encoding="utf-8") + parsed = ingest_file(str(feed)) + self.assertEqual(parsed["source_id"], ELIXIR_SOURCE_ID) + database = VulnerabilityDatabase(str(root / "db.sqlite")) + try: + report = ingest_file_to_database(str(feed), database, "elixir-1") + self.assertEqual(report["state"], "staged") + affected = database.evaluate_component(Component(ecosystem="Hex", name="plug", version="1.15.2")) + fixed = database.evaluate_component(Component(ecosystem="Hex", name="plug", version="1.15.3")) + self.assertEqual(affected["status"], "affected") + self.assertEqual(fixed["status"], "not_affected") + finally: + database.close() diff --git a/tests/test_v13_haskell_source.py b/tests/test_v13_haskell_source.py new file mode 100644 index 0000000..e1dc9dc --- /dev/null +++ b/tests/test_v13_haskell_source.py @@ -0,0 +1,28 @@ +import tempfile +import unittest +from pathlib import Path + +from src.vulnerability.database import VulnerabilityDatabase +from src.vulnerability.models import Component +from src.vulnerability.sources.haskell import HASKELL_SOURCE_ID, ingest_file, ingest_file_to_database + + +class TestV13HaskellSource(unittest.TestCase): + def test_haskell_fixture_staging_and_fixed_boundary(self): + payload = '{"schema":"coderisktools.vulnerability.haskell-feed","version":1,"advisories":[{"id":"CVE-2026-7","package":"aeson","fixed":"2.2.1.1","severity":"high"}]}' + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + feed = root / "haskell.json" + feed.write_text(payload, encoding="utf-8") + parsed = ingest_file(str(feed)) + self.assertEqual(parsed["source_id"], HASKELL_SOURCE_ID) + database = VulnerabilityDatabase(str(root / "db.sqlite")) + try: + report = ingest_file_to_database(str(feed), database, "haskell-1") + self.assertEqual(report["state"], "staged") + affected = database.evaluate_component(Component(ecosystem="Hackage", name="aeson", version="2.2.1.0")) + fixed = database.evaluate_component(Component(ecosystem="Hackage", name="aeson", version="2.2.1.1")) + self.assertEqual(affected["status"], "affected") + self.assertEqual(fixed["status"], "not_affected") + finally: + database.close() diff --git a/tests/test_v13_maven_source.py b/tests/test_v13_maven_source.py new file mode 100644 index 0000000..256434d --- /dev/null +++ b/tests/test_v13_maven_source.py @@ -0,0 +1,28 @@ +import tempfile +import unittest +from pathlib import Path + +from src.vulnerability.database import VulnerabilityDatabase +from src.vulnerability.models import Component +from src.vulnerability.sources.maven import MAVEN_SOURCE_ID, ingest_file, ingest_file_to_database + + +class TestV13MavenSource(unittest.TestCase): + def test_maven_fixture_staging_and_fixed_boundary(self): + payload = '{"schema":"coderisktools.vulnerability.maven-feed","version":1,"advisories":[{"id":"CVE-2026-1","package":"org.example:demo","fixed":"2.4.1","severity":"high"}]}' + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + feed = root / "maven.json" + feed.write_text(payload, encoding="utf-8") + parsed = ingest_file(str(feed)) + self.assertEqual(parsed["source_id"], MAVEN_SOURCE_ID) + database = VulnerabilityDatabase(str(root / "db.sqlite")) + try: + report = ingest_file_to_database(str(feed), database, "maven-1") + self.assertEqual(report["state"], "staged") + affected = database.evaluate_component(Component(ecosystem="Maven", name="org.example:demo", version="2.4.0")) + fixed = database.evaluate_component(Component(ecosystem="Maven", name="org.example:demo", version="2.4.1")) + self.assertEqual(affected["status"], "affected") + self.assertEqual(fixed["status"], "not_affected") + finally: + database.close() diff --git a/tests/test_v13_nuget_source.py b/tests/test_v13_nuget_source.py new file mode 100644 index 0000000..9f0d52f --- /dev/null +++ b/tests/test_v13_nuget_source.py @@ -0,0 +1,28 @@ +import tempfile +import unittest +from pathlib import Path + +from src.vulnerability.database import VulnerabilityDatabase +from src.vulnerability.models import Component +from src.vulnerability.sources.nuget import NUGET_SOURCE_ID, ingest_file, ingest_file_to_database + + +class TestV13NugetSource(unittest.TestCase): + def test_nuget_fixture_staging_and_fixed_boundary(self): + payload = '{"schema":"coderisktools.vulnerability.nuget-feed","version":1,"advisories":[{"id":"CVE-2026-2","package":"Example.Core","fixed":"8.0.4","severity":"high"}]}' + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + feed = root / "nuget.json" + feed.write_text(payload, encoding="utf-8") + parsed = ingest_file(str(feed)) + self.assertEqual(parsed["source_id"], NUGET_SOURCE_ID) + database = VulnerabilityDatabase(str(root / "db.sqlite")) + try: + report = ingest_file_to_database(str(feed), database, "nuget-1") + self.assertEqual(report["state"], "staged") + affected = database.evaluate_component(Component(ecosystem="NuGet", name="Example.Core", version="8.0.3")) + fixed = database.evaluate_component(Component(ecosystem="NuGet", name="Example.Core", version="8.0.4")) + self.assertEqual(affected["status"], "affected") + self.assertEqual(fixed["status"], "not_affected") + finally: + database.close() diff --git a/tests/test_v13_r_source.py b/tests/test_v13_r_source.py new file mode 100644 index 0000000..83beb0d --- /dev/null +++ b/tests/test_v13_r_source.py @@ -0,0 +1,28 @@ +import tempfile +import unittest +from pathlib import Path + +from src.vulnerability.database import VulnerabilityDatabase +from src.vulnerability.models import Component +from src.vulnerability.sources.r import R_SOURCE_ID, ingest_file, ingest_file_to_database + + +class TestV13RSource(unittest.TestCase): + def test_r_fixture_staging_and_fixed_boundary(self): + payload = '{"schema":"coderisktools.vulnerability.r-feed","version":1,"advisories":[{"id":"CVE-2026-8","package":"jsonlite","fixed":"1.8.8","severity":"high"}]}' + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + feed = root / "r.json" + feed.write_text(payload, encoding="utf-8") + parsed = ingest_file(str(feed)) + self.assertEqual(parsed["source_id"], R_SOURCE_ID) + database = VulnerabilityDatabase(str(root / "db.sqlite")) + try: + report = ingest_file_to_database(str(feed), database, "r-1") + self.assertEqual(report["state"], "staged") + affected = database.evaluate_component(Component(ecosystem="CRAN", name="jsonlite", version="1.8.7")) + fixed = database.evaluate_component(Component(ecosystem="CRAN", name="jsonlite", version="1.8.8")) + self.assertEqual(affected["status"], "affected") + self.assertEqual(fixed["status"], "not_affected") + finally: + database.close() diff --git a/tests/test_v13_rubygems_source.py b/tests/test_v13_rubygems_source.py new file mode 100644 index 0000000..ec3861c --- /dev/null +++ b/tests/test_v13_rubygems_source.py @@ -0,0 +1,28 @@ +import tempfile +import unittest +from pathlib import Path + +from src.vulnerability.database import VulnerabilityDatabase +from src.vulnerability.models import Component +from src.vulnerability.sources.rubygems import RUBYGEMS_SOURCE_ID, ingest_file, ingest_file_to_database + + +class TestV13RubygemsSource(unittest.TestCase): + def test_rubygems_fixture_staging_and_fixed_boundary(self): + payload = '{"schema":"coderisktools.vulnerability.rubygems-feed","version":1,"advisories":[{"id":"CVE-2026-3","package":"rack","fixed":"3.0.9","severity":"high"}]}' + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + feed = root / "rubygems.json" + feed.write_text(payload, encoding="utf-8") + parsed = ingest_file(str(feed)) + self.assertEqual(parsed["source_id"], RUBYGEMS_SOURCE_ID) + database = VulnerabilityDatabase(str(root / "db.sqlite")) + try: + report = ingest_file_to_database(str(feed), database, "rubygems-1") + self.assertEqual(report["state"], "staged") + affected = database.evaluate_component(Component(ecosystem="RubyGems", name="rack", version="3.0.8")) + fixed = database.evaluate_component(Component(ecosystem="RubyGems", name="rack", version="3.0.9")) + self.assertEqual(affected["status"], "affected") + self.assertEqual(fixed["status"], "not_affected") + finally: + database.close() diff --git a/tests/test_v13_swift_source.py b/tests/test_v13_swift_source.py new file mode 100644 index 0000000..9971b15 --- /dev/null +++ b/tests/test_v13_swift_source.py @@ -0,0 +1,28 @@ +import tempfile +import unittest +from pathlib import Path + +from src.vulnerability.database import VulnerabilityDatabase +from src.vulnerability.models import Component +from src.vulnerability.sources.swift import SWIFT_SOURCE_ID, ingest_file, ingest_file_to_database + + +class TestV13SwiftSource(unittest.TestCase): + def test_swift_fixture_staging_and_fixed_boundary(self): + payload = '{"schema":"coderisktools.vulnerability.swift-feed","version":1,"advisories":[{"id":"CVE-2026-4","package":"swift-nio","fixed":"2.65.0","severity":"high"}]}' + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + feed = root / "swift.json" + feed.write_text(payload, encoding="utf-8") + parsed = ingest_file(str(feed)) + self.assertEqual(parsed["source_id"], SWIFT_SOURCE_ID) + database = VulnerabilityDatabase(str(root / "db.sqlite")) + try: + report = ingest_file_to_database(str(feed), database, "swift-1") + self.assertEqual(report["state"], "staged") + affected = database.evaluate_component(Component(ecosystem="Swift", name="swift-nio", version="2.64.0")) + fixed = database.evaluate_component(Component(ecosystem="Swift", name="swift-nio", version="2.65.0")) + self.assertEqual(affected["status"], "affected") + self.assertEqual(fixed["status"], "not_affected") + finally: + database.close() diff --git a/tests/test_v13_vcpkg_source.py b/tests/test_v13_vcpkg_source.py new file mode 100644 index 0000000..0d0ea55 --- /dev/null +++ b/tests/test_v13_vcpkg_source.py @@ -0,0 +1,28 @@ +import tempfile +import unittest +from pathlib import Path + +from src.vulnerability.database import VulnerabilityDatabase +from src.vulnerability.models import Component +from src.vulnerability.sources.vcpkg import VCPKG_SOURCE_ID, ingest_file, ingest_file_to_database + + +class TestV13VcpkgSource(unittest.TestCase): + def test_vcpkg_fixture_staging_and_fixed_boundary(self): + payload = '{"schema":"coderisktools.vulnerability.vcpkg-feed","version":1,"advisories":[{"id":"CVE-2026-10","package":"openssl","fixed":"3.3.1","severity":"high"}]}' + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + feed = root / "vcpkg.json" + feed.write_text(payload, encoding="utf-8") + parsed = ingest_file(str(feed)) + self.assertEqual(parsed["source_id"], VCPKG_SOURCE_ID) + database = VulnerabilityDatabase(str(root / "db.sqlite")) + try: + report = ingest_file_to_database(str(feed), database, "vcpkg-1") + self.assertEqual(report["state"], "staged") + affected = database.evaluate_component(Component(ecosystem="vcpkg", name="openssl", version="3.3.0")) + fixed = database.evaluate_component(Component(ecosystem="vcpkg", name="openssl", version="3.3.1")) + self.assertEqual(affected["status"], "affected") + self.assertEqual(fixed["status"], "not_affected") + finally: + database.close() diff --git a/tests/test_v14_csaf_quality.py b/tests/test_v14_csaf_quality.py new file mode 100644 index 0000000..7e83fa3 --- /dev/null +++ b/tests/test_v14_csaf_quality.py @@ -0,0 +1,23 @@ +import unittest + +from src.vulnerability.sources.csaf_quality import csaf_quality_gate + + +class TestV14CsafQuality(unittest.TestCase): + def test_quality_gate_accepts_valid_and_reports_provider_extensions(self): + report = csaf_quality_gate({ + "source_id": "csaf-security", + "source_digest": "sha256:abc", + "advisories": [{ + "id": "CVE-2026-1", + "affected_product_ids": ["p1"], + "not_affected_product_ids": [], + "products": {"p1": ("PyPI", "example", "1.0.0")}, + "vendor_status": {"p1": {"status": "under_investigation", "provider_extension": {"foo": "bar"}}}, + "remediations": [{"category": "vendor_fix", "details": "upgrade", "product_ids": ["p1"]}], + }], + "provenance": {"schema": "csaf_security_advisory", "version": "2.0"}, + }) + self.assertTrue(report["passed"]) + self.assertEqual(report["provider_extension_count"], 1) + self.assertEqual(report["remediation_count"], 1) diff --git a/tests/test_v14_csaf_registry.py b/tests/test_v14_csaf_registry.py new file mode 100644 index 0000000..07bf363 --- /dev/null +++ b/tests/test_v14_csaf_registry.py @@ -0,0 +1,16 @@ +import unittest + +from src.vulnerability.sources.csaf_registry import CsafProviderRegistry + + +class TestV14CsafRegistry(unittest.TestCase): + def test_provider_registry_and_health_are_deterministic(self): + registry = CsafProviderRegistry() + registry.register("example", "https://example.invalid/csaf", enabled=True) + registry.record_result("example", success=True, records=3, digest="sha256:abc") + report = registry.health_report("example") + self.assertEqual(report["provider_id"], "example") + self.assertEqual(report["status"], "healthy") + self.assertEqual(report["last_records"], 3) + self.assertEqual(report["last_digest"], "sha256:abc") + self.assertEqual(registry.list_providers()[0]["provider_id"], "example") diff --git a/tests/test_v14_csaf_remediations.py b/tests/test_v14_csaf_remediations.py new file mode 100644 index 0000000..e25c806 --- /dev/null +++ b/tests/test_v14_csaf_remediations.py @@ -0,0 +1,25 @@ +import tempfile +import unittest +from pathlib import Path + +from src.vulnerability.database import VulnerabilityDatabase +from src.vulnerability.sources.csaf import ingest_file, ingest_file_to_database + + +class TestV14CsafRemediations(unittest.TestCase): + def test_remediation_and_vendor_status_are_preserved(self): + payload = '{"csaf_version":"2.0","document":{"category":"csaf_security_advisory","title":"Vendor advisory","tracking":{"id":"CSAF-2026-2"}},"product_tree":{"branches":[{"product":{"product_id":"pkg-1","product_identification_helper":{"purl":"pkg:pypi/example@1.0.0"}}}]},"vulnerabilities":[{"cve":"CVE-2026-12","product_status":{"known_affected":["pkg-1"],"vendor_specific":{"pkg-1":{"status":"under_investigation"}}},"remediations":[{"category":"vendor_fix","details":"Upgrade to 1.0.1","product_ids":["pkg-1"]}]}]}' + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + feed = root / "csaf.json" + feed.write_text(payload, encoding="utf-8") + parsed = ingest_file(str(feed)) + advisory = parsed["advisories"][0] + self.assertEqual(advisory["remediations"][0]["category"], "vendor_fix") + self.assertEqual(advisory["vendor_status"]["pkg-1"]["status"], "under_investigation") + database = VulnerabilityDatabase(str(root / "db.sqlite")) + try: + report = ingest_file_to_database(str(feed), database, "csaf-2") + self.assertEqual(report["state"], "staged") + finally: + database.close() diff --git a/tests/test_v14_csaf_source.py b/tests/test_v14_csaf_source.py new file mode 100644 index 0000000..86c8980 --- /dev/null +++ b/tests/test_v14_csaf_source.py @@ -0,0 +1,27 @@ +import tempfile +import unittest +from pathlib import Path + +from src.vulnerability.database import VulnerabilityDatabase +from src.vulnerability.models import Component +from src.vulnerability.sources.csaf import CSAF_SOURCE_ID, ingest_file, ingest_file_to_database + + +class TestV14CsafSource(unittest.TestCase): + def test_generic_csaf_product_tree_and_status_staging(self): + payload = '{"csaf_version":"2.0","document":{"category":"csaf_security_advisory","title":"Example advisory","tracking":{"id":"CSAF-2026-1","current_release_date":"2026-01-01T00:00:00Z"}},"product_tree":{"branches":[{"name":"product","product":{"product_id":"pkg-1","product_identification_helper":{"purl":"pkg:pypi/example@1.0.0"}}}]},"vulnerabilities":[{"cve":"CVE-2026-11","product_status":{"known_affected":["pkg-1"]},"notes":[{"category":"description","text":"Example"}]}]}' + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + feed = root / "csaf.json" + feed.write_text(payload, encoding="utf-8") + parsed = ingest_file(str(feed)) + self.assertEqual(parsed["source_id"], CSAF_SOURCE_ID) + self.assertEqual(parsed["advisory_count"], 1) + database = VulnerabilityDatabase(str(root / "db.sqlite")) + try: + report = ingest_file_to_database(str(feed), database, "csaf-1") + self.assertEqual(report["state"], "staged") + result = database.evaluate_component(Component(ecosystem="PyPI", name="example", version="1.0.0")) + self.assertEqual(result["status"], "affected") + finally: + database.close() diff --git a/tests/test_v15_acceptance.py b/tests/test_v15_acceptance.py new file mode 100644 index 0000000..7fb42e4 --- /dev/null +++ b/tests/test_v15_acceptance.py @@ -0,0 +1,21 @@ +import unittest + +from src.vulnerability.benchmark import run_fixture_benchmark + + +class TestV15Acceptance(unittest.TestCase): + def test_runner_passes_golden_acceptance_gates(self): + report = run_fixture_benchmark( + "tests/fixtures/v15_public_benchmark.json", + min_precision=1.0, + min_recall=0.5, + min_f1=0.66, + max_p95_ms=3.0, + ) + self.assertTrue(report["passed"]) + self.assertEqual(report["acceptance"]["failed"], []) + + def test_runner_reports_failed_gate_without_raising(self): + report = run_fixture_benchmark("tests/fixtures/v15_public_benchmark.json", min_recall=0.9) + self.assertFalse(report["passed"]) + self.assertIn("recall", report["acceptance"]["failed"]) diff --git a/tests/test_v15_benchmark.py b/tests/test_v15_benchmark.py new file mode 100644 index 0000000..8c9bda7 --- /dev/null +++ b/tests/test_v15_benchmark.py @@ -0,0 +1,20 @@ +import unittest + +from src.vulnerability.benchmark import BenchmarkCase, evaluate_cases, summarize_metrics + + +class TestV15Benchmark(unittest.TestCase): + def test_precision_recall_and_latency_summary(self): + cases = [ + BenchmarkCase("affected", expected=True, observed=True, elapsed_ms=2.0), + BenchmarkCase("clean", expected=False, observed=False, elapsed_ms=1.0), + BenchmarkCase("miss", expected=True, observed=False, elapsed_ms=3.0), + ] + metrics = evaluate_cases(cases) + summary = summarize_metrics(metrics, cases) + self.assertEqual(metrics["true_positive"], 1) + self.assertEqual(metrics["false_negative"], 1) + self.assertEqual(summary["precision"], 1.0) + self.assertEqual(summary["recall"], 0.5) + self.assertEqual(summary["f1"], 2 / 3) + self.assertEqual(summary["latency_ms"]["p95"], 3.0) diff --git a/tests/test_v15_database_benchmark.py b/tests/test_v15_database_benchmark.py new file mode 100644 index 0000000..2444c82 --- /dev/null +++ b/tests/test_v15_database_benchmark.py @@ -0,0 +1,24 @@ +import tempfile +import unittest +from pathlib import Path + +from src.vulnerability.benchmark import run_database_benchmark +from src.vulnerability.database import VulnerabilityDatabase + + +class TestV15DatabaseBenchmark(unittest.TestCase): + def test_database_adapter_uses_real_evaluate_component(self): + with tempfile.TemporaryDirectory() as directory: + database = VulnerabilityDatabase(str(Path(directory) / "db.sqlite")) + try: + database.import_osv_records([{"id": "CVE-2026-99", "affected": [{"package": {"ecosystem": "PyPI", "name": "example"}, "ranges": [{"type": "ECOSYSTEM", "events": [{"introduced": "0"}]}]}]}], source="fixture") + cases = [ + {"case_id": "affected", "ecosystem": "PyPI", "name": "example", "version": "1.0.0", "expected": True}, + {"case_id": "clean", "ecosystem": "PyPI", "name": "other", "version": "1.0.0", "expected": False}, + ] + report = run_database_benchmark(database, cases) + self.assertEqual(report["metrics"]["true_positive"], 1) + self.assertEqual(report["metrics"]["true_negative"], 1) + self.assertEqual(report["summary"]["recall"], 1.0) + finally: + database.close() diff --git a/tests/test_v15_external_comparisons.py b/tests/test_v15_external_comparisons.py new file mode 100644 index 0000000..b674fc7 --- /dev/null +++ b/tests/test_v15_external_comparisons.py @@ -0,0 +1,19 @@ +import json +import unittest +from pathlib import Path + +from src.vulnerability.benchmark import compare_external_evidence + + +class TestExternalComparisons(unittest.TestCase): + def test_comparison_classifies_differences_without_merging_results(self): + fixture = json.loads((Path(__file__).parent / "fixtures" / "v15_external_comparison.json").read_text(encoding="utf-8")) + report = compare_external_evidence( + set(fixture["internal"]), + {tool: set(ids) for tool, ids in fixture["tools"].items()}, + ) + self.assertEqual(report["tools"], ["Grype", "OSV-Scanner", "Trivy"]) + self.assertEqual(report["tools"][0], "Grype") + self.assertIn("CVE-3", report["differences"]["OSV-Scanner"]["external_only"]) + self.assertIn("CVE-4", report["differences"]["Grype"]["external_only"]) + self.assertEqual(report["differences"]["Trivy"]["status"], "aligned") diff --git a/tests/test_v15_final_report.py b/tests/test_v15_final_report.py new file mode 100644 index 0000000..19cea79 --- /dev/null +++ b/tests/test_v15_final_report.py @@ -0,0 +1,21 @@ +import tempfile +import unittest +from pathlib import Path + +from src.vulnerability.benchmark import build_final_benchmark_report +from src.vulnerability.database import VulnerabilityDatabase + + +class TestV15FinalReport(unittest.TestCase): + def test_final_report_combines_quality_performance_and_ecosystems(self): + with tempfile.TemporaryDirectory() as directory: + database = VulnerabilityDatabase(str(Path(directory) / "db.sqlite")) + try: + database.import_osv_records([{"id": "CVE-2026-101", "affected": [{"package": {"ecosystem": "PyPI", "name": "example"}, "ranges": [{"type": "ECOSYSTEM", "events": [{"introduced": "0"}]}]}]}], source="fixture") + report = build_final_benchmark_report(database, {"PyPI": [{"case_id": "affected", "ecosystem": "PyPI", "name": "example", "version": "1.0.0", "expected": True}, {"case_id": "clean", "ecosystem": "PyPI", "name": "other", "version": "1.0.0", "expected": False}]}) + self.assertTrue(report["passed"]) + self.assertEqual(report["ecosystems"], ["PyPI"]) + self.assertEqual(report["summary"]["recall"], 1.0) + self.assertIn("performance", report) + finally: + database.close() diff --git a/tests/test_v15_memory.py b/tests/test_v15_memory.py new file mode 100644 index 0000000..0fd4ea4 --- /dev/null +++ b/tests/test_v15_memory.py @@ -0,0 +1,21 @@ +import tempfile +import unittest +from pathlib import Path + +from src.vulnerability.benchmark import run_database_memory_benchmark +from src.vulnerability.database import VulnerabilityDatabase + + +class TestV15Memory(unittest.TestCase): + def test_memory_benchmark_reports_peak_and_acceptance(self): + with tempfile.TemporaryDirectory() as directory: + database = VulnerabilityDatabase(str(Path(directory) / "db.sqlite")) + try: + database.import_osv_records([{"id": "CVE-2026-MEM", "affected": [{"package": {"ecosystem": "PyPI", "name": "memory-example"}, "ranges": [{"type": "ECOSYSTEM", "events": [{"introduced": "0"}]}]}]}], source="fixture") + cases = [{"case_id": "affected", "ecosystem": "PyPI", "name": "memory-example", "version": "1.0.0", "expected": True}] + report = run_database_memory_benchmark(database, cases, repetitions=2, max_peak_kib=1024 * 1024) + self.assertTrue(report["passed"]) + self.assertEqual(report["repetitions"], 2) + self.assertGreaterEqual(report["peak_kib"], 0.0) + finally: + database.close() diff --git a/tests/test_v15_multiecosystem.py b/tests/test_v15_multiecosystem.py new file mode 100644 index 0000000..5083954 --- /dev/null +++ b/tests/test_v15_multiecosystem.py @@ -0,0 +1,24 @@ +import tempfile +import unittest +from pathlib import Path + +from src.vulnerability.benchmark import load_benchmark_fixture, run_database_benchmark +from src.vulnerability.database import VulnerabilityDatabase + + +class TestV15Multiecosystem(unittest.TestCase): + def test_multiecosystem_fixture_meets_golden_thresholds(self): + fixture = load_benchmark_fixture(Path("tests/fixtures/v15_multiecosystem.json")) + with tempfile.TemporaryDirectory() as directory: + database = VulnerabilityDatabase(str(Path(directory) / "db.sqlite")) + try: + records = [] + for item in fixture["records"]: + records.append({"id": item["id"], "affected": [{"package": {"ecosystem": item["ecosystem"], "name": item["name"]}, "ranges": [{"type": "ECOSYSTEM", "events": [{"introduced": "0"}, {"fixed": item["fixed"]}]}]}]}) + database.import_osv_records(records, source="v15-fixture") + report = run_database_benchmark(database, fixture["cases"]) + self.assertGreaterEqual(report["summary"]["precision"], fixture["golden"]["min_precision"]) + self.assertGreaterEqual(report["summary"]["recall"], fixture["golden"]["min_recall"]) + self.assertGreaterEqual(report["summary"]["f1"], fixture["golden"]["min_f1"]) + finally: + database.close() diff --git a/tests/test_v15_performance.py b/tests/test_v15_performance.py new file mode 100644 index 0000000..7d65a59 --- /dev/null +++ b/tests/test_v15_performance.py @@ -0,0 +1,22 @@ +import tempfile +import unittest +from pathlib import Path + +from src.vulnerability.benchmark import run_database_benchmark_repeated +from src.vulnerability.database import VulnerabilityDatabase + + +class TestV15Performance(unittest.TestCase): + def test_repeated_database_benchmark_reports_stability(self): + with tempfile.TemporaryDirectory() as directory: + database = VulnerabilityDatabase(str(Path(directory) / "db.sqlite")) + try: + database.import_osv_records([{"id": "CVE-2026-100", "affected": [{"package": {"ecosystem": "PyPI", "name": "example"}, "ranges": [{"type": "ECOSYSTEM", "events": [{"introduced": "0"}]}]}]}], source="fixture") + cases = [{"case_id": "affected", "ecosystem": "PyPI", "name": "example", "version": "1.0.0", "expected": True}] + report = run_database_benchmark_repeated(database, cases, repetitions=3, max_p95_ms=1000.0) + self.assertTrue(report["passed"]) + self.assertEqual(report["repetitions"], 3) + self.assertEqual(len(report["runs"]), 3) + self.assertIn("p95", report["latency_ms"]) + finally: + database.close() diff --git a/tests/test_v15_public_fixtures.py b/tests/test_v15_public_fixtures.py new file mode 100644 index 0000000..5796dc6 --- /dev/null +++ b/tests/test_v15_public_fixtures.py @@ -0,0 +1,15 @@ +import json +import unittest +from pathlib import Path + +from src.vulnerability.benchmark import load_benchmark_fixture + + +class TestV15PublicFixtures(unittest.TestCase): + def test_versioned_public_fixture_has_golden_results(self): + fixture = load_benchmark_fixture(Path("tests/fixtures/v15_public_benchmark.json")) + self.assertEqual(fixture["schema"], "coderisktools.vulnerability.benchmark") + self.assertEqual(fixture["version"], 1) + self.assertEqual(len(fixture["cases"]), 3) + self.assertEqual(fixture["golden"]["precision"], 1.0) + self.assertEqual(fixture["golden"]["recall"], 0.5) diff --git a/tests/test_v15_quality_suites.py b/tests/test_v15_quality_suites.py new file mode 100644 index 0000000..0906ee9 --- /dev/null +++ b/tests/test_v15_quality_suites.py @@ -0,0 +1,27 @@ +import unittest + +from src.vulnerability.benchmark import build_quality_suites_report, benchmark_regression_gate + + +class TestV15QualitySuites(unittest.TestCase): + def test_quality_suites_partition_results_and_regression_gate(self): + cases = [ + {"case_id": "tp", "expected": True, "observed": True, "elapsed_ms": 1.0}, + {"case_id": "tn", "expected": False, "observed": False, "elapsed_ms": 1.0}, + {"case_id": "fp", "expected": False, "observed": True, "elapsed_ms": 1.0}, + {"case_id": "fn", "expected": True, "observed": False, "elapsed_ms": 1.0}, + ] + report = build_quality_suites_report(cases) + self.assertEqual(report["precision_suite"]["count"], 2) + self.assertEqual(report["recall_suite"]["count"], 2) + self.assertEqual(report["false_positive_suite"]["count"], 1) + self.assertEqual(report["false_negative_suite"]["count"], 1) + gate = benchmark_regression_gate(report, report) + self.assertTrue(gate["passed"]) + + def test_regression_gate_rejects_new_false_negative(self): + baseline = {"false_positive_suite": {"case_ids": []}, "false_negative_suite": {"case_ids": []}} + current = {"false_positive_suite": {"case_ids": []}, "false_negative_suite": {"case_ids": ["new-fn"]}} + gate = benchmark_regression_gate(current, baseline) + self.assertFalse(gate["passed"]) + self.assertEqual(gate["failed"], ["false_negative_regression"]) diff --git a/tests/test_v16_final_snapshot.py b/tests/test_v16_final_snapshot.py new file mode 100644 index 0000000..64e3dfc --- /dev/null +++ b/tests/test_v16_final_snapshot.py @@ -0,0 +1,26 @@ +import unittest + +from src.vulnerability.release import build_production_snapshot_report + + +class TestV16FinalSnapshotReport(unittest.TestCase): + def test_final_report_combines_release_gates(self): + report = build_production_snapshot_report( + { + "build_id": "build-1", + "source_digest": "sha256:abc", + "manifest_digest": "sha256:def", + "reproducibility": {"python": "3.11", "platform": "linux", "deterministic": True}, + "licenses": ["MIT"], + "attributions": ["OSV"], + "manifest_signature": {"verified": True}, + "airgap_bundle": {"verified": True}, + }, + {"snapshot_id": "snap-1", "status": "active", "records": 10, "errors": [], "quality": {"invalid": 0}}, + {"OSV": {"records": 10, "advisories": 5}}, + rollback={"ready": True, "applied": False}, + disaster_recovery={"verified": True, "activated": False}, + ) + self.assertTrue(report["ready"]) + self.assertEqual(report["failed"], []) + self.assertTrue(report["health"]["healthy"]) diff --git a/tests/test_v16_health_coverage.py b/tests/test_v16_health_coverage.py new file mode 100644 index 0000000..1b34ba1 --- /dev/null +++ b/tests/test_v16_health_coverage.py @@ -0,0 +1,13 @@ +import unittest + +from src.vulnerability.release import database_health_report, source_coverage_report + + +class TestV16HealthCoverage(unittest.TestCase): + def test_health_and_coverage_reports(self): + health = database_health_report({"snapshot_id": "snap-1", "status": "active", "records": 10, "errors": [], "quality": {"invalid": 0}}) + self.assertTrue(health["healthy"]) + coverage = source_coverage_report({"OSV": {"records": 8, "advisories": 4}, "CSAF": {"records": 2, "advisories": 1}}) + self.assertEqual(coverage["total_records"], 10) + self.assertEqual(coverage["sources"][0]["source_id"], "CSAF") + self.assertEqual(coverage["source_count"], 2) diff --git a/tests/test_v16_production_gate.py b/tests/test_v16_production_gate.py new file mode 100644 index 0000000..d08370d --- /dev/null +++ b/tests/test_v16_production_gate.py @@ -0,0 +1,24 @@ +import unittest + +from src.vulnerability.release import production_snapshot_gate + + +class TestV16ProductionGate(unittest.TestCase): + def test_complete_snapshot_passes_readiness_gate(self): + report = production_snapshot_gate({ + "build_id": "build-1", + "source_digest": "sha256:abc", + "manifest_digest": "sha256:def", + "reproducibility": {"python": "3.11", "platform": "linux", "deterministic": True}, + "licenses": ["MIT"], + "attributions": ["OSV"], + "manifest_signature": {"verified": True}, + "airgap_bundle": {"verified": True}, + }) + self.assertTrue(report["passed"]) + self.assertEqual(report["failed"], []) + + def test_incomplete_snapshot_reports_missing_requirements(self): + report = production_snapshot_gate({"build_id": "build-2"}) + self.assertFalse(report["passed"]) + self.assertIn("source_digest", report["failed"]) diff --git a/tests/test_v16_recovery.py b/tests/test_v16_recovery.py new file mode 100644 index 0000000..906b3d6 --- /dev/null +++ b/tests/test_v16_recovery.py @@ -0,0 +1,31 @@ +import tempfile +import unittest +from pathlib import Path + +from src.vulnerability.airgap import export_air_gap_bundle, import_air_gap_bundle +from src.vulnerability.database import VulnerabilityDatabase +from src.vulnerability.release import rollback_readiness_report + + +class TestV16Recovery(unittest.TestCase): + def test_rollback_plan_is_non_destructive_until_apply(self): + database = VulnerabilityDatabase(":memory:") + try: + report = rollback_readiness_report(database, {"keep-1"}) + self.assertTrue(report["ready"]) + self.assertFalse(report["applied"]) + finally: + database.close() + + def test_airgap_bundle_can_be_restored(self): + with tempfile.TemporaryDirectory() as directory: + source_path = Path(directory) / "source.sqlite" + bundle = Path(directory) / "snapshot.tar.gz" + target = Path(directory) / "restored.sqlite" + database = VulnerabilityDatabase(str(source_path)) + database.import_osv_records([{"id": "CVE-2026-DR", "affected": []}], source="fixture") + export_air_gap_bundle(database, bundle) + database.close() + result = import_air_gap_bundle(bundle, target) + self.assertFalse(result["activated"]) + self.assertTrue(target.is_file()) diff --git a/tests/test_v16_release_docs.py b/tests/test_v16_release_docs.py new file mode 100644 index 0000000..fc016d2 --- /dev/null +++ b/tests/test_v16_release_docs.py @@ -0,0 +1,14 @@ +from pathlib import Path +import unittest + + +class TestV16ReleaseDocumentation(unittest.TestCase): + def test_release_boundary_documents_exist_and_state_limits(self): + root = Path(__file__).parents[1] / "docs" + release = (root / "RELEASE-NOTES-3.0.1.md").read_text(encoding="utf-8") + limits = (root / "KNOWN-LIMITATIONS.md").read_text(encoding="utf-8") + self.assertIn("# CodeRiskTools Scanner 3.0.1", release) + self.assertIn("# Known limitations", limits) + self.assertIn("bounded", limits) + self.assertIn("not a security audit", limits) + self.assertIn("does not execute", release) diff --git a/tests/test_v16_reproducibility.py b/tests/test_v16_reproducibility.py new file mode 100644 index 0000000..4710328 --- /dev/null +++ b/tests/test_v16_reproducibility.py @@ -0,0 +1,29 @@ +import tempfile +import unittest +from pathlib import Path + +from src.vulnerability.release import reproducibility_report + + +class TestV16Reproducibility(unittest.TestCase): + def test_identical_artifact_sets_are_reproducible(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + (root / "manifest.json").write_text('{"version":1}\n', encoding="utf-8") + (root / "snapshot.sqlite").write_bytes(b"snapshot") + report = reproducibility_report([root / "manifest.json", root / "snapshot.sqlite"], [root / "manifest.json", root / "snapshot.sqlite"]) + self.assertTrue(report["reproducible"]) + self.assertEqual(report["failed"], []) + + def test_changed_artifact_is_reported(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + first = root / "first" / "artifact.bin" + second = root / "second" / "artifact.bin" + first.parent.mkdir() + second.parent.mkdir() + first.write_bytes(b"one") + second.write_bytes(b"two") + report = reproducibility_report([first], [second]) + self.assertFalse(report["reproducible"]) + self.assertIn("digest_mismatch", report["failed"]) diff --git a/tests/test_wave1_enrichment_feeds.py b/tests/test_wave1_enrichment_feeds.py new file mode 100644 index 0000000..3b3c238 --- /dev/null +++ b/tests/test_wave1_enrichment_feeds.py @@ -0,0 +1,34 @@ +import json +import tempfile +import unittest +from pathlib import Path +from unittest.mock import Mock + +from src.vulnerability.sources.epss import ingest_file as ingest_epss +from src.vulnerability.sources.ghsa import ingest_file as ingest_ghsa +from src.vulnerability.sources.kev import ingest_file as ingest_kev +from src.vulnerability.database import ImportStats + + +class TestWave1EnrichmentFeeds(unittest.TestCase): + def test_kev_epss_and_ghsa_file_contracts_are_bounded_and_explicit(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + kev_path = root / "kev.json" + kev_path.write_text(json.dumps({"vulnerabilities": [{}]}), encoding="utf-8") + epss_path = root / "epss.json" + epss_path.write_text(json.dumps({"data": [{}]}), encoding="utf-8") + ghsa_path = root / "ghsa.json" + ghsa_path.write_text(json.dumps({"advisories": [{}]}), encoding="utf-8") + db = Mock() + db.import_kev_json.return_value = ImportStats(1, 1, 0) + db.import_epss_json.return_value = ImportStats(1, 1, 0) + db.import_ghsa_json.return_value = ImportStats(1, 1, 1) + reports = [ + ingest_kev(str(kev_path), db, "snapshot-kev"), + ingest_epss(str(epss_path), db, "snapshot-epss"), + ingest_ghsa(str(ghsa_path), db, "snapshot-ghsa"), + ] + self.assertEqual([report["state"] for report in reports], ["staged"] * 3) + self.assertTrue(all(not report["activated"] for report in reports)) + self.assertTrue(all(report["source_digest"].startswith("sha256:") for report in reports))