diff --git a/CHANGELOG.md b/CHANGELOG.md index bca35f6..7e2058b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,30 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] - Upcoming changes... +## [0.15.0] - 2026-08-17 +### Changed +- OSV vulnerabilities are read from the `osv` and `osv_severity` tables instead of the `api.osv.dev` HTTP API. The response is unchanged: same fields, same `source`, same URL construction, and the `cvss` array still carries every vector +- Removed the OSV HTTP client along with `getRepoURL` and the GIT-ecosystem fallback. The table stores `pkg:github` purls directly, so a component is looked up by its purl with no translation to a repository URL and no retry +- `packageurl-go` is no longer a direct dependency +- A failed OSV lookup is now reported as `Failed to query OSV data` rather than `No vulnerabilities found`, so a broken query is no longer indistinguishable from a component with no vulnerabilities +- OSV use case tests no longer reach the network; they run on SQLite against a fixture covering both version-matching mechanisms, multi-vector CVSS and non-CVSS scores +- Advisories published under repackager ecosystems (`TuxCare:Maven`, `Echo:npm` and the like) are excluded. They share the purl of the upstream package, so a lookup by purl alone picked them up while the OSV API, which filters by ecosystem, does not return them. Their fixed versions are unreachable from the upstream registry, they carry no CVE aliases, and their `introduced_version` is `0`, so they attached to every version of every affected component. This brought `pkg:maven/org.apache.logging.log4j/log4j-core@2.0.0` from 18 advisories back to the 8 the API returns. Note the filter is by vendor prefix, not by "ecosystem contains a colon": legitimate distro ecosystems are versioned that way (`Ubuntu:22.04:LTS`, `Debian:12`) and cover 69% of the table + +### Added +- `pkg/models/osv.go`, reading OSV data with one portable query per engine and collapsing the table's per-range rows into one entry per vulnerability +- `pkg/models/osv_array.go`, parsing the PostgreSQL array literal form of the list columns. 1,082 production rows have a quoted element and some contain a comma, so splitting on commas alone would invent versions +- `TestOSVParity`, comparing the model against the live OSV API. Skipped unless `PG_DSN` and `OSV_PARITY` are set + +### Removed +- `VULN_OSV_API_BASE_URL`. It is no longer read, and the config no longer rejects an empty value, which used to prevent startup for a setting that did nothing + +### Fixed +- README documented `OSV_ENABLED`, `OSV_API_BASE_URL` and `OSV_VULNERABILITY_INFO_BASE_URL`, none of which match the environment variables the service actually reads + +### Deployment +- Requires the `osv` and `osv_severity` tables. `osv` must carry every OSV `affected` entry, including those whose ranges are of type `ECOSYSTEM` (93% of them); a load that keeps only `SEMVER` ranges silently drops vulnerabilities +- Known limitation: OSV marks retracted entries with `withdrawn` and the table has no such column, so 43,739 retracted vulnerabilities across 192,159 rows are reported as live. Adding the column is pending + ## [0.14.0] - 2026-08-11 ### Added - SQLite support alongside PostgreSQL: set `DB_DRIVER=sqlite` and `DB_DSN` to a database file diff --git a/README.md b/README.md index 919f04a..a9b81ea 100644 --- a/README.md +++ b/README.md @@ -50,13 +50,18 @@ DB_SSL_MODE=disable # DB_DSN=/path/to/vulnerabilities.db # Vulnerability data sources -OSV_ENABLED=true # Enable/disable OSV (Open Source Vulnerabilities) database -OSV_API_BASE_URL=https://api.osv.dev/v1 -OSV_VULNERABILITY_INFO_BASE_URL=https://osv.dev/vulnerability +VULN_OSV_SOURCE_ENABLED=true # Enable/disable OSV (Open Source Vulnerabilities) data +VULN_OSV_INFO_BASE_URL=https://osv.dev/vulnerability # Builds the URL of each vulnerability returned +VULN_OSV_API_WORKERS=5 # Components looked up concurrently -SCANOSS_ENABLED=true # Enable/disable SCANOSS vulnerability database +VULN_SCANOSS_SOURCE_ENABLED=true # Enable/disable SCANOSS vulnerability data +VULN_SCANOSS_WORKERS=5 # Components looked up concurrently ``` +Both vulnerability sources are read from the database. OSV is no longer queried over +HTTP, so the service needs the `osv` and `osv_severity` tables to be present and +populated; without them, OSV lookups report no vulnerabilities. + ## Docker Environment The vulnerability server can be deployed as a Docker container. diff --git a/go.mod b/go.mod index 4872ca5..32b1ec6 100644 --- a/go.mod +++ b/go.mod @@ -8,7 +8,6 @@ require ( github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 github.com/jmoiron/sqlx v1.4.0 github.com/lib/pq v1.12.3 - github.com/package-url/packageurl-go v0.1.5 github.com/pandatix/go-cvss v0.6.2 github.com/scanoss/go-component-helper v0.6.0 github.com/scanoss/go-grpc-helper v0.16.0 @@ -36,6 +35,7 @@ require ( github.com/mattn/go-isatty v0.0.20 // indirect github.com/mattn/go-sqlite3 v1.14.42 // indirect github.com/ncruces/go-strftime v1.0.0 // indirect + github.com/package-url/packageurl-go v0.1.5 // indirect github.com/phuslu/iploc v1.0.20230201 // indirect github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect github.com/scanoss/go-models v0.8.0 // indirect diff --git a/pkg/config/server_config.go b/pkg/config/server_config.go index c3aa0f5..c5504f3 100644 --- a/pkg/config/server_config.go +++ b/pkg/config/server_config.go @@ -73,10 +73,13 @@ type ServerConfig struct { } Source struct { OSV struct { - APIBaseURL string `env:"VULN_OSV_API_BASE_URL"` + // InfoBaseURL builds the URL of each returned vulnerability. There is no API + // base URL any more: OSV data is read from the database, not from api.osv.dev. InfoBaseURL string `env:"VULN_OSV_INFO_BASE_URL"` Enabled bool `env:"VULN_OSV_SOURCE_ENABLED"` - APIWorkers int `env:"VULN_OSV_API_WORKERS"` + // APIWorkers caps how many components are looked up concurrently. The name is + // kept so existing deployments keep their setting. + APIWorkers int `env:"VULN_OSV_API_WORKERS"` } SCANOSS struct { Enabled bool `env:"VULN_SCANOSS_SOURCE_ENABLED"` @@ -120,7 +123,6 @@ func setServerConfigDefaults(cfg *ServerConfig) { cfg.Telemetry.Enabled = false cfg.Telemetry.OltpExporter = "0.0.0.0:4317" // Default OTEL OLTP gRPC Exporter endpoint cfg.Components.CommitMissing = false - cfg.Source.OSV.APIBaseURL = "https://api.osv.dev/v1" cfg.Source.OSV.InfoBaseURL = "https://osv.dev/vulnerability" cfg.Source.OSV.Enabled = true cfg.Source.OSV.APIWorkers = 5 @@ -136,9 +138,6 @@ func IsValidConfig(cfg *ServerConfig) error { // Check OSV source config if cfg.Source.OSV.Enabled { - if cfg.Source.OSV.APIBaseURL == "" { - return errors.New("OSV API Base URL cannot be empty") - } if cfg.Source.OSV.InfoBaseURL == "" { return errors.New("OSV Info Base URL cannot be empty") } diff --git a/pkg/config/server_config_test.go b/pkg/config/server_config_test.go index b4e3949..0034377 100644 --- a/pkg/config/server_config_test.go +++ b/pkg/config/server_config_test.go @@ -104,9 +104,9 @@ func TestConfigValidation(t *testing.T) { expectError: true, }, { - name: "invalid with empty OSV API base URL", + name: "invalid with empty OSV info base URL", modifyConf: func(c *ServerConfig) { - c.Source.OSV.APIBaseURL = "" + c.Source.OSV.InfoBaseURL = "" }, expectError: true, }, diff --git a/pkg/models/common.go b/pkg/models/common.go index c63cf21..4c46812 100644 --- a/pkg/models/common.go +++ b/pkg/models/common.go @@ -81,6 +81,7 @@ var testDataFiles = []string{ "../models/tests/projects.sql", "../models/tests/epss.sql", "../models/tests/vulns_scenario.sql", + "../models/tests/osv_scenario.sql", } // LoadTestSchema creates the production schema in the supplied DB. Call this before diff --git a/pkg/models/osv.go b/pkg/models/osv.go new file mode 100644 index 0000000..0cb5b19 --- /dev/null +++ b/pkg/models/osv.go @@ -0,0 +1,287 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (C) 2018-2025 SCANOSS.COM + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 2 of the License, or + * (at your option) any later version. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +// Reading OSV vulnerabilities from the database instead of the OSV HTTP API. +// +// Two things the API did for us have to happen here instead: +// +// - Version matching. The API took a version and returned only the applicable +// vulnerabilities. The table exposes the raw bounds, and OSV uses two mechanisms +// that both have to be honoured: an explicit list in affected_versions, and a +// range built from introduced_version / fixed_version / last_affected. Which one a +// row uses varies by ecosystem - pypi is almost all lists, golang almost all ranges. +// +// - One row per vulnerability. The table is keyed by +// (id, ecosystem, purl, introduced_version, fixed_version), so it averages 15.6 +// rows per vulnerability, one per affected range. The API returns a single entry, +// so rows are collapsed by id here. +// +// CVSS vectors live in osv_severity, one row per vector, because a vulnerability can +// carry up to five. They are fetched in a second query rather than joined, to avoid +// multiplying the already-duplicated rows of the main query. + +package models + +import ( + "context" + "errors" + "fmt" + "strings" + "time" + + "github.com/jmoiron/sqlx" + "go.uber.org/zap" + "scanoss.com/vulnerabilities/pkg/utils" +) + +// OSVModel queries the OSV tables. +type OSVModel struct { + db *sqlx.DB + s *zap.SugaredLogger +} + +// OSVVulnerability is one OSV vulnerability, already collapsed to a single entry and +// carrying every CVSS vector recorded for it. +type OSVVulnerability struct { + ID string + Aliases []string + Summary string + Severity string + Published utils.OnlyDate + Modified utils.OnlyDate + Severities []OSVSeverity +} + +// OSVSeverity is a single scoring entry. Type is kept alongside Score because not +// everything OSV publishes is a CVSS vector: 54,565 rows are of type "Ubuntu" with +// values like "medium". Callers decide what to do with those. +type OSVSeverity struct { + Type string `db:"type"` + Score string `db:"score"` +} + +// osvQueryWithArgs builds the lookup, appending one exclusion per repackager ecosystem. +// Placeholders are neutral and rebound by the driver, so the same builder serves both +// engines. +func osvQueryWithArgs(purl string) (string, []interface{}) { + query := osvByPurlQuery + args := []interface{}{purl} + for _, pattern := range osvRepackagerEcosystems { + query += "\n AND o.ecosystem NOT LIKE ?" + args = append(args, pattern) + } + return query + "\nORDER BY o.id", args +} + +// osvRow is one (vulnerability, affected range) pair as stored. +type osvRow struct { + ID string `db:"id"` + Aliases string `db:"aliases"` + Summary string `db:"summary"` + Severity string `db:"severity"` + Published utils.OnlyDate `db:"published"` + Modified utils.OnlyDate `db:"modified"` + IntroducedVersion string `db:"introduced_version"` + LastAffected string `db:"last_affected"` + FixedVersion string `db:"fixed_version"` + AffectedVersions string `db:"affected_versions"` +} + +// Repackager ecosystems, excluded from results. OSV publishes advisories for vendors +// that rebuild libraries under their own ecosystem name, following the pattern +// Vendor:LanguageEcosystem - TuxCare:Maven, Echo:npm and so on. Their rows sit under the +// same purl as the upstream package, so a lookup by purl alone picks them up while the +// OSV API, which filters by ecosystem, does not return them. +// +// They are dropped because their fixed versions are unreachable from the upstream +// registry (log4j-core "2.22.1-tuxcare.2" does not exist in Maven Central), they carry +// no aliases tying them to a CVE, and they use introduced_version 0, so they would +// attach to every version of every affected component. +// +// TO RESTORE THEM: empty this list. Nothing else depends on it. Doing so adds 10 +// advisories to pkg:maven/org.apache.logging.log4j/log4j-core, for example, and there is +// a test pinning the current behaviour that would need updating. +// +// Note that the filter must not be "ecosystem contains a colon": legitimate distro +// ecosystems are versioned that way (Ubuntu:22.04:LTS, Debian:12) and that pattern +// covers 3,770,300 of 5,480,673 rows - excluding them would drop every Debian and Ubuntu +// package vulnerability. The repackagers are 31,093 rows, 0.57%. +var osvRepackagerEcosystems = []string{"TuxCare:%", "Echo:%"} + +// osvByPurlQuery returns every stored range for a purl. The CAST on the array columns +// is what keeps this portable: PostgreSQL renders them as {a,b}, and the SQLite export +// stores that same text, so one query serves both engines. COALESCE guards the columns +// the schema allows to be null. +const osvByPurlQuery = `SELECT + o.id, + CAST(o.aliases AS TEXT) AS aliases, + COALESCE(o.summary, '') AS summary, + COALESCE(o.severity, '') AS severity, + o.published, + o.modified, + COALESCE(o.introduced_version, '') AS introduced_version, + COALESCE(o.last_affected, '') AS last_affected, + COALESCE(o.fixed_version, '') AS fixed_version, + CAST(o.affected_versions AS TEXT) AS affected_versions +FROM osv o +WHERE o.purl = ?` + +// NewOSVModel creates a new instance of the OSV Model. +func NewOSVModel(s *zap.SugaredLogger, db *sqlx.DB) *OSVModel { + return &OSVModel{db: db, s: s} +} + +// GetVulnsByPurl returns the OSV vulnerabilities affecting the given purl. An empty +// version means every vulnerability recorded for the purl, matching what the API +// returns when no version is supplied. +func (m *OSVModel) GetVulnsByPurl(ctx context.Context, purl string, version string) ([]OSVVulnerability, error) { + if len(strings.TrimSpace(purl)) == 0 { + m.s.Error("Please specify a valid Purl to query") + return nil, errors.New("please specify a valid Purl to query") + } + query, args := osvQueryWithArgs(strings.TrimSpace(purl)) + var rows []osvRow + err := m.db.SelectContext(ctx, &rows, m.db.Rebind(query), args...) + if err != nil { + m.s.Errorf("Failed to query the osv table for %v: %v", purl, err) + return nil, fmt.Errorf("failed to query the osv table: %v", err) + } + vulns := collapseOSVRows(rows, version) + if len(vulns) == 0 { + return vulns, nil + } + if err = m.attachSeverities(ctx, vulns); err != nil { + // The vulnerabilities themselves are still usable without their vectors. + m.s.Warnf("Failed to load CVSS vectors for %v: %v", purl, err) + } + m.s.Debugf("Found %v OSV vulnerabilities for %v (version %v)", len(vulns), purl, version) + return vulns, nil +} + +// collapseOSVRows reduces the rows to one entry per vulnerability, keeping only the +// vulnerabilities that have at least one range covering the requested version. Order +// follows the query, so the result is stable. +// +// Which row supplies the vulnerability's own fields is not arbitrary. Those fields are +// meant to be identical across the rows of one id, but in practice they are not: +// 137,302 (id, purl) pairs disagree on at least one of them, overwhelmingly on +// modified. The most recently modified row is the one that agrees with osv_json, and +// therefore with what the API returns, so that is the row used - even when a different +// row is the one that matched the version. +func collapseOSVRows(rows []osvRow, version string) []OSVVulnerability { + type candidate struct { + newest osvRow + matched bool + } + byID := make(map[string]*candidate, len(rows)) + ids := make([]string, 0, len(rows)) + for _, row := range rows { + found, ok := byID[row.ID] + if !ok { + byID[row.ID] = &candidate{newest: row, matched: row.affects(version)} + ids = append(ids, row.ID) + continue + } + if time.Time(row.Modified).After(time.Time(found.newest.Modified)) { + found.newest = row + } + if row.affects(version) { + found.matched = true + } + } + vulns := make([]OSVVulnerability, 0, len(ids)) + for _, id := range ids { + found := byID[id] + if !found.matched { + continue + } + vulns = append(vulns, OSVVulnerability{ + ID: found.newest.ID, + Aliases: osvArrayValues(found.newest.Aliases), + Summary: found.newest.Summary, + Severity: found.newest.Severity, + Published: found.newest.Published, + Modified: found.newest.Modified, + }) + } + return vulns +} + +// affects reports whether this stored range covers the given version. An empty version +// matches, so a caller that does not know the version still gets the vulnerability. +func (r osvRow) affects(version string) bool { + version = strings.TrimSpace(version) + if len(version) == 0 { + return true + } + // OSV allows an affected entry to carry both an explicit version list and ranges; + // either one matching is enough. + if osvArrayContains(r.AffectedVersions, version) { + return true + } + introduced := strings.TrimSpace(r.IntroducedVersion) + fixed := strings.TrimSpace(r.FixedVersion) + lastAffected := strings.TrimSpace(r.LastAffected) + // With no bounds at all and no list, the row says nothing about versions. Treat a + // bare list that did not match as a miss, rather than as an unbounded range. + if len(introduced) == 0 && len(fixed) == 0 && len(lastAffected) == 0 { + return len(osvArrayValues(r.AffectedVersions)) == 0 + } + key := naturalSortKey(version) + // "0" is OSV's way of saying "since the beginning". + if len(introduced) > 0 && introduced != "0" && key < naturalSortKey(introduced) { + return false + } + // fixed_version is exclusive: the fix landed in it, so it is not affected. + if len(fixed) > 0 && key >= naturalSortKey(fixed) { + return false + } + // last_affected is inclusive. + if len(lastAffected) > 0 && key > naturalSortKey(lastAffected) { + return false + } + return true +} + +// attachSeverities fills in the CVSS vectors for the given vulnerabilities. +func (m *OSVModel) attachSeverities(ctx context.Context, vulns []OSVVulnerability) error { + ids := make([]string, 0, len(vulns)) + for _, v := range vulns { + ids = append(ids, v.ID) + } + type severityRow struct { + ID string `db:"id"` + Type string `db:"type"` + Score string `db:"score"` + } + query, args, err := sqlx.In( + "SELECT id, COALESCE(type, '') AS type, COALESCE(score, '') AS score FROM osv_severity WHERE id IN (?)", ids) + if err != nil { + return err + } + var rows []severityRow + if err = m.db.SelectContext(ctx, &rows, m.db.Rebind(query), args...); err != nil { + return err + } + byID := make(map[string][]OSVSeverity, len(rows)) + for _, row := range rows { + byID[row.ID] = append(byID[row.ID], OSVSeverity{Type: row.Type, Score: row.Score}) + } + for i := range vulns { + vulns[i].Severities = byID[vulns[i].ID] + } + return nil +} diff --git a/pkg/models/osv_array.go b/pkg/models/osv_array.go new file mode 100644 index 0000000..0171379 --- /dev/null +++ b/pkg/models/osv_array.go @@ -0,0 +1,92 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (C) 2018-2025 SCANOSS.COM + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 2 of the License, or + * (at your option) any later version. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +// Reading the list-valued columns of the osv table. +// +// osv.affected_versions, aliases, upstream and related are PostgreSQL text arrays. +// SQLite has no array type, so the queries cast them to text and the SQLite export +// stores that same text form. Both engines therefore hand us PostgreSQL's array +// literal syntax, which this file parses. +// +// The quoting rules matter and are not hypothetical: 1,082 rows in production have a +// quoted element in affected_versions, and some of those elements contain a comma +// (`{"v1,1",v1.1}`). Splitting on commas alone would silently invent versions. + +package models + +import "strings" + +// osvArrayValues parses a PostgreSQL array literal into its elements. An empty array +// ({}), an empty string or a NULL-ish value all yield no elements. +func osvArrayValues(raw string) []string { + raw = strings.TrimSpace(raw) + if len(raw) == 0 || raw == "{}" || raw == "NULL" { + return nil + } + // Tolerate a value that arrives without the braces. + if strings.HasPrefix(raw, "{") && strings.HasSuffix(raw, "}") { + raw = raw[1 : len(raw)-1] + } + if len(raw) == 0 { + return nil + } + var ( + values []string + current strings.Builder + inQuotes bool + escaped bool + ) + flush := func() { + value := current.String() + current.Reset() + // Unquoted elements carry no significant surrounding space; quoted ones do, and + // have already been emitted verbatim. + values = append(values, value) + } + for i := 0; i < len(raw); i++ { + char := raw[i] + switch { + case escaped: + current.WriteByte(char) + escaped = false + case char == '\\': + escaped = true + case char == '"': + inQuotes = !inQuotes + case char == ',' && !inQuotes: + flush() + default: + current.WriteByte(char) + } + } + flush() + // Trim only the elements that were not quoted; a quoted element keeps its spaces. + out := make([]string, 0, len(values)) + for _, value := range values { + out = append(out, strings.TrimSpace(value)) + } + return out +} + +// osvArrayContains reports whether the array literal holds the given value. +func osvArrayContains(raw, value string) bool { + for _, candidate := range osvArrayValues(raw) { + if candidate == value { + return true + } + } + return false +} diff --git a/pkg/models/osv_array_test.go b/pkg/models/osv_array_test.go new file mode 100644 index 0000000..232a6e2 --- /dev/null +++ b/pkg/models/osv_array_test.go @@ -0,0 +1,118 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (C) 2018-2025 SCANOSS.COM + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 2 of the License, or + * (at your option) any later version. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package models + +import ( + "reflect" + "testing" +) + +func TestOSVArrayValues(t *testing.T) { + tests := []struct { + name string + raw string + want []string + }{ + {name: "empty array", raw: "{}", want: nil}, + {name: "empty string", raw: "", want: nil}, + {name: "single value", raw: "{0.0.1}", want: []string{"0.0.1"}}, + { + name: "several values", + raw: "{v1.0.0,v1.0.1,v1.1.0}", + want: []string{"v1.0.0", "v1.0.1", "v1.1.0"}, + }, + { + name: "single alias", + raw: "{CVE-2018-16342}", + want: []string{"CVE-2018-16342"}, + }, + { + name: "two aliases, as the cve field is derived from the first", + raw: "{CVE-2026-56812,GHSA-63mc-hw7g-86rr}", + want: []string{"CVE-2026-56812", "GHSA-63mc-hw7g-86rr"}, + }, + { + // production row: the quoted element contains a comma, so splitting on commas + // alone would yield "v1" and "1" - two versions that do not exist + name: "quoted value containing a comma", + raw: `{"v1,1",v1.1}`, + want: []string{"v1,1", "v1.1"}, + }, + { + // production row from affected_versions + name: "quoted value with spaces and operators", + raw: `{3.2.0-beta1,"beta: <= 3.2.0.beta2",v3.1.1}`, + want: []string{"3.2.0-beta1", "beta: <= 3.2.0.beta2", "v3.1.1"}, + }, + { + name: "quoted value with an equals sign", + raw: `{"= 1.6.0",v1.6.0}`, + want: []string{"= 1.6.0", "v1.6.0"}, + }, + { + name: "escaped quote inside a value", + raw: `{"a\"b",c}`, + want: []string{`a"b`, "c"}, + }, + { + name: "escaped backslash", + raw: `{"a\\b"}`, + want: []string{`a\b`}, + }, + {name: "value without braces is tolerated", raw: "1.0.0", want: []string{"1.0.0"}}, + {name: "NULL text", raw: "NULL", want: nil}, + { + name: "surrounding whitespace on unquoted values", + raw: "{ 1.0.0 , 2.0.0 }", + want: []string{"1.0.0", "2.0.0"}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := osvArrayValues(tt.raw); !reflect.DeepEqual(got, tt.want) { + t.Errorf("osvArrayValues(%q) = %#v, want %#v", tt.raw, got, tt.want) + } + }) + } +} + +func TestOSVArrayContains(t *testing.T) { + tests := []struct { + name string + raw string + value string + want bool + }{ + {name: "present", raw: "{1.0.0,2.0.0}", value: "2.0.0", want: true}, + {name: "absent", raw: "{1.0.0,2.0.0}", value: "3.0.0", want: false}, + {name: "empty array", raw: "{}", value: "1.0.0", want: false}, + {name: "exact match only, not a prefix", raw: "{1.0.0}", value: "1.0", want: false}, + {name: "quoted value with comma", raw: `{"v1,1",v1.1}`, value: "v1,1", want: true}, + { + name: "a comma inside a quoted value is not a separator", + raw: `{"v1,1"}`, value: "v1", want: false, + }, + {name: "v prefix is significant", raw: "{v1.0.0}", value: "1.0.0", want: false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := osvArrayContains(tt.raw, tt.value); got != tt.want { + t.Errorf("osvArrayContains(%q, %q) = %v, want %v", tt.raw, tt.value, got, tt.want) + } + }) + } +} diff --git a/pkg/models/osv_parity_test.go b/pkg/models/osv_parity_test.go new file mode 100644 index 0000000..26423ca --- /dev/null +++ b/pkg/models/osv_parity_test.go @@ -0,0 +1,194 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (C) 2018-2025 SCANOSS.COM + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 2 of the License, or + * (at your option) any later version. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +// Parity check between OSVModel and the OSV HTTP API it replaces. This is what +// validates the version matching in osv.go: the API used to decide which +// vulnerabilities applied to a version, and now we do. +// +// Skipped unless both PG_DSN and OSV_PARITY are set, so it never runs in CI and never +// reaches the network unless asked: +// +// PG_DSN='postgres://user:pass@host:5432/db?sslmode=disable' OSV_PARITY=1 \ +// go test ./pkg/models/ -run TestOSVParity -v -timeout 900s +// +// Anything the API reports that the model misses is a false negative: a vulnerability +// the service would stop reporting. Those are the ones that matter. + +package models + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "net/http" + "os" + "sort" + "testing" + "time" + + "github.com/jmoiron/sqlx" + _ "github.com/lib/pq" + zlog "github.com/scanoss/zap-logging-helper/pkg/logger" +) + +// osvAPIQuery asks the live OSV API which vulnerabilities affect a purl at a version. +func osvAPIQuery(t *testing.T, client *http.Client, purl, version string) []string { + t.Helper() + body, err := json.Marshal(map[string]interface{}{ + "package": map[string]string{"purl": purl}, + "version": version, + }) + if err != nil { + t.Fatalf("marshal: %v", err) + } + req, err := http.NewRequestWithContext(context.Background(), + http.MethodPost, "https://api.osv.dev/v1/query", bytes.NewBuffer(body)) + if err != nil { + t.Fatalf("request: %v", err) + } + req.Header.Set("Content-Type", "application/json") + resp, err := client.Do(req) + if err != nil { + t.Logf(" API call failed for %v@%v: %v", purl, version, err) + return nil + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + t.Logf(" API returned %d for %v@%v", resp.StatusCode, purl, version) + return nil + } + var decoded struct { + Vulns []struct { + ID string `json:"id"` + } `json:"vulns"` + } + if err = json.NewDecoder(resp.Body).Decode(&decoded); err != nil { + t.Logf(" decode failed for %v@%v: %v", purl, version, err) + return nil + } + ids := make([]string, 0, len(decoded.Vulns)) + for _, v := range decoded.Vulns { + ids = append(ids, v.ID) + } + sort.Strings(ids) + return ids +} + +func TestOSVParity(t *testing.T) { + dsn := os.Getenv("PG_DSN") + if dsn == "" || os.Getenv("OSV_PARITY") == "" { + t.Skip("PG_DSN and OSV_PARITY not both set") + } + _ = zlog.NewSugaredDevLogger() + defer zlog.SyncZap() + ctx := context.Background() + db, err := sqlx.Connect("postgres", dsn) + if err != nil { + t.Fatalf("connect: %v", err) + } + defer db.Close() + model := NewOSVModel(zlog.S, db) + client := &http.Client{Timeout: 30 * time.Second} + + // Sample real (purl, version) pairs across ecosystems, drawn from stored ranges so + // the version is one the data actually says something about. + type sample struct { + Purl string `db:"purl"` + Version string `db:"version"` + } + var samples []sample + err = db.SelectContext(ctx, &samples, ` + SELECT purl, version FROM ( + SELECT o.purl, + coalesce(nullif(o.affected_versions[1], ''), nullif(o.introduced_version,''), '') AS version, + row_number() OVER (PARTITION BY split_part(o.purl,'/',1) ORDER BY o.id) AS rn + FROM osv o + WHERE split_part(o.purl,'/',1) IN ('pkg:npm','pkg:pypi','pkg:maven','pkg:golang','pkg:cargo','pkg:gem') + AND o.purl NOT LIKE '%25%' + ) t WHERE version <> '' AND version <> '0' AND rn <= 4 + ORDER BY purl`) + if err != nil { + t.Fatalf("sampling failed: %v", err) + } + t.Logf("comparing %d purl/version pairs", len(samples)) + + var checked, agree, falseNegatives, falsePositives int + for _, s := range samples { + apiIDs := osvAPIQuery(t, client, s.Purl, s.Version) + vulns, err := model.GetVulnsByPurl(ctx, s.Purl, s.Version) + if err != nil { + t.Errorf("%v@%v: model failed: %v", s.Purl, s.Version, err) + continue + } + dbIDs := make([]string, 0, len(vulns)) + for _, v := range vulns { + dbIDs = append(dbIDs, v.ID) + } + sort.Strings(dbIDs) + + inAPI := map[string]bool{} + for _, id := range apiIDs { + inAPI[id] = true + } + inDB := map[string]bool{} + for _, id := range dbIDs { + inDB[id] = true + } + var missing, extra []string + for _, id := range apiIDs { + if !inDB[id] { + missing = append(missing, id) + } + } + for _, id := range dbIDs { + if !inAPI[id] { + extra = append(extra, id) + } + } + checked++ + switch { + case len(missing) == 0 && len(extra) == 0: + agree++ + fmt.Printf(" OK %-45s %-12s api=%-3d db=%-3d\n", s.Purl, s.Version, len(apiIDs), len(dbIDs)) + default: + if len(missing) > 0 { + falseNegatives++ + } + if len(extra) > 0 { + falsePositives++ + } + fmt.Printf(" DIFF %-45s %-12s api=%-3d db=%-3d missing=%d extra=%d\n", + s.Purl, s.Version, len(apiIDs), len(dbIDs), len(missing), len(extra)) + if len(missing) > 0 { + fmt.Printf(" missing (API has, model does not): %v\n", missing) + } + if len(extra) > 0 { + fmt.Printf(" extra (model has, API does not): %v\n", extra) + } + } + } + fmt.Printf("\nchecked=%d agree=%d with_false_negatives=%d with_false_positives=%d\n", + checked, agree, falseNegatives, falsePositives) + if checked == 0 { + t.Errorf("no pairs were compared") + } + // False negatives are the failure that matters: vulnerabilities the service would + // stop reporting relative to the API. + if falseNegatives > 0 { + t.Errorf("%d of %d pairs miss vulnerabilities the API reports", falseNegatives, checked) + } +} diff --git a/pkg/models/osv_test.go b/pkg/models/osv_test.go new file mode 100644 index 0000000..d3a9af2 --- /dev/null +++ b/pkg/models/osv_test.go @@ -0,0 +1,282 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (C) 2018-2025 SCANOSS.COM + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 2 of the License, or + * (at your option) any later version. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package models + +import ( + "testing" + "time" + + zlog "github.com/scanoss/zap-logging-helper/pkg/logger" + + "scanoss.com/vulnerabilities/pkg/utils" +) + +func onlyDate(value string) utils.OnlyDate { + return utils.OnlyDate(utils.ParseTime(value)) +} + +func TestOSVRowAffects(t *testing.T) { + tests := []struct { + name string + row osvRow + version string + want bool + why string + }{ + { + name: "explicit version list, present", + row: osvRow{AffectedVersions: "{5.1.11,5.1.12,5.1.13}"}, + version: "5.1.12", want: true, + why: "pypi and npm rows mostly use an explicit list", + }, + { + name: "explicit version list, absent", + row: osvRow{AffectedVersions: "{5.1.11,5.1.12}"}, + version: "5.1.99", want: false, + why: "a list that does not contain the version is a miss, not an open range", + }, + { + name: "introduced 0 means from the beginning", + row: osvRow{IntroducedVersion: "0", FixedVersion: "3.39.9"}, + version: "1.0.0", want: true, + why: "0 is how OSV expresses an unbounded lower end", + }, + { + name: "fixed version is exclusive", + row: osvRow{IntroducedVersion: "0", FixedVersion: "3.39.9"}, + version: "3.39.9", want: false, + why: "the fix landed in that version, so it is not affected", + }, + { + name: "just below the fix", + row: osvRow{IntroducedVersion: "0", FixedVersion: "3.39.9"}, + version: "3.39.8", want: true, + }, + { + name: "below the introduced bound", + row: osvRow{IntroducedVersion: "1.2.0", FixedVersion: "1.5.15"}, + version: "1.1.0", want: false, + }, + { + name: "inside a closed range", + row: osvRow{IntroducedVersion: "1.2.0", FixedVersion: "1.5.15"}, + version: "1.3.0", want: true, + }, + { + name: "at the introduced bound, which is inclusive", + row: osvRow{IntroducedVersion: "1.2.0", FixedVersion: "1.5.15"}, + version: "1.2.0", want: true, + }, + { + name: "last_affected is inclusive", + row: osvRow{IntroducedVersion: "1.0.0", LastAffected: "2.0.0"}, + version: "2.0.0", want: true, + }, + { + name: "beyond last_affected", + row: osvRow{IntroducedVersion: "1.0.0", LastAffected: "2.0.0"}, + version: "2.0.1", want: false, + }, + { + name: "open ended range with only introduced", + row: osvRow{IntroducedVersion: "2.0.0"}, + version: "99.0.0", want: true, + }, + { + name: "no bounds and no list affects everything", + row: osvRow{}, + version: "1.0.0", want: true, + why: "rows like this exist and mean the package is affected outright", + }, + { + name: "empty version matches, so a caller without one still sees the vuln", + row: osvRow{IntroducedVersion: "1.0.0", FixedVersion: "2.0.0"}, + version: "", want: true, + }, + { + name: "list and range together, matched by the list", + row: osvRow{AffectedVersions: "{9.9.9}", IntroducedVersion: "1.0.0", FixedVersion: "2.0.0"}, + version: "9.9.9", want: true, + why: "OSV allows both on one entry; either matching is enough", + }, + { + name: "list and range together, matched by the range", + row: osvRow{AffectedVersions: "{9.9.9}", IntroducedVersion: "1.0.0", FixedVersion: "2.0.0"}, + version: "1.5.0", want: true, + }, + { + name: "numeric ordering, not string ordering", + row: osvRow{IntroducedVersion: "0", FixedVersion: "10.0.0"}, + version: "9.0.0", want: true, + why: "a plain string compare would place 9.0.0 after 10.0.0 and miss it", + }, + { + name: "version with a v prefix, as stored in affected_versions", + row: osvRow{AffectedVersions: "{v1.0.0,v1.0.1}"}, + version: "v1.0.1", want: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := tt.row.affects(tt.version); got != tt.want { + t.Errorf("affects(%q) = %v, want %v (%s)", tt.version, got, tt.want, tt.why) + } + }) + } +} + +// TestCollapseOSVRowsDeduplicates covers the row-per-range shape of the table: the API +// returns one entry per vulnerability, the table averages 15.6 rows. +func TestCollapseOSVRowsDeduplicates(t *testing.T) { + rows := []osvRow{ + {ID: "GHSA-1", IntroducedVersion: "1.0.0", FixedVersion: "1.5.0", Summary: "one"}, + {ID: "GHSA-1", IntroducedVersion: "2.0.0", FixedVersion: "2.5.0", Summary: "one"}, + {ID: "GHSA-1", IntroducedVersion: "3.0.0", FixedVersion: "3.5.0", Summary: "one"}, + } + got := collapseOSVRows(rows, "2.1.0") + if len(got) != 1 { + t.Fatalf("collapseOSVRows() returned %d entries, want 1", len(got)) + } + if got[0].ID != "GHSA-1" { + t.Errorf("ID = %q, want %q", got[0].ID, "GHSA-1") + } +} + +// TestCollapseOSVRowsMatchesOnAnyRange checks a vulnerability is kept when any of its +// ranges covers the version, even if earlier rows do not. +func TestCollapseOSVRowsMatchesOnAnyRange(t *testing.T) { + rows := []osvRow{ + {ID: "GHSA-1", IntroducedVersion: "1.0.0", FixedVersion: "1.5.0"}, + {ID: "GHSA-1", IntroducedVersion: "3.0.0", FixedVersion: "3.5.0"}, + } + if got := collapseOSVRows(rows, "3.1.0"); len(got) != 1 { + t.Errorf("a version matching only the second range returned %d entries, want 1", len(got)) + } + if got := collapseOSVRows(rows, "2.0.0"); len(got) != 0 { + t.Errorf("a version matching no range returned %d entries, want 0", len(got)) + } +} + +// TestCollapseOSVRowsUsesNewestRow pins which row supplies the vulnerability fields. +// 137,302 (id, purl) pairs in production disagree across rows, almost always on +// modified, and the newest row is the one that agrees with osv_json and the API. +func TestCollapseOSVRowsUsesNewestRow(t *testing.T) { + rows := []osvRow{ + { + ID: "GHSA-2g4f-4pwh-qvx6", Summary: "stale text", Severity: "LOW", + Modified: onlyDate("2026-02-19"), Aliases: "{CVE-OLD}", + IntroducedVersion: "0", FixedVersion: "9.9.9", + }, + { + ID: "GHSA-2g4f-4pwh-qvx6", Summary: "current text", Severity: "HIGH", + Modified: onlyDate("2026-03-04"), Aliases: "{CVE-NEW}", + IntroducedVersion: "0", FixedVersion: "9.9.9", + }, + } + got := collapseOSVRows(rows, "1.0.0") + if len(got) != 1 { + t.Fatalf("returned %d entries, want 1", len(got)) + } + if got[0].Summary != "current text" { + t.Errorf("Summary = %q, want the newest row's value %q", got[0].Summary, "current text") + } + if got[0].Severity != "HIGH" { + t.Errorf("Severity = %q, want %q", got[0].Severity, "HIGH") + } + if len(got[0].Aliases) != 1 || got[0].Aliases[0] != "CVE-NEW" { + t.Errorf("Aliases = %v, want [CVE-NEW]", got[0].Aliases) + } + if want := utils.ParseTime("2026-03-04"); !want.Equal(time.Time(got[0].Modified)) { + t.Errorf("Modified = %v, want %v", time.Time(got[0].Modified), want) + } +} + +// TestCollapseOSVRowsNewestRowWinsRegardlessOfWhichMatched checks the newest row supplies +// the fields even when a different row is the one covering the version. +func TestCollapseOSVRowsNewestRowWinsRegardlessOfWhichMatched(t *testing.T) { + rows := []osvRow{ + // this row matches the version + { + ID: "GHSA-1", Summary: "old", Modified: onlyDate("2026-01-01"), + IntroducedVersion: "1.0.0", FixedVersion: "2.0.0", + }, + // this one does not, but is newer + { + ID: "GHSA-1", Summary: "new", Modified: onlyDate("2026-06-01"), + IntroducedVersion: "5.0.0", FixedVersion: "6.0.0", + }, + } + got := collapseOSVRows(rows, "1.5.0") + if len(got) != 1 { + t.Fatalf("returned %d entries, want 1", len(got)) + } + if got[0].Summary != "new" { + t.Errorf("Summary = %q, want %q: the newest row supplies the fields even when "+ + "another row matched the version", got[0].Summary, "new") + } +} + +// TestCollapseOSVRowsPreservesOrder guards a stable response ordering. +func TestCollapseOSVRowsPreservesOrder(t *testing.T) { + rows := []osvRow{ + {ID: "GHSA-a"}, {ID: "GHSA-b"}, {ID: "GHSA-a"}, {ID: "GHSA-c"}, + } + got := collapseOSVRows(rows, "1.0.0") + want := []string{"GHSA-a", "GHSA-b", "GHSA-c"} + if len(got) != len(want) { + t.Fatalf("returned %d entries, want %d", len(got), len(want)) + } + for i := range want { + if got[i].ID != want[i] { + t.Errorf("entry %d = %q, want %q", i, got[i].ID, want[i]) + } + } +} + +// TestGetVulnsByPurlExcludesRepackagers pins the ecosystem filtering. Repackager +// advisories (Vendor:LanguageEcosystem, such as TuxCare:npm) sit under the same purl as +// the upstream package and are excluded; distro ecosystems also contain a colon +// (Ubuntu:22.04:LTS) and must be kept. +// +// This is the test to update if osvRepackagerEcosystems is ever emptied to bring them +// back. Filtering on "contains a colon" instead would drop 69% of the production table, +// every Debian and Ubuntu advisory included. +func TestGetVulnsByPurlExcludesRepackagers(t *testing.T) { + db, ctx := newScenarioDB(t) + model := NewOSVModel(zlog.S, db) + + vulns, err := model.GetVulnsByPurl(ctx, "pkg:npm/testosv", "1.0.0") + if err != nil { + t.Fatalf("GetVulnsByPurl() unexpected error: %v", err) + } + var sawRepackager, sawDistro bool + for _, v := range vulns { + switch v.ID { + case "CLSA-TEST-0001": + sawRepackager = true + case "UBUNTU-TEST-0001": + sawDistro = true + } + } + if sawRepackager { + t.Errorf("a TuxCare:npm advisory was returned; repackager ecosystems must be excluded") + } + if !sawDistro { + t.Errorf("the Ubuntu:22.04:LTS advisory was not returned; distro ecosystems contain a " + + "colon too and must not be filtered out") + } +} diff --git a/pkg/models/test_schema.go b/pkg/models/test_schema.go index 2be8405..6ed0e7e 100644 --- a/pkg/models/test_schema.go +++ b/pkg/models/test_schema.go @@ -96,4 +96,15 @@ CREATE INDEX idx_ruby_dependencies_purl_name_version ON ruby_dependencies (purl_ CREATE TABLE epss_data (cve TEXT, epss TEXT, percentile TEXT); CREATE INDEX idx_epss_data_cve ON epss_data (cve); CREATE INDEX idx_nmci_short_cpe_id ON nvd_match_criteria_ids (short_cpe_id); +CREATE TABLE osv ( + id TEXT, ecosystem TEXT, purl TEXT, + introduced_version TEXT, last_affected TEXT, fixed_version TEXT, + affected_versions TEXT, aliases TEXT, + summary TEXT, severity TEXT, + published TEXT, modified TEXT, indexed_date TEXT, + upstream TEXT, related TEXT +); +CREATE INDEX idx_osv_purl ON osv (purl); +CREATE TABLE osv_severity (id TEXT, type TEXT, score TEXT); +CREATE INDEX idx_osv_severity_id ON osv_severity (id); ` diff --git a/pkg/models/tests/osv_scenario.sql b/pkg/models/tests/osv_scenario.sql new file mode 100644 index 0000000..03ea0b1 --- /dev/null +++ b/pkg/models/tests/osv_scenario.sql @@ -0,0 +1,98 @@ +-- Test data only. The schema comes from testSchemaDDL in pkg/models/test_schema.go; do not add DDL here. +-- +-- A deterministic OSV scenario for one component, covering the shapes the real table +-- takes. Versions and vectors are made up, but the shapes are drawn from production: +-- +-- * both matching mechanisms: an explicit affected_versions list, and a range built +-- from introduced_version / fixed_version / last_affected +-- * several rows per vulnerability, because the table is keyed by range and averages +-- 15.6 rows per vulnerability while the response carries one entry +-- * rows of the same vulnerability disagreeing on modified, which is what decides +-- whose summary and aliases win (the newest) +-- * multiple CVSS vectors on one vulnerability, and a non-CVSS score of type Ubuntu +-- that the service is expected to skip +-- +-- Component under test: pkg:npm/testosv + +-- OSV-TEST-0001: single range, 1.0.0 <= v < 2.0.0. Two CVSS vectors. +INSERT INTO osv (id, ecosystem, purl, introduced_version, last_affected, fixed_version, + affected_versions, aliases, summary, severity, published, modified, indexed_date) +VALUES ('OSV-TEST-0001', 'npm', 'pkg:npm/testosv', '1.0.0', '', '2.0.0', + '{}', '{CVE-2026-0001,GHSA-test-0001}', 'affects 1.x only', 'HIGH', + '2026-01-15', '2026-02-20', '2026-03-01'); + +-- OSV-TEST-0002: explicit version list, no range at all. This is how 70.7% of +-- production rows look. +INSERT INTO osv (id, ecosystem, purl, introduced_version, last_affected, fixed_version, + affected_versions, aliases, summary, severity, published, modified, indexed_date) +VALUES ('OSV-TEST-0002', 'npm', 'pkg:npm/testosv', '', '', '', + '{3.0.0,3.0.1}', '{CVE-2026-0002}', 'affects exactly 3.0.0 and 3.0.1', 'MODERATE', + '2026-02-15', '2026-03-20', '2026-03-21'); + +-- OSV-TEST-0003: two rows, one per range. 1.0.0 <= v < 1.5.0 and 4.0.0 <= v < 4.2.0. +-- A version matching either range must return the vulnerability once. +INSERT INTO osv (id, ecosystem, purl, introduced_version, last_affected, fixed_version, + affected_versions, aliases, summary, severity, published, modified, indexed_date) +VALUES ('OSV-TEST-0003', 'npm', 'pkg:npm/testosv', '0', '', '1.5.0', + '{}', '{CVE-2026-0003}', 'affects early and late', 'CRITICAL', + '2026-03-15', '2026-04-20', '2026-04-21'); +INSERT INTO osv (id, ecosystem, purl, introduced_version, last_affected, fixed_version, + affected_versions, aliases, summary, severity, published, modified, indexed_date) +VALUES ('OSV-TEST-0003', 'npm', 'pkg:npm/testosv', '4.0.0', '', '4.2.0', + '{}', '{CVE-2026-0003}', 'affects early and late', 'CRITICAL', + '2026-03-15', '2026-04-20', '2026-04-21'); + +-- OSV-TEST-0004: last_affected instead of fixed_version, so 5.0.0 itself is affected. +INSERT INTO osv (id, ecosystem, purl, introduced_version, last_affected, fixed_version, + affected_versions, aliases, summary, severity, published, modified, indexed_date) +VALUES ('OSV-TEST-0004', 'npm', 'pkg:npm/testosv', '4.5.0', '5.0.0', '', + '{}', '{}', 'affects up to and including 5.0.0', 'LOW', + '2026-04-15', '2026-05-20', '2026-05-21'); + +-- OSV-TEST-0005: two rows that disagree on modified, summary, severity and aliases. +-- 137,302 production (id, purl) pairs do this. The newest row must win. +INSERT INTO osv (id, ecosystem, purl, introduced_version, last_affected, fixed_version, + affected_versions, aliases, summary, severity, published, modified, indexed_date) +VALUES ('OSV-TEST-0005', 'npm', 'pkg:npm/testosv', '0', '', '9.9.9', + '{}', '{CVE-STALE}', 'stale summary', 'LOW', + '2026-05-15', '2026-05-16', '2026-05-17'); +INSERT INTO osv (id, ecosystem, purl, introduced_version, last_affected, fixed_version, + affected_versions, aliases, summary, severity, published, modified, indexed_date) +VALUES ('OSV-TEST-0005', 'npm', 'pkg:npm/testosv', '0', '', '9.9.9', + '{}', '{CVE-CURRENT}', 'current summary', 'HIGH', + '2026-05-15', '2026-07-30', '2026-07-31'); + +-- A different component: must never appear in results for pkg:npm/testosv. +INSERT INTO osv (id, ecosystem, purl, introduced_version, last_affected, fixed_version, + affected_versions, aliases, summary, severity, published, modified, indexed_date) +VALUES ('OSV-TEST-9999', 'npm', 'pkg:npm/unrelated', '0', '', '9.9.9', + '{}', '{CVE-2026-9999}', 'unrelated component', 'HIGH', + '2026-06-15', '2026-07-20', '2026-07-21'); + +-- CVSS vectors. OSV-TEST-0001 carries two, which is why they live in their own table. +INSERT INTO osv_severity (id, type, score) +VALUES ('OSV-TEST-0001', 'CVSS_V3', 'CVSS:3.1/AV:N/AC:L/PR:N/UI:R/S:C/C:L/I:L/A:N'); +INSERT INTO osv_severity (id, type, score) +VALUES ('OSV-TEST-0001', 'CVSS_V4', 'CVSS:4.0/AV:N/AC:L/AT:N/PR:N/UI:P/VC:L/VI:L/VA:N/SC:L/SI:L/SA:N'); +INSERT INTO osv_severity (id, type, score) +VALUES ('OSV-TEST-0002', 'CVSS_V3', 'CVSS:3.1/AV:L/AC:L/PR:N/UI:N/S:U/C:N/I:N/A:H'); +-- Not a CVSS vector: the service must skip it rather than fail, as it did with the API. +INSERT INTO osv_severity (id, type, score) VALUES ('OSV-TEST-0004', 'Ubuntu', 'medium'); + +-- Repackager advisory: same purl as the upstream package, but published under a vendor +-- ecosystem (Vendor:LanguageEcosystem). The service excludes these, so it must never +-- appear in results. Production has 31,093 such rows across TuxCare:* and Echo:*. +-- Note the fixed version is a vendor build, unreachable from the upstream registry, and +-- introduced_version 0 means it would otherwise attach to every version. +INSERT INTO osv (id, ecosystem, purl, introduced_version, last_affected, fixed_version, + affected_versions, aliases, summary, severity, published, modified, indexed_date) +VALUES ('CLSA-TEST-0001', 'TuxCare:npm', 'pkg:npm/testosv', '0', '', '9.9.9-tuxcare.1', + '{}', '{}', 'TuxCare security update for testosv', 'HIGH', + '2026-06-15', '2026-07-20', '2026-07-21'); +-- A distro ecosystem also contains a colon (Ubuntu:22.04:LTS, Debian:12) and must NOT be +-- filtered: those are legitimate and cover 69% of the table. +INSERT INTO osv (id, ecosystem, purl, introduced_version, last_affected, fixed_version, + affected_versions, aliases, summary, severity, published, modified, indexed_date) +VALUES ('UBUNTU-TEST-0001', 'Ubuntu:22.04:LTS', 'pkg:npm/testosv', '0', '', '9.9.9', + '{}', '{CVE-2026-7777}', 'distro advisory, must be returned', 'MEDIUM', + '2026-06-15', '2026-07-20', '2026-07-21'); diff --git a/pkg/usecase/OSV_use_case.go b/pkg/usecase/OSV_use_case.go index 6712557..c1db49a 100644 --- a/pkg/usecase/OSV_use_case.go +++ b/pkg/usecase/OSV_use_case.go @@ -15,326 +15,166 @@ */ // Package usecase implements the vulnerabilities service business logic. +// +// OSV vulnerabilities are read from the database rather than from api.osv.dev. The HTTP +// client, its worker pool sizing and the GIT-ecosystem fallback are gone: the osv table +// stores purls directly, including pkg:github ones, so a component is looked up by its +// purl with no translation to a repository URL and no retry. +// +// The response shape is unchanged. VULN_OSV_API_BASE_URL is no longer used; +// VULN_OSV_VULNERABILITY_INFO_BASE_URL still is, because it builds the URL of each +// returned vulnerability. VULN_OSV_API_WORKERS keeps its role as the concurrency limit, +// now over database lookups instead of HTTP calls. package usecase import ( - "bytes" "context" - "encoding/json" - "fmt" - "net/http" - "net/url" "time" - "github.com/package-url/packageurl-go" + "github.com/jmoiron/sqlx" compHelper "github.com/scanoss/go-component-helper/componenthelper" "github.com/scanoss/go-grpc-helper/pkg/grpc/domain" - zlog "github.com/scanoss/zap-logging-helper/pkg/logger" "go.uber.org/zap" "scanoss.com/vulnerabilities/pkg/config" "scanoss.com/vulnerabilities/pkg/dtos" + "scanoss.com/vulnerabilities/pkg/models" "scanoss.com/vulnerabilities/pkg/utils" ) -type OSVPackageRequest struct { - Purl string `json:"purl,omitempty"` - Name string `json:"name,omitempty"` - Ecosystem string `json:"ecosystem,omitempty"` -} - -type OSVRequest struct { - Version string `json:"version,omitempty"` - Package OSVPackageRequest `json:"package"` - Requirement string `json:"-"` - OriginalPurl string `json:"-"` - FallbackPackage *OSVPackageRequest `json:"-"` -} +// osvSource is the value reported in the source field of every OSV vulnerability. +const osvSource = "OSV" type OSVUseCase struct { - OSVAPIBaseURL string OSVInfoBaseURL string - client *http.Client // Single shared - MaxAPIWorkers int + maxWorkers int + model *models.OSVModel s *zap.SugaredLogger } -func NewOSVUseCase(s *zap.SugaredLogger, config *config.ServerConfig) *OSVUseCase { +func NewOSVUseCase(s *zap.SugaredLogger, config *config.ServerConfig, db *sqlx.DB) *OSVUseCase { + workers := config.Source.OSV.APIWorkers + if workers < 1 { + workers = 1 + } return &OSVUseCase{ - OSVAPIBaseURL: config.Source.OSV.APIBaseURL, OSVInfoBaseURL: config.Source.OSV.InfoBaseURL, - client: &http.Client{ - Timeout: 15 * time.Second, - }, - MaxAPIWorkers: config.Source.OSV.APIWorkers, - s: s, - } -} - -// getRepoURL converts a PURL string into a Git repository URL if the PURL refers to a known git host. -// -// It supports two resolution strategies: -// -// 1. repository_url qualifier: If the PURL contains a "repository_url" qualifier, its value is used directly. -// This is the standard mechanism for hosts without a dedicated PURL type (e.g., pkg:/...?repository_url=https://gitlab.gnome.org/GNOME/gimp). -// -// 2. Direct type-based: For PURL types that have a spec-defined default repository URL, the host is resolved -// from the type (e.g., pkg:github/owner/repo -> https://github.com/owner/repo). -// -// Supported PURL types with default URLs (defined in spec): -// - github: https://github.com (see: https://github.com/package-url/purl-spec/blob/main/types-doc/github-definition.md) -// - bitbucket: https://bitbucket.org (see: https://github.com/package-url/purl-spec/blob/main/types-doc/bitbucket-definition.md) -// -// Supported PURL types with default URLs (not yet in spec): -// - gitlab: https://gitlab.com (candidate: https://github.com/package-url/purl-spec/blob/main/docs/candidate-purl-types.md) -// - gitee: https://gitee.com (not in spec or candidates) -// -// Not handled: git hosts without a dedicated PURL type and without a "repository_url" qualifier -// (e.g., gitlab.gnome.org, gitlab.freedesktop.org, gitlab.xiph.org, vcgit.hhi.fraunhofer.de, -// git.codelinaro.org, yoctoproject.org, trustedfirmware.org, sourceware.org, gitcode.com, -// eclipse.org, invent.kde.org). These hosts have no defined PURL type in the spec. -// -// Reference: https://github.com/package-url/purl-spec -// -// Returns a pointer to the repository URL string, or nil if the PURL is invalid or does not match any known git host. -func (us OSVUseCase) getRepoURL(purlString string) *string { - // Parse PURL to check if it's a git-based package - purl, err := packageurl.FromString(purlString) - if err != nil { - return nil - } - repoURL := purl.Qualifiers.Map()["repository_url"] - if repoURL != "" { - decoded, errUnescape := url.QueryUnescape(repoURL) - if errUnescape != nil { - return nil - } - return &decoded + maxWorkers: workers, + model: models.NewOSVModel(s, db), + s: s, } - - // Default URLs by purl type - gitHosts := map[string]string{ - "github": "https://github.com", - "gitlab": "https://gitlab.com", // not defined in the purl spec. See: https://github.com/package-url/purl-spec/blob/main/docs/candidate-purl-types.md - "bitbucket": "https://bitbucket.org", - "gitee": "https://gitee.com", - } - host, hostFound := gitHosts[purl.Type] - namespace := purl.Namespace - if hostFound { - built := fmt.Sprintf("%s/%s/%s", host, namespace, purl.Name) - return &built - } - return nil -} - -// getOSVRequestsFromDTO converts a slice of ComponentDTOs into OSVRequest objects. -// For git-based packages (GitHub, GitLab, Bitbucket), it constructs a repository URL -// and sets the ecosystem to "GIT", with the original PURL as a fallback. -// For all other packages, the PURL is used directly. -func (us OSVUseCase) getOSVRequestsFromDTO(componentDTOs []compHelper.Component) []OSVRequest { - var osvRequests []OSVRequest - for _, c := range componentDTOs { - osvRequest := OSVRequest{ - Version: c.Version, - Requirement: c.Requirement, - OriginalPurl: c.Purl, - } - // Parse PURL to check if it's a git-based package - repoURL := us.getRepoURL(c.Purl) - - if repoURL != nil { - osvRequest.Package = OSVPackageRequest{ - Name: *repoURL, - Ecosystem: "GIT", - } - fallback := OSVPackageRequest{ - Purl: c.Purl, - } - osvRequest.FallbackPackage = &fallback - } - if osvRequest.Package == (OSVPackageRequest{}) { - // For other packages, use the PURL directly - osvRequest.Package = OSVPackageRequest{ - Purl: c.Purl, - } - } - osvRequests = append(osvRequests, osvRequest) - } - return osvRequests } func (us OSVUseCase) Execute(ctx context.Context, components []compHelper.Component) dtos.VulnerabilityOutput { - osvRequests := us.getOSVRequestsFromDTO(components) - return us.processRequests(ctx, osvRequests) -} - -func (us OSVUseCase) processRequests(ctx context.Context, requests []OSVRequest) dtos.VulnerabilityOutput { - numJobs := len(requests) - jobs := make(chan OSVRequest, numJobs) + numJobs := len(components) + response := dtos.VulnerabilityOutput{Components: []dtos.VulnerabilityComponentOutput{}} + if numJobs == 0 { + return response + } ctx, cancel := context.WithTimeout(ctx, 3*time.Minute) defer cancel() + jobs := make(chan compHelper.Component, numJobs) results := make(chan dtos.VulnerabilityComponentOutput, numJobs) - workers := min(us.MaxAPIWorkers, numJobs) + workers := min(us.maxWorkers, numJobs) for i := 0; i < workers; i++ { - go us.processRequest(ctx, jobs, results) + go us.processComponent(ctx, jobs, results) } - for _, r := range requests { - jobs <- r + for _, c := range components { + jobs <- c } close(jobs) - // Collect all results into a slice - var response = dtos.VulnerabilityOutput{ - Components: []dtos.VulnerabilityComponentOutput{}, - } for i := 0; i < numJobs; i++ { - result := <-results - response.Components = append(response.Components, result) + response.Components = append(response.Components, <-results) } return response } -// processRequest is a worker function that processes OSV vulnerability requests concurrently. -// It reads requests from the jobs channel, queries the OSV API for each request, and sends -// the results to the results channel. The worker terminates when the jobs channel is closed -// or when the context is cancelled. -func (us OSVUseCase) processRequest(ctx context.Context, jobs chan OSVRequest, results chan dtos.VulnerabilityComponentOutput) { +// processComponent is a worker that looks up each component and sends the result on. +func (us OSVUseCase) processComponent(ctx context.Context, jobs chan compHelper.Component, + results chan dtos.VulnerabilityComponentOutput) { for { select { - case j, ok := <-jobs: + case c, ok := <-jobs: if !ok { - return // Channel closed, stop worker - } - response := dtos.VulnerabilityComponentOutput{ - Purl: j.OriginalPurl, - Requirement: j.Requirement, - Version: j.Version, - ComponentStatus: domain.ComponentStatus{ - Message: "", - StatusCode: domain.Success, - }, + return // channel closed, stop worker } - response.Vulnerabilities = us.queryOSV(ctx, j) - - // Fallback: if GIT ecosystem returned no results, retry with the PURL directly - if len(response.Vulnerabilities) == 0 && j.FallbackPackage != nil { - us.s.Debugf("No vulnerabilities found for GIT ecosystem, falling back to PURL query for: %s", j.OriginalPurl) - fallbackReq := OSVRequest{ - Version: j.Version, - Package: *j.FallbackPackage, - OriginalPurl: j.OriginalPurl, - } - fallbackVulns := us.queryOSV(ctx, fallbackReq) - if fallbackVulns != nil { - response.Vulnerabilities = fallbackVulns - } else { - response.ComponentStatus = domain.ComponentStatus{ - Message: "No vulnerabilities found for: " + j.OriginalPurl, - StatusCode: domain.NoInfo, - } - } - } else if len(response.Vulnerabilities) == 0 { - response.ComponentStatus = domain.ComponentStatus{ - Message: "No vulnerabilities found for: " + j.OriginalPurl, - StatusCode: domain.NoInfo, - } - } - results <- response + results <- us.lookup(ctx, c) case <-ctx.Done(): - // Cancellation signal received: stop working and return immediately us.s.Debugf("Worker: Cancellation signal received, stopping.") return } } } -// queryOSV performs a single OSV API query and returns mapped vulnerabilities, or nil on error. -func (us OSVUseCase) queryOSV(ctx context.Context, r OSVRequest) []dtos.VulnerabilitiesOutput { - out, err := json.Marshal(struct { - Version string `json:"version,omitempty"` - Package OSVPackageRequest `json:"package"` - }{ - Version: r.Version, - Package: r.Package, - }) - if err != nil { - us.s.Errorf("Failed to marshal request: %s", err) - return nil - } - req, err := http.NewRequestWithContext(ctx, http.MethodPost, us.OSVAPIBaseURL+"/query", bytes.NewBuffer(out)) - if err != nil { - us.s.Errorf("Failed to create HTTP request: %s", err) - return nil +// lookup queries the OSV tables for one component. +func (us OSVUseCase) lookup(ctx context.Context, c compHelper.Component) dtos.VulnerabilityComponentOutput { + response := dtos.VulnerabilityComponentOutput{ + Purl: c.Purl, + Requirement: c.Requirement, + Version: c.Version, + ComponentStatus: domain.ComponentStatus{ + Message: "", + StatusCode: domain.Success, + }, } - req.Header.Set("Content-Type", "application/json") - resp, err := us.client.Do(req) + // The stored purls carry no version, so strip one if the caller left it in. + purl := utils.PurlRemoveFromVersionComponent(c.Purl) + vulns, err := us.model.GetVulnsByPurl(ctx, purl, c.Version) if err != nil { - us.s.Errorf("HTTP request failed: %s", err) - return nil - } - defer func() { - if closeErr := resp.Body.Close(); closeErr != nil { - us.s.Warnf("Failed to close response body: %s", closeErr) + // Reported as a lookup failure rather than as "nothing found", so a broken query + // cannot be mistaken for a clean component. + us.s.Errorf("Failed to get OSV vulnerabilities for %v: %v", c.Purl, err) + response.ComponentStatus = domain.ComponentStatus{ + Message: "Failed to query OSV data for: " + c.Purl, + StatusCode: domain.NoInfo, } - }() - - if resp.StatusCode != http.StatusOK { - us.s.Errorf("Unexpected HTTP status: %d", resp.StatusCode) - return nil + return response } - - var osvResponse dtos.OSVResponseDTO - err = json.NewDecoder(resp.Body).Decode(&osvResponse) - if err != nil { - us.s.Errorf("Failed to decode response: %s", err) - return nil + response.Vulnerabilities = us.mapVulnerabilities(vulns) + if len(response.Vulnerabilities) == 0 { + response.ComponentStatus = domain.ComponentStatus{ + Message: "No vulnerabilities found for: " + c.Purl, + StatusCode: domain.NoInfo, + } } - return us.mapOSVVulnerabilities(osvResponse.Vulns) + return response } -// mapOSVVulnerabilities converts OSV vulnerabilities to the required DTO structure. -func (us OSVUseCase) mapOSVVulnerabilities(vulns []dtos.Entry) []dtos.VulnerabilitiesOutput { - vulnerabilities := make([]dtos.VulnerabilitiesOutput, 0, len(vulns)) - for _, vul := range vulns { - // Select CVE or use the ID as fallback - cve := vul.ID - if len(vul.Aliases) > 0 { - cve = vul.Aliases[0] - } - - // Determine severity - severity := "" - if vul.DatabaseSpecific.Severity != "" { - severity = vul.DatabaseSpecific.Severity +// mapVulnerabilities converts stored OSV vulnerabilities into the response DTO. +func (us OSVUseCase) mapVulnerabilities(vulns []models.OSVVulnerability) []dtos.VulnerabilitiesOutput { + out := make([]dtos.VulnerabilitiesOutput, 0, len(vulns)) + for _, vuln := range vulns { + // Prefer the first alias, which is where the CVE lands when OSV has one. + cve := vuln.ID + if len(vuln.Aliases) > 0 { + cve = vuln.Aliases[0] } - var cvss []dtos.CVSS - if vul.Severity != nil { - for _, s := range vul.Severity { - cvssResult, err := utils.GetCVSS(s.Score) - if err != nil { - zlog.S.Warnf("Failed to get CVSS severity and score from: %v, %v", s, err) - continue - } - cvss = append(cvss, dtos.CVSS{ - Cvss: s.Score, - CvssSeverity: cvssResult.Severity, - CvssScore: cvssResult.Score, - }) + for _, severity := range vuln.Severities { + // Not every score is a CVSS vector - 54,565 rows are Ubuntu severities like + // "medium". Those are skipped, exactly as they were when the API returned them. + parsed, err := utils.GetCVSS(severity.Score) + if err != nil { + us.s.Warnf("Failed to get CVSS severity and score from %v (%v): %v", + severity.Score, severity.Type, err) + continue } + cvss = append(cvss, dtos.CVSS{ + Cvss: severity.Score, + CvssSeverity: parsed.Severity, + CvssScore: parsed.Score, + }) } - - // Map to VulnerabilitiesOutput DTO - vulnerabilities = append(vulnerabilities, dtos.VulnerabilitiesOutput{ - ID: vul.ID, + out = append(out, dtos.VulnerabilitiesOutput{ + ID: vuln.ID, Cve: cve, - Summary: vul.Summary, - Severity: severity, - Published: utils.OnlyDate(vul.Published), - Modified: utils.OnlyDate(vul.Modified), - Source: "OSV", + Summary: vuln.Summary, + Severity: vuln.Severity, + Published: vuln.Published, + Modified: vuln.Modified, + Source: osvSource, URL: us.OSVInfoBaseURL + "/" + cve, Cvss: cvss, }) } - return vulnerabilities + return out } diff --git a/pkg/usecase/OSV_use_case_test.go b/pkg/usecase/OSV_use_case_test.go index 3bfb46d..f857c0a 100644 --- a/pkg/usecase/OSV_use_case_test.go +++ b/pkg/usecase/OSV_use_case_test.go @@ -14,209 +14,318 @@ * along with this program. If not, see . */ +// These tests used to call api.osv.dev over the network, so they needed internet and +// exercised whatever OSV happened to return that day. They now run against the osv +// tables in SQLite, seeded from pkg/models/tests/osv_scenario.sql. +// +// TestGetRepoURL is gone along with getRepoURL: the table stores pkg:github purls +// directly, so nothing translates a purl into a repository URL any more. + package usecase import ( "context" + "sort" "testing" - compHelper "github.com/scanoss/go-component-helper/componenthelper" - "github.com/grpc-ecosystem/go-grpc-middleware/logging/zap/ctxzap" + "github.com/jmoiron/sqlx" + compHelper "github.com/scanoss/go-component-helper/componenthelper" "github.com/scanoss/go-grpc-helper/pkg/grpc/domain" - zlog "github.com/scanoss/zap-logging-helper/pkg/logger" + _ "modernc.org/sqlite" "scanoss.com/vulnerabilities/pkg/config" + "scanoss.com/vulnerabilities/pkg/models" ) -func TestOSVUseCase(t *testing.T) { - err := zlog.NewSugaredDevLogger() - if err != nil { - t.Fatalf("an error '%s' was not expected when opening a sugared logger", err) +const osvScenarioPurl = "pkg:npm/testosv" + +func newOSVUseCase(t *testing.T) (*OSVUseCase, context.Context) { + t.Helper() + if err := zlog.NewSugaredDevLogger(); err != nil { + t.Fatalf("failed to open a sugared logger: %v", err) } - defer zlog.SyncZap() + t.Cleanup(zlog.SyncZap) ctx := ctxzap.ToContext(context.Background(), zlog.L) s := ctxzap.Extract(ctx).Sugar() - - serverConfig, err := config.NewServerConfig(nil) + db, err := sqlx.Connect("sqlite", ":memory:") if err != nil { - t.Fatalf("failed to load Config: %v", err) - } - - testCases := []struct { - name string - input []compHelper.Component - }{ - { - name: "OSV Use Case Test", - input: []compHelper.Component{ - { - Purl: "pkg:pypi/mlflow", - Requirement: "2.3.0", - Version: "2.3.0", - Status: domain.ComponentStatus{ - Message: "", - StatusCode: domain.Success, - }, - }, - { - Purl: "pkg:golang/github.com/navidrome/navidrome", - Status: domain.ComponentStatus{ - Message: "", - StatusCode: domain.Success, - }, - }, - }, - }, + t.Fatalf("failed to open a stub database connection: %v", err) } - OSVUseCase := NewOSVUseCase(s, serverConfig) - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - r := OSVUseCase.Execute(ctx, tc.input) - if len(r.Components) == 0 { - t.Errorf("Expected Purls to have elements, got empty slice") - } - }) + db.SetMaxOpenConns(1) + t.Cleanup(func() { models.CloseDB(db) }) + if err = models.LoadTestSQLData(db, ctx, nil); err != nil { + t.Fatalf("failed to load test data: %v", err) } -} - -func TestGetRepoURL(t *testing.T) { - err := zlog.NewSugaredDevLogger() - if err != nil { - t.Fatalf("an error '%s' was not expected when opening a sugared logger", err) - } - defer zlog.SyncZap() - ctx := ctxzap.ToContext(context.Background(), zlog.L) - s := ctxzap.Extract(ctx).Sugar() - serverConfig, err := config.NewServerConfig(nil) if err != nil { - t.Fatalf("failed to load Config: %v", err) + t.Fatalf("failed to load config: %v", err) } + return NewOSVUseCase(s, serverConfig, db), ctx +} - us := NewOSVUseCase(s, serverConfig) +func osvCveNames(vulns []struct { + Cve string +}) []string { + out := make([]string, 0, len(vulns)) + for _, v := range vulns { + out = append(out, v.Cve) + } + sort.Strings(out) + return out +} +// TestOSVUseCaseVersionMatching walks the version bounds the scenario encodes. +// +// UBUNTU-TEST-0001 appears in every expectation: it spans 0 to 9.9.9, so it covers all +// of these versions. It is in the fixture to prove distro ecosystems survive the +// repackager filter, and CLSA-TEST-0001 is absent from every expectation because it does +// not. +func TestOSVUseCaseVersionMatching(t *testing.T) { + us, ctx := newOSVUseCase(t) tests := []struct { - name string - purl string - expected *string + version string + wantIDs []string + reason string }{ - // Direct type-based hosts - { - name: "GitHub PURL", - purl: "pkg:github/owner/repo@v1.0.0", - expected: strPtr("https://github.com/owner/repo"), - }, - { - name: "GitLab PURL", - purl: "pkg:gitlab/owner/repo@v1.0.0", - expected: strPtr("https://gitlab.com/owner/repo"), - }, - { - name: "Bitbucket PURL", - purl: "pkg:bitbucket/owner/repo@v1.0.0", - expected: strPtr("https://bitbucket.org/owner/repo"), - }, - { - name: "Gitee PURL", - purl: "pkg:gitee/owner/repo@v1.0.0", - expected: strPtr("https://gitee.com/owner/repo"), - }, - // repository_url qualifier-based hosts - { - name: "GNOME GitLab via repository_url", - purl: "pkg:generic/gnome.org/GNOME/gimp@GIMP_2_10_36?repository_url=https://gitlab.gnome.org/GNOME/gimp", - expected: strPtr("https://gitlab.gnome.org/GNOME/gimp"), - }, - { - name: "Freedesktop GitLab via repository_url", - purl: "pkg:generic/freedesktop.org/mesa/mesa@mesa-24.0.0?repository_url=https://gitlab.freedesktop.org/mesa/mesa", - expected: strPtr("https://gitlab.freedesktop.org/mesa/mesa"), - }, - { - name: "Xiph GitLab via repository_url", - purl: "pkg:generic/xiph.org/xiph/opus@v1.6?repository_url=https://gitlab.xiph.org/xiph/opus", - expected: strPtr("https://gitlab.xiph.org/xiph/opus"), - }, - { - name: "Fraunhofer HHI via repository_url", - purl: "pkg:generic/vcgit.hhi.fraunhofer.de/jvet/VVCSoftware_VTM@VTM-15.0?repository_url=https://vcgit.hhi.fraunhofer.de/jvet/VVCSoftware_VTM", - expected: strPtr("https://vcgit.hhi.fraunhofer.de/jvet/VVCSoftware_VTM"), - }, - { - name: "CodeLinaro via repository_url", - purl: "pkg:generic/codelinaro.org/linaro/qcomlt/kernel@v6.0?repository_url=https://git.codelinaro.org/linaro/qcomlt/kernel", - expected: strPtr("https://git.codelinaro.org/linaro/qcomlt/kernel"), - }, - { - name: "Yocto Project via repository_url", - purl: "pkg:generic/yoctoproject.org/poky@yocto-4.0?repository_url=https://git.yoctoproject.org/poky", - expected: strPtr("https://git.yoctoproject.org/poky"), - }, { - name: "Trusted Firmware via repository_url", - purl: "pkg:generic/trustedfirmware.org/TF-A/trusted-firmware-a@lts-v2.12?repository_url=https://git.trustedfirmware.org/TF-A/trusted-firmware-a.git", - expected: strPtr("https://git.trustedfirmware.org/TF-A/trusted-firmware-a.git"), + version: "1.2.0", + wantIDs: []string{"OSV-TEST-0001", "OSV-TEST-0003", "OSV-TEST-0005", "UBUNTU-TEST-0001"}, + reason: "inside 1.0.0-2.0.0, inside 0-1.5.0, and inside the open 0-9.9.9", }, { - name: "Sourceware via repository_url", - purl: "pkg:generic/sourceware.org/glibc@glibc-2.39?repository_url=https://sourceware.org/git/glibc.git", - expected: strPtr("https://sourceware.org/git/glibc.git"), + version: "2.0.0", + wantIDs: []string{"OSV-TEST-0005", "UBUNTU-TEST-0001"}, + reason: "fixed_version 2.0.0 is exclusive, and 2.0.0 is past 1.5.0", }, { - name: "GitCode via repository_url", - purl: "pkg:generic/gitcode.com/openharmony/docs@v1.0.0?repository_url=https://gitcode.com/openharmony/docs", - expected: strPtr("https://gitcode.com/openharmony/docs"), + version: "3.0.1", + wantIDs: []string{"OSV-TEST-0002", "OSV-TEST-0005", "UBUNTU-TEST-0001"}, + reason: "3.0.1 is in the explicit affected_versions list", }, { - name: "Eclipse via repository_url", - purl: "pkg:generic/eclipse.org/jgit/jgit@v7.0.0?repository_url=https://git.eclipse.org/c/jgit/jgit.git", - expected: strPtr("https://git.eclipse.org/c/jgit/jgit.git"), + version: "3.0.2", + wantIDs: []string{"OSV-TEST-0005", "UBUNTU-TEST-0001"}, + reason: "a version list that does not contain it is a miss, not an open range", }, { - name: "KDE Invent via repository_url", - purl: "pkg:generic/invent.kde.org/plasma/plasma-desktop@v6.0.0?repository_url=https://invent.kde.org/plasma/plasma-desktop", - expected: strPtr("https://invent.kde.org/plasma/plasma-desktop"), + version: "4.1.0", + wantIDs: []string{"OSV-TEST-0003", "OSV-TEST-0005", "UBUNTU-TEST-0001"}, + reason: "matches the second range of 0003; 0004 only starts at 4.5.0", }, { - name: "Gitee via repository_url", - purl: "pkg:generic/openharmony/docs@v5.0.0?repository_url=https://gitee.com/openharmony/docs", - expected: strPtr("https://gitee.com/openharmony/docs"), + version: "4.6.0", + wantIDs: []string{"OSV-TEST-0004", "OSV-TEST-0005", "UBUNTU-TEST-0001"}, + reason: "inside 4.5.0 to 5.0.0, and past the 4.2.0 fix of 0003's second range", }, - // Non-git PURL returns nil { - name: "PyPI PURL returns nil", - purl: "pkg:pypi/requests@2.28.0", - expected: nil, + version: "5.0.0", + wantIDs: []string{"OSV-TEST-0004", "OSV-TEST-0005", "UBUNTU-TEST-0001"}, + reason: "last_affected 5.0.0 is inclusive", }, - // Invalid PURL returns nil { - name: "Invalid PURL returns nil", - purl: "not-a-purl", - expected: nil, + version: "5.0.1", + wantIDs: []string{"OSV-TEST-0005", "UBUNTU-TEST-0001"}, + reason: "past last_affected", }, } - - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - result := us.getRepoURL(tc.purl) - if tc.expected == nil { - if result != nil { - t.Errorf("expected nil, got %s", *result) - } - return + for _, tt := range tests { + t.Run(tt.version, func(t *testing.T) { + out := us.Execute(ctx, []compHelper.Component{ + {Purl: osvScenarioPurl, Version: tt.version, Status: domain.ComponentStatus{StatusCode: domain.Success}}, + }) + if len(out.Components) != 1 { + t.Fatalf("returned %d components, want 1", len(out.Components)) } - if result == nil { - t.Errorf("expected %s, got nil", *tc.expected) + var got []string + for _, v := range out.Components[0].Vulnerabilities { + got = append(got, v.ID) + } + sort.Strings(got) + if len(got) != len(tt.wantIDs) { + t.Errorf("version %v returned %v, want %v (%s)", tt.version, got, tt.wantIDs, tt.reason) return } - if *result != *tc.expected { - t.Errorf("expected %s, got %s", *tc.expected, *result) + for i := range tt.wantIDs { + if got[i] != tt.wantIDs[i] { + t.Errorf("version %v returned %v, want %v (%s)", tt.version, got, tt.wantIDs, tt.reason) + return + } } }) } } -func strPtr(s string) *string { - return &s +// TestOSVUseCaseOutputFields checks every field of the response, which is what has to +// stay identical now that the data comes from the database. +func TestOSVUseCaseOutputFields(t *testing.T) { + us, ctx := newOSVUseCase(t) + out := us.Execute(ctx, []compHelper.Component{ + {Purl: osvScenarioPurl, Version: "1.2.0", Requirement: "1.2.0"}, + }) + if len(out.Components) != 1 { + t.Fatalf("returned %d components, want 1", len(out.Components)) + } + component := out.Components[0] + if component.Purl != osvScenarioPurl { + t.Errorf("Purl = %q, want %q", component.Purl, osvScenarioPurl) + } + if component.ComponentStatus.StatusCode != domain.Success { + t.Errorf("StatusCode = %v, want Success", component.ComponentStatus.StatusCode) + } + var found bool + for _, v := range component.Vulnerabilities { + if v.ID != "OSV-TEST-0001" { + continue + } + found = true + if v.Cve != "CVE-2026-0001" { + t.Errorf("Cve = %q, want the first alias %q", v.Cve, "CVE-2026-0001") + } + if v.Severity != "HIGH" { + t.Errorf("Severity = %q, want %q", v.Severity, "HIGH") + } + if v.Summary != "affects 1.x only" { + t.Errorf("Summary = %q, want %q", v.Summary, "affects 1.x only") + } + if v.Source != "OSV" { + t.Errorf("Source = %q, want %q", v.Source, "OSV") + } + if want := us.OSVInfoBaseURL + "/CVE-2026-0001"; v.URL != want { + t.Errorf("URL = %q, want %q", v.URL, want) + } + // two vectors, which is why they live in their own table + if len(v.Cvss) != 2 { + t.Errorf("Cvss has %d entries, want 2: %+v", len(v.Cvss), v.Cvss) + } + for _, c := range v.Cvss { + if c.CvssScore == 0 { + t.Errorf("vector %q parsed to a zero score", c.Cvss) + } + if c.CvssSeverity == "" { + t.Errorf("vector %q parsed to an empty severity", c.Cvss) + } + } + } + if !found { + t.Errorf("OSV-TEST-0001 missing from the response") + } +} + +// TestOSVUseCaseFallsBackToIDWhenNoAlias covers the cve field when OSV has no alias. +func TestOSVUseCaseFallsBackToIDWhenNoAlias(t *testing.T) { + us, ctx := newOSVUseCase(t) + out := us.Execute(ctx, []compHelper.Component{{Purl: osvScenarioPurl, Version: "5.0.0"}}) + if len(out.Components) != 1 { + t.Fatalf("returned %d components, want 1", len(out.Components)) + } + var found bool + for _, v := range out.Components[0].Vulnerabilities { + if v.ID != "OSV-TEST-0004" { + continue + } + found = true + if v.Cve != "OSV-TEST-0004" { + t.Errorf("Cve = %q, want the id itself when there is no alias", v.Cve) + } + // its only score is an Ubuntu severity, not a CVSS vector, so it is skipped + if len(v.Cvss) != 0 { + t.Errorf("Cvss = %+v, want empty: a non-CVSS score must be skipped", v.Cvss) + } + } + if !found { + t.Errorf("OSV-TEST-0004 missing from the response") + } +} + +// TestOSVUseCaseUsesNewestRow pins which row wins when rows of one vulnerability +// disagree, as 137,302 production pairs do. +func TestOSVUseCaseUsesNewestRow(t *testing.T) { + us, ctx := newOSVUseCase(t) + out := us.Execute(ctx, []compHelper.Component{{Purl: osvScenarioPurl, Version: "1.0.0"}}) + if len(out.Components) != 1 { + t.Fatalf("returned %d components, want 1", len(out.Components)) + } + var found bool + for _, v := range out.Components[0].Vulnerabilities { + if v.ID != "OSV-TEST-0005" { + continue + } + found = true + if v.Summary != "current summary" { + t.Errorf("Summary = %q, want the newest row's %q", v.Summary, "current summary") + } + if v.Cve != "CVE-CURRENT" { + t.Errorf("Cve = %q, want %q", v.Cve, "CVE-CURRENT") + } + if v.Severity != "HIGH" { + t.Errorf("Severity = %q, want %q", v.Severity, "HIGH") + } + } + if !found { + t.Errorf("OSV-TEST-0005 missing from the response") + } +} + +// TestOSVUseCaseIgnoresOtherComponents guards against leaking another purl's data. +func TestOSVUseCaseIgnoresOtherComponents(t *testing.T) { + us, ctx := newOSVUseCase(t) + out := us.Execute(ctx, []compHelper.Component{{Purl: osvScenarioPurl, Version: "1.0.0"}}) + for _, v := range out.Components[0].Vulnerabilities { + if v.ID == "OSV-TEST-9999" { + t.Errorf("response leaked a vulnerability belonging to another component") + } + } +} + +// TestOSVUseCaseNoVulnerabilities checks the status when a component is clean. +func TestOSVUseCaseNoVulnerabilities(t *testing.T) { + us, ctx := newOSVUseCase(t) + out := us.Execute(ctx, []compHelper.Component{{Purl: "pkg:npm/nothing-here", Version: "1.0.0"}}) + if len(out.Components) != 1 { + t.Fatalf("returned %d components, want 1", len(out.Components)) + } + if len(out.Components[0].Vulnerabilities) != 0 { + t.Errorf("returned %d vulnerabilities, want 0", len(out.Components[0].Vulnerabilities)) + } + if out.Components[0].ComponentStatus.StatusCode != domain.NoInfo { + t.Errorf("StatusCode = %v, want NoInfo", out.Components[0].ComponentStatus.StatusCode) + } +} + +// TestOSVUseCaseHandlesSeveralComponents exercises the worker pool. +func TestOSVUseCaseHandlesSeveralComponents(t *testing.T) { + us, ctx := newOSVUseCase(t) + out := us.Execute(ctx, []compHelper.Component{ + {Purl: osvScenarioPurl, Version: "1.2.0"}, + {Purl: "pkg:npm/unrelated", Version: "1.0.0"}, + {Purl: "pkg:npm/nothing-here", Version: "1.0.0"}, + }) + if len(out.Components) != 3 { + t.Errorf("returned %d components, want 3", len(out.Components)) + } +} + +// TestOSVUseCaseEmptyInput checks the empty case returns an empty response, not a panic. +func TestOSVUseCaseEmptyInput(t *testing.T) { + us, ctx := newOSVUseCase(t) + out := us.Execute(ctx, nil) + if len(out.Components) != 0 { + t.Errorf("returned %d components, want 0", len(out.Components)) + } +} + +// TestOSVUseCaseStripsEmbeddedVersion checks a purl carrying its version still matches +// the stored purls, which never carry one. +func TestOSVUseCaseStripsEmbeddedVersion(t *testing.T) { + us, ctx := newOSVUseCase(t) + out := us.Execute(ctx, []compHelper.Component{ + {Purl: osvScenarioPurl + "@1.2.0", Version: "1.2.0"}, + }) + if len(out.Components) != 1 { + t.Fatalf("returned %d components, want 1", len(out.Components)) + } + if len(out.Components[0].Vulnerabilities) == 0 { + t.Errorf("a purl with an embedded version returned nothing; it must be stripped before the lookup") + } } diff --git a/pkg/usecase/vulnerability_use_case.go b/pkg/usecase/vulnerability_use_case.go index 598a8fc..02a1d7e 100644 --- a/pkg/usecase/vulnerability_use_case.go +++ b/pkg/usecase/vulnerability_use_case.go @@ -98,7 +98,7 @@ func (us VulnerabilityUseCase) Execute(ctx context.Context, componentDTOs []comp go func() { defer wg.Done() us.s.Debugf("vulnerabilities: OSV enabled") - osvUseCase := NewOSVUseCase(us.s, us.config) + osvUseCase := NewOSVUseCase(us.s, us.config, us.db) osvVulnerabilities = osvUseCase.Execute(ctx, validComponents) }() }