From 959200e6514f0322937db7bb736f19feb668cefe Mon Sep 17 00:00:00 2001 From: "containarium-ci[bot]" <287705844+containarium-ci[bot]@users.noreply.github.com> Date: Fri, 30 Oct 2026 08:44:17 +0000 Subject: [PATCH] feat(vuln): add NVD CPE fallback, KEV and EPSS enrichment, ranking (#11) Co-authored-by: hsinatfootprintai <149764663+hsinatfootprintai@users.noreply.github.com> --- cmd/touchstone/main.go | 19 +- internal/service/scanner.go | 68 +++--- internal/service/scanner_test.go | 105 +++++++- internal/vuln/epss.go | 78 ++++++ internal/vuln/feeds_test.go | 225 ++++++++++++++++++ internal/vuln/http.go | 44 ++++ internal/vuln/kev.go | 98 ++++++++ internal/vuln/live_test.go | 46 ++++ internal/vuln/match.go | 218 +++++++++++++++++ internal/vuln/match_test.go | 189 +++++++++++++++ internal/vuln/nvd.go | 150 ++++++++++++ internal/vuln/osv.go | 2 +- internal/vuln/testdata/epss/epss.json | 14 ++ .../kev/known_exploited_vulnerabilities.json | 34 +++ .../vuln/testdata/nvd/cves-nginx-1.18.0.json | 53 +++++ internal/vuln/vuln.go | 9 +- 16 files changed, 1292 insertions(+), 60 deletions(-) create mode 100644 internal/vuln/epss.go create mode 100644 internal/vuln/feeds_test.go create mode 100644 internal/vuln/http.go create mode 100644 internal/vuln/kev.go create mode 100644 internal/vuln/live_test.go create mode 100644 internal/vuln/match.go create mode 100644 internal/vuln/match_test.go create mode 100644 internal/vuln/nvd.go create mode 100644 internal/vuln/testdata/epss/epss.json create mode 100644 internal/vuln/testdata/kev/known_exploited_vulnerabilities.json create mode 100644 internal/vuln/testdata/nvd/cves-nginx-1.18.0.json diff --git a/cmd/touchstone/main.go b/cmd/touchstone/main.go index a4e9956..b0efc86 100644 --- a/cmd/touchstone/main.go +++ b/cmd/touchstone/main.go @@ -1,8 +1,8 @@ // Command touchstone scans a vendor URL and prints an evidence-graded // ScanReport as protojson (--format json, the default) or as Markdown // (--format md), or serves the web page and POST /v1/scans (serve, slice -// S9). The scan fetches the page and its scripts passively and -// looks components up in OSV (slice S2). +// S9). The scan fetches the page and its scripts passively, looks +// components up in OSV and NVD, and ranks them with KEV and EPSS. package main import ( @@ -30,7 +30,7 @@ type newService func(fetch.Options) touchstonev1.ScanServiceServer func main() { newSvc := func(o fetch.Options) touchstonev1.ScanServiceServer { - return &service.Scanner{Fetch: o, Vulns: vuln.NewOSV()} + return &service.Scanner{Fetch: o, Vulns: newMatcher(os.Getenv("NVD_API_KEY"))} } if err := run(context.Background(), os.Args[1:], os.Stdout, newSvc); err != nil { fmt.Fprintln(os.Stderr, "touchstone:", err) @@ -38,6 +38,19 @@ func main() { } } +// newMatcher wires the public vulnerability feeds: OSV by purl, NVD by CPE +// (capped per scan without an API key), then KEV and EPSS. One Matcher +// serves every scan of the process, so the KEV cache is shared. +func newMatcher(nvdAPIKey string) *vuln.Matcher { + nvd := vuln.NewNVD(nvdAPIKey) + return &vuln.Matcher{ + Purl: vuln.NewOSV(), + CPE: nvd, + CPELimit: nvd.PerScanLimit(), + Enrichers: []vuln.Enricher{vuln.NewKEV(), vuln.NewEPSS()}, + } +} + func run(ctx context.Context, args []string, out io.Writer, newSvc newService) error { if len(args) > 0 && args[0] == "serve" { return serve(ctx, args[1:], out, os.Stderr, newSvc) diff --git a/internal/service/scanner.go b/internal/service/scanner.go index 6607cea..06fd461 100644 --- a/internal/service/scanner.go +++ b/internal/service/scanner.go @@ -19,9 +19,9 @@ import ( // Scanner is the walking-skeleton ScanService (slice S2). It fetches the // target page and the scripts it loads through a PassiveFetcher, -// fingerprints them, looks each purl up through vuln.Source, and returns -// the report. Merging across tiers and the NVD, KEV and EPSS lookups come -// in later slices. +// fingerprints them, looks each component up through vuln.Matcher (OSV by +// purl, NVD by CPE, enriched with KEV and EPSS), and returns the report. +// Merging across tiers comes in a later slice. type Scanner struct { touchstonev1.UnimplementedScanServiceServer @@ -30,8 +30,8 @@ type Scanner struct { // ScanRequest. Fetch fetch.Options - // Vulns looks components up by purl (OSV in the CLI). - Vulns vuln.Source + // Vulns looks components up and ranks their vulnerabilities. + Vulns *vuln.Matcher // Now returns the scan's start and completion time; time.Now if nil. Now func() time.Time @@ -126,38 +126,27 @@ func collect(ctx context.Context, f fetch.Fetcher, page *fetch.Response) ([]pipe return merge(comps), errors.Join(errs...) } -// lookup asks vuln.Source about every component that has a purl and -// converts the components to proto. It returns the lookup status, which is -// COMPLETE only if every lookup succeeded. +// lookup asks the Matcher about every component (by purl, else CPE) and +// converts the components, with their ranked vulnerabilities, to proto. +// It returns the lookup status, which is COMPLETE only if every lookup +// and enrichment succeeded. func (s *Scanner) lookup(ctx context.Context, rep *touchstonev1.ScanReport, comps []pipeline.Component) (touchstonev1.LookupStatus, []*touchstonev1.Component) { - var out []*touchstonev1.Component - var ok, failed int - var errs []error - for _, c := range comps { - var findings []vuln.Finding - if c.Purl != "" { - var err error - findings, err = s.Vulns.Lookup(ctx, []vuln.Query{{Purl: c.Purl}}) - if err != nil { - failed++ - errs = append(errs, err) - } else { - ok++ - } - } - out = append(out, toProto(c, findings)) + queries := make([]vuln.Query, len(comps)) + for i, c := range comps { + queries[i] = vuln.Query{Purl: c.Purl, CPE: c.CPE} } - if ok+failed > 0 { - rep.Sources = append(rep.Sources, &touchstonev1.SourceStatus{ - Source: s.Vulns.Name(), - Ok: failed == 0, - Error: errText(errors.Join(errs...)), - }) + res := s.Vulns.Match(ctx, queries) + for _, st := range res.Sources { + rep.Sources = append(rep.Sources, &touchstonev1.SourceStatus{Source: st.Source, Ok: st.OK, Error: st.Error}) + } + out := make([]*touchstonev1.Component, len(comps)) + for i, c := range comps { + out[i] = toProto(c, res.Matches[i]) } - switch { - case failed == 0: + switch res.Status { + case vuln.StatusComplete: return touchstonev1.LookupStatus_LOOKUP_STATUS_COMPLETE, out - case ok > 0: + case vuln.StatusPartial: return touchstonev1.LookupStatus_LOOKUP_STATUS_PARTIAL, out default: return touchstonev1.LookupStatus_LOOKUP_STATUS_FAILED, out @@ -199,7 +188,7 @@ func merge(comps []pipeline.Component) []pipeline.Component { // toProto converts one component. The pipeline's Confidence and Kind // strings are the proto enum names without their type prefix. -func toProto(c pipeline.Component, findings []vuln.Finding) *touchstonev1.Component { +func toProto(c pipeline.Component, matches []vuln.Match) *touchstonev1.Component { out := &touchstonev1.Component{ Name: c.Name, Version: c.Version, @@ -215,11 +204,14 @@ func toProto(c pipeline.Component, findings []vuln.Finding) *touchstonev1.Compon Detail: e.Detail, }) } - for _, f := range findings { + for _, m := range matches { out.Vulnerabilities = append(out.Vulnerabilities, &touchstonev1.Vulnerability{ - Id: f.ID, - Aliases: f.Aliases, - Summary: f.Summary, + Id: m.ID, + Aliases: m.Aliases, + Summary: m.Summary, + Kev: m.KEV, + Epss: m.EPSS, + CvssBase: m.CVSSBase, }) } return out diff --git a/internal/service/scanner_test.go b/internal/service/scanner_test.go index d73098b..7ae63c1 100644 --- a/internal/service/scanner_test.go +++ b/internal/service/scanner_test.go @@ -68,16 +68,47 @@ func recordedOSVSite(t *testing.T) *httptest.Server { return srv } -func newTestScanner(osvURL string) *Scanner { +// recordedFeed serves one file recorded in internal/vuln/testdata for a GET +// of path. +func recordedFeed(t *testing.T, path, file string) *httptest.Server { + t.Helper() + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + b, err := os.ReadFile(filepath.Join("..", "vuln", "testdata", file)) + if r.Method != http.MethodGet || r.URL.Path != path || err != nil { + http.NotFound(w, r) + return + } + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write(b) + })) + t.Cleanup(srv.Close) + return srv +} + +// newTestScanner looks components up in OSV at osvURL and in recorded +// NVD, KEV and EPSS replies. +func newTestScanner(t *testing.T, osvURL string) *Scanner { + t.Helper() + nvd := recordedFeed(t, "/rest/json/cves/2.0", filepath.Join("nvd", "cves-nginx-1.18.0.json")) + kev := recordedFeed(t, "/kev.json", filepath.Join("kev", "known_exploited_vulnerabilities.json")) + epss := recordedFeed(t, "/data/v1/epss", filepath.Join("epss", "epss.json")) return &Scanner{ Fetch: fetch.Options{AllowPrivate: true, Version: ToolVersion}, - Vulns: &vuln.OSV{BaseURL: osvURL, Client: http.DefaultClient}, + Vulns: &vuln.Matcher{ + Purl: &vuln.OSV{BaseURL: osvURL, Client: http.DefaultClient}, + CPE: &vuln.NVD{BaseURL: nvd.URL, Client: http.DefaultClient}, + CPELimit: vuln.UnauthenticatedNVDLimit, + Enrichers: []vuln.Enricher{ + &vuln.KEV{URL: kev.URL + "/kev.json", Client: http.DefaultClient}, + &vuln.EPSS{BaseURL: epss.URL, Client: http.DefaultClient}, + }, + }, } } func TestScannerReportsJQueryWithOSVIDs(t *testing.T) { site := vendorSite(t) - rep, err := newTestScanner(recordedOSVSite(t).URL).Scan(context.Background(), + rep, err := newTestScanner(t, recordedOSVSite(t).URL).Scan(context.Background(), &touchstonev1.ScanRequest{Url: site.URL + "/"}) if err != nil { t.Fatalf("Scan: %v", err) @@ -121,9 +152,55 @@ func TestScannerReportsJQueryWithOSVIDs(t *testing.T) { } } +// The test KEV feed lists CVE-2020-11022, so it ranks above +// CVE-2019-11358 despite the lower EPSS; the nginx banner is matched by +// CPE in NVD. +func TestScannerRanksAndEnriches(t *testing.T) { + site := vendorSite(t) + rep, err := newTestScanner(t, recordedOSVSite(t).URL).Scan(context.Background(), + &touchstonev1.ScanRequest{Url: site.URL + "/"}) + if err != nil { + t.Fatalf("Scan: %v", err) + } + if rep.GetLookupStatus() != touchstonev1.LookupStatus_LOOKUP_STATUS_COMPLETE { + t.Fatalf("lookup_status = %v, want COMPLETE: %v", rep.GetLookupStatus(), rep.GetSources()) + } + byName := map[string]*touchstonev1.Component{} + for _, c := range rep.GetComponents() { + byName[c.GetName()] = c + } + + jq := byName["jquery"].GetVulnerabilities() + if len(jq) != 2 { + t.Fatalf("jquery vulnerabilities = %v, want 2", jq) + } + if jq[0].GetId() != "CVE-2020-11022" || !jq[0].GetKev() || jq[0].Epss == nil { + t.Errorf("first jquery vulnerability = %v, want the KEV-listed CVE-2020-11022 with an EPSS score", jq[0]) + } + if jq[1].GetId() != "CVE-2019-11358" || jq[1].GetKev() || jq[1].GetEpss() != 0.01422 { + t.Errorf("second jquery vulnerability = %v, want CVE-2019-11358, not KEV, EPSS 0.01422", jq[1]) + } + + ng := byName["nginx"].GetVulnerabilities() + if len(ng) != 1 || ng[0].GetId() != "CVE-2021-23017" || ng[0].GetCvssBase() != 7.7 || ng[0].Epss == nil { + t.Errorf("nginx vulnerabilities = %v, want CVE-2021-23017 from NVD with CVSS 7.7 and EPSS", ng) + } + + var sources []string + for _, s := range rep.GetSources() { + if !s.GetOk() { + t.Errorf("source %s failed: %s", s.GetSource(), s.GetError()) + } + sources = append(sources, s.GetSource()) + } + if strings.Join(sources, ",") != "fingerprint,OSV,NVD,KEV,EPSS" { + t.Errorf("sources = %v, want fingerprint,OSV,NVD,KEV,EPSS", sources) + } +} + func TestScannerCheckedListMatchesRequests(t *testing.T) { site := vendorSite(t) - rep, err := newTestScanner(recordedOSVSite(t).URL).Scan(context.Background(), + rep, err := newTestScanner(t, recordedOSVSite(t).URL).Scan(context.Background(), &touchstonev1.ScanRequest{Url: site.URL + "/"}) if err != nil { t.Fatalf("Scan: %v", err) @@ -152,7 +229,7 @@ func TestScannerCheckedListMatchesRequests(t *testing.T) { func TestScannerRecordsScriptRefusedByBudget(t *testing.T) { site := vendorSite(t) - rep, err := newTestScanner(recordedOSVSite(t).URL).Scan(context.Background(), + rep, err := newTestScanner(t, recordedOSVSite(t).URL).Scan(context.Background(), &touchstonev1.ScanRequest{Url: site.URL + "/", RequestBudget: 1}) if err != nil { t.Fatalf("Scan: %v", err) @@ -172,7 +249,7 @@ func TestScannerRecordsScriptRefusedByBudget(t *testing.T) { func TestScannerTargetUnreachable(t *testing.T) { site := httptest.NewServer(http.NotFoundHandler()) site.Close() - rep, err := newTestScanner(recordedOSVSite(t).URL).Scan(context.Background(), + rep, err := newTestScanner(t, recordedOSVSite(t).URL).Scan(context.Background(), &touchstonev1.ScanRequest{Url: site.URL + "/"}) if err != nil { t.Fatalf("Scan: an unreachable target is a report, not an RPC error: %v", err) @@ -188,23 +265,25 @@ func TestScannerTargetUnreachable(t *testing.T) { } } -func TestScannerOSVDownIsFailedLookup(t *testing.T) { +// With OSV down but NVD up, the nginx banner (CPE only) is still looked +// up, so the report is PARTIAL, never COMPLETE. +func TestScannerOSVDownIsPartialLookup(t *testing.T) { osv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { http.Error(w, "down", http.StatusServiceUnavailable) })) t.Cleanup(osv.Close) site := vendorSite(t) - rep, err := newTestScanner(osv.URL).Scan(context.Background(), + rep, err := newTestScanner(t, osv.URL).Scan(context.Background(), &touchstonev1.ScanRequest{Url: site.URL + "/"}) if err != nil { t.Fatalf("Scan: %v", err) } - if rep.GetLookupStatus() != touchstonev1.LookupStatus_LOOKUP_STATUS_FAILED { - t.Errorf("lookup_status = %v, want FAILED", rep.GetLookupStatus()) + if rep.GetLookupStatus() != touchstonev1.LookupStatus_LOOKUP_STATUS_PARTIAL { + t.Errorf("lookup_status = %v, want PARTIAL", rep.GetLookupStatus()) } for _, c := range rep.GetComponents() { - if len(c.GetVulnerabilities()) != 0 { - t.Errorf("component %s has vulnerabilities despite the failed lookup", c.GetName()) + if c.GetName() == "jquery" && len(c.GetVulnerabilities()) != 0 { + t.Errorf("jquery has vulnerabilities despite the failed OSV lookup") } } var osvStatus *touchstonev1.SourceStatus @@ -219,7 +298,7 @@ func TestScannerOSVDownIsFailedLookup(t *testing.T) { } func TestScannerRejectsNonHTTPTarget(t *testing.T) { - _, err := newTestScanner("http://unused").Scan(context.Background(), + _, err := newTestScanner(t, "http://unused").Scan(context.Background(), &touchstonev1.ScanRequest{Url: "ftp://vendor.example/"}) if err == nil { t.Fatal("Scan: want an InvalidArgument error for an ftp URL") diff --git a/internal/vuln/epss.go b/internal/vuln/epss.go new file mode 100644 index 0000000..9ede5e7 --- /dev/null +++ b/internal/vuln/epss.go @@ -0,0 +1,78 @@ +package vuln + +import ( + "context" + "errors" + "fmt" + "net/http" + "net/url" + "strconv" + "strings" + "time" +) + +// DefaultEPSSURL is FIRST's EPSS API. +const DefaultEPSSURL = "https://api.first.org" + +// epssBatch is how many CVEs one EPSS request asks for; it is the API's +// default page size, so one page always holds the whole answer. +const epssBatch = 100 + +const maxEPSSResponseBytes = 2 << 20 + +// EPSS is the vuln.Enricher that adds FIRST's exploit prediction score to +// each finding with a CVE id. +type EPSS struct { + BaseURL string + Client *http.Client +} + +// NewEPSS returns an EPSS enricher for the public API with a 10 s timeout +// per request. +func NewEPSS() *EPSS { + return &EPSS{BaseURL: DefaultEPSSURL, Client: &http.Client{Timeout: 10 * time.Second}} +} + +var _ Enricher = (*EPSS)(nil) + +func (e *EPSS) Name() string { return "EPSS" } + +// epssResponse is the wire shape of GET /data/v1/epss. The API sends the +// score as a decimal string. +type epssResponse struct { + Data []struct { + CVE string `json:"cve"` + EPSS string `json:"epss"` + } `json:"data"` +} + +// Enrich returns the EPSS score of each CVE id that has one. Ids that are +// not CVEs (an OSV or GHSA id with no CVE alias) are never sent. +func (e *EPSS) Enrich(ctx context.Context, findingIDs []string) (map[string]Enrichment, error) { + var cves []string + for _, id := range findingIDs { + if strings.HasPrefix(id, "CVE-") { + cves = append(cves, id) + } + } + out := map[string]Enrichment{} + for start := 0; start < len(cves); start += epssBatch { + batch := cves[start:min(start+epssBatch, len(cves))] + v := url.Values{"cve": {strings.Join(batch, ",")}} + var resp epssResponse + if err := getJSON(ctx, e.Client, "epss", e.BaseURL+"/data/v1/epss?"+v.Encode(), nil, maxEPSSResponseBytes, &resp); err != nil { + return nil, err + } + for _, d := range resp.Data { + if d.CVE == "" { + return nil, errors.New("epss: result without a cve") + } + score, err := strconv.ParseFloat(d.EPSS, 64) + if err != nil || score < 0 || score > 1 { + return nil, fmt.Errorf("epss: %s: bad score %q", d.CVE, d.EPSS) + } + out[d.CVE] = Enrichment{EPSS: &score} + } + } + return out, nil +} diff --git a/internal/vuln/feeds_test.go b/internal/vuln/feeds_test.go new file mode 100644 index 0000000..338cb9e --- /dev/null +++ b/internal/vuln/feeds_test.go @@ -0,0 +1,225 @@ +package vuln + +import ( + "context" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "sync" + "testing" + "time" +) + +const nginxCPE = "cpe:2.3:a:f5:nginx:1.18.0:*:*:*:*:*:*:*" + +// recorded serves one file under testdata for path, records each request, +// and answers anything else with 404 and a test failure. +func recorded(t *testing.T, path, file string) (*httptest.Server, *[]*http.Request) { + t.Helper() + var mu sync.Mutex + var reqs []*http.Request + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + mu.Lock() + reqs = append(reqs, r) + mu.Unlock() + if r.Method != http.MethodGet || r.URL.Path != path { + t.Errorf("unexpected request %s %s", r.Method, r.URL.Path) + http.NotFound(w, r) + return + } + b, err := os.ReadFile(filepath.Join("testdata", file)) + if err != nil { + t.Errorf("read fixture: %v", err) + http.NotFound(w, r) + return + } + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write(b) + })) + t.Cleanup(srv.Close) + return srv, &reqs +} + +func failing(t *testing.T) *httptest.Server { + t.Helper() + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + http.Error(w, "unavailable", http.StatusServiceUnavailable) + })) + t.Cleanup(srv.Close) + return srv +} + +func TestNVDLookupByCPE(t *testing.T) { + srv, reqs := recorded(t, "/rest/json/cves/2.0", "nvd/cves-nginx-1.18.0.json") + n := &NVD{BaseURL: srv.URL, Client: srv.Client()} + got, err := n.Lookup(context.Background(), []Query{{CPE: nginxCPE}}) + if err != nil { + t.Fatalf("Lookup: %v", err) + } + if len(got) != 1 || got[0].ID != "CVE-2021-23017" { + t.Fatalf("Lookup = %+v, want CVE-2021-23017", got) + } + if !strings.HasPrefix(got[0].Summary, "A security issue in nginx resolver") { + t.Errorf("summary = %q, want the English description", got[0].Summary) + } + if got[0].CVSSBase == nil || *got[0].CVSSBase != 7.7 { + t.Errorf("cvss = %v, want the v3.1 primary score 7.7", got[0].CVSSBase) + } + if len(*reqs) != 1 { + t.Fatalf("requests = %d, want 1", len(*reqs)) + } + r := (*reqs)[0] + if r.URL.Query().Get("cpeName") != nginxCPE { + t.Errorf("cpeName = %q, want %q", r.URL.Query().Get("cpeName"), nginxCPE) + } + if r.Header.Get("apiKey") != "" { + t.Errorf("apiKey header sent without a key") + } +} + +func TestNVDSendsAPIKey(t *testing.T) { + srv, reqs := recorded(t, "/rest/json/cves/2.0", "nvd/cves-nginx-1.18.0.json") + n := &NVD{BaseURL: srv.URL, APIKey: "k", Client: srv.Client()} + if _, err := n.Lookup(context.Background(), []Query{{CPE: nginxCPE}}); err != nil { + t.Fatalf("Lookup: %v", err) + } + if got := (*reqs)[0].Header.Get("apiKey"); got != "k" { + t.Errorf("apiKey header = %q, want k", got) + } +} + +func TestNVDPerScanLimit(t *testing.T) { + if got := NewNVD("").PerScanLimit(); got != UnauthenticatedNVDLimit { + t.Errorf("without a key: limit = %d, want %d", got, UnauthenticatedNVDLimit) + } + if got := NewNVD("k").PerScanLimit(); got != 0 { + t.Errorf("with a key: limit = %d, want 0 (no cap)", got) + } +} + +func TestNVDIgnoresPurlQueries(t *testing.T) { + srv, reqs := recorded(t, "/rest/json/cves/2.0", "nvd/cves-nginx-1.18.0.json") + n := &NVD{BaseURL: srv.URL, Client: srv.Client()} + got, err := n.Lookup(context.Background(), []Query{{Purl: "pkg:npm/jquery@1.12.4", CPE: "cpe:2.3:a:jquery:jquery:1.12.4:*:*:*:*:*:*:*"}}) + if err != nil || len(got) != 0 || len(*reqs) != 0 { + t.Fatalf("Lookup = %v, %v with %d requests; want nothing asked for a purl query", got, err, len(*reqs)) + } +} + +func TestNVDErrors(t *testing.T) { + for name, body := range map[string]string{ + "no totalResults": `{"vulnerabilities":[]}`, + "truncated results": `{"totalResults":3,"vulnerabilities":[{"cve":{"id":"CVE-2021-23017"}}]}`, + "missing id": `{"totalResults":1,"vulnerabilities":[{"cve":{}}]}`, + } { + t.Run(name, func(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = w.Write([]byte(body)) + })) + t.Cleanup(srv.Close) + n := &NVD{BaseURL: srv.URL, Client: srv.Client()} + if got, err := n.Lookup(context.Background(), []Query{{CPE: nginxCPE}}); err == nil || got != nil { + t.Fatalf("Lookup = %v, %v; want an error and no findings", got, err) + } + }) + } + t.Run("non-200", func(t *testing.T) { + n := &NVD{BaseURL: failing(t).URL, Client: http.DefaultClient} + if _, err := n.Lookup(context.Background(), []Query{{CPE: nginxCPE}}); err == nil { + t.Fatal("Lookup: want an error for HTTP 503") + } + }) +} + +func TestKEVFlagsListedCVEs(t *testing.T) { + srv, _ := recorded(t, "/kev.json", "kev/known_exploited_vulnerabilities.json") + k := &KEV{URL: srv.URL + "/kev.json", Client: srv.Client()} + got, err := k.Enrich(context.Background(), []string{"CVE-2020-11022", "CVE-2019-11358"}) + if err != nil { + t.Fatalf("Enrich: %v", err) + } + if !got["CVE-2020-11022"].KEV { + t.Errorf("CVE-2020-11022 not flagged: %v", got) + } + if _, ok := got["CVE-2019-11358"]; ok { + t.Errorf("CVE-2019-11358 is not in the catalog but was returned: %v", got) + } +} + +func TestKEVCachesForTTL(t *testing.T) { + srv, reqs := recorded(t, "/kev.json", "kev/known_exploited_vulnerabilities.json") + now := time.Date(2026, 10, 30, 0, 0, 0, 0, time.UTC) + k := &KEV{URL: srv.URL + "/kev.json", Client: srv.Client(), Now: func() time.Time { return now }} + ctx := context.Background() + for _, step := range []struct { + advance time.Duration + fetches int + }{ + {0, 1}, + {KEVTTL - time.Minute, 1}, // still fresh: served from memory + {2 * time.Minute, 2}, // past the TTL: fetched again + } { + now = now.Add(step.advance) + if _, err := k.Enrich(ctx, []string{"CVE-2020-11022"}); err != nil { + t.Fatalf("Enrich: %v", err) + } + if len(*reqs) != step.fetches { + t.Errorf("after +%v: fetches = %d, want %d", step.advance, len(*reqs), step.fetches) + } + } +} + +func TestKEVErrors(t *testing.T) { + k := &KEV{URL: failing(t).URL, Client: http.DefaultClient} + if _, err := k.Enrich(context.Background(), []string{"CVE-2020-11022"}); err == nil { + t.Error("Enrich: want an error for HTTP 503") + } + empty := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = w.Write([]byte(`{"vulnerabilities":[]}`)) + })) + t.Cleanup(empty.Close) + k = &KEV{URL: empty.URL, Client: empty.Client()} + if _, err := k.Enrich(context.Background(), []string{"CVE-2020-11022"}); err == nil { + t.Error("Enrich: want an error for an empty catalog, not \"nothing is exploited\"") + } +} + +func TestEPSSScores(t *testing.T) { + srv, reqs := recorded(t, "/data/v1/epss", "epss/epss.json") + e := &EPSS{BaseURL: srv.URL, Client: srv.Client()} + got, err := e.Enrich(context.Background(), []string{"CVE-2020-11022", "GHSA-xxxx-xxxx-xxxx"}) + if err != nil { + t.Fatalf("Enrich: %v", err) + } + if s := got["CVE-2020-11022"].EPSS; s == nil || *s != 0.00281 { + t.Errorf("CVE-2020-11022 epss = %v, want 0.00281", s) + } + if len(*reqs) != 1 || (*reqs)[0].URL.Query().Get("cve") != "CVE-2020-11022" { + t.Errorf("requests = %v, want one asking only for the CVE id", *reqs) + } +} + +func TestEPSSSkipsRequestWithoutCVEs(t *testing.T) { + e := &EPSS{BaseURL: "http://unused.invalid", Client: http.DefaultClient} + got, err := e.Enrich(context.Background(), []string{"GHSA-xxxx-xxxx-xxxx"}) + if err != nil || len(got) != 0 { + t.Fatalf("Enrich = %v, %v; want nothing and no request", got, err) + } +} + +func TestEPSSErrors(t *testing.T) { + e := &EPSS{BaseURL: failing(t).URL, Client: http.DefaultClient} + if _, err := e.Enrich(context.Background(), []string{"CVE-2020-11022"}); err == nil { + t.Error("Enrich: want an error for HTTP 503") + } + bad := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = w.Write([]byte(`{"data":[{"cve":"CVE-2020-11022","epss":"high"}]}`)) + })) + t.Cleanup(bad.Close) + e = &EPSS{BaseURL: bad.URL, Client: bad.Client()} + if _, err := e.Enrich(context.Background(), []string{"CVE-2020-11022"}); err == nil { + t.Error("Enrich: want an error for a non-numeric score") + } +} diff --git a/internal/vuln/http.go b/internal/vuln/http.go new file mode 100644 index 0000000..eeff2e3 --- /dev/null +++ b/internal/vuln/http.go @@ -0,0 +1,44 @@ +package vuln + +import ( + "context" + "encoding/json" + "fmt" + "io" + "net/http" +) + +// getJSON sends one GET to a vulnerability feed and decodes the reply into +// out. Like OSV.do, any non-200 status is an error, so a failed lookup is +// never read as "no vulnerabilities". name prefixes every error. +func getJSON(ctx context.Context, client *http.Client, name, rawURL string, header http.Header, maxBytes int64, out any) error { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, rawURL, nil) + if err != nil { + return err + } + req.Header.Set("Accept", "application/json") + for k, vs := range header { + for _, v := range vs { + req.Header.Add(k, v) + } + } + resp, err := client.Do(req) + if err != nil { + return fmt.Errorf("%s: GET: %w", name, err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("%s: GET: HTTP %d", name, resp.StatusCode) + } + b, err := io.ReadAll(io.LimitReader(resp.Body, maxBytes+1)) + if err != nil { + return fmt.Errorf("%s: GET: %w", name, err) + } + if int64(len(b)) > maxBytes { + return fmt.Errorf("%s: GET: response exceeds %d bytes", name, maxBytes) + } + if err := json.Unmarshal(b, out); err != nil { + return fmt.Errorf("%s: GET: decode: %w", name, err) + } + return nil +} diff --git a/internal/vuln/kev.go b/internal/vuln/kev.go new file mode 100644 index 0000000..e4c6645 --- /dev/null +++ b/internal/vuln/kev.go @@ -0,0 +1,98 @@ +package vuln + +import ( + "context" + "errors" + "net/http" + "sync" + "time" +) + +// DefaultKEVURL is CISA's Known Exploited Vulnerabilities catalog. +const DefaultKEVURL = "https://www.cisa.gov/sites/default/files/feeds/known_exploited_vulnerabilities.json" + +// KEVTTL is how long a fetched KEV catalog is reused. The catalog is the +// only state Touchstone keeps between scans (see +// docs/architecture/touchstone.md). +const KEVTTL = 6 * time.Hour + +// maxKEVResponseBytes caps the catalog download (about 1.5 MiB today). +const maxKEVResponseBytes = 32 << 20 + +// KEV is the vuln.Enricher that flags findings listed in CISA KEV. The +// feed is fetched on first use and cached in memory for KEVTTL; a KEV is +// safe for concurrent scans. +type KEV struct { + URL string + Client *http.Client + + // Now returns the current time for the cache; time.Now if nil. + Now func() time.Time + + mu sync.Mutex + cves map[string]bool + fetchedAt time.Time +} + +// NewKEV returns a KEV enricher for CISA's feed with a 30 s timeout. +func NewKEV() *KEV { + return &KEV{URL: DefaultKEVURL, Client: &http.Client{Timeout: 30 * time.Second}} +} + +var _ Enricher = (*KEV)(nil) + +func (k *KEV) Name() string { return "KEV" } + +// kevCatalog is the wire shape of the KEV feed. Only the fields Touchstone +// reads are declared. +type kevCatalog struct { + Vulnerabilities []struct { + CVEID string `json:"cveID"` + } `json:"vulnerabilities"` +} + +// Enrich returns {KEV: true} for each id in the catalog; other ids are +// left out. +func (k *KEV) Enrich(ctx context.Context, findingIDs []string) (map[string]Enrichment, error) { + cves, err := k.catalog(ctx) + if err != nil { + return nil, err + } + out := map[string]Enrichment{} + for _, id := range findingIDs { + if cves[id] { + out[id] = Enrichment{KEV: true} + } + } + return out, nil +} + +// catalog returns the cached CVE set, fetching it when it is missing or +// older than KEVTTL. A failed refresh is an error, not a stale answer. +func (k *KEV) catalog(ctx context.Context) (map[string]bool, error) { + now := time.Now + if k.Now != nil { + now = k.Now + } + k.mu.Lock() + defer k.mu.Unlock() + if k.cves != nil && now().Sub(k.fetchedAt) < KEVTTL { + return k.cves, nil + } + var c kevCatalog + if err := getJSON(ctx, k.Client, "kev", k.URL, nil, maxKEVResponseBytes, &c); err != nil { + return nil, err + } + if len(c.Vulnerabilities) == 0 { + return nil, errors.New("kev: catalog lists no vulnerabilities") + } + cves := make(map[string]bool, len(c.Vulnerabilities)) + for _, v := range c.Vulnerabilities { + if v.CVEID == "" { + return nil, errors.New("kev: catalog entry without a cveID") + } + cves[v.CVEID] = true + } + k.cves, k.fetchedAt = cves, now() + return cves, nil +} diff --git a/internal/vuln/live_test.go b/internal/vuln/live_test.go new file mode 100644 index 0000000..35242b5 --- /dev/null +++ b/internal/vuln/live_test.go @@ -0,0 +1,46 @@ +//go:build live + +// The live smoke test asks the real OSV, NVD, KEV and EPSS services. It is +// opt-in (go test -tags live ./internal/vuln) and meant for the nightly +// job, never for pull requests. NVD_API_KEY is used when set. + +package vuln + +import ( + "context" + "os" + "testing" + "time" +) + +func TestLiveMatch(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) + defer cancel() + nvd := NewNVD(os.Getenv("NVD_API_KEY")) + m := &Matcher{ + Purl: NewOSV(), + CPE: nvd, + CPELimit: nvd.PerScanLimit(), + Enrichers: []Enricher{NewKEV(), NewEPSS()}, + } + res := m.Match(ctx, []Query{ + {Purl: "pkg:npm/jquery@1.12.4"}, + {CPE: "cpe:2.3:a:f5:nginx:1.18.0:*:*:*:*:*:*:*"}, + {Purl: "pkg:maven/org.apache.logging.log4j/log4j-core@2.14.1"}, + }) + if res.Status != StatusComplete { + t.Fatalf("Status = %v, want complete: %+v", res.Status, res.Sources) + } + for i, name := range []string{"jquery (OSV)", "nginx (NVD)", "log4j-core (OSV)"} { + if len(res.Matches[i]) == 0 { + t.Errorf("%s: no vulnerabilities", name) + } + } + // Log4Shell is in KEV, so it ranks first for log4j-core and has a score. + if len(res.Matches[2]) > 0 { + top := res.Matches[2][0] + if !top.KEV || top.EPSS == nil { + t.Errorf("log4j-core top match = %+v, want a KEV-listed CVE with an EPSS score", top) + } + } +} diff --git a/internal/vuln/match.go b/internal/vuln/match.go new file mode 100644 index 0000000..57a6310 --- /dev/null +++ b/internal/vuln/match.go @@ -0,0 +1,218 @@ +package vuln + +import ( + "cmp" + "context" + "errors" + "fmt" + "slices" + "strings" +) + +// Match is one finding with the enrichment attached to it. +type Match struct { + Finding + Enrichment +} + +// Rank orders matches for display: KEV-listed first, then EPSS descending, +// then CVSS base descending. A missing EPSS or CVSS sorts after every +// present one; ties keep id order so the report is stable. +func Rank(ms []Match) { + slices.SortStableFunc(ms, func(a, b Match) int { + if a.KEV != b.KEV { + if a.KEV { + return -1 + } + return 1 + } + if c := descMissingLast(a.EPSS, b.EPSS); c != 0 { + return c + } + if c := descMissingLast(a.CVSSBase, b.CVSSBase); c != 0 { + return c + } + return cmp.Compare(a.ID, b.ID) + }) +} + +func descMissingLast(a, b *float64) int { + switch { + case a == nil && b == nil: + return 0 + case a == nil: + return 1 + case b == nil: + return -1 + default: + return cmp.Compare(*b, *a) + } +} + +// Status is how complete a Matcher's answer is. Only StatusComplete may be +// shown as a vulnerability count. +type Status int + +const ( + StatusComplete Status = iota + 1 // every lookup and enrichment succeeded + StatusPartial // some lookups or an enrichment failed or were skipped + StatusFailed // every lookup attempted failed or was skipped +) + +// SourceStatus is one vulnerability source's or enricher's outcome. +type SourceStatus struct { + Source string + OK bool + Error string +} + +// Result is a Matcher's answer: Matches[i] holds the ranked matches for +// the i-th query. +type Result struct { + Matches [][]Match + Status Status + Sources []SourceStatus +} + +// Matcher looks up a scan's components: purls in Purl (OSV), components +// with only a CPE in CPE (NVD), then enriches every finding (KEV, EPSS) +// and ranks each component's matches. A nil Purl or CPE source means that +// kind of query is not looked up. +type Matcher struct { + Purl Source + CPE Source + + // CPELimit caps the CPE lookups in one scan; 0 is no cap. Queries + // past the cap are skipped and reported in the CPE source's status. + CPELimit int + Enrichers []Enricher +} + +// sourceTally counts one source's outcomes within a Match call. +type sourceTally struct { + ok, failed, skipped int + errs []error +} + +// Match looks up every query. A source failing never stops the others; it +// is recorded in Sources and makes Status PARTIAL or FAILED. +func (m *Matcher) Match(ctx context.Context, queries []Query) Result { + res := Result{Matches: make([][]Match, len(queries))} + var purl, cpe sourceTally + cpeTried := 0 + for i, q := range queries { + var src Source + var t *sourceTally + switch { + case q.Purl != "" && m.Purl != nil: + src, t = m.Purl, &purl + case q.Purl == "" && q.CPE != "" && m.CPE != nil: + if m.CPELimit > 0 && cpeTried >= m.CPELimit { + cpe.skipped++ + continue + } + cpeTried++ + src, t = m.CPE, &cpe + default: + continue + } + findings, err := src.Lookup(ctx, []Query{q}) + if err != nil { + t.failed++ + t.errs = append(t.errs, err) + continue + } + t.ok++ + for _, f := range findings { + res.Matches[i] = append(res.Matches[i], Match{Finding: f}) + } + } + if m.Purl != nil { + res.addSource(m.Purl.Name(), purl, 0) + } + if m.CPE != nil { + res.addSource(m.CPE.Name(), cpe, m.CPELimit) + } + + enrichFailed := m.enrich(ctx, &res) + for _, ms := range res.Matches { + Rank(ms) + } + + ok := purl.ok + cpe.ok + bad := purl.failed + purl.skipped + cpe.failed + cpe.skipped + switch { + case bad == 0 && !enrichFailed: + res.Status = StatusComplete + case bad > 0 && ok == 0: + res.Status = StatusFailed + default: + res.Status = StatusPartial + } + return res +} + +// addSource records a source's status if it was asked anything or had +// queries skipped by the cap. +func (r *Result) addSource(name string, t sourceTally, limit int) { + if t.ok+t.failed+t.skipped == 0 { + return + } + errs := t.errs + if t.skipped > 0 { + errs = append(errs, fmt.Errorf("%s: %d CPE lookups skipped: capped at %d per scan without an API key (set NVD_API_KEY)", + strings.ToLower(name), t.skipped, limit)) + } + r.Sources = append(r.Sources, SourceStatus{ + Source: name, + OK: t.failed+t.skipped == 0, + Error: errText(errors.Join(errs...)), + }) +} + +// enrich runs every enricher over the distinct finding ids and merges +// their answers into the matches. It reports whether any enricher failed. +func (m *Matcher) enrich(ctx context.Context, res *Result) bool { + var ids []string + seen := map[string]bool{} + for _, ms := range res.Matches { + for _, x := range ms { + if !seen[x.ID] { + seen[x.ID] = true + ids = append(ids, x.ID) + } + } + } + if len(ids) == 0 { + return false + } + failed := false + for _, e := range m.Enrichers { + got, err := e.Enrich(ctx, ids) + res.Sources = append(res.Sources, SourceStatus{Source: e.Name(), OK: err == nil, Error: errText(err)}) + if err != nil { + failed = true + continue + } + for _, ms := range res.Matches { + for j := range ms { + en, ok := got[ms[j].ID] + if !ok { + continue + } + ms[j].KEV = ms[j].KEV || en.KEV + if ms[j].EPSS == nil { + ms[j].EPSS = en.EPSS + } + } + } + } + return failed +} + +func errText(err error) string { + if err == nil { + return "" + } + return strings.TrimSpace(err.Error()) +} diff --git a/internal/vuln/match_test.go b/internal/vuln/match_test.go new file mode 100644 index 0000000..cd3a3c0 --- /dev/null +++ b/internal/vuln/match_test.go @@ -0,0 +1,189 @@ +package vuln + +import ( + "context" + "errors" + "strings" + "testing" +) + +func f64(v float64) *float64 { return &v } + +func TestRank(t *testing.T) { + for _, tc := range []struct { + name string + in []Match + want []string + }{ + { + name: "KEV first even with a lower EPSS", + in: []Match{ + {Finding: Finding{ID: "A"}, Enrichment: Enrichment{EPSS: f64(0.9)}}, + {Finding: Finding{ID: "B"}, Enrichment: Enrichment{KEV: true, EPSS: f64(0.01)}}, + }, + want: []string{"B", "A"}, + }, + { + name: "EPSS descending", + in: []Match{ + {Finding: Finding{ID: "A"}, Enrichment: Enrichment{EPSS: f64(0.1)}}, + {Finding: Finding{ID: "B"}, Enrichment: Enrichment{EPSS: f64(0.5)}}, + {Finding: Finding{ID: "C"}, Enrichment: Enrichment{EPSS: f64(0.3)}}, + }, + want: []string{"B", "C", "A"}, + }, + { + name: "missing EPSS sorts last even with a high CVSS", + in: []Match{ + {Finding: Finding{ID: "A", CVSSBase: f64(9.8)}}, + {Finding: Finding{ID: "B", CVSSBase: f64(2.0)}, Enrichment: Enrichment{EPSS: f64(0.0001)}}, + }, + want: []string{"B", "A"}, + }, + { + name: "CVSS breaks EPSS ties, missing CVSS last", + in: []Match{ + {Finding: Finding{ID: "A"}, Enrichment: Enrichment{EPSS: f64(0.2)}}, + {Finding: Finding{ID: "B", CVSSBase: f64(5.0)}, Enrichment: Enrichment{EPSS: f64(0.2)}}, + {Finding: Finding{ID: "C", CVSSBase: f64(7.5)}, Enrichment: Enrichment{EPSS: f64(0.2)}}, + }, + want: []string{"C", "B", "A"}, + }, + { + name: "missing EPSS ordered by CVSS", + in: []Match{ + {Finding: Finding{ID: "A", CVSSBase: f64(4.0)}}, + {Finding: Finding{ID: "B", CVSSBase: f64(8.0)}}, + }, + want: []string{"B", "A"}, + }, + { + name: "full ties by id", + in: []Match{ + {Finding: Finding{ID: "CVE-2"}}, + {Finding: Finding{ID: "CVE-1"}}, + }, + want: []string{"CVE-1", "CVE-2"}, + }, + { + name: "KEV group ranked by EPSS within itself", + in: []Match{ + {Finding: Finding{ID: "A"}, Enrichment: Enrichment{KEV: true}}, + {Finding: Finding{ID: "B"}, Enrichment: Enrichment{EPSS: f64(0.99)}}, + {Finding: Finding{ID: "C"}, Enrichment: Enrichment{KEV: true, EPSS: f64(0.4)}}, + }, + want: []string{"C", "A", "B"}, + }, + } { + t.Run(tc.name, func(t *testing.T) { + Rank(tc.in) + var got []string + for _, m := range tc.in { + got = append(got, m.ID) + } + if strings.Join(got, ",") != strings.Join(tc.want, ",") { + t.Errorf("Rank = %v, want %v", got, tc.want) + } + }) + } +} + +func TestLookupStatus(t *testing.T) { + down := errors.New("down") + osvOK := &FakeSource{SourceName: "OSV", Findings: []Finding{{ID: "CVE-1"}}} + osvDown := &FakeSource{SourceName: "OSV", Err: down} + nvdOK := &FakeSource{SourceName: "NVD", Findings: []Finding{{ID: "CVE-2"}}} + nvdDown := &FakeSource{SourceName: "NVD", Err: down} + kevOK := &FakeEnricher{EnricherName: "KEV"} + kevDown := &FakeEnricher{EnricherName: "KEV", Err: down} + both := []Query{{Purl: "pkg:npm/a@1"}, {CPE: "cpe:2.3:a:b:b:1:*:*:*:*:*:*:*"}} + + for _, tc := range []struct { + name string + m Matcher + queries []Query + want Status + }{ + {"all succeed", Matcher{Purl: osvOK, CPE: nvdOK, Enrichers: []Enricher{kevOK}}, both, StatusComplete}, + {"nothing to look up", Matcher{Purl: osvOK, CPE: nvdOK}, nil, StatusComplete}, + {"OSV down, NVD up", Matcher{Purl: osvDown, CPE: nvdOK}, both, StatusPartial}, + {"NVD down, OSV up", Matcher{Purl: osvOK, CPE: nvdDown}, both, StatusPartial}, + {"every source down", Matcher{Purl: osvDown, CPE: nvdDown}, both, StatusFailed}, + {"only source down", Matcher{Purl: osvDown}, both[:1], StatusFailed}, + {"enricher down", Matcher{Purl: osvOK, CPE: nvdOK, Enrichers: []Enricher{kevDown}}, both, StatusPartial}, + {"CPE cap skips a lookup", Matcher{Purl: osvOK, CPE: nvdOK, CPELimit: 1}, + append(both, Query{CPE: "cpe:2.3:a:c:c:1:*:*:*:*:*:*:*"}), StatusPartial}, + {"CPE cap with only CPE queries", Matcher{CPE: nvdOK, CPELimit: 1}, + []Query{{CPE: "cpe:2.3:a:b:b:1:*:*:*:*:*:*:*"}, {CPE: "cpe:2.3:a:c:c:1:*:*:*:*:*:*:*"}}, StatusPartial}, + } { + t.Run(tc.name, func(t *testing.T) { + res := tc.m.Match(context.Background(), tc.queries) + if res.Status != tc.want { + t.Errorf("Status = %v, want %v (sources %+v)", res.Status, tc.want, res.Sources) + } + if res.Status != StatusComplete { + failed := false + for _, s := range res.Sources { + if !s.OK && s.Error != "" { + failed = true + } + } + if !failed { + t.Errorf("Status %v but no SourceStatus carries an error: %+v", res.Status, res.Sources) + } + } + }) + } +} + +func TestMatcherRoutesQueriesAndEnriches(t *testing.T) { + m := Matcher{ + Purl: &FakeSource{SourceName: "OSV", Findings: []Finding{{ID: "CVE-1", CVSSBase: f64(9.0)}, {ID: "CVE-2"}}}, + CPE: &FakeSource{SourceName: "NVD", Findings: []Finding{{ID: "CVE-3"}}}, + Enrichers: []Enricher{ + &FakeEnricher{EnricherName: "KEV", Enrichments: map[string]Enrichment{"CVE-2": {KEV: true}}}, + &FakeEnricher{EnricherName: "EPSS", Enrichments: map[string]Enrichment{"CVE-2": {EPSS: f64(0.3)}, "CVE-3": {EPSS: f64(0.1)}}}, + }, + } + res := m.Match(context.Background(), []Query{ + {Purl: "pkg:npm/a@1", CPE: "cpe:2.3:a:a:a:1:*:*:*:*:*:*:*"}, + {CPE: "cpe:2.3:a:b:b:1:*:*:*:*:*:*:*"}, + {}, + }) + if res.Status != StatusComplete { + t.Fatalf("Status = %v, want complete: %+v", res.Status, res.Sources) + } + if len(res.Matches) != 3 || len(res.Matches[0]) != 2 || len(res.Matches[1]) != 1 || len(res.Matches[2]) != 0 { + t.Fatalf("Matches = %+v; want OSV's two, NVD's one, nothing for the empty query", res.Matches) + } + top := res.Matches[0][0] + if top.ID != "CVE-2" || !top.KEV || top.EPSS == nil || *top.EPSS != 0.3 { + t.Errorf("top match = %+v, want CVE-2 with KEV and EPSS 0.3", top) + } + if e := res.Matches[1][0].EPSS; e == nil || *e != 0.1 { + t.Errorf("NVD match epss = %v, want 0.1", e) + } + var names []string + for _, s := range res.Sources { + names = append(names, s.Source) + } + if strings.Join(names, ",") != "OSV,NVD,KEV,EPSS" { + t.Errorf("sources = %v, want OSV,NVD,KEV,EPSS", names) + } +} + +func TestMatcherReportsCPECap(t *testing.T) { + m := Matcher{CPE: &FakeSource{SourceName: "NVD"}, CPELimit: 2} + var qs []Query + for _, p := range []string{"a", "b", "c", "d"} { + qs = append(qs, Query{CPE: "cpe:2.3:a:" + p + ":" + p + ":1:*:*:*:*:*:*:*"}) + } + res := m.Match(context.Background(), qs) + if len(res.Sources) != 1 { + t.Fatalf("sources = %+v, want the NVD status", res.Sources) + } + s := res.Sources[0] + if s.OK || !strings.Contains(s.Error, "2 CPE lookups skipped") || !strings.Contains(s.Error, "capped at 2") { + t.Errorf("NVD status = %+v, want the cap and the skipped count reported", s) + } +} diff --git a/internal/vuln/nvd.go b/internal/vuln/nvd.go new file mode 100644 index 0000000..1fec967 --- /dev/null +++ b/internal/vuln/nvd.go @@ -0,0 +1,150 @@ +package vuln + +import ( + "context" + "fmt" + "net/http" + "net/url" + "time" +) + +// DefaultNVDURL is the public NVD API. +const DefaultNVDURL = "https://services.nvd.nist.gov" + +// UnauthenticatedNVDLimit is how many CPE lookups one scan makes without +// NVD_API_KEY. NVD allows about 5 unauthenticated requests per 30 s. +const UnauthenticatedNVDLimit = 5 + +// maxNVDResponseBytes caps one NVD response; a popular CPE can match a +// few hundred CVEs. +const maxNVDResponseBytes = 16 << 20 + +// nvdPageSize is the most results NVD returns in one page. +const nvdPageSize = 2000 + +// NVD is the vuln.Source that asks NVD 2.0 for the CVEs that apply to a +// CPE. It is used for components that have no purl. Like OSV, it calls +// the API with its own client, outside the vendor's request budget. +type NVD struct { + BaseURL string + APIKey string // optional; sent as the apiKey header + Client *http.Client +} + +// NewNVD returns an NVD source for the public API with a 20 s timeout per +// request. apiKey may be empty. +func NewNVD(apiKey string) *NVD { + return &NVD{BaseURL: DefaultNVDURL, APIKey: apiKey, Client: &http.Client{Timeout: 20 * time.Second}} +} + +var _ Source = (*NVD)(nil) + +func (n *NVD) Name() string { return "NVD" } + +// PerScanLimit is how many CPE lookups a scan may make: 0 (no cap) with an +// API key, UnauthenticatedNVDLimit without one. +func (n *NVD) PerScanLimit() int { + if n.APIKey != "" { + return 0 + } + return UnauthenticatedNVDLimit +} + +// nvdResponse is the wire shape of GET /rest/json/cves/2.0. Only the +// fields Touchstone reads are declared. +type nvdResponse struct { + TotalResults *int `json:"totalResults"` + Vulnerabilities []struct { + CVE nvdCVE `json:"cve"` + } `json:"vulnerabilities"` +} + +type nvdCVE struct { + ID string `json:"id"` + Descriptions []struct { + Lang string `json:"lang"` + Value string `json:"value"` + } `json:"descriptions"` + Metrics struct { + V31 []nvdMetric `json:"cvssMetricV31"` + V30 []nvdMetric `json:"cvssMetricV30"` + V2 []nvdMetric `json:"cvssMetricV2"` + } `json:"metrics"` +} + +type nvdMetric struct { + Type string `json:"type"` // Primary or Secondary + CVSSData struct { + BaseScore *float64 `json:"baseScore"` + } `json:"cvssData"` +} + +// Lookup returns the CVEs for every query that carries a CPE and no purl +// (purl queries are OSV's). Each CPE is one request. +func (n *NVD) Lookup(ctx context.Context, queries []Query) ([]Finding, error) { + var header http.Header + if n.APIKey != "" { + header = http.Header{"apiKey": {n.APIKey}} + } + var out []Finding + seen := map[string]bool{} + for _, q := range queries { + if q.CPE == "" || q.Purl != "" { + continue + } + v := url.Values{"cpeName": {q.CPE}, "resultsPerPage": {fmt.Sprint(nvdPageSize)}} + var resp nvdResponse + if err := getJSON(ctx, n.Client, "nvd", n.BaseURL+"/rest/json/cves/2.0?"+v.Encode(), header, maxNVDResponseBytes, &resp); err != nil { + return nil, err + } + if resp.TotalResults == nil { + return nil, fmt.Errorf("nvd: %s: response has no totalResults", q.CPE) + } + if *resp.TotalResults > len(resp.Vulnerabilities) { + return nil, fmt.Errorf("nvd: %s: %d results, only %d returned", q.CPE, *resp.TotalResults, len(resp.Vulnerabilities)) + } + for _, r := range resp.Vulnerabilities { + if r.CVE.ID == "" { + return nil, fmt.Errorf("nvd: %s: result without a CVE id", q.CPE) + } + if !seen[r.CVE.ID] { + seen[r.CVE.ID] = true + out = append(out, findingFromNVD(r.CVE)) + } + } + } + return out, nil +} + +// findingFromNVD takes the English description as the summary and the +// newest CVSS version's base score, preferring NVD's own (Primary) metric. +func findingFromNVD(c nvdCVE) Finding { + f := Finding{ID: c.ID} + for _, d := range c.Descriptions { + if d.Lang == "en" { + f.Summary = d.Value + break + } + } + for _, metrics := range [][]nvdMetric{c.Metrics.V31, c.Metrics.V30, c.Metrics.V2} { + if s := baseScore(metrics); s != nil { + f.CVSSBase = s + break + } + } + return f +} + +func baseScore(metrics []nvdMetric) *float64 { + for _, m := range metrics { + if m.Type == "Primary" && m.CVSSData.BaseScore != nil { + return m.CVSSData.BaseScore + } + } + for _, m := range metrics { + if m.CVSSData.BaseScore != nil { + return m.CVSSData.BaseScore + } + } + return nil +} diff --git a/internal/vuln/osv.go b/internal/vuln/osv.go index 994df00..9d08779 100644 --- a/internal/vuln/osv.go +++ b/internal/vuln/osv.go @@ -65,7 +65,7 @@ type osvVuln struct { } // Lookup returns the advisories for every query that carries a purl. CPE -// queries are ignored (NVD handles those, slice S3). Advisories are fetched +// queries are ignored (NVD handles those). Advisories are fetched // with one querybatch call, then one GET per distinct id for details. func (o *OSV) Lookup(ctx context.Context, queries []Query) ([]Finding, error) { var req osvBatchRequest diff --git a/internal/vuln/testdata/epss/epss.json b/internal/vuln/testdata/epss/epss.json new file mode 100644 index 0000000..9831e9a --- /dev/null +++ b/internal/vuln/testdata/epss/epss.json @@ -0,0 +1,14 @@ +{ + "status": "OK", + "status-code": 200, + "version": "1.0", + "access": "public, no authentication required", + "total": 3, + "offset": 0, + "limit": 100, + "data": [ + {"cve": "CVE-2019-11358", "epss": "0.014220000", "percentile": "0.802000000", "date": "2026-10-29"}, + {"cve": "CVE-2020-11022", "epss": "0.002810000", "percentile": "0.512000000", "date": "2026-10-29"}, + {"cve": "CVE-2021-23017", "epss": "0.035470000", "percentile": "0.874000000", "date": "2026-10-29"} + ] +} diff --git a/internal/vuln/testdata/kev/known_exploited_vulnerabilities.json b/internal/vuln/testdata/kev/known_exploited_vulnerabilities.json new file mode 100644 index 0000000..a251d87 --- /dev/null +++ b/internal/vuln/testdata/kev/known_exploited_vulnerabilities.json @@ -0,0 +1,34 @@ +{ + "title": "CISA Catalog of Known Exploited Vulnerabilities (Touchstone test fixture: trimmed, entries chosen for tests)", + "catalogVersion": "2026.10.29", + "dateReleased": "2026-10-29T00:00:00.000Z", + "count": 2, + "vulnerabilities": [ + { + "cveID": "CVE-2020-11022", + "vendorProject": "JQuery", + "product": "JQuery", + "vulnerabilityName": "JQuery Cross-Site Scripting (XSS) Vulnerability", + "dateAdded": "2025-01-23", + "shortDescription": "Test fixture entry.", + "requiredAction": "Apply mitigations per vendor instructions or discontinue use of the product if mitigations are unavailable.", + "dueDate": "2025-02-13", + "knownRansomwareCampaignUse": "Unknown", + "notes": "", + "cwes": ["CWE-79"] + }, + { + "cveID": "CVE-2021-44228", + "vendorProject": "Apache", + "product": "Log4j2", + "vulnerabilityName": "Apache Log4j2 Remote Code Execution Vulnerability", + "dateAdded": "2021-12-10", + "shortDescription": "Test fixture entry.", + "requiredAction": "Apply updates per vendor instructions.", + "dueDate": "2021-12-24", + "knownRansomwareCampaignUse": "Known", + "notes": "", + "cwes": ["CWE-20"] + } + ] +} diff --git a/internal/vuln/testdata/nvd/cves-nginx-1.18.0.json b/internal/vuln/testdata/nvd/cves-nginx-1.18.0.json new file mode 100644 index 0000000..36316f2 --- /dev/null +++ b/internal/vuln/testdata/nvd/cves-nginx-1.18.0.json @@ -0,0 +1,53 @@ +{ + "resultsPerPage": 1, + "startIndex": 0, + "totalResults": 1, + "format": "NVD_CVE", + "version": "2.0", + "timestamp": "2026-10-29T00:00:00.000", + "vulnerabilities": [ + { + "cve": { + "id": "CVE-2021-23017", + "sourceIdentifier": "f5sirt@f5.com", + "published": "2021-06-01T13:15:07.853", + "vulnStatus": "Analyzed", + "descriptions": [ + { + "lang": "en", + "value": "A security issue in nginx resolver was identified, which might allow an attacker who is able to forge UDP packets from the DNS server to cause 1-byte memory overwrite, resulting in worker process crash or potential other impact." + }, + { + "lang": "es", + "value": "Se ha identificado un problema de seguridad en el resolver de nginx." + } + ], + "metrics": { + "cvssMetricV31": [ + { + "source": "nvd@nist.gov", + "type": "Primary", + "cvssData": { + "version": "3.1", + "vectorString": "CVSS:3.1/AV:N/AC:H/PR:N/UI:N/S:U/C:H/I:L/A:H", + "baseScore": 7.7, + "baseSeverity": "HIGH" + } + } + ], + "cvssMetricV2": [ + { + "source": "nvd@nist.gov", + "type": "Primary", + "cvssData": { + "version": "2.0", + "vectorString": "AV:N/AC:L/Au:N/C:P/I:P/A:P", + "baseScore": 7.5 + } + } + ] + } + } + } + ] +} diff --git a/internal/vuln/vuln.go b/internal/vuln/vuln.go index e693a33..a71d17c 100644 --- a/internal/vuln/vuln.go +++ b/internal/vuln/vuln.go @@ -1,8 +1,7 @@ -// Package vuln defines the Source and Enricher seams used to look up and -// enrich vulnerabilities for a component. Real implementations (OSV, NVD, -// KEV, EPSS; see docs/architecture/touchstone.md) land in slice S3; this -// package only fixes the interfaces so S0 callers and their tests can be -// written against them now. +// Package vuln looks up and enriches vulnerabilities for components: the +// Source seam (OSV by purl, NVD by CPE), the Enricher seam (KEV, EPSS), +// Rank, and the Matcher that combines them and decides the lookup status +// (see docs/architecture/touchstone.md). package vuln import "context"