feat(vuln): add NVD CPE fallback, KEV and EPSS enrichment, ranking (#11)

cmd/touchstone/main.go

@@ -1,8 +1,8 @@
 // Command touchstone scans a vendor URL and prints an evidence-graded
 // ScanReport as protojson (--format json, the default) or as Markdown
 // (--format md), or serves the web page and POST /v1/scans (serve, slice
-// S9). The scan fetches the page and its scripts passively and
-// looks components up in OSV (slice S2).
+// S9). The scan fetches the page and its scripts passively, looks
+// components up in OSV and NVD, and ranks them with KEV and EPSS.
 package main
 
 import (
@@ -30,14 +30,27 @@ type newService func(fetch.Options) touchstonev1.ScanServiceServer
 
 func main() {
 	newSvc := func(o fetch.Options) touchstonev1.ScanServiceServer {
-		return &service.Scanner{Fetch: o, Vulns: vuln.NewOSV()}
+		return &service.Scanner{Fetch: o, Vulns: newMatcher(os.Getenv("NVD_API_KEY"))}
 	}
 	if err := run(context.Background(), os.Args[1:], os.Stdout, newSvc); err != nil {
 		fmt.Fprintln(os.Stderr, "touchstone:", err)
 		os.Exit(2)
 	}
 }
 
+// newMatcher wires the public vulnerability feeds: OSV by purl, NVD by CPE
+// (capped per scan without an API key), then KEV and EPSS. One Matcher
+// serves every scan of the process, so the KEV cache is shared.
+func newMatcher(nvdAPIKey string) *vuln.Matcher {
+	nvd := vuln.NewNVD(nvdAPIKey)
+	return &vuln.Matcher{
+		Purl:      vuln.NewOSV(),
+		CPE:       nvd,
+		CPELimit:  nvd.PerScanLimit(),
+		Enrichers: []vuln.Enricher{vuln.NewKEV(), vuln.NewEPSS()},
+	}
+}
+
 func run(ctx context.Context, args []string, out io.Writer, newSvc newService) error {
 	if len(args) > 0 && args[0] == "serve" {
 		return serve(ctx, args[1:], out, os.Stderr, newSvc)

internal/service/scanner.go

@@ -19,9 +19,9 @@ import (
 
 // Scanner is the walking-skeleton ScanService (slice S2). It fetches the
 // target page and the scripts it loads through a PassiveFetcher,
-// fingerprints them, looks each purl up through vuln.Source, and returns
-// the report. Merging across tiers and the NVD, KEV and EPSS lookups come
-// in later slices.
+// fingerprints them, looks each component up through vuln.Matcher (OSV by
+// purl, NVD by CPE, enriched with KEV and EPSS), and returns the report.
+// Merging across tiers comes in a later slice.
 type Scanner struct {
 	touchstonev1.UnimplementedScanServiceServer
 
@@ -30,8 +30,8 @@ type Scanner struct {
 	// ScanRequest.
 	Fetch fetch.Options
 
-	// Vulns looks components up by purl (OSV in the CLI).
-	Vulns vuln.Source
+	// Vulns looks components up and ranks their vulnerabilities.
+	Vulns *vuln.Matcher
 
 	// Now returns the scan's start and completion time; time.Now if nil.
 	Now func() time.Time
@@ -126,38 +126,27 @@ func collect(ctx context.Context, f fetch.Fetcher, page *fetch.Response) ([]pipe
 	return merge(comps), errors.Join(errs...)
 }
 
-// lookup asks vuln.Source about every component that has a purl and
-// converts the components to proto. It returns the lookup status, which is
-// COMPLETE only if every lookup succeeded.
+// lookup asks the Matcher about every component (by purl, else CPE) and
+// converts the components, with their ranked vulnerabilities, to proto.
+// It returns the lookup status, which is COMPLETE only if every lookup
+// and enrichment succeeded.
 func (s *Scanner) lookup(ctx context.Context, rep *touchstonev1.ScanReport, comps []pipeline.Component) (touchstonev1.LookupStatus, []*touchstonev1.Component) {
-	var out []*touchstonev1.Component
-	var ok, failed int
-	var errs []error
-	for _, c := range comps {
-		var findings []vuln.Finding
-		if c.Purl != "" {
-			var err error
-			findings, err = s.Vulns.Lookup(ctx, []vuln.Query{{Purl: c.Purl}})
-			if err != nil {
-				failed++
-				errs = append(errs, err)
-			} else {
-				ok++
-			}
-		}
-		out = append(out, toProto(c, findings))
+	queries := make([]vuln.Query, len(comps))
+	for i, c := range comps {
+		queries[i] = vuln.Query{Purl: c.Purl, CPE: c.CPE}
 	}
-	if ok+failed > 0 {
-		rep.Sources = append(rep.Sources, &touchstonev1.SourceStatus{
-			Source: s.Vulns.Name(),
-			Ok:     failed == 0,
-			Error:  errText(errors.Join(errs...)),
-		})
+	res := s.Vulns.Match(ctx, queries)
+	for _, st := range res.Sources {
+		rep.Sources = append(rep.Sources, &touchstonev1.SourceStatus{Source: st.Source, Ok: st.OK, Error: st.Error})
+	}
+	out := make([]*touchstonev1.Component, len(comps))
+	for i, c := range comps {
+		out[i] = toProto(c, res.Matches[i])
 	}
-	switch {
-	case failed == 0:
+	switch res.Status {
+	case vuln.StatusComplete:
 		return touchstonev1.LookupStatus_LOOKUP_STATUS_COMPLETE, out
-	case ok > 0:
+	case vuln.StatusPartial:
 		return touchstonev1.LookupStatus_LOOKUP_STATUS_PARTIAL, out
 	default:
 		return touchstonev1.LookupStatus_LOOKUP_STATUS_FAILED, out
@@ -199,7 +188,7 @@ func merge(comps []pipeline.Component) []pipeline.Component {
 
 // toProto converts one component. The pipeline's Confidence and Kind
 // strings are the proto enum names without their type prefix.
-func toProto(c pipeline.Component, findings []vuln.Finding) *touchstonev1.Component {
+func toProto(c pipeline.Component, matches []vuln.Match) *touchstonev1.Component {
 	out := &touchstonev1.Component{
 		Name:       c.Name,
 		Version:    c.Version,
@@ -215,11 +204,14 @@ func toProto(c pipeline.Component, findings []vuln.Finding) *touchstonev1.Compon
 			Detail:   e.Detail,
 		})
 	}
-	for _, f := range findings {
+	for _, m := range matches {
 		out.Vulnerabilities = append(out.Vulnerabilities, &touchstonev1.Vulnerability{
-			Id:      f.ID,
-			Aliases: f.Aliases,
-			Summary: f.Summary,
+			Id:       m.ID,
+			Aliases:  m.Aliases,
+			Summary:  m.Summary,
+			Kev:      m.KEV,
+			Epss:     m.EPSS,
+			CvssBase: m.CVSSBase,
 		})
 	}
 	return out

internal/service/scanner_test.go

@@ -68,16 +68,47 @@ func recordedOSVSite(t *testing.T) *httptest.Server {
 	return srv
 }
 
-func newTestScanner(osvURL string) *Scanner {
+// recordedFeed serves one file recorded in internal/vuln/testdata for a GET
+// of path.
+func recordedFeed(t *testing.T, path, file string) *httptest.Server {
+	t.Helper()
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		b, err := os.ReadFile(filepath.Join("..", "vuln", "testdata", file))
+		if r.Method != http.MethodGet || r.URL.Path != path || err != nil {
+			http.NotFound(w, r)
+			return
+		}
+		w.Header().Set("Content-Type", "application/json")
+		_, _ = w.Write(b)
+	}))
+	t.Cleanup(srv.Close)
+	return srv
+}
+
+// newTestScanner looks components up in OSV at osvURL and in recorded
+// NVD, KEV and EPSS replies.
+func newTestScanner(t *testing.T, osvURL string) *Scanner {
+	t.Helper()
+	nvd := recordedFeed(t, "/rest/json/cves/2.0", filepath.Join("nvd", "cves-nginx-1.18.0.json"))
+	kev := recordedFeed(t, "/kev.json", filepath.Join("kev", "known_exploited_vulnerabilities.json"))
+	epss := recordedFeed(t, "/data/v1/epss", filepath.Join("epss", "epss.json"))
 	return &Scanner{
 		Fetch: fetch.Options{AllowPrivate: true, Version: ToolVersion},
-		Vulns: &vuln.OSV{BaseURL: osvURL, Client: http.DefaultClient},
+		Vulns: &vuln.Matcher{
+			Purl:     &vuln.OSV{BaseURL: osvURL, Client: http.DefaultClient},
+			CPE:      &vuln.NVD{BaseURL: nvd.URL, Client: http.DefaultClient},
+			CPELimit: vuln.UnauthenticatedNVDLimit,
+			Enrichers: []vuln.Enricher{
+				&vuln.KEV{URL: kev.URL + "/kev.json", Client: http.DefaultClient},
+				&vuln.EPSS{BaseURL: epss.URL, Client: http.DefaultClient},
+			},
+		},
 	}
 }
 
 func TestScannerReportsJQueryWithOSVIDs(t *testing.T) {
 	site := vendorSite(t)
-	rep, err := newTestScanner(recordedOSVSite(t).URL).Scan(context.Background(),
+	rep, err := newTestScanner(t, recordedOSVSite(t).URL).Scan(context.Background(),
 		&touchstonev1.ScanRequest{Url: site.URL + "/"})
 	if err != nil {
 		t.Fatalf("Scan: %v", err)
@@ -121,9 +152,55 @@ func TestScannerReportsJQueryWithOSVIDs(t *testing.T) {
 	}
 }
 
+// The test KEV feed lists CVE-2020-11022, so it ranks above
+// CVE-2019-11358 despite the lower EPSS; the nginx banner is matched by
+// CPE in NVD.
+func TestScannerRanksAndEnriches(t *testing.T) {
+	site := vendorSite(t)
+	rep, err := newTestScanner(t, recordedOSVSite(t).URL).Scan(context.Background(),
+		&touchstonev1.ScanRequest{Url: site.URL + "/"})
+	if err != nil {
+		t.Fatalf("Scan: %v", err)
+	}
+	if rep.GetLookupStatus() != touchstonev1.LookupStatus_LOOKUP_STATUS_COMPLETE {
+		t.Fatalf("lookup_status = %v, want COMPLETE: %v", rep.GetLookupStatus(), rep.GetSources())
+	}
+	byName := map[string]*touchstonev1.Component{}
+	for _, c := range rep.GetComponents() {
+		byName[c.GetName()] = c
+	}
+
+	jq := byName["jquery"].GetVulnerabilities()
+	if len(jq) != 2 {
+		t.Fatalf("jquery vulnerabilities = %v, want 2", jq)
+	}
+	if jq[0].GetId() != "CVE-2020-11022" || !jq[0].GetKev() || jq[0].Epss == nil {
+		t.Errorf("first jquery vulnerability = %v, want the KEV-listed CVE-2020-11022 with an EPSS score", jq[0])
+	}
+	if jq[1].GetId() != "CVE-2019-11358" || jq[1].GetKev() || jq[1].GetEpss() != 0.01422 {
+		t.Errorf("second jquery vulnerability = %v, want CVE-2019-11358, not KEV, EPSS 0.01422", jq[1])
+	}
+
+	ng := byName["nginx"].GetVulnerabilities()
+	if len(ng) != 1 || ng[0].GetId() != "CVE-2021-23017" || ng[0].GetCvssBase() != 7.7 || ng[0].Epss == nil {
+		t.Errorf("nginx vulnerabilities = %v, want CVE-2021-23017 from NVD with CVSS 7.7 and EPSS", ng)
+	}
+
+	var sources []string
+	for _, s := range rep.GetSources() {
+		if !s.GetOk() {
+			t.Errorf("source %s failed: %s", s.GetSource(), s.GetError())
+		}
+		sources = append(sources, s.GetSource())
+	}
+	if strings.Join(sources, ",") != "fingerprint,OSV,NVD,KEV,EPSS" {
+		t.Errorf("sources = %v, want fingerprint,OSV,NVD,KEV,EPSS", sources)
+	}
+}
+
 func TestScannerCheckedListMatchesRequests(t *testing.T) {
 	site := vendorSite(t)
-	rep, err := newTestScanner(recordedOSVSite(t).URL).Scan(context.Background(),
+	rep, err := newTestScanner(t, recordedOSVSite(t).URL).Scan(context.Background(),
 		&touchstonev1.ScanRequest{Url: site.URL + "/"})
 	if err != nil {
 		t.Fatalf("Scan: %v", err)
@@ -152,7 +229,7 @@ func TestScannerCheckedListMatchesRequests(t *testing.T) {
 
 func TestScannerRecordsScriptRefusedByBudget(t *testing.T) {
 	site := vendorSite(t)
-	rep, err := newTestScanner(recordedOSVSite(t).URL).Scan(context.Background(),
+	rep, err := newTestScanner(t, recordedOSVSite(t).URL).Scan(context.Background(),
 		&touchstonev1.ScanRequest{Url: site.URL + "/", RequestBudget: 1})
 	if err != nil {
 		t.Fatalf("Scan: %v", err)
@@ -172,7 +249,7 @@ func TestScannerRecordsScriptRefusedByBudget(t *testing.T) {
 func TestScannerTargetUnreachable(t *testing.T) {
 	site := httptest.NewServer(http.NotFoundHandler())
 	site.Close()
-	rep, err := newTestScanner(recordedOSVSite(t).URL).Scan(context.Background(),
+	rep, err := newTestScanner(t, recordedOSVSite(t).URL).Scan(context.Background(),
 		&touchstonev1.ScanRequest{Url: site.URL + "/"})
 	if err != nil {
 		t.Fatalf("Scan: an unreachable target is a report, not an RPC error: %v", err)
@@ -188,23 +265,25 @@ func TestScannerTargetUnreachable(t *testing.T) {
 	}
 }
 
-func TestScannerOSVDownIsFailedLookup(t *testing.T) {
+// With OSV down but NVD up, the nginx banner (CPE only) is still looked
+// up, so the report is PARTIAL, never COMPLETE.
+func TestScannerOSVDownIsPartialLookup(t *testing.T) {
 	osv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		http.Error(w, "down", http.StatusServiceUnavailable)
 	}))
 	t.Cleanup(osv.Close)
 	site := vendorSite(t)
-	rep, err := newTestScanner(osv.URL).Scan(context.Background(),
+	rep, err := newTestScanner(t, osv.URL).Scan(context.Background(),
 		&touchstonev1.ScanRequest{Url: site.URL + "/"})
 	if err != nil {
 		t.Fatalf("Scan: %v", err)
 	}
-	if rep.GetLookupStatus() != touchstonev1.LookupStatus_LOOKUP_STATUS_FAILED {
-		t.Errorf("lookup_status = %v, want FAILED", rep.GetLookupStatus())
+	if rep.GetLookupStatus() != touchstonev1.LookupStatus_LOOKUP_STATUS_PARTIAL {
+		t.Errorf("lookup_status = %v, want PARTIAL", rep.GetLookupStatus())
 	}
 	for _, c := range rep.GetComponents() {
-		if len(c.GetVulnerabilities()) != 0 {
-			t.Errorf("component %s has vulnerabilities despite the failed lookup", c.GetName())
+		if c.GetName() == "jquery" && len(c.GetVulnerabilities()) != 0 {
+			t.Errorf("jquery has vulnerabilities despite the failed OSV lookup")
 		}
 	}
 	var osvStatus *touchstonev1.SourceStatus
@@ -219,7 +298,7 @@ func TestScannerOSVDownIsFailedLookup(t *testing.T) {
 }
 
 func TestScannerRejectsNonHTTPTarget(t *testing.T) {
-	_, err := newTestScanner("http://unused").Scan(context.Background(),
+	_, err := newTestScanner(t, "http://unused").Scan(context.Background(),
 		&touchstonev1.ScanRequest{Url: "ftp://vendor.example/"})
 	if err == nil {
 		t.Fatal("Scan: want an InvalidArgument error for an ftp URL")

internal/vuln/epss.go

@@ -0,0 +1,78 @@
+package vuln
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"net/http"
+	"net/url"
+	"strconv"
+	"strings"
+	"time"
+)
+
+// DefaultEPSSURL is FIRST's EPSS API.
+const DefaultEPSSURL = "https://api.first.org"
+
+// epssBatch is how many CVEs one EPSS request asks for; it is the API's
+// default page size, so one page always holds the whole answer.
+const epssBatch = 100
+
+const maxEPSSResponseBytes = 2 << 20
+
+// EPSS is the vuln.Enricher that adds FIRST's exploit prediction score to
+// each finding with a CVE id.
+type EPSS struct {
+	BaseURL string
+	Client  *http.Client
+}
+
+// NewEPSS returns an EPSS enricher for the public API with a 10 s timeout
+// per request.
+func NewEPSS() *EPSS {
+	return &EPSS{BaseURL: DefaultEPSSURL, Client: &http.Client{Timeout: 10 * time.Second}}
+}
+
+var _ Enricher = (*EPSS)(nil)
+
+func (e *EPSS) Name() string { return "EPSS" }
+
+// epssResponse is the wire shape of GET /data/v1/epss. The API sends the
+// score as a decimal string.
+type epssResponse struct {
+	Data []struct {
+		CVE  string `json:"cve"`
+		EPSS string `json:"epss"`
+	} `json:"data"`
+}
+
+// Enrich returns the EPSS score of each CVE id that has one. Ids that are
+// not CVEs (an OSV or GHSA id with no CVE alias) are never sent.
+func (e *EPSS) Enrich(ctx context.Context, findingIDs []string) (map[string]Enrichment, error) {
+	var cves []string
+	for _, id := range findingIDs {
+		if strings.HasPrefix(id, "CVE-") {
+			cves = append(cves, id)
+		}
+	}
+	out := map[string]Enrichment{}
+	for start := 0; start < len(cves); start += epssBatch {
+		batch := cves[start:min(start+epssBatch, len(cves))]
+		v := url.Values{"cve": {strings.Join(batch, ",")}}
+		var resp epssResponse
+		if err := getJSON(ctx, e.Client, "epss", e.BaseURL+"/data/v1/epss?"+v.Encode(), nil, maxEPSSResponseBytes, &resp); err != nil {
+			return nil, err
+		}
+		for _, d := range resp.Data {
+			if d.CVE == "" {
+				return nil, errors.New("epss: result without a cve")
+			}
+			score, err := strconv.ParseFloat(d.EPSS, 64)
+			if err != nil || score < 0 || score > 1 {
+				return nil, fmt.Errorf("epss: %s: bad score %q", d.CVE, d.EPSS)
+			}
+			out[d.CVE] = Enrichment{EPSS: &score}
+		}
+	}
+	return out, nil
+}