From cd02869c0a3e9257a274cdb222a0778bcab77bfc Mon Sep 17 00:00:00 2001 From: "containarium-ci[bot]" <287705844+containarium-ci[bot]@users.noreply.github.com> Date: Sat, 24 Oct 2026 09:34:14 +0000 Subject: [PATCH 01/12] feat(scan): walking skeleton from page fetch to OSV-backed report (#8) --- cmd/touchstone/main.go | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/cmd/touchstone/main.go b/cmd/touchstone/main.go index b8fddd7..19cf4e7 100644 --- a/cmd/touchstone/main.go +++ b/cmd/touchstone/main.go @@ -1,7 +1,7 @@ // Command touchstone scans a vendor URL and prints an evidence-graded // ScanReport as protojson (--format json, the default) or as Markdown -// (--format md). In slice S0 the scan is served by the stub ScanService, -// so the report is canned. +// (--format md). The scan fetches the page and its scripts passively and +// looks components up in OSV (slice S2). package main import ( @@ -16,6 +16,7 @@ import ( "github.com/FootprintAI/touchstone/internal/fetch" "github.com/FootprintAI/touchstone/internal/report" "github.com/FootprintAI/touchstone/internal/service" + "github.com/FootprintAI/touchstone/internal/vuln" "google.golang.org/protobuf/encoding/protojson" ) @@ -27,7 +28,9 @@ const usage = "usage: touchstone scan [--allow-private] [--format json|md] type newService func(fetch.Options) touchstonev1.ScanServiceServer func main() { - newSvc := func(fetch.Options) touchstonev1.ScanServiceServer { return &service.Stub{} } + newSvc := func(o fetch.Options) touchstonev1.ScanServiceServer { + return &service.Scanner{Fetch: o, Vulns: vuln.NewOSV()} + } if err := run(context.Background(), os.Args[1:], os.Stdout, newSvc); err != nil { fmt.Fprintln(os.Stderr, "touchstone:", err) os.Exit(2) From 32a764c1d84cc3c19ae1cb54e8686bafb5e1b928 Mon Sep 17 00:00:00 2001 From: "containarium-ci[bot]" <287705844+containarium-ci[bot]@users.noreply.github.com> Date: Sat, 24 Oct 2026 09:34:16 +0000 Subject: [PATCH 02/12] feat(scan): walking skeleton from page fetch to OSV-backed report (#8) --- go.mod | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index bf9ff3d..c2ed19f 100644 --- a/go.mod +++ b/go.mod @@ -3,15 +3,16 @@ module github.com/FootprintAI/touchstone go 1.26.0 require ( + github.com/dlclark/regexp2 v1.11.0 github.com/grpc-ecosystem/grpc-gateway/v2 v2.32.0 github.com/santhosh-tekuri/jsonschema/v6 v6.0.3 + golang.org/x/net v0.61.0 google.golang.org/genproto/googleapis/api v0.0.0-20261019180522-ce908d29299b google.golang.org/grpc v1.84.0 google.golang.org/protobuf v1.36.12 ) require ( - golang.org/x/net v0.61.0 // indirect golang.org/x/sys v0.49.0 // indirect golang.org/x/text v0.43.0 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20261012220534-4341edf3f941 // indirect From 124c7c9ea4b0ca30e86a2552e5846fba84d9640e Mon Sep 17 00:00:00 2001 From: "containarium-ci[bot]" <287705844+containarium-ci[bot]@users.noreply.github.com> Date: Sat, 24 Oct 2026 09:34:17 +0000 Subject: [PATCH 03/12] feat(scan): walking skeleton from page fetch to OSV-backed report (#8) --- internal/fingerprint/data/signatures.json | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 internal/fingerprint/data/signatures.json diff --git a/internal/fingerprint/data/signatures.json b/internal/fingerprint/data/signatures.json new file mode 100644 index 0000000..571c088 --- /dev/null +++ b/internal/fingerprint/data/signatures.json @@ -0,0 +1,14 @@ +{ + "_comment": "Retire.js-style signatures, uri and filename extractors only. Hand-curated for slice S2 (jQuery only); to be replaced by a pinned snapshot of Retire.js jsrepository in S7. {version} in purl and cpe is replaced by the captured version.", + "libraries": [ + { + "name": "jquery", + "purl": "pkg:npm/jquery@{version}", + "cpe": "cpe:2.3:a:jquery:jquery:{version}:*:*:*:*:*:*:*", + "extractors": { + "uri": ["jquery[-.]v?([0-9]+(?:\\.[0-9]+)+)(?:\\.min|\\.slim|\\.slim\\.min)?\\.js"], + "filename": ["^jquery[-.]v?([0-9]+(?:\\.[0-9]+)+)(?:\\.min|\\.slim|\\.slim\\.min)?\\.js$"] + } + } + ] +} From 1391877a0d6e90f5004e6ab4e708ccf6c123e36e Mon Sep 17 00:00:00 2001 From: "containarium-ci[bot]" <287705844+containarium-ci[bot]@users.noreply.github.com> Date: Sat, 24 Oct 2026 09:34:19 +0000 Subject: [PATCH 04/12] feat(scan): walking skeleton from page fetch to OSV-backed report (#8) --- internal/fingerprint/fingerprint.go | 238 ++++++++++++++++++++++++++++ 1 file changed, 238 insertions(+) create mode 100644 internal/fingerprint/fingerprint.go diff --git a/internal/fingerprint/fingerprint.go b/internal/fingerprint/fingerprint.go new file mode 100644 index 0000000..5aaedd6 --- /dev/null +++ b/internal/fingerprint/fingerprint.go @@ -0,0 +1,238 @@ +// Package fingerprint detects libraries from the URLs of fetched scripts, +// from response headers, and from the + + + + + + `) + got, err := ScriptURLs(page, body) + if err != nil { + t.Fatalf("ScriptURLs: %v", err) + } + want := []string{ + "https://vendor.example/app/js/jquery-1.12.4.min.js", + "https://vendor.example/static/app.js?v=2", + "https://cdn.example/lib/x.js", + "https://vendor.example/app/main.mjs", + } + if !reflect.DeepEqual(got, want) { + t.Errorf("ScriptURLs = %q\nwant %q", got, want) + } +} + +func TestFromHeaders(t *testing.T) { + page := "https://vendor.example/" + tests := []struct { + name string + header http.Header + want []string // "name version" + }{ + {"apache", http.Header{"Server": {"Apache/2.4.29 (Ubuntu)"}}, []string{"apache http_server 2.4.29"}}, + {"nginx", http.Header{"Server": {"nginx/1.18.0"}}, []string{"nginx 1.18.0"}}, + {"php", http.Header{"X-Powered-By": {"PHP/7.2.34"}}, []string{"php 7.2.34"}}, + {"unknown banner", http.Header{"Server": {"cloudflare"}}, nil}, + {"no headers", http.Header{}, nil}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := FromHeaders(page, tt.header) + var names []string + for _, c := range got { + names = append(names, c.Name+" "+c.Version) + if c.Purl != "" { + t.Errorf("banner component has purl %q; banners carry CPE only", c.Purl) + } + if !contains(c.Caveats, "MAY_BE_BACKPORTED") { + t.Errorf("caveats = %v, want MAY_BE_BACKPORTED", c.Caveats) + } + if len(c.Evidence) != 1 || c.Evidence[0].Kind != "FINGERPRINT_HEADER" || c.Evidence[0].Location != page { + t.Errorf("evidence = %+v, want header evidence at %s", c.Evidence, page) + } + if !strings.HasPrefix(c.CPE, "cpe:2.3:a:") { + t.Errorf("cpe = %q", c.CPE) + } + } + if !reflect.DeepEqual(names, tt.want) { + t.Errorf("components = %q, want %q", names, tt.want) + } + }) + } +} + +func contains(list []string, want string) bool { + for _, s := range list { + if s == want { + return true + } + } + return false +} From f7fe1d40ed39a8fa3f5c007edd3b3b606e256f70 Mon Sep 17 00:00:00 2001 From: "containarium-ci[bot]" <287705844+containarium-ci[bot]@users.noreply.github.com> Date: Sat, 24 Oct 2026 09:34:22 +0000 Subject: [PATCH 06/12] feat(scan): walking skeleton from page fetch to OSV-backed report (#8) --- internal/service/scanner.go | 242 ++++++++++++++++++++++++++++++++++++ 1 file changed, 242 insertions(+) create mode 100644 internal/service/scanner.go diff --git a/internal/service/scanner.go b/internal/service/scanner.go new file mode 100644 index 0000000..76c139e --- /dev/null +++ b/internal/service/scanner.go @@ -0,0 +1,242 @@ +package service + +import ( + "context" + "errors" + "net/url" + "strings" + "time" + + touchstonev1 "github.com/FootprintAI/touchstone/gen/go/touchstone/v1" + "github.com/FootprintAI/touchstone/internal/fetch" + "github.com/FootprintAI/touchstone/internal/fingerprint" + "github.com/FootprintAI/touchstone/internal/pipeline" + "github.com/FootprintAI/touchstone/internal/vuln" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + "google.golang.org/protobuf/types/known/timestamppb" +) + +// Scanner is the walking-skeleton ScanService (slice S2). It fetches the +// target page and the scripts it loads through a PassiveFetcher, +// fingerprints them, looks each purl up through vuln.Source, and returns +// the report. Merging across tiers and the NVD, KEV and EPSS lookups come +// in later slices. +type Scanner struct { + touchstonev1.UnimplementedScanServiceServer + + // Fetch carries the settings that are not per request: AllowPrivate + // (CLI only) and Version. The request budget comes from each + // ScanRequest. + Fetch fetch.Options + + // Vulns looks components up by purl (OSV in the CLI). + Vulns vuln.Source + + // Now returns the scan's start and completion time; time.Now if nil. + Now func() time.Time +} + +var _ touchstonev1.ScanServiceServer = (*Scanner)(nil) + +// Scan runs one scan. A target that cannot be fetched is not an RPC error: +// the report says TARGET_UNREACHABLE and lists the attempt in checked. +func (s *Scanner) Scan(ctx context.Context, req *touchstonev1.ScanRequest) (*touchstonev1.ScanReport, error) { + target, err := absoluteURL(req.GetUrl()) + if err != nil { + return nil, err + } + opts := s.Fetch + opts.Budget = req.GetRequestBudget() + f, err := fetch.New(target.String(), opts) + if err != nil { + return nil, status.Error(codes.InvalidArgument, err.Error()) + } + + now := time.Now + if s.Now != nil { + now = s.Now + } + rep := &touchstonev1.ScanReport{ + SchemaVersion: "1", + TargetUrl: target.String(), + ToolVersion: ToolVersion, + StartedAt: timestamppb.New(now().UTC()), + RequestBudget: f.RequestBudget(), + Disclaimer: Disclaimer, + } + + page, err := f.Get(ctx, target.String(), fetch.ReasonTarget) + if err != nil { + rep.ScanStatus = touchstonev1.ScanStatus_SCAN_STATUS_TARGET_UNREACHABLE + s.finish(rep, f, now) + return rep, nil + } + rep.ScanStatus = touchstonev1.ScanStatus_SCAN_STATUS_OK + + comps, fpErr := collect(ctx, f, page) + rep.Sources = append(rep.Sources, &touchstonev1.SourceStatus{ + Source: "fingerprint", + Ok: fpErr == nil, + Error: errText(fpErr), + }) + + rep.LookupStatus, rep.Components = s.lookup(ctx, rep, comps) + s.finish(rep, f, now) + return rep, nil +} + +// collect fingerprints the target page's headers and the scripts it +// references. Each script is fetched as SUBRESOURCE, as a browser would +// load it, and only a 2xx script counts as loaded. +func collect(ctx context.Context, f fetch.Fetcher, page *fetch.Response) ([]pipeline.Component, error) { + comps := fingerprint.FromHeaders(page.URL, page.Header) + pageURL, err := url.Parse(page.URL) + if err != nil { + return comps, err + } + scripts, err := fingerprint.ScriptURLs(pageURL, page.Body) + if err != nil { + return comps, err + } + var errs []error + for _, src := range scripts { + resp, err := f.Get(ctx, src, fetch.ReasonSubresource) + if err != nil { + // The refusal or failure is already in the checked list. + continue + } + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + continue + } + found, err := fingerprint.FromURL(src) + comps = append(comps, found...) + if err != nil { + errs = append(errs, err) + } + } + return merge(comps), errors.Join(errs...) +} + +// lookup asks vuln.Source about every component that has a purl and +// converts the components to proto. It returns the lookup status, which is +// COMPLETE only if every lookup succeeded. +func (s *Scanner) lookup(ctx context.Context, rep *touchstonev1.ScanReport, comps []pipeline.Component) (touchstonev1.LookupStatus, []*touchstonev1.Component) { + var out []*touchstonev1.Component + var ok, failed int + var errs []error + for _, c := range comps { + var findings []vuln.Finding + if c.Purl != "" { + var err error + findings, err = s.Vulns.Lookup(ctx, []vuln.Query{{Purl: c.Purl}}) + if err != nil { + failed++ + errs = append(errs, err) + } else { + ok++ + } + } + out = append(out, toProto(c, findings)) + } + if ok+failed > 0 { + rep.Sources = append(rep.Sources, &touchstonev1.SourceStatus{ + Source: s.Vulns.Name(), + Ok: failed == 0, + Error: errText(errors.Join(errs...)), + }) + } + switch { + case failed == 0: + return touchstonev1.LookupStatus_LOOKUP_STATUS_COMPLETE, out + case ok > 0: + return touchstonev1.LookupStatus_LOOKUP_STATUS_PARTIAL, out + default: + return touchstonev1.LookupStatus_LOOKUP_STATUS_FAILED, out + } +} + +// finish fills in the request accounting and the completion time. +func (s *Scanner) finish(rep *touchstonev1.ScanReport, f *fetch.PassiveFetcher, now func() time.Time) { + rep.Checked = f.Checked() + rep.RequestsUsed = f.RequestsUsed() + rep.CompletedAt = timestamppb.New(now().UTC()) +} + +// merge combines components that share a key (purl, else name): the same +// library loaded twice is one component with both evidence items. +func merge(comps []pipeline.Component) []pipeline.Component { + var out []pipeline.Component + index := map[string]int{} + for _, c := range comps { + key := c.Purl + if key == "" { + key = c.Name + } + i, seen := index[key] + if !seen { + index[key] = len(out) + out = append(out, c) + continue + } + out[i].Evidence = append(out[i].Evidence, c.Evidence...) + for _, caveat := range c.Caveats { + if !contains(out[i].Caveats, caveat) { + out[i].Caveats = append(out[i].Caveats, caveat) + } + } + } + return out +} + +// toProto converts one component. The pipeline's Confidence and Kind +// strings are the proto enum names without their type prefix. +func toProto(c pipeline.Component, findings []vuln.Finding) *touchstonev1.Component { + out := &touchstonev1.Component{ + Name: c.Name, + Version: c.Version, + Purl: c.Purl, + Cpe: c.CPE, + Confidence: touchstonev1.Confidence(touchstonev1.Confidence_value["CONFIDENCE_"+c.Confidence]), + Caveats: c.Caveats, + } + for _, e := range c.Evidence { + out.Evidence = append(out.Evidence, &touchstonev1.Evidence{ + Kind: touchstonev1.EvidenceKind(touchstonev1.EvidenceKind_value["EVIDENCE_KIND_"+e.Kind]), + Location: e.Location, + Detail: e.Detail, + }) + } + for _, f := range findings { + out.Vulnerabilities = append(out.Vulnerabilities, &touchstonev1.Vulnerability{ + Id: f.ID, + Aliases: f.Aliases, + Summary: f.Summary, + }) + } + return out +} + +func absoluteURL(raw string) (*url.URL, error) { + u, err := url.Parse(raw) + if err != nil || (u.Scheme != "http" && u.Scheme != "https") || u.Host == "" { + return nil, status.Errorf(codes.InvalidArgument, "url must be an absolute http(s) URL, got %q", raw) + } + return u, nil +} + +func errText(err error) string { + if err == nil { + return "" + } + return strings.TrimSpace(err.Error()) +} + +func contains(list []string, want string) bool { + for _, s := range list { + if s == want { + return true + } + } + return false +} From 5d704509ec5338c62e1ceafa760ff00056d8b475 Mon Sep 17 00:00:00 2001 From: "containarium-ci[bot]" <287705844+containarium-ci[bot]@users.noreply.github.com> Date: Sat, 24 Oct 2026 09:34:24 +0000 Subject: [PATCH 07/12] feat(scan): walking skeleton from page fetch to OSV-backed report (#8) --- internal/service/scanner_test.go | 227 +++++++++++++++++++++++++++++++ 1 file changed, 227 insertions(+) create mode 100644 internal/service/scanner_test.go diff --git a/internal/service/scanner_test.go b/internal/service/scanner_test.go new file mode 100644 index 0000000..d73098b --- /dev/null +++ b/internal/service/scanner_test.go @@ -0,0 +1,227 @@ +package service + +import ( + "context" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "testing" + + touchstonev1 "github.com/FootprintAI/touchstone/gen/go/touchstone/v1" + "github.com/FootprintAI/touchstone/internal/fetch" + "github.com/FootprintAI/touchstone/internal/vuln" +) + +const fixturePage = ` + +vendor` + +// vendorSite serves the fixture page and its jQuery script. The page is +// served with a Server banner so the header rule is exercised too. +func vendorSite(t *testing.T) *httptest.Server { + t.Helper() + mux := http.NewServeMux() + mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/" { + http.NotFound(w, r) + return + } + w.Header().Set("Server", "nginx/1.18.0") + w.Header().Set("Content-Type", "text/html") + _, _ = w.Write([]byte(fixturePage)) + }) + mux.HandleFunc("/js/jquery-1.12.4.min.js", func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/javascript") + _, _ = w.Write([]byte("/*! jQuery v1.12.4 */")) + }) + srv := httptest.NewServer(mux) + t.Cleanup(srv.Close) + return srv +} + +// recordedOSVSite replays the OSV responses recorded in internal/vuln. +func recordedOSVSite(t *testing.T) *httptest.Server { + t.Helper() + dir := filepath.Join("..", "vuln", "testdata", "osv") + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + var file string + switch { + case r.Method == http.MethodPost && r.URL.Path == "/v1/querybatch": + file = filepath.Join(dir, "querybatch-jquery-1.12.4.json") + case r.Method == http.MethodGet && strings.HasPrefix(r.URL.Path, "/v1/vulns/"): + file = filepath.Join(dir, "vulns", strings.TrimPrefix(r.URL.Path, "/v1/vulns/")+".json") + default: + http.NotFound(w, r) + return + } + b, err := os.ReadFile(file) + if err != nil { + http.NotFound(w, r) + return + } + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write(b) + })) + t.Cleanup(srv.Close) + return srv +} + +func newTestScanner(osvURL string) *Scanner { + return &Scanner{ + Fetch: fetch.Options{AllowPrivate: true, Version: ToolVersion}, + Vulns: &vuln.OSV{BaseURL: osvURL, Client: http.DefaultClient}, + } +} + +func TestScannerReportsJQueryWithOSVIDs(t *testing.T) { + site := vendorSite(t) + rep, err := newTestScanner(recordedOSVSite(t).URL).Scan(context.Background(), + &touchstonev1.ScanRequest{Url: site.URL + "/"}) + if err != nil { + t.Fatalf("Scan: %v", err) + } + + if rep.GetScanStatus() != touchstonev1.ScanStatus_SCAN_STATUS_OK { + t.Fatalf("scan_status = %v, want OK", rep.GetScanStatus()) + } + if rep.GetLookupStatus() != touchstonev1.LookupStatus_LOOKUP_STATUS_COMPLETE { + t.Errorf("lookup_status = %v, want COMPLETE", rep.GetLookupStatus()) + } + if len(rep.GetComponents()) != 2 { + t.Fatalf("components = %v, want jquery and the nginx banner", rep.GetComponents()) + } + + var jq *touchstonev1.Component + for _, c := range rep.GetComponents() { + if c.GetName() == "jquery" { + jq = c + } + } + if jq == nil { + t.Fatalf("no jquery component in %v", rep.GetComponents()) + } + if jq.GetVersion() != "1.12.4" || jq.GetPurl() != "pkg:npm/jquery@1.12.4" { + t.Errorf("jquery = version %q purl %q", jq.GetVersion(), jq.GetPurl()) + } + if jq.GetConfidence() != touchstonev1.Confidence_CONFIDENCE_LOW { + t.Errorf("confidence = %v, want LOW", jq.GetConfidence()) + } + if len(jq.GetEvidence()) != 1 || jq.GetEvidence()[0].GetLocation() != site.URL+"/js/jquery-1.12.4.min.js" || + jq.GetEvidence()[0].GetKind() != touchstonev1.EvidenceKind_EVIDENCE_KIND_FINGERPRINT_ASSET { + t.Errorf("evidence = %v, want the script URL as asset evidence", jq.GetEvidence()) + } + var ids []string + for _, v := range jq.GetVulnerabilities() { + ids = append(ids, v.GetId()) + } + if !contains(ids, "CVE-2020-11022") { + t.Errorf("vulnerability ids = %v, want CVE-2020-11022", ids) + } +} + +func TestScannerCheckedListMatchesRequests(t *testing.T) { + site := vendorSite(t) + rep, err := newTestScanner(recordedOSVSite(t).URL).Scan(context.Background(), + &touchstonev1.ScanRequest{Url: site.URL + "/"}) + if err != nil { + t.Fatalf("Scan: %v", err) + } + checked := rep.GetChecked() + if len(checked) != 2 { + t.Fatalf("checked = %v, want the target and the script", checked) + } + want := []struct{ url, reason string }{ + {site.URL + "/", "TARGET"}, + {site.URL + "/js/jquery-1.12.4.min.js", "SUBRESOURCE"}, + } + for i, w := range want { + if checked[i].GetUrl() != w.url || checked[i].GetReason() != w.reason || + checked[i].GetResult() != touchstonev1.CheckedLocation_RESULT_FOUND { + t.Errorf("checked[%d] = %v, want %s %s FOUND", i, checked[i], w.reason, w.url) + } + } + if rep.GetRequestsUsed() != uint32(len(checked)) { + t.Errorf("requests_used = %d, want %d", rep.GetRequestsUsed(), len(checked)) + } + if rep.GetRequestBudget() != fetch.DefaultBudget { + t.Errorf("request_budget = %d, want %d", rep.GetRequestBudget(), fetch.DefaultBudget) + } +} + +func TestScannerRecordsScriptRefusedByBudget(t *testing.T) { + site := vendorSite(t) + rep, err := newTestScanner(recordedOSVSite(t).URL).Scan(context.Background(), + &touchstonev1.ScanRequest{Url: site.URL + "/", RequestBudget: 1}) + if err != nil { + t.Fatalf("Scan: %v", err) + } + checked := rep.GetChecked() + if len(checked) != 2 || checked[1].GetResult() != touchstonev1.CheckedLocation_RESULT_REFUSED { + t.Fatalf("checked = %v, want the script refused by the budget", checked) + } + if rep.GetRequestsUsed() != 1 { + t.Errorf("requests_used = %d, want 1", rep.GetRequestsUsed()) + } + if len(rep.GetComponents()) != 1 || rep.GetComponents()[0].GetName() != "nginx" { + t.Errorf("components = %v, want only the banner (the script was never loaded)", rep.GetComponents()) + } +} + +func TestScannerTargetUnreachable(t *testing.T) { + site := httptest.NewServer(http.NotFoundHandler()) + site.Close() + rep, err := newTestScanner(recordedOSVSite(t).URL).Scan(context.Background(), + &touchstonev1.ScanRequest{Url: site.URL + "/"}) + if err != nil { + t.Fatalf("Scan: an unreachable target is a report, not an RPC error: %v", err) + } + if rep.GetScanStatus() != touchstonev1.ScanStatus_SCAN_STATUS_TARGET_UNREACHABLE { + t.Errorf("scan_status = %v, want TARGET_UNREACHABLE", rep.GetScanStatus()) + } + if len(rep.GetComponents()) != 0 || rep.GetLookupStatus() != touchstonev1.LookupStatus_LOOKUP_STATUS_UNSPECIFIED { + t.Errorf("report carries components or a lookup status: %v", rep) + } + if len(rep.GetChecked()) != 1 || rep.GetRequestsUsed() != 1 { + t.Errorf("checked = %v, requests_used = %d; want the one attempt", rep.GetChecked(), rep.GetRequestsUsed()) + } +} + +func TestScannerOSVDownIsFailedLookup(t *testing.T) { + osv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + http.Error(w, "down", http.StatusServiceUnavailable) + })) + t.Cleanup(osv.Close) + site := vendorSite(t) + rep, err := newTestScanner(osv.URL).Scan(context.Background(), + &touchstonev1.ScanRequest{Url: site.URL + "/"}) + if err != nil { + t.Fatalf("Scan: %v", err) + } + if rep.GetLookupStatus() != touchstonev1.LookupStatus_LOOKUP_STATUS_FAILED { + t.Errorf("lookup_status = %v, want FAILED", rep.GetLookupStatus()) + } + for _, c := range rep.GetComponents() { + if len(c.GetVulnerabilities()) != 0 { + t.Errorf("component %s has vulnerabilities despite the failed lookup", c.GetName()) + } + } + var osvStatus *touchstonev1.SourceStatus + for _, s := range rep.GetSources() { + if s.GetSource() == "OSV" { + osvStatus = s + } + } + if osvStatus == nil || osvStatus.GetOk() || osvStatus.GetError() == "" { + t.Errorf("OSV source status = %v, want ok=false with an error", osvStatus) + } +} + +func TestScannerRejectsNonHTTPTarget(t *testing.T) { + _, err := newTestScanner("http://unused").Scan(context.Background(), + &touchstonev1.ScanRequest{Url: "ftp://vendor.example/"}) + if err == nil { + t.Fatal("Scan: want an InvalidArgument error for an ftp URL") + } +} From 0643ae609ca12caad517f0f4925bf614b8d4e466 Mon Sep 17 00:00:00 2001 From: "containarium-ci[bot]" <287705844+containarium-ci[bot]@users.noreply.github.com> Date: Sat, 24 Oct 2026 09:34:25 +0000 Subject: [PATCH 08/12] feat(scan): walking skeleton from page fetch to OSV-backed report (#8) --- internal/vuln/osv.go | 166 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 166 insertions(+) create mode 100644 internal/vuln/osv.go diff --git a/internal/vuln/osv.go b/internal/vuln/osv.go new file mode 100644 index 0000000..994df00 --- /dev/null +++ b/internal/vuln/osv.go @@ -0,0 +1,166 @@ +package vuln + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "net/url" + "strings" + "time" +) + +// DefaultOSVURL is the public OSV.dev API. +const DefaultOSVURL = "https://api.osv.dev" + +// maxOSVResponseBytes caps one OSV response; the API's replies are small. +const maxOSVResponseBytes = 2 << 20 + +// OSV is the vuln.Source that asks OSV.dev for vulnerabilities by purl. +// It calls the API with its own client, so its requests don't count +// against the vendor's request budget (see docs/architecture/touchstone.md). +type OSV struct { + BaseURL string + Client *http.Client +} + +// NewOSV returns an OSV source for the public API with a 10 s timeout per +// request. +func NewOSV() *OSV { + return &OSV{BaseURL: DefaultOSVURL, Client: &http.Client{Timeout: 10 * time.Second}} +} + +var _ Source = (*OSV)(nil) + +func (o *OSV) Name() string { return "OSV" } + +// osvBatchRequest and osvBatchResponse are the wire shapes of +// POST /v1/querybatch. Only the fields Touchstone reads are declared. +type osvBatchRequest struct { + Queries []osvQuery `json:"queries"` +} + +type osvQuery struct { + Package osvPackage `json:"package"` +} + +type osvPackage struct { + Purl string `json:"purl"` +} + +type osvBatchResponse struct { + Results []struct { + Vulns []osvVuln `json:"vulns"` + } `json:"results"` +} + +// osvVuln is one advisory as returned by /v1/querybatch (id only) and by +// GET /v1/vulns/{id} (everything else). +type osvVuln struct { + ID string `json:"id"` + Aliases []string `json:"aliases"` + Summary string `json:"summary"` +} + +// Lookup returns the advisories for every query that carries a purl. CPE +// queries are ignored (NVD handles those, slice S3). Advisories are fetched +// with one querybatch call, then one GET per distinct id for details. +func (o *OSV) Lookup(ctx context.Context, queries []Query) ([]Finding, error) { + var req osvBatchRequest + for _, q := range queries { + if q.Purl != "" { + req.Queries = append(req.Queries, osvQuery{Package: osvPackage{Purl: q.Purl}}) + } + } + if len(req.Queries) == 0 { + return nil, nil + } + + var batch osvBatchResponse + if err := o.do(ctx, http.MethodPost, "/v1/querybatch", req, &batch); err != nil { + return nil, err + } + if len(batch.Results) != len(req.Queries) { + return nil, fmt.Errorf("osv: querybatch returned %d results for %d queries", len(batch.Results), len(req.Queries)) + } + + seen := map[string]bool{} + var ids []string + for _, r := range batch.Results { + for _, v := range r.Vulns { + if v.ID != "" && !seen[v.ID] { + seen[v.ID] = true + ids = append(ids, v.ID) + } + } + } + + var out []Finding + for _, id := range ids { + var v osvVuln + if err := o.do(ctx, http.MethodGet, "/v1/vulns/"+url.PathEscape(id), nil, &v); err != nil { + return nil, err + } + out = append(out, findingFromOSV(id, v)) + } + return out, nil +} + +// findingFromOSV uses the CVE alias as the finding id when there is one, +// as the report's Vulnerability.id contract requires, else the OSV id. +func findingFromOSV(id string, v osvVuln) Finding { + f := Finding{ID: id, Aliases: v.Aliases, Summary: v.Summary} + for _, a := range v.Aliases { + if strings.HasPrefix(a, "CVE-") { + f.ID = a + break + } + } + if f.ID != id { + f.Aliases = append([]string{id}, v.Aliases...) + } + return f +} + +// do sends one JSON request to the OSV API and decodes the reply into out. +// Any non-200 status is an error, so a failed lookup is never read as "no +// vulnerabilities". +func (o *OSV) do(ctx context.Context, method, path string, body any, out any) error { + var r io.Reader + if body != nil { + b, err := json.Marshal(body) + if err != nil { + return err + } + r = bytes.NewReader(b) + } + req, err := http.NewRequestWithContext(ctx, method, o.BaseURL+path, r) + if err != nil { + return err + } + req.Header.Set("Accept", "application/json") + if body != nil { + req.Header.Set("Content-Type", "application/json") + } + resp, err := o.Client.Do(req) + if err != nil { + return fmt.Errorf("osv: %s %s: %w", method, path, err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("osv: %s %s: HTTP %d", method, path, resp.StatusCode) + } + b, err := io.ReadAll(io.LimitReader(resp.Body, maxOSVResponseBytes+1)) + if err != nil { + return fmt.Errorf("osv: %s %s: %w", method, path, err) + } + if len(b) > maxOSVResponseBytes { + return fmt.Errorf("osv: %s %s: response exceeds %d bytes", method, path, maxOSVResponseBytes) + } + if err := json.Unmarshal(b, out); err != nil { + return fmt.Errorf("osv: %s %s: decode: %w", method, path, err) + } + return nil +} From f835356868d0224a221dec9e0b2bb5a32d6ea5d3 Mon Sep 17 00:00:00 2001 From: "containarium-ci[bot]" <287705844+containarium-ci[bot]@users.noreply.github.com> Date: Sat, 24 Oct 2026 09:34:27 +0000 Subject: [PATCH 09/12] feat(scan): walking skeleton from page fetch to OSV-backed report (#8) --- internal/vuln/osv_test.go | 122 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 122 insertions(+) create mode 100644 internal/vuln/osv_test.go diff --git a/internal/vuln/osv_test.go b/internal/vuln/osv_test.go new file mode 100644 index 0000000..e00ec16 --- /dev/null +++ b/internal/vuln/osv_test.go @@ -0,0 +1,122 @@ +package vuln + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "sync" + "testing" +) + +// recordedOSV serves the recorded responses under testdata/osv: the +// querybatch reply for POST /v1/querybatch and vulns/.json for +// GET /v1/vulns/. Any other request is a test failure. +func recordedOSV(t *testing.T) (*httptest.Server, *[]string) { + t.Helper() + var mu sync.Mutex + var calls []string + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + mu.Lock() + calls = append(calls, r.Method+" "+r.URL.Path) + mu.Unlock() + var file string + switch { + case r.Method == http.MethodPost && r.URL.Path == "/v1/querybatch": + file = "querybatch-jquery-1.12.4.json" + case r.Method == http.MethodGet && strings.HasPrefix(r.URL.Path, "/v1/vulns/"): + file = filepath.Join("vulns", strings.TrimPrefix(r.URL.Path, "/v1/vulns/")+".json") + default: + t.Errorf("unexpected request %s %s", r.Method, r.URL.Path) + http.NotFound(w, r) + return + } + b, err := os.ReadFile(filepath.Join("testdata", "osv", file)) + if err != nil { + http.NotFound(w, r) + return + } + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write(b) + })) + t.Cleanup(srv.Close) + return srv, &calls +} + +func osvFor(srv *httptest.Server) *OSV { + return &OSV{BaseURL: srv.URL, Client: srv.Client()} +} + +func TestOSVLookupByPurl(t *testing.T) { + srv, calls := recordedOSV(t) + got, err := osvFor(srv).Lookup(context.Background(), []Query{{Purl: "pkg:npm/jquery@1.12.4"}}) + if err != nil { + t.Fatalf("Lookup: %v", err) + } + if len(got) != 2 { + t.Fatalf("Lookup: got %d findings, want 2: %+v", len(got), got) + } + // The CVE alias is the id; the OSV id moves to aliases. + if got[0].ID != "CVE-2019-11358" || got[1].ID != "CVE-2020-11022" { + t.Errorf("ids = %q, %q; want CVE ids", got[0].ID, got[1].ID) + } + if got[1].Summary != "Potential XSS vulnerability in jQuery.htmlPrefilter" { + t.Errorf("summary = %q", got[1].Summary) + } + if !contains(got[1].Aliases, "GHSA-gxr4-xjj5-5px2") { + t.Errorf("aliases = %v, want the GHSA id kept", got[1].Aliases) + } + want := []string{ + "POST /v1/querybatch", + "GET /v1/vulns/GHSA-6c3j-c64m-qhgq", + "GET /v1/vulns/GHSA-gxr4-xjj5-5px2", + } + if strings.Join(*calls, "|") != strings.Join(want, "|") { + t.Errorf("calls = %v, want %v", *calls, want) + } +} + +func TestOSVIgnoresCPEOnlyQueries(t *testing.T) { + srv, calls := recordedOSV(t) + got, err := osvFor(srv).Lookup(context.Background(), []Query{{CPE: "cpe:2.3:a:jquery:jquery:1.12.4:*:*:*:*:*:*:*"}}) + if err != nil || len(got) != 0 { + t.Fatalf("Lookup = %v, %v; want no findings and no error", got, err) + } + if len(*calls) != 0 { + t.Errorf("made requests for CPE-only query: %v", *calls) + } +} + +func TestOSVNonOKStatusIsError(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + http.Error(w, "unavailable", http.StatusServiceUnavailable) + })) + t.Cleanup(srv.Close) + got, err := osvFor(srv).Lookup(context.Background(), []Query{{Purl: "pkg:npm/jquery@1.12.4"}}) + if err == nil || got != nil { + t.Fatalf("Lookup = %v, %v; want an error and no findings", got, err) + } +} + +func TestOSVResultCountMismatchIsError(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _ = json.NewEncoder(w).Encode(map[string]any{"results": []any{}}) + })) + t.Cleanup(srv.Close) + _, err := osvFor(srv).Lookup(context.Background(), []Query{{Purl: "pkg:npm/jquery@1.12.4"}}) + if err == nil { + t.Fatal("Lookup: want an error when querybatch returns fewer results than queries") + } +} + +func contains(list []string, want string) bool { + for _, s := range list { + if s == want { + return true + } + } + return false +} From 2b51013f4235578d887c8e148a2f314bb16ba0ae Mon Sep 17 00:00:00 2001 From: "containarium-ci[bot]" <287705844+containarium-ci[bot]@users.noreply.github.com> Date: Sat, 24 Oct 2026 09:34:28 +0000 Subject: [PATCH 10/12] feat(scan): walking skeleton from page fetch to OSV-backed report (#8) --- .../testdata/osv/querybatch-jquery-1.12.4.json | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 internal/vuln/testdata/osv/querybatch-jquery-1.12.4.json diff --git a/internal/vuln/testdata/osv/querybatch-jquery-1.12.4.json b/internal/vuln/testdata/osv/querybatch-jquery-1.12.4.json new file mode 100644 index 0000000..077ce39 --- /dev/null +++ b/internal/vuln/testdata/osv/querybatch-jquery-1.12.4.json @@ -0,0 +1,16 @@ +{ + "results": [ + { + "vulns": [ + { + "id": "GHSA-6c3j-c64m-qhgq", + "modified": "2024-11-08T00:00:00Z" + }, + { + "id": "GHSA-gxr4-xjj5-5px2", + "modified": "2024-11-08T00:00:00Z" + } + ] + } + ] +} From f23c91f2c118fb152e173575c70a60dd2b976e59 Mon Sep 17 00:00:00 2001 From: "containarium-ci[bot]" <287705844+containarium-ci[bot]@users.noreply.github.com> Date: Sat, 24 Oct 2026 09:34:30 +0000 Subject: [PATCH 11/12] feat(scan): walking skeleton from page fetch to OSV-backed report (#8) --- .../testdata/osv/vulns/GHSA-6c3j-c64m-qhgq.json | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 internal/vuln/testdata/osv/vulns/GHSA-6c3j-c64m-qhgq.json diff --git a/internal/vuln/testdata/osv/vulns/GHSA-6c3j-c64m-qhgq.json b/internal/vuln/testdata/osv/vulns/GHSA-6c3j-c64m-qhgq.json new file mode 100644 index 0000000..3ec5ac5 --- /dev/null +++ b/internal/vuln/testdata/osv/vulns/GHSA-6c3j-c64m-qhgq.json @@ -0,0 +1,13 @@ +{ + "id": "GHSA-6c3j-c64m-qhgq", + "aliases": ["CVE-2019-11358"], + "summary": "jQuery prototype pollution in extend", + "affected": [ + { + "package": {"ecosystem": "npm", "name": "jquery", "purl": "pkg:npm/jquery"}, + "ranges": [ + {"type": "ECOSYSTEM", "events": [{"introduced": "1.0.3"}, {"fixed": "3.4.0"}]} + ] + } + ] +} From f36f877d49b8e2b10732dc2bdc06046a63605d75 Mon Sep 17 00:00:00 2001 From: "containarium-ci[bot]" <287705844+containarium-ci[bot]@users.noreply.github.com> Date: Sat, 24 Oct 2026 09:34:31 +0000 Subject: [PATCH 12/12] feat(scan): walking skeleton from page fetch to OSV-backed report (#8) --- .../testdata/osv/vulns/GHSA-gxr4-xjj5-5px2.json | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 internal/vuln/testdata/osv/vulns/GHSA-gxr4-xjj5-5px2.json diff --git a/internal/vuln/testdata/osv/vulns/GHSA-gxr4-xjj5-5px2.json b/internal/vuln/testdata/osv/vulns/GHSA-gxr4-xjj5-5px2.json new file mode 100644 index 0000000..06f2af8 --- /dev/null +++ b/internal/vuln/testdata/osv/vulns/GHSA-gxr4-xjj5-5px2.json @@ -0,0 +1,13 @@ +{ + "id": "GHSA-gxr4-xjj5-5px2", + "aliases": ["CVE-2020-11022"], + "summary": "Potential XSS vulnerability in jQuery.htmlPrefilter", + "affected": [ + { + "package": {"ecosystem": "npm", "name": "jquery", "purl": "pkg:npm/jquery"}, + "ranges": [ + {"type": "ECOSYSTEM", "events": [{"introduced": "1.2"}, {"fixed": "3.5.0"}]} + ] + } + ] +}