Agent proposal: [S7] Fingerprint complete: Retire.js filecontent/hashes via regexp2 with timeout, header CPE table,… (#15)

NOTICE

@@ -0,0 +1,7 @@
+Touchstone
+Copyright FootprintAI
+
+This product includes signature data derived from Retire.js
+(https://github.com/RetireJS/retire.js), jsrepository-v4.json,
+licensed under the Apache License, Version 2.0
+(http://www.apache.org/licenses/LICENSE-2.0).

internal/fingerprint/data/signatures.json

@@ -1,5 +1,5 @@
 {
-  "_comment": "Retire.js-style signatures, uri and filename extractors only. Hand-curated for slice S2 (jQuery only); to be replaced by a pinned snapshot of Retire.js jsrepository in S7. {version} in purl and cpe is replaced by the captured version.",
+  "_comment": "Retire.js-style signatures (uri, filename, filecontent, hashes). Hand-curated subset in the Retire.js jsrepository shape; to be replaced by a pinned snapshot of jsrepository-v4.json (Apache-2.0, see NOTICE). {version} in purl and cpe is replaced by the captured version; §§version§§ in a filecontent pattern is the version capture.",
   "libraries": [
     {
       "name": "jquery",
@@ -9,6 +9,27 @@
         "uri": ["jquery[-.]v?([0-9]+(?:\\.[0-9]+)+)(?:\\.min|\\.slim|\\.slim\\.min)?\\.js"],
         "filename": ["^jquery[-.]v?([0-9]+(?:\\.[0-9]+)+)(?:\\.min|\\.slim|\\.slim\\.min)?\\.js$"]
       }
+    },
+    {
+      "name": "react",
+      "purl": "pkg:npm/react@{version}",
+      "cpe": "cpe:2.3:a:facebook:react:{version}:*:*:*:*:*:*:*",
+      "extractors": {
+        "filecontent": [
+          "/\\*\\*?\\s*@license React v§§version§§"
+        ]
+      }
+    },
+    {
+      "name": "vue",
+      "purl": "pkg:npm/vue@{version}",
+      "cpe": "cpe:2.3:a:vuejs:vue:{version}:*:*:*:*:*:*:*",
+      "extractors": {
+        "filecontent": [
+          "/\\*!\\s*\\*\\s*Vue\\.js v§§version§§",
+          "Vue\\.version\\s*=\\s*[\"']§§version§§[\"']"
+        ]
+      }
     }
   ]
 }

internal/fingerprint/fingerprint.go

@@ -6,7 +6,9 @@ package fingerprint
 
 import (
 	"bytes"
+	"crypto/sha1"
 	_ "embed"
+	"encoding/hex"
 	"encoding/json"
 	"errors"
 	"fmt"
@@ -36,16 +38,21 @@ type signatureFile struct {
 		Purl       string `json:"purl"`
 		CPE        string `json:"cpe"`
 		Extractors struct {
-			URI      []string `json:"uri"`
-			Filename []string `json:"filename"`
+			URI         []string          `json:"uri"`
+			Filename    []string          `json:"filename"`
+			FileContent []string          `json:"filecontent"`
+			Hashes      map[string]string `json:"hashes"`
 		} `json:"extractors"`
 	} `json:"libraries"`
 }
 
-// library is one compiled signature entry.
+// library is one compiled signature entry. hashes maps the lower-case hex
+// SHA-1 of a whole asset to the version it is.
 type library struct {
 	name, purl, cpe string
 	uri, filename   []*regexp2.Regexp
+	filecontent     []*regexp2.Regexp
+	hashes          map[string]string
 }
 
 // libraries is compiled once at start-up from the embedded signatures. A
@@ -68,14 +75,20 @@ func compile(data []byte) ([]library, error) {
 	}
 	var out []library
 	for _, l := range sf.Libraries {
-		lib := library{name: l.Name, purl: l.Purl, cpe: l.CPE}
+		lib := library{name: l.Name, purl: l.Purl, cpe: l.CPE, hashes: map[string]string{}}
 		var err error
 		if lib.uri, err = compileAll(l.Name, "uri", l.Extractors.URI); err != nil {
 			return nil, err
 		}
 		if lib.filename, err = compileAll(l.Name, "filename", l.Extractors.Filename); err != nil {
 			return nil, err
 		}
+		if lib.filecontent, err = compileAll(l.Name, "filecontent", l.Extractors.FileContent); err != nil {
+			return nil, err
+		}
+		for digest, version := range l.Extractors.Hashes {
+			lib.hashes[strings.ToLower(digest)] = version
+		}
 		out = append(out, lib)
 	}
 	return out, nil
@@ -131,6 +144,45 @@ func FromURL(assetURL string) ([]pipeline.Component, error) {
 	return out, errors.Join(errs...)
 }
 
+// FromContent matches the bytes of a script that FromURL did not recognise:
+// the filecontent extractors run over the text, and the hashes extractors
+// over the SHA-1 of the whole asset. Each library it recognises yields one
+// Low-confidence component, with the asset URL as evidence.
+func FromContent(assetURL string, body []byte) ([]pipeline.Component, error) {
+	return matchContent(libraries, assetURL, body)
+}
+
+func matchContent(libs []library, assetURL string, body []byte) ([]pipeline.Component, error) {
+	sum := sha1.Sum(body)
+	digest := hex.EncodeToString(sum[:])
+	text := string(body)
+	var out []pipeline.Component
+	var errs []error
+	for _, lib := range libs {
+		if version, ok := lib.hashes[digest]; ok {
+			c := lib.component(version)
+			c.Evidence = []pipeline.Evidence{{Kind: "FINGERPRINT_ASSET", Location: assetURL, Detail: "sha1 " + digest}}
+			out = append(out, c)
+			continue
+		}
+		c, ok, err := lib.match(text, lib.filecontent)
+		if err != nil {
+			errs = append(errs, err)
+			continue
+		}
+		if !ok {
+			continue
+		}
+		c.Evidence = []pipeline.Evidence{{
+			Kind:     "FINGERPRINT_ASSET",
+			Location: assetURL,
+			Detail:   c.Evidence[0].Detail,
+		}}
+		out = append(out, c)
+	}
+	return out, errors.Join(errs...)
+}
+
 // match runs res against input and, on the first hit, builds a component
 // with the captured version. The evidence detail is the matched text.
 func (l library) match(input string, res []*regexp2.Regexp) (pipeline.Component, bool, error) {
@@ -142,20 +194,25 @@ func (l library) match(input string, res []*regexp2.Regexp) (pipeline.Component,
 		if m == nil {
 			continue
 		}
-		version := m.GroupByNumber(1).String()
-		c := pipeline.Component{
-			Name:       l.name,
-			Version:    version,
-			Purl:       strings.ReplaceAll(l.purl, "{version}", version),
-			CPE:        strings.ReplaceAll(l.cpe, "{version}", version),
-			Confidence: "LOW",
-			Evidence:   []pipeline.Evidence{{Detail: m.String()}},
-		}
+		c := l.component(m.GroupByNumber(1).String())
+		c.Evidence = []pipeline.Evidence{{Detail: m.String()}}
 		return c, true, nil
 	}
 	return pipeline.Component{}, false, nil
 }
 
+// component builds the library's component at version, with its purl and
+// CPE filled in. The caller adds the evidence.
+func (l library) component(version string) pipeline.Component {
+	return pipeline.Component{
+		Name:       l.name,
+		Version:    version,
+		Purl:       strings.ReplaceAll(l.purl, "{version}", version),
+		CPE:        strings.ReplaceAll(l.cpe, "{version}", version),
+		Confidence: "LOW",
+	}
+}
+
 // headerRule maps one response header value to a component. Banners
 // describe the server software, which distributions may patch without
 // changing the version string, hence the MAY_BE_BACKPORTED caveat.
@@ -169,6 +226,7 @@ type headerRule struct {
 var headerRules = []headerRule{
 	{"Server", regexp.MustCompile(`^Apache/([0-9]+(?:\.[0-9]+)+)`), "apache http_server", "cpe:2.3:a:apache:http_server:%s:*:*:*:*:*:*:*"},
 	{"Server", regexp.MustCompile(`^nginx/([0-9]+(?:\.[0-9]+)+)`), "nginx", "cpe:2.3:a:f5:nginx:%s:*:*:*:*:*:*:*"},
+	{"Server", regexp.MustCompile(`^Microsoft-IIS/([0-9]+(?:\.[0-9]+)+)`), "iis", "cpe:2.3:a:microsoft:internet_information_services:%s:*:*:*:*:*:*:*"},
 	{"X-Powered-By", regexp.MustCompile(`^PHP/([0-9]+(?:\.[0-9]+)+)`), "php", "cpe:2.3:a:php:php:%s:*:*:*:*:*:*:*"},
 }
 

internal/fingerprint/fingerprint_test.go

@@ -6,6 +6,7 @@ import (
 	"reflect"
 	"strings"
 	"testing"
+	"time"
 )
 
 func TestSignaturesCompile(t *testing.T) {
@@ -98,6 +99,7 @@ func TestFromHeaders(t *testing.T) {
 	}{
 		{"apache", http.Header{"Server": {"Apache/2.4.29 (Ubuntu)"}}, []string{"apache http_server 2.4.29"}},
 		{"nginx", http.Header{"Server": {"nginx/1.18.0"}}, []string{"nginx 1.18.0"}},
+		{"iis", http.Header{"Server": {"Microsoft-IIS/10.0"}}, []string{"iis 10.0"}},
 		{"php", http.Header{"X-Powered-By": {"PHP/7.2.34"}}, []string{"php 7.2.34"}},
 		{"unknown banner", http.Header{"Server": {"cloudflare"}}, nil},
 		{"no headers", http.Header{}, nil},
@@ -128,6 +130,75 @@ func TestFromHeaders(t *testing.T) {
 	}
 }
 
+// testSignatures has one library with a content pattern and one with a
+// SHA-1 entry. sha1("abc") is a9993e36 4706816a ba3e2571 7850c26c 9cd0d89d.
+const testSignatures = `{"libraries":[
+	{"name":"fakelib","purl":"pkg:npm/fakelib@{version}","cpe":"cpe:2.3:a:fake:fakelib:{version}:*:*:*:*:*:*:*",
+	 "extractors":{"filecontent":["/\\*! FakeLib v([0-9]+\\.[0-9]+\\.[0-9]+)"]}},
+	{"name":"hashed","purl":"pkg:npm/hashed@{version}","cpe":"","extractors":{"hashes":{"A9993E364706816ABA3E25717850C26C9CD0D89D":"1.0.0"}}}
+]}`
+
+func TestFromContent(t *testing.T) {
+	libs, err := compile([]byte(testSignatures))
+	if err != nil {
+		t.Fatalf("compile: %v", err)
+	}
+	const asset = "https://vendor.example/static/bundle.js"
+	tests := []struct {
+		name   string
+		body   string
+		want   string // "name version"; "" = no component
+		detail string
+	}{
+		{"banner in content", "/*! FakeLib v2.3.4 */ var x = 1;", "fakelib 2.3.4", "/*! FakeLib v2.3.4"},
+		{"sha1 of whole asset", "abc", "hashed 1.0.0", "sha1 a9993e364706816aba3e25717850c26c9cd0d89d"},
+		{"no match", "var x = 1;", "", ""},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got, err := matchContent(libs, asset, []byte(tt.body))
+			if err != nil {
+				t.Fatalf("matchContent: %v", err)
+			}
+			if tt.want == "" {
+				if len(got) != 0 {
+					t.Fatalf("matchContent = %+v, want no components", got)
+				}
+				return
+			}
+			if len(got) != 1 {
+				t.Fatalf("matchContent = %+v, want one component", got)
+			}
+			c := got[0]
+			if c.Name+" "+c.Version != tt.want || c.Confidence != "LOW" {
+				t.Errorf("component = %q %q confidence %q; want %q LOW", c.Name, c.Version, c.Confidence, tt.want)
+			}
+			if len(c.Evidence) != 1 || c.Evidence[0].Kind != "FINGERPRINT_ASSET" ||
+				c.Evidence[0].Location != asset || c.Evidence[0].Detail != tt.detail {
+				t.Errorf("evidence = %+v, want asset evidence at %s detail %q", c.Evidence, asset, tt.detail)
+			}
+		})
+	}
+}
+
+// A catastrophic pattern must be stopped by MatchTimeout and reported as an
+// error, not left to run for the length of the input.
+func TestMatchTimeout(t *testing.T) {
+	libs, err := compile([]byte(`{"libraries":[{"name":"redos","purl":"","cpe":"","extractors":{"filecontent":["^(a+)+$"]}}]}`))
+	if err != nil {
+		t.Fatalf("compile: %v", err)
+	}
+	body := []byte(strings.Repeat("a", 40) + "!")
+	start := time.Now()
+	_, err = matchContent(libs, "https://vendor.example/x.js", body)
+	if err == nil {
+		t.Fatal("matchContent returned no error; want a timeout")
+	}
+	if elapsed := time.Since(start); elapsed > 5*time.Second {
+		t.Errorf("matchContent took %v; want it stopped near the 100ms timeout", elapsed)
+	}
+}
+
 func contains(list []string, want string) bool {
 	for _, s := range list {
 		if s == want {

internal/service/scanner.go

@@ -110,10 +110,18 @@ func collect(ctx context.Context, f fetch.Fetcher, page *fetch.Response) ([]pipe
 			continue
 		}
 		found, err := fingerprint.FromURL(src)
-		comps = append(comps, found...)
 		if err != nil {
 			errs = append(errs, err)
 		}
+		if len(found) == 0 {
+			// Content is matched only when the URL names no library, so one
+			// script yields at most one match per library.
+			found, err = fingerprint.FromContent(src, resp.Body)
+			if err != nil {
+				errs = append(errs, err)
+			}
+		}
+		comps = append(comps, found...)
 	}
 	return merge(comps), errors.Join(errs...)
 }