patch: extended support for SI spec v2.1.0

CHANGELOG.md

@@ -1,5 +1,17 @@
 # Changelog
 
+## v2.3.0
+
+This release expands on the Security Insights v2.1.0 specification by adding a new micro-CLI for non-library use cases. It also contains discovery and fetching improvements to the library.
+
+### Features
+
+- `si.Discover` finds a repository's Security Insights file among the known filename and location variants using at most two GitHub API calls
+- `si.Fetch` returns the raw bytes of a file in a public GitHub repository; `si.ErrNotFound` identifies a missing file
+- `si fetch` command (`v2/cmd/si`) runs discovery and parsing over many targets and prints machine-readable JSON, for catalogues and dashboards built on Security Insights data. Each result carries a status of `ok`, `not-found`, `invalid` or `error`; the command exits 2 when any target hit a transport `error`, so an incomplete batch is never mistaken for a clean one
+- `si.Load` returns `si.ErrParentUnavailable` when the parent named in `project-si-source` cannot be fetched for a reason unrelated to its content; `si fetch` reports that as `error`, not `invalid`
+- All GitHub API calls, including the existing `si.Read`, now authenticate with `GITHUB_TOKEN` when set and honour `GITHUB_API_URL`; all requests time out after 30 seconds
+
 ## v2.2.1
 
 ### Features

v2/README.md

@@ -19,6 +19,64 @@ func main() {
 }
 ```
 
+### Discovering the file
+
+Repositories keep the file under a handful of names and locations. `si.Discover` lists the root and `.github/` directories (at most two API calls) and returns the first known filename it finds; `si.Fetch` returns the raw bytes of a known path.
+
+```go
+ctx := context.Background()
+path, err := si.Discover(ctx, "ossf", "scorecard")   // ".github/security-insights.yml"
+raw, err := si.Fetch(ctx, "ossf", "scorecard", path)
+insights, err := si.Load(raw)
+```
+
+Set `GITHUB_TOKEN` to authenticate GitHub API calls (unauthenticated requests are limited to 60 per hour). `GITHUB_API_URL`, which GitHub Actions exports, is honoured for GitHub Enterprise. Both apply to `si.Read` as well. Requests time out after 30 seconds.
+
+## Command line
+
+`si fetch` runs discovery and parsing over any number of targets and prints a JSON array, in input order, for other tools to consume:
+
+```sh
+cd v2
+go run ./cmd/si fetch ossf/scorecard mindersec/minder \
+  https://github.com/openbao/openbao/blob/main/.github/security-insights.yml
+```
+
+Targets may be `owner/repo` (discover the path), `owner/repo/<path>`, a `github.com/.../blob/...` URL, or a `raw.githubusercontent.com` URL. URLs are read from the default branch; the ref in the URL is ignored and must be a single path segment.
+
+Each result looks like:
+
+```json
+{
+  "target": "ossf/scorecard",
+  "owner": "ossf",
+  "repo": "scorecard",
+  "path": ".github/security-insights.yml",
+  "status": "ok",
+  "schema-version": "2.0.0",
+  "insights": { "header": { ... }, "project": { ... }, "repository": { ... } }
+}
+```
+
+`status` is one of:
+
+| status | meaning |
+|---|---|
+| `ok` | the file was fetched and parsed; `insights` is populated |
+| `not-found` | no file at the path, or discovery found nothing |
+| `invalid` | the file exists but does not parse as Security Insights v2 (`error` says why; `schema-version` is still reported when the header declares one), or the target string is malformed |
+| `error` | the file, or the parent it names in `project-si-source`, could not be fetched (rate limit, server error, network, bad token); nothing is known about it |
+
+The full array is always printed. Exit codes:
+
+| exit | when |
+|---|---|
+| 0 | every target was processed; statuses are `ok`, `not-found` or `invalid` |
+| 1 | the JSON could not be written |
+| 2 | usage error, or at least one target has status `error`, so the output is incomplete and should not be committed as a snapshot |
+
+Progress is written to stderr.
+
 ## Schema version support
 
 The module supports Security Insights schema version 2.x, including v2.2.0:

v2/cmd/si/fetch.go

@@ -0,0 +1,134 @@
+package main
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"net/url"
+	"strings"
+
+	"github.com/goccy/go-yaml"
+	"github.com/ossf/si-tooling/v2/si"
+)
+
+const (
+	statusOK       = "ok"        // file fetched and parsed
+	statusNotFound = "not-found" // no file at the path, or discovery found nothing
+	statusInvalid  = "invalid"   // file fetched but si.Load rejected it, or the target string is malformed
+	statusError    = "error"     // transport failure (rate limit, 5xx, network, bad token, unreachable parent): the file's state is unknown
+)
+
+// result is one element of the JSON array `si fetch` prints.
+type result struct {
+	Target        string               `json:"target"`
+	Owner         string               `json:"owner,omitempty"`
+	Repo          string               `json:"repo,omitempty"`
+	Path          string               `json:"path,omitempty"`
+	Status        string               `json:"status"`
+	Error         string               `json:"error,omitempty"`
+	SchemaVersion string               `json:"schema-version,omitempty"`
+	Insights      *si.SecurityInsights `json:"insights,omitempty"`
+}
+
+// urlForms are the URL shapes parseTarget accepts. After owner and repo, a
+// marker segment (if any) and a single ref segment precede the file path. The
+// ref is ignored; the default branch is read.
+var urlForms = []struct{ prefix, marker, shape string }{
+	{"https://github.com/", "blob", "https://github.com/owner/repo/blob/<ref>/<path>"},
+	{"https://raw.githubusercontent.com/", "", "https://raw.githubusercontent.com/owner/repo/<ref>/<path>"},
+}
+
+// parseTarget splits a target into owner, repo and path. An empty path means
+// the caller should discover it.
+func parseTarget(target string) (owner, repo, path string, err error) {
+	parts := strings.Split(strings.Trim(target, "/"), "/")
+	if strings.Contains(target, "://") {
+		if parts, err = parseURLTarget(target); err != nil {
+			return "", "", "", err
+		}
+	}
+	if len(parts) < 2 || parts[0] == "" || parts[1] == "" {
+		return "", "", "", fmt.Errorf("expected owner/repo[/path], got %s", target)
+	}
+	return parts[0], parts[1], strings.Join(parts[2:], "/"), nil
+}
+
+// parseURLTarget reduces a supported URL to its owner, repo and path segments.
+func parseURLTarget(target string) ([]string, error) {
+	for _, f := range urlForms {
+		if !strings.HasPrefix(target, f.prefix) {
+			continue
+		}
+		u, err := url.Parse(target)
+		if err != nil {
+			return nil, err
+		}
+		parts := strings.Split(strings.Trim(u.Path, "/"), "/")
+		skip := 1 // the ref
+		if f.marker != "" {
+			skip = 2 // the marker and the ref
+		}
+		if len(parts) < 3+skip || (f.marker != "" && parts[2] != f.marker) {
+			return nil, fmt.Errorf("expected %s, got %s", f.shape, target)
+		}
+		return append(parts[:2], parts[2+skip:]...), nil
+	}
+	return nil, fmt.Errorf("unsupported URL %s: only github.com blob and raw.githubusercontent.com URLs are accepted", target)
+}
+
+func fetch(ctx context.Context, target string) result {
+	r := result{Target: target}
+	owner, repo, path, err := parseTarget(target)
+	if err != nil {
+		r.Status, r.Error = statusInvalid, err.Error()
+		return r
+	}
+	r.Owner, r.Repo, r.Path = owner, repo, path
+	if r.Path == "" {
+		if r.Path, err = si.Discover(ctx, owner, repo); err != nil {
+			return fail(r, err)
+		}
+	}
+	raw, err := si.Fetch(ctx, owner, repo, r.Path)
+	if err != nil {
+		return fail(r, err)
+	}
+	r.SchemaVersion = schemaVersion(raw)
+	insights, err := si.Load(raw)
+	if errors.Is(err, si.ErrParentUnavailable) {
+		return fail(r, err)
+	}
+	if err != nil {
+		r.Status, r.Error = statusInvalid, err.Error()
+		return r
+	}
+	r.Status, r.Insights = statusOK, insights
+	return r
+}
+
+// fail records a Discover/Fetch failure. Only a 404 says anything about the
+// file; every other failure is the transport's, and must not be mistaken for a
+// bad or missing file by consumers that act on not-found/invalid.
+func fail(r result, err error) result {
+	r.Error = err.Error()
+	if errors.Is(err, si.ErrNotFound) {
+		r.Status = statusNotFound
+	} else {
+		r.Status = statusError
+	}
+	return r
+}
+
+// schemaVersion leniently reads header.schema-version so a file si.Load
+// rejects (a v1 file, an unknown field) can still report which schema it claims.
+// It deliberately does not reuse si.Header: a malformed sibling field would
+// then lose the version too.
+func schemaVersion(raw []byte) string {
+	var doc struct {
+		Header struct {
+			SchemaVersion string `yaml:"schema-version"`
+		} `yaml:"header"`
+	}
+	_ = yaml.Unmarshal(raw, &doc)
+	return doc.Header.SchemaVersion
+}

v2/cmd/si/fetch_test.go

@@ -0,0 +1,133 @@
+package main
+
+import (
+	"context"
+	"errors"
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"testing"
+
+	"github.com/ossf/si-tooling/v2/internal/ghtest"
+	"github.com/ossf/si-tooling/v2/si"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestParseTarget(t *testing.T) {
+	tests := []struct {
+		target            string
+		owner, repo, path string
+		wantErr           bool
+	}{
+		{target: "ossf/scorecard", owner: "ossf", repo: "scorecard"},
+		{target: "ossf/scorecard/.github/security-insights.yml", owner: "ossf", repo: "scorecard", path: ".github/security-insights.yml"},
+		{target: "https://github.com/ossf/scorecard/blob/main/.github/security-insights.yml", owner: "ossf", repo: "scorecard", path: ".github/security-insights.yml"},
+		{target: "https://raw.githubusercontent.com/ossf/scorecard/main/.github/security-insights.yml", owner: "ossf", repo: "scorecard", path: ".github/security-insights.yml"},
+		{target: "https://github.com/ossf/scorecard", wantErr: true},
+		{target: "https://github.com/ossf/scorecard/tree/main/.github", wantErr: true},
+		{target: "https://raw.githubusercontent.com/ossf/scorecard", wantErr: true},
+		{target: "https://gitlab.com/o/r/-/blob/main/x.yml", wantErr: true},
+		{target: "scorecard", wantErr: true},
+		{target: "", wantErr: true},
+	}
+	for _, tt := range tests {
+		t.Run(tt.target, func(t *testing.T) {
+			owner, repo, path, err := parseTarget(tt.target)
+			if tt.wantErr {
+				assert.Error(t, err)
+				return
+			}
+			require.NoError(t, err)
+			assert.Equal(t, [3]string{tt.owner, tt.repo, tt.path}, [3]string{owner, repo, path})
+		})
+	}
+}
+
+func TestSchemaVersion(t *testing.T) {
+	assert.Equal(t, "1.0.0", schemaVersion([]byte("header:\n  schema-version: 1.0.0\n  commit-hash: abc\n")))
+	assert.Equal(t, "", schemaVersion([]byte("not: yaml: at: all")))
+}
+
+func TestFail(t *testing.T) {
+	assert.Equal(t, statusNotFound, fail(result{}, si.ErrNotFound).Status)
+	assert.Equal(t, statusError, fail(result{}, errors.New("boom")).Status)
+}
+
+func TestFetchTransportError(t *testing.T) {
+	ctx := context.Background()
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+		w.Header().Set("X-RateLimit-Remaining", "0")
+		w.WriteHeader(http.StatusForbidden)
+		_, _ = w.Write([]byte(`{"message":"API rate limit exceeded"}`))
+	}))
+	t.Cleanup(srv.Close)
+	t.Setenv("GITHUB_API_URL", srv.URL)
+
+	for _, target := range []string{"o/r", "o/r/security-insights.yml"} {
+		r := fetch(ctx, target)
+		assert.Equal(t, statusError, r.Status, target)
+		assert.Contains(t, r.Error, "rate limit", target)
+		assert.Nil(t, r.Insights)
+	}
+
+	srv500 := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+		w.WriteHeader(http.StatusInternalServerError)
+	}))
+	t.Cleanup(srv500.Close)
+	t.Setenv("GITHUB_API_URL", srv500.URL)
+	assert.Equal(t, statusError, fetch(ctx, "o/r").Status)
+}
+
+func TestFetchParentUnavailable(t *testing.T) {
+	parent := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+		w.WriteHeader(http.StatusBadGateway)
+	}))
+	t.Cleanup(parent.Close)
+	ghtest.Serve(t, map[string]string{
+		"security-insights.yml": "header:\n  schema-version: 2.0.0\n  project-si-source: " + parent.URL + "\n",
+	}, nil)
+
+	r := fetch(context.Background(), "o/r/security-insights.yml")
+	assert.Equal(t, statusError, r.Status, "an unreachable parent says nothing about the file")
+	assert.Equal(t, "2.0.0", r.SchemaVersion)
+	assert.Nil(t, r.Insights)
+}
+
+func TestFetchBadTarget(t *testing.T) {
+	r := fetch(context.Background(), "nonsense")
+	assert.Equal(t, statusInvalid, r.Status)
+	assert.Equal(t, "nonsense", r.Target)
+	assert.Nil(t, r.Insights)
+}
+
+// Discovery itself is tested in package si; here the fake has no root listing,
+// so an owner/repo target is not-found.
+func TestFetch(t *testing.T) {
+	ctx := context.Background()
+	minimal, err := os.ReadFile("../../si/test_data/minimal-v2.2.0.yml")
+	require.NoError(t, err)
+	ghtest.Serve(t, map[string]string{
+		"SECURITY-INSIGHTS.yml": string(minimal),
+		"v1.yml":                "header:\n  schema-version: 1.0.0\n  commit-hash: abc\n",
+	}, nil)
+
+	exact := fetch(ctx, "https://github.com/o/r/blob/main/SECURITY-INSIGHTS.yml")
+	assert.Equal(t, statusOK, exact.Status)
+	assert.Equal(t, "o", exact.Owner)
+	assert.Equal(t, "2.2.0", exact.SchemaVersion)
+	require.NotNil(t, exact.Insights)
+
+	invalid := fetch(ctx, "o/r/v1.yml")
+	assert.Equal(t, statusInvalid, invalid.Status)
+	assert.Equal(t, "1.0.0", invalid.SchemaVersion)
+	assert.Nil(t, invalid.Insights)
+
+	missing := fetch(ctx, "o/r/nope.yml")
+	assert.Equal(t, statusNotFound, missing.Status)
+	assert.Equal(t, "nope.yml", missing.Path)
+
+	undiscoverable := fetch(ctx, "o/r")
+	assert.Equal(t, statusNotFound, undiscoverable.Status)
+	assert.Equal(t, "", undiscoverable.Path)
+}