feat(sqlite): support SQLite alongside PostgreSQL in the vulnerability queries

CHANGELOG.md

@@ -8,6 +8,29 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ## [Unreleased]
 - Upcoming changes...
 
+## [0.14.0] - 2026-08-11
+### Added
+- SQLite support alongside PostgreSQL: set `DB_DRIVER=sqlite` and `DB_DSN` to a database file
+- `sql.Scanner` and `driver.Valuer` on `utils.OnlyDate`, so a date reads from a `TEXT` column (SQLite) or a `date` column (PostgreSQL)
+- `pkg/models/test_schema.go` holding the production schema the tests build their database from
+- `pkg/models/tests/vulns_scenario.sql`, a deterministic fixture whose match criteria cover every version-bound combination
+- `pkg/models/pg_parity_test.go`, comparing the rewritten queries against the ones they replace on a real PostgreSQL database. Skipped unless `PG_DSN` is set
+
+### Changed
+- Rewrote both vulnerability queries as portable SQL, valid on PostgreSQL and SQLite
+- Moved version range matching out of SQL into `pkg/models/version_range.go`, porting the `natural_sort_order` PostgreSQL function so both engines agree on which vulnerabilities apply to a version
+- Test fixtures under `pkg/models/tests/` now carry data only. They used to define their own tables, describing a schema that does not exist, which is what let queries referencing non-existent columns pass CI
+- Unified the test driver on `modernc.org/sqlite`, the one the server uses; `mattn/go-sqlite3` is no longer a direct dependency and CGO is not required
+
+### Fixed
+- `GetVulnsByPurlName` joined `cpes.id` and `nvd_match_criteria_ids.cpe_ids`, matching numeric CPE ids against a UUID. It returned no vulnerabilities at all on PostgreSQL and could not run on SQLite
+- `GetVulnsByPurlVersion` relied on `array_agg`, the `&&` array overlap operator and the custom `natural_sort_order` function, none available in SQLite
+- `saveLicense` inserted `is_sanitized`, which is not a column in the `licenses` table
+- `saveVersion` passed four arguments to an insert with two placeholders
+
+### Deployment
+- Requires an `epss_data` table (`cve`, `epss`, `percentile`) plus an index on `cve`. Without it the EPSS lookup fails on every request and every vulnerability is returned with `epss.probability` and `epss.percentile` reading `0`, indistinguishable from a genuine zero
+
 ## [0.13.0] - 2026-06-22
 ### Added
 - `/health` liveness endpoint (GET) on the REST gateway

README.md

@@ -37,13 +37,18 @@ APP_PORT=50052
 APP_MODE=dev
 APP_DEBUG=false
 
-DB_DRIVER=postgres
+DB_DRIVER=postgres                  # postgres or sqlite
 DB_HOST=localhost
 DB_USER=scanoss
 DB_PASSWD=
 DB_SCHEMA=vulnerabilities
 DB_SSL_MODE=disable
 
+# To run against SQLite instead, point DB_DSN at the database file.
+# The remaining DB_* values are only used to build a DSN when DB_DSN is empty.
+# DB_DRIVER=sqlite
+# DB_DSN=/path/to/vulnerabilities.db
+
 # Vulnerability data sources
 OSV_ENABLED=true                    # Enable/disable OSV (Open Source Vulnerabilities) database
 OSV_API_BASE_URL=https://api.osv.dev/v1

go.mod

@@ -8,7 +8,6 @@ require (
 	github.com/grpc-ecosystem/go-grpc-middleware v1.4.0
 	github.com/jmoiron/sqlx v1.4.0
 	github.com/lib/pq v1.12.3
-	github.com/mattn/go-sqlite3 v1.14.42
 	github.com/package-url/packageurl-go v0.1.5
 	github.com/pandatix/go-cvss v0.6.2
 	github.com/scanoss/go-component-helper v0.6.0
@@ -35,6 +34,7 @@ require (
 	github.com/google/uuid v1.6.0 // indirect
 	github.com/grpc-ecosystem/grpc-gateway/v2 v2.28.0 // indirect
 	github.com/mattn/go-isatty v0.0.20 // indirect
+	github.com/mattn/go-sqlite3 v1.14.42 // indirect
 	github.com/ncruces/go-strftime v1.0.0 // indirect
 	github.com/phuslu/iploc v1.0.20230201 // indirect
 	github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect

pkg/adapters/vulnerability_support_test.go

@@ -22,9 +22,9 @@ import (
 
 	common "github.com/scanoss/papi/api/commonv2"
 
-	_ "github.com/mattn/go-sqlite3"
 	pb "github.com/scanoss/papi/api/vulnerabilitiesv2"
 	zlog "github.com/scanoss/zap-logging-helper/pkg/logger"
+	_ "modernc.org/sqlite"
 
 	"scanoss.com/vulnerabilities/pkg/dtos"
 )

pkg/models/common.go

@@ -22,6 +22,7 @@ import (
 	"context"
 	"fmt"
 	"os"
+	"strings"
 
 	"github.com/jmoiron/sqlx"
 	zlog "github.com/scanoss/zap-logging-helper/pkg/logger"
@@ -34,34 +35,70 @@ func loadSQLData(db *sqlx.DB, ctx context.Context, conn *sqlx.Conn, filename str
 	if err != nil {
 		return err
 	}
+	return execSQL(db, ctx, conn, string(file))
+}
+
+// execSQL runs the given SQL against either the supplied connection or the DB pool.
+func execSQL(db *sqlx.DB, ctx context.Context, conn *sqlx.Conn, sql string) error {
+	var err error
 	if conn != nil {
-		_, err = conn.ExecContext(ctx, string(file))
+		_, err = conn.ExecContext(ctx, sql)
 	} else {
-		_, err = db.Exec(string(file))
+		_, err = db.Exec(sql)
 	}
-	if err != nil {
-		return err
+	return err
+}
+
+// idempotentDDL rewrites CREATE TABLE/INDEX into their IF NOT EXISTS form so a test
+// can load the schema more than once. testSchemaDDL transcribes production and must not
+// carry test-only clauses, so the rewrite happens here instead of in the constant.
+func idempotentDDL(sql string) string {
+	for _, kind := range []string{"TABLE", "INDEX"} {
+		bare := "CREATE " + kind + " "
+		guarded := bare + "IF NOT EXISTS "
+		// Normalise first, so an already-guarded statement is not double-guarded.
+		sql = strings.ReplaceAll(sql, guarded, bare)
+		sql = strings.ReplaceAll(sql, bare, guarded)
+	}
+	return sql
+}
+
+// testDataFiles are the data fixtures loaded on top of the schema. They carry data
+// only; the tables come from testSchemaDDL in test_schema.go. Paths are relative to a
+// package directory under pkg/.
+var testDataFiles = []string{
+	"../models/tests/cpe.sql",
+	"../models/tests/cve.sql",
+	"../models/tests/purl.sql",
+	"../models/tests/short_cpe_purl.sql",
+	"../models/tests/short_cpe.sql",
+	"../models/tests/versions.sql",
+	"../models/tests/ndv_match_criteria_ids.sql",
+	"../models/tests/all_urls.sql",
+	"../models/tests/mines.sql",
+	"../models/tests/licenses.sql",
+	"../models/tests/golang_projects.sql",
+	"../models/tests/projects.sql",
+	"../models/tests/epss.sql",
+	"../models/tests/vulns_scenario.sql",
+}
+
+// LoadTestSchema creates the production schema in the supplied DB. Call this before
+// loading any data fixture, since the fixtures do not define their own tables.
+// It is safe to call more than once on the same DB.
+func LoadTestSchema(db *sqlx.DB, ctx context.Context, conn *sqlx.Conn) error {
+	if err := execSQL(db, ctx, conn, idempotentDDL(testSchemaDDL)); err != nil {
+		return fmt.Errorf("failed to load the test schema: %v", err)
 	}
 	return nil
 }
 
-// LoadTestSQLData loads all the required test SQL files.
+// LoadTestSQLData loads the production schema plus all the test data fixtures.
 func LoadTestSQLData(db *sqlx.DB, ctx context.Context, conn *sqlx.Conn) error {
-	files := []string{
-		"../models/tests/cpe.sql",
-		"../models/tests/cpe_cve.sql",
-		"../models/tests/cve.sql",
-		"../models/tests/purl.sql",
-		"../models/tests/short_cpe_purl.sql",
-		"../models/tests/short_cpe.sql",
-		"../models/tests/versions.sql",
-		"../models/tests/ndv_match_criteria_ids.sql",
-		"../models/tests/all_urls.sql",
-		"../models/tests/mines.sql",
-		"../models/tests/licenses.sql",
-		"../models/tests/golang_projects.sql",
+	if err := LoadTestSchema(db, ctx, conn); err != nil {
+		return err
 	}
-	return loadTestSQLDataFiles(db, ctx, conn, files)
+	return loadTestSQLDataFiles(db, ctx, conn, testDataFiles)
 }
 
 // loadTestSQLDataFiles loads a list of test SQL files.

pkg/models/common_helpers_test.go

@@ -0,0 +1,37 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ * Copyright (C) 2018-2025 SCANOSS.COM
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 2 of the License, or
+ * (at your option) any later version.
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+// Helpers shared by the tests in this package. They live in a _test.go file so they do
+// not ship in the binary; LoadTestSchema and LoadTestSQLData stay in common.go because
+// tests in other packages call them.
+
+package models
+
+import (
+	"context"
+
+	"github.com/jmoiron/sqlx"
+)
+
+// loadTestSQLDataFilesWithSchema loads the production schema followed by the given data
+// fixtures. Use this instead of loadTestSQLDataFiles when a test only needs a subset of
+// the fixtures, since the fixtures do not create their own tables.
+func loadTestSQLDataFilesWithSchema(db *sqlx.DB, ctx context.Context, conn *sqlx.Conn, files []string) error {
+	if err := LoadTestSchema(db, ctx, conn); err != nil {
+		return err
+	}
+	return loadTestSQLDataFiles(db, ctx, conn, files)
+}

pkg/models/common_test.go

@@ -22,7 +22,7 @@ import (
 	zlog "github.com/scanoss/zap-logging-helper/pkg/logger"
 
 	"github.com/jmoiron/sqlx"
-	_ "github.com/mattn/go-sqlite3"
+	_ "modernc.org/sqlite"
 )
 
 func TestDbLoad(t *testing.T) {
@@ -32,12 +32,12 @@ func TestDbLoad(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	db, err := sqlx.Connect("sqlite3", ":memory:")
+	db, err := sqlx.Connect("sqlite", ":memory:")
 	if err != nil {
 		t.Fatalf("an error '%s' was not expected when opening a stub database connection", err)
 	}
 	defer CloseDB(db)
-	err = loadSQLData(db, nil, nil, "./tests/mines.sql")
+	err = loadTestSQLDataFilesWithSchema(db, nil, nil, []string{"./tests/mines.sql"})
 	if err != nil {
 		t.Errorf("failed to load SQL test data: %v", err)
 	}

pkg/models/cpe_purl_test.go

@@ -50,7 +50,7 @@ func setupTest(t *testing.T) (*sqlx.Conn, *CpePurlModel) {
 	}
 	defer zlog.SyncZap()
 
-	db, err := sqlx.Connect("sqlite3", ":memory:")
+	db, err := sqlx.Connect("sqlite", ":memory:")
 	if err != nil {
 		t.Fatalf("an error '%s' was not expected when opening a stub database connection", err)
 	}
@@ -160,7 +160,7 @@ func TestGetCpesByPurlString(t *testing.T) {
 		t.Fatalf("an error '%s' was not expected when opening a sugared logger", err)
 	}
 	defer zlog.SyncZap()
-	db, err := sqlx.Connect("sqlite3", ":memory:")
+	db, err := sqlx.Connect("sqlite", ":memory:")
 	if err != nil {
 		t.Fatalf("an error '%s' was not expected when opening a stub database connection", err)
 	}
@@ -235,7 +235,7 @@ func TestGetCpesByPurlStringVersion(t *testing.T) {
 		t.Fatalf("an error '%s' was not expected when opening a sugared logger", err)
 	}
 	defer zlog.SyncZap()
-	db, err := sqlx.Connect("sqlite3", ":memory:")
+	db, err := sqlx.Connect("sqlite", ":memory:")
 	if err != nil {
 		t.Fatalf("an error '%s' was not expected when opening a stub database connection", err)
 	}

pkg/models/epss_test.go

@@ -28,7 +28,7 @@ import (
 	zlog "github.com/scanoss/zap-logging-helper/pkg/logger"
 
 	"github.com/jmoiron/sqlx"
-	_ "github.com/mattn/go-sqlite3"
+	_ "modernc.org/sqlite"
 )
 
 func TestGetEPSSByCVEs(t *testing.T) {
@@ -41,13 +41,13 @@ func TestGetEPSSByCVEs(t *testing.T) {
 
 	s := ctxzap.Extract(ctx).Sugar()
 
-	db, err := sqlx.Connect("sqlite3", ":memory:")
+	db, err := sqlx.Connect("sqlite", ":memory:")
 	if err != nil {
 		t.Fatalf("an error '%s' was not expected when opening a stub database connection", err)
 	}
 	defer CloseDB(db)
 
-	err = loadSQLData(db, nil, nil, "./tests/epss.sql")
+	err = loadTestSQLDataFilesWithSchema(db, nil, nil, []string{"./tests/epss.sql"})
 	if err != nil {
 		t.Fatalf("failed to load SQL test data: %v", err)
 	}
@@ -68,6 +68,26 @@ func TestGetEPSSByCVEs(t *testing.T) {
 	if len(results) != 3 {
 		t.Errorf("GetEPSSByCVEs() expected 3 results, got %d", len(results))
 	}
+	// The scores must survive the scan, not just the row count. epss_data columns are
+	// TEXT like the rest of the schema, so this covers the TEXT to float32 conversion.
+	want := map[string][2]float32{
+		"CVE-2017-9302":  {0.00143, 0.5124},
+		"CVE-2015-0269":  {0.00285, 0.6832},
+		"CVE-2018-10083": {0.00891, 0.8215},
+	}
+	for _, got := range results {
+		exp, ok := want[got.Cve]
+		if !ok {
+			t.Errorf("GetEPSSByCVEs() returned unexpected CVE %v", got.Cve)
+			continue
+		}
+		if got.Epss != exp[0] {
+			t.Errorf("GetEPSSByCVEs() %v epss = %v, want %v", got.Cve, got.Epss, exp[0])
+		}
+		if got.Percentile != exp[1] {
+			t.Errorf("GetEPSSByCVEs() %v percentile = %v, want %v", got.Cve, got.Percentile, exp[1])
+		}
+	}
 }
 
 func TestGetEPSSByCVEsEmpty(t *testing.T) {
@@ -79,7 +99,7 @@ func TestGetEPSSByCVEsEmpty(t *testing.T) {
 	defer zlog.SyncZap()
 	s := ctxzap.Extract(ctx).Sugar()
 
-	db, err := sqlx.Connect("sqlite3", ":memory:")
+	db, err := sqlx.Connect("sqlite", ":memory:")
 	if err != nil {
 		t.Fatalf("an error '%s' was not expected when opening a stub database connection", err)
 	}
@@ -90,7 +110,7 @@ func TestGetEPSSByCVEsEmpty(t *testing.T) {
 		t.Fatalf("failed to load Config: %v", err)
 	}
 
-	err = loadSQLData(db, nil, nil, "./tests/epss.sql")
+	err = loadTestSQLDataFilesWithSchema(db, nil, nil, []string{"./tests/epss.sql"})
 	if err != nil {
 		t.Fatalf("failed to load SQL test data: %v", err)
 	}
@@ -116,7 +136,7 @@ func TestGetEPSSByCVEsNotFound(t *testing.T) {
 	defer zlog.SyncZap()
 	s := ctxzap.Extract(ctx).Sugar()
 
-	db, err := sqlx.Connect("sqlite3", ":memory:")
+	db, err := sqlx.Connect("sqlite", ":memory:")
 	if err != nil {
 		t.Fatalf("an error '%s' was not expected when opening a stub database connection", err)
 	}
@@ -127,7 +147,7 @@ func TestGetEPSSByCVEsNotFound(t *testing.T) {
 		t.Fatalf("failed to load Config: %v", err)
 	}
 
-	err = loadSQLData(db, nil, nil, "./tests/epss.sql")
+	err = loadTestSQLDataFilesWithSchema(db, nil, nil, []string{"./tests/epss.sql"})
 	if err != nil {
 		t.Fatalf("failed to load SQL test data: %v", err)
 	}
@@ -154,7 +174,7 @@ func TestGetEPSSByCVEsSingleCVE(t *testing.T) {
 	defer zlog.SyncZap()
 	s := ctxzap.Extract(ctx).Sugar()
 
-	db, err := sqlx.Connect("sqlite3", ":memory:")
+	db, err := sqlx.Connect("sqlite", ":memory:")
 	if err != nil {
 		t.Fatalf("an error '%s' was not expected when opening a stub database connection", err)
 	}
@@ -165,7 +185,7 @@ func TestGetEPSSByCVEsSingleCVE(t *testing.T) {
 		t.Fatalf("failed to load Config: %v", err)
 	}
 
-	err = loadSQLData(db, nil, nil, "./tests/epss.sql")
+	err = loadTestSQLDataFilesWithSchema(db, nil, nil, []string{"./tests/epss.sql"})
 	if err != nil {
 		t.Fatalf("failed to load SQL test data: %v", err)
 	}