diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b385bd9..0846a55 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -65,7 +65,7 @@ jobs: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - - run: shellcheck -x upgrade.sh bump.sh sigstore.sh interop.sh + - run: shellcheck -x upgrade.sh bump.sh sigstore.sh interop.sh suite/run.sh - name: actionlint run: | bash <(curl -sSfL https://raw.githubusercontent.com/rhysd/actionlint/914e7df21a07ef503a81201c76d2b11c789d3fca/scripts/download-actionlint.bash) 1.7.12 @@ -211,6 +211,30 @@ jobs: # The crates.io release carries the same bytes until a bump, so only the build log shows which copy was compiled. - run: grep -rqsF "rerun-if-changed=$GITHUB_WORKSPACE/sqlite3mc/" "$RUSQLITE_DIR/target/debug/build" + suite: + name: SQLite3MC's and SQLite's tests pass on the shipped amalgamation + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: sigstore/cosign-installer@6f9f17788090df1f26f669e9d70d6ae9567deba6 # v4.1.2 + - run: sudo apt-get install -y --no-install-recommends tcl-dev + - run: ./suite/run.sh + + sanitizers: + name: SQLite3MC's and SQLite's tests are clean under ASAN and UBSan + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: sigstore/cosign-installer@6f9f17788090df1f26f669e9d70d6ae9567deba6 # v4.1.2 + - run: sudo apt-get install -y --no-install-recommends tcl-dev + - run: ./suite/run.sh + env: + SANITIZE: "1" + release-bytes: name: Vendored files match the release runs-on: ubuntu-latest diff --git a/README.md b/README.md index 9ad083e..01ef963 100644 --- a/README.md +++ b/README.md @@ -20,3 +20,5 @@ The version encodes the release, so `205.1.x` is SQLite3MC 2.5.1. A `205.1` requ SQLite3MC is MIT licensed. The amalgamation also carries public-domain code (SQLite among it), a password-hashing file under CC0-1.0, a block under the Unlicense, and Argon2 under CC0-1.0 or Apache-2.0. A daily workflow in the [repository](https://github.com/LucaCappelletti94/sqlite3mc-src) opens a pull request for each new SQLite3MC release, taking the archive's checksum only from the release's Sigstore-signed `SHA256SUMS`. CI re-runs `upgrade.sh`, which checks that signature again before trusting the pinned checksum, to prove the vendored bytes match the pinned release. It also compiles the packaged sources natively and through `sqlite-wasm-rs`, and checks that each build opens the other's files in every cipher, under Node and in headless Chrome and Firefox, on OPFS too. + +CI also runs SQLite3MC's own tests, a rekey of every cipher and SQLite's TCL suite through the shipped amalgamation, once plainly and once under the address and undefined-behaviour sanitizers, with every expected failure named and explained. diff --git a/suite/expected.txt b/suite/expected.txt new file mode 100644 index 0000000..d0ddcd9 --- /dev/null +++ b/suite/expected.txt @@ -0,0 +1,95 @@ +== test1 +sqlcipher +4 +ok +1|Alf +2|Bert +3|Cecil +4|Donald +== test2 +sqlcipher +ok +1|Alf +2|Bert +3|Cecil +4|Donald +5|Ernie +== test3 +aegis +ok +200 +Alfa|Romeo +Ammonium|Sylfita +Bolibar|Augusta +Bonaventura|Julia +Marinata|Pia +Mortadella|Mona +Peschbal|Hannibal +Rauchschwalbe|Remigius +Schmidt|Siglinde +Walter|Magdalena +== test4 +ascon128 +ok +200 +Alfa|Romeo +Ammonium|Sylfita +Bolibar|Augusta +Bonaventura|Julia +Marinata|Pia +Mortadella|Mona +Peschbal|Hannibal +Rauchschwalbe|Remigius +Schmidt|Siglinde +Walter|Magdalena +== sqlciphertest +sqlcipher +1 +ok +75709 +1|1 +one|one +1|2 +one|two +1|2 +sqlcipher +2 +4000 +0 +ok +38768 +test-0-0|test-0-1 +test-1-0|test-1-1 +sqlcipher +2 +2 +ok +78536 +1|1 +one|one +1|2 +one|two +sqlcipher +2 +ok +78536 +1|1 +one|one +1|2 +one|two +sqlcipher +3 +ok +78536 +1|1 +one|one +1|2 +one|two +sqlcipher +4 +ok +78536 +1|1 +one|one +1|2 +one|two diff --git a/suite/run.sh b/suite/run.sh new file mode 100755 index 0000000..8f3c65f --- /dev/null +++ b/suite/run.sh @@ -0,0 +1,180 @@ +#!/bin/sh -e +# Runs SQLite3MC's own tests, a rekey of every cipher, and SQLite's TCL suite against the shipped amalgamation. + +cd "$(dirname "$0")/.." +ROOT=$(pwd) +SHIPPED="$ROOT/sqlite3mc/sqlite3mc_amalgamation.c" +HEADER="$ROOT/sqlite3mc/sqlite3mc_amalgamation.h" +# shellcheck source=sigstore.sh +. "$ROOT/sigstore.sh" +WORK=$(mktemp -d) +trap 'rm -rf "$WORK"' EXIT + +# SQLite test cases that fail by SQLite3MC's design or through a named SQLite3MC defect, each with its reason. +EXPECTED_FAILURES=$(cat <<'EOF' +mutex1.2.singlethread.4 SQLite3MC's VFS guards its open files with a recursive mutex of its own +mutex1.2.multithread.4 SQLite3MC's VFS guards its open files with a recursive mutex of its own +mutex1.3.2 a -nomutex connection still takes that process-wide VFS mutex +memsubsys2-4.1 SQLite3MC's cipher tables stay allocated between sqlite3_initialize and sqlite3_shutdown +memsubsys2-4.3 SQLite3MC's cipher tables stay allocated between sqlite3_initialize and sqlite3_shutdown +memsubsys2-4.4 SQLite3MC's cipher tables stay allocated between sqlite3_initialize and sqlite3_shutdown +memsubsys2-4.11 SQLite3MC's cipher tables stay allocated between sqlite3_initialize and sqlite3_shutdown +backup2-10 mcIoRead drops read errors of the real file, fixed by SQLite3MC pull request 272 after 2.5.1 +EOF +) +# SQLite test files that crash through a named SQLite3MC defect, each with its reason. +EXPECTED_CRASHES=$(cat <<'EOF' +init sqlite3mcRegisterCipher writes through an unchecked sqlite3_malloc when an allocation fails +quota sqlite3mcCloneCodecParameterTable writes through an unchecked sqlite3_malloc when an allocation fails +multiplex sqlite3mcCloneCodecParameterTable writes through an unchecked sqlite3_malloc when an allocation fails +EOF +) +# SQLite test files not run, each with its reason. +SKIP_FILES="" + +# SANITIZE=1 builds under clang with AddressSanitizer and UBSan, where any report ends the process. +CC=cc +OPT="-O2" +if [ -n "${SANITIZE:-}" ]; then + CC=clang + OPT="-O1 -g -fno-omit-frame-pointer -fsanitize=address,undefined -fno-sanitize-recover=all" + ASAN_OPTIONS="detect_leaks=1:abort_on_error=1" + UBSAN_OPTIONS="print_stacktrace=1:halt_on_error=1" + export ASAN_OPTIONS UBSAN_OPTIONS + SKIP_FILES=$(cat <<'EOF' +crash8 its simulated crashes end child processes holding memory, which LeakSanitizer reports on SQLite alone too +EOF +) +fi + +# A clean sanitizer run only counts if the binary is really instrumented. +instrumented() { + [ -z "${SANITIZE:-}" ] || { nm "$1" | grep -q __asan_report_load && nm "$1" | grep -q __ubsan_handle; } || + { echo "$1 is not instrumented" >&2; exit 1; } +} + +# SQLite3MC's shell from the signed release archive, over the shipped amalgamation and header. +version=$(sed -n 's/^#define SQLITE3MC_VERSION_STRING *"SQLite3 Multiple Ciphers \(.*\)"$/\1/p' "$HEADER") +sqlite_version=$(sed -n 's/^#define SQLITE_VERSION *"\(.*\)"$/\1/p' "$HEADER") +archive="sqlite3mc-${version}-sqlite-${sqlite_version}-amalgamation.zip" +fetch_signed_sums "$version" "$WORK" +curl -sfL -o "$WORK/$archive" "https://github.com/utelle/SQLite3MultipleCiphers/releases/download/v${version}/$archive" +(cd "$WORK" && grep -F " $archive" SHA256SUMS | shasum -a 256 -c -) +unzip -q -d "$WORK" "$WORK/$archive" shell3mc_amalgamation.c +printf '#include "%s"\n' "$HEADER" > "$WORK/sqlite3.h" +# shellcheck disable=SC2086 +$CC $OPT -I"$WORK" -o "$WORK/sqlite3mc" "$WORK/shell3mc_amalgamation.c" "$SHIPPED" -lm +instrumented "$WORK/sqlite3mc" + +# SQLite3MC's test directory at the commit its signing certificate names, run as its own CI runs it. +commit=$(base64 -d "$WORK/SHA256SUMS.pem" | openssl x509 -noout -text | + awk '/1\.3\.6\.1\.4\.1\.57264\.1\.3:/ { getline; gsub(/ /, ""); print }') +git init --quiet "$WORK/mc" +git -C "$WORK/mc" fetch --quiet --depth 1 https://github.com/utelle/SQLite3MultipleCiphers.git "refs/tags/v${version}:refs/tags/v${version}" +[ "$(git -C "$WORK/mc" rev-parse "v${version}^{commit}")" = "$commit" ] || + { echo "SQLite3MC v${version} does not point at the signed commit ${commit}" >&2; exit 1; } +git -C "$WORK/mc" archive "$commit" test | tar x -C "$WORK/mc" +failed="" +# Upstream never checks the output, so its results, without the echoed script lines, are compared here. +while read -r name db script; do + echo "== $name" + (cd "$WORK/mc" && "$WORK/sqlite3mc" "$db" ".read test/$script" 2>&1 | grep -vxF -f "test/$script") || true +done > "$WORK/sqlite3mc.out" <<'EOF' +test1 test1.db3 test1.sql +test2 test2.db3 test2.sql +test3 test/persons-aegis-testkey.db3 test3.sql +test4 test/persons-ascon128-testkey.db3 test4.sql +sqlciphertest dummy.db3 sqlciphertest.sql +EOF +if diff -u "$ROOT/suite/expected.txt" "$WORK/sqlite3mc.out"; then echo "pass SQLite3MC's tests"; else failed=" sqlite3mc"; fi + +# Every built-in cipher encrypts, rekeys, refuses the old key and no key, and decrypts back to plaintext. +rows="WITH RECURSIVE n(x) AS (SELECT 1 UNION ALL SELECT x + 1 FROM n WHERE x < 1000) + INSERT INTO t SELECT x, printf('%.100c', char(65 + x % 26)) FROM n" +check="SELECT count(*), sum(a), sum(length(b)) FROM t" +mc() { "$WORK/sqlite3mc" "$db" "$@"; } +for cipher in aes128cbc aes256cbc chacha20 sqlcipher rc4 ascon128 aegis; do + db="$WORK/rekey-$cipher.db" + { + mc "PRAGMA cipher='$cipher'" "PRAGMA key='before'" "CREATE TABLE t(a INTEGER PRIMARY KEY, b TEXT)" "$rows" "PRAGMA rekey='after'" + mc "PRAGMA cipher='$cipher'" "PRAGMA key='after'" "$check" "PRAGMA integrity_check" + mc "PRAGMA cipher='$cipher'" "PRAGMA key='before'" "$check" + mc "$check" + mc "PRAGMA cipher='$cipher'" "PRAGMA key='after'" "PRAGMA rekey=''" + mc "$check" "PRAGMA integrity_check" + } > "$WORK/rekey.out" 2> "$WORK/rekey.err" || true + # Each invocation writes its error before its buffered results, so the two streams are compared apart. + echo "-- errors" >> "$WORK/rekey.out" + cat "$WORK/rekey.err" >> "$WORK/rekey.out" + cat > "$WORK/rekey.want" <&2 + exit 1 +fi +# Fossil names older files by SHA1 and newer ones by SHA3-256. +{ + awk '$1 == "F" && length($3) == 40 { print $2 }' manifest | xargs openssl dgst -sha1 -r + awk '$1 == "F" && length($3) == 64 { print $2 }' manifest | xargs openssl dgst -sha3-256 -r + find . -type f ! -name 'manifest*' | sed 's|^\./|file |' +} | sort > "$WORK/tree" +awk '$1 == "F" { print $3 " *" $2; print "file " $2 }' manifest | sort | cmp -s - "$WORK/tree" || + { echo "SQLite's source tree differs from its manifest" >&2; exit 1; } +# TCL_LIB names the tclConfig.sh directory when configure cannot find it. +CC="$CC" ./configure ${TCL_LIB:+--with-tcl="$TCL_LIB"} > configure.log +make sqlite3.c > make.log +# testfixture reaches SQLite internals through -DSQLITE_PRIVATE="", which the shipped amalgamation rejects by +# defining sqlite3mcCodecAttach static against its SQLITE_PRIVATE declaration. The copy drops that one static, +# as SQLite3MC pull request 276 proposes, and the run stops once a release no longer carries it. +line=$(grep -n -x 'sqlite3mcCodecAttach(sqlite3\* db, int nDb, const char\* zPath, const void\* zKey, int nKey)' "$SHIPPED" | cut -d: -f1) +if [ -z "$line" ] || [ "$(sed -n "$((line - 1))p" "$SHIPPED")" != "static int" ]; then + echo "sqlite3mcCodecAttach is no longer defined static, so drop its edit from suite/run.sh" >&2 + exit 1 +fi +sed "$((line - 1))s/^static int$/SQLITE_PRIVATE int/" "$SHIPPED" > sqlite3.c +[ "$(diff "$SHIPPED" sqlite3.c | grep -c '^[<>]')" -eq 2 ] || { echo "the testfixture copy differs by more than one line" >&2; exit 1; } +make testfixture CC="$CC" CFLAGS="$OPT" LDFLAGS="${SANITIZE:+$OPT}" > testfixture.log 2>&1 +instrumented testfixture +printf 'sqlite3 db :memory:\nputs [db one {SELECT sqlite3mc_version()}]\n' > probe.tcl +# A testfixture that cannot open a database runs no test, so its own report ends the run. +tested=$(./testfixture probe.tcl) || { echo "Failed:$failed testfixture" >&2; exit 1; } +echo "Testing $tested with SQLite's veryquick suite" + +skips=$(echo "$SKIP_FILES" | awk 'NF { printf " ~%s.test", $1 }') +# shellcheck disable=SC2086 +./testfixture test/testrunner.tcl veryquick --jobs "$(nproc)" $skips > testrunner.out 2>&1 || + { tail -n 40 testrunner.out; echo "Failed:$failed testrunner" >&2; exit 1; } +for name in $(echo "$SKIP_FILES" | awk '{ print $1 }'); do echo "skip $name"; done +awk -v failures="$(echo "$EXPECTED_FAILURES" | awk '{ print $1 }')" \ + -v crashes="$(echo "$EXPECTED_CRASHES" | awk '{ print $1 }')" \ + -f "$ROOT/suite/verdict.awk" testrunner.log testrunner.log || failed="$failed sqlite" + +[ -z "$failed" ] || { echo "Failed:$failed" >&2; exit 1; } diff --git a/suite/verdict.awk b/suite/verdict.awk new file mode 100644 index 0000000..c0c08ad --- /dev/null +++ b/suite/verdict.awk @@ -0,0 +1,84 @@ +# Judges SQLite's testrunner.log, read twice. The first pass finds the unfreed memory most files report, which is +# SQLite3MC's global cipher tables, and the second holds every file to it, to its summary line, and to the +# expected failures and crashes named in the variables failures and crashes. + +BEGIN { + n = split(failures, list) + for (i = 1; i <= n; i++) expected[list[i]] = 1 + n = split(crashes, list) + for (i = 1; i <= n; i++) crash[list[i]] = 1 +} + +FNR == 1 { pass++ } + +pass == 1 { + if (/^Unfreed memory: /) leaks[$0]++ + next +} + +FNR == 1 { + for (line in leaks) if (leaks[line] > leaks[baseline]) baseline = line + print "baseline " (baseline == "" ? "All memory allocations freed" : baseline) +} + +# A file that returns early, as the Windows-only ones do here, ends done without a summary. A crash ends failed. +/^### / { finish(); file = $2; state = $NF; next } +/^[0-9]+ errors out of [0-9]+ tests/ { errors = $1; total = $5; summary = 1 } +/^!Failures on these tests:/ { listed = substr($0, length("!Failures on these tests:") + 1) } +/^==[0-9]+==(ERROR|WARNING): |runtime error: / { if (sanitizer == "") sanitizer = $0 } +/^Unfreed memory: / { leak = $0 } +/ files were left open$/ { open_files = $0 } +/^!|^==[0-9]+==|^SUMMARY: |runtime error: / { if (lines++ < 20) details = details " " $0 "\n" } + +function finish( name, problem, n, i, names) { + if (file == "") return + name = file + sub(/.*\//, "", name) + sub(/\.test$/, "", name) + files++ + if (name in crash) { + crashed[name] = 1 + if (summary || state != "(failed)") { + print "FAIL " name " no longer crashes, so drop it from EXPECTED_CRASHES" + bad = 1 + } else { + print "crash " name " as expected" + } + } else if (!summary && state == "(done)" && sanitizer == "") { + returned = returned " " name + } else { + problem = "" + if (!summary) problem = ", no summary line" + n = split(listed, names) + if (summary && errors != n) problem = problem ", " errors " errors but " n " named failures" + for (i = 1; i <= n; i++) { + if (names[i] in expected) failed[names[i]] = 1 + else problem = problem ", unexpected failure " names[i] + } + if (sanitizer != "") problem = problem ", " sanitizer + if (leak != "" && leak != baseline) problem = problem ", " leak + if (open_files != "") problem = problem ", " open_files + if (problem != "") { + print "FAIL " name " " substr(problem, 3) + printf "%s", details + bad = 1 + } else { + tests += total + } + } + file = state = listed = sanitizer = leak = open_files = details = "" + summary = lines = 0 +} + +END { + finish() + for (name in expected) { + if (name in failed) print "known " name + else { print "FAIL " name " passes now, so drop it from EXPECTED_FAILURES"; bad = 1 } + } + for (name in crash) if (!(name in crashed)) { print "FAIL " name " did not run"; bad = 1 } + if (files == 0) { print "FAIL testrunner ran no files"; bad = 1 } + if (returned != "") print "returned before their summary:" returned + print files " files, " tests " tests in the files that passed" + exit bad +}