Check native and Wasm SQLite3MC open each other's files, under Node and in browsers

.github/workflows/ci.yml

@@ -65,7 +65,7 @@ jobs:
       - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
         with:
           persist-credentials: false
-      - run: shellcheck -x upgrade.sh bump.sh sigstore.sh
+      - run: shellcheck -x upgrade.sh bump.sh sigstore.sh interop.sh
       - name: actionlint
         run: |
           bash <(curl -sSfL https://raw.githubusercontent.com/rhysd/actionlint/914e7df21a07ef503a81201c76d2b11c789d3fca/scripts/download-actionlint.bash) 1.7.12
@@ -121,8 +121,8 @@ jobs:
       - run: cargo run --manifest-path "$SMOKE_MANIFEST"
         shell: bash
 
-  wasm:
-    name: sqlite-wasm-rs builds the packaged amalgamation for Wasm
+  interop:
+    name: Native and Wasm SQLite3MC open each other's files
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
@@ -136,19 +136,44 @@ jobs:
       - uses: taiki-e/install-action@7623a79cdfecb99d681017af368ca353d9f49bb5 # v2.87.19
         with:
           tool: wasm-pack
-      # A C library function the released sqlite-wasm-rs does not provide fails this link.
+      # Both sides compile the unpacked .crate, and a C library function the released sqlite-wasm-rs does not provide fails the Wasm link.
       - name: Package and unpack
         run: |
           set -euo pipefail
           cargo package
           crate=$(echo target/package/sqlite3mc-src-*.crate)
+          dir="$PWD/target/package/$(basename "$crate" .crate)"
           tar -xzf "$crate" -C target/package
-          echo "SQLITE_WASM_RS_SOURCE_DIR=$PWD/target/package/$(basename "$crate" .crate)/sqlite3mc" >> "$GITHUB_ENV"
-      - run: wasm-pack test --node --release
-        working-directory: wasm
+          cp -r smoke "$dir/smoke"
+          echo "PACKAGE_DIR=$dir" >> "$GITHUB_ENV"
+      - run: ./interop.sh node
+        env:
+          INTEROP_CRATE: ${{ env.PACKAGE_DIR }}
+          WASM_BINDGEN_TEST_TIMEOUT: "300"
       # sqlite-wasm-rs vendors the same release, so only its build log shows which copy it compiled.
-      - run: grep -rqsF "rerun-if-changed=$SQLITE_WASM_RS_SOURCE_DIR" wasm/target/wasm32-unknown-unknown/release/build
+      - run: grep -rqsF "rerun-if-changed=$PACKAGE_DIR/sqlite3mc" wasm/target/wasm32-unknown-unknown/release/build
       - run: cargo clippy --manifest-path wasm/Cargo.toml --target wasm32-unknown-unknown --all-targets -- -D warnings
+        env:
+          SQLITE_WASM_RS_SOURCE_DIR: ${{ env.PACKAGE_DIR }}/sqlite3mc
+
+  browsers:
+    name: Wasm SQLite3MC in Chrome and Firefox opens native files and writes files native opens, on OPFS too
+    runs-on: ubuntu-latest
+    env:
+      WASM_BINDGEN_TEST_TIMEOUT: "300"
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          persist-credentials: false
+      - uses: dtolnay/rust-toolchain@02cb101ec7c40f2c49e1d9714d64511d8e1b74de # master
+        with:
+          toolchain: stable
+          targets: wasm32-unknown-unknown
+      - uses: taiki-e/install-action@7623a79cdfecb99d681017af368ca353d9f49bb5 # v2.87.19
+        with:
+          tool: wasm-pack
+      - run: ./interop.sh chrome
+      - run: ./interop.sh firefox
 
   rusqlite:
     name: rusqlite's SQLite3MC tests pass on these sources

README.md

@@ -19,4 +19,4 @@ The version encodes the release, so `205.1.x` is SQLite3MC 2.5.1. A `205.1` requ
 
 SQLite3MC is MIT licensed. The amalgamation also carries public-domain code (SQLite among it), a password-hashing file under CC0-1.0, a block under the Unlicense, and Argon2 under CC0-1.0 or Apache-2.0.
 
-A daily workflow in the [repository](https://github.com/LucaCappelletti94/sqlite3mc-src) opens a pull request for each new SQLite3MC release, taking the archive's checksum only from the release's Sigstore-signed `SHA256SUMS`. CI re-runs `upgrade.sh`, which checks that signature again before trusting the pinned checksum, to prove the vendored bytes match the pinned release.
+A daily workflow in the [repository](https://github.com/LucaCappelletti94/sqlite3mc-src) opens a pull request for each new SQLite3MC release, taking the archive's checksum only from the release's Sigstore-signed `SHA256SUMS`. CI re-runs `upgrade.sh`, which checks that signature again before trusting the pinned checksum, to prove the vendored bytes match the pinned release. It also compiles the packaged sources natively and through `sqlite-wasm-rs`, and checks that each build opens the other's files in every cipher, under Node and in headless Chrome and Firefox, on OPFS too.

interop.sh

@@ -0,0 +1,43 @@
+#!/bin/sh -e
+
+# Native SQLite3MC writes a file per cipher. Each runtime named (node when none is, chrome, firefox) runs the
+# Wasm tests, which read those files and print each file they write, and native reads the printed files.
+# Both sides compile the sources of $INTEROP_CRATE, an unpacked .crate with smoke/ copied in, or of this checkout.
+cd "$(dirname "$0")"
+crate=$(cd "${INTEROP_CRATE:-.}" && pwd)
+# wasm/build.rs embeds everything in $fixtures, so what Wasm hands back goes to $returned.
+fixtures="$PWD/target/interop"
+returned="$PWD/target/interop-wasm"
+export SQLITE_WASM_RS_SOURCE_DIR="$crate/sqlite3mc"
+native() {
+    cargo run --release --manifest-path "$crate/smoke/Cargo.toml" --bin interop -- "$@"
+}
+
+[ $# -gt 0 ] || set -- node
+native write "$fixtures"
+rm -rf "$returned" && mkdir -p "$returned"
+for runtime; do
+    log="$returned/$runtime.log"
+    # sahpool skips itself under Node, which has no OPFS.
+    case $runtime in
+    node)
+        vfses=memvfs
+        run() { wasm-pack test --node --release --test encryption --test sahpool -- --nocapture; }
+        ;;
+    chrome | firefox)
+        vfses="memvfs opfs"
+        run() { WASM_BINDGEN_USE_BROWSER=1 wasm-pack test --headless "--$runtime" --release --test encryption --test sahpool -- --nocapture; }
+        ;;
+    *) echo "unknown runtime $runtime" >&2 && exit 1 ;;
+    esac
+    (cd wasm && run) >"$log" 2>&1 || { cat "$log" && exit 1; }
+    grep -v sqlite3mc-interop-file "$log"
+    # A file whose line is missing or cut short fails the native read.
+    sed -n 's/^.*sqlite3mc-interop-file \([a-z0-9-]*\) \([A-Za-z0-9+/=]*\) end$/\1 \2/p' "$log" |
+        while read -r name data; do
+            printf '%s' "$data" | base64 -d >"$returned/$runtime-$name.db"
+        done
+    for vfs in $vfses; do
+        native read "$returned" "$runtime-$vfs"
+    done
+done

smoke/Cargo.toml

@@ -4,6 +4,7 @@ version = "0.0.0"
 edition = "2021"
 publish = false
 description = "Compiles and links the vendored amalgamation as a -sys crate would"
+default-run = "sqlite3mc-src-smoke"
 
 [dependencies]
 sqlite3mc-src = { path = ".." }

smoke/src/bin/interop.rs

@@ -0,0 +1,77 @@
+//! `write DIR` creates `native-<cipher>.db` for every cipher, `read DIR PREFIX` checks the `PREFIX-<cipher>.db` Wasm wrote.
+
+#[path = "../exchange.rs"]
+mod exchange;
+
+#[expect(
+    non_camel_case_types,
+    reason = "SQLite's C names, as sqlite-wasm-rs spells them"
+)]
+mod ffi {
+    use std::ffi::{c_char, c_int, c_uchar, c_void};
+
+    pub enum sqlite3 {}
+    pub enum sqlite3_stmt {}
+
+    pub const SQLITE_OK: c_int = 0;
+    pub const SQLITE_NOTADB: c_int = 26;
+    pub const SQLITE_ROW: c_int = 100;
+    pub const SQLITE_OPEN_READWRITE: c_int = 2;
+    pub const SQLITE_OPEN_CREATE: c_int = 4;
+
+    type ExecCallback =
+        unsafe extern "C" fn(*mut c_void, c_int, *mut *mut c_char, *mut *mut c_char) -> c_int;
+
+    unsafe extern "C" {
+        pub fn sqlite3_open_v2(
+            filename: *const c_char,
+            db: *mut *mut sqlite3,
+            flags: c_int,
+            vfs: *const c_char,
+        ) -> c_int;
+        pub fn sqlite3_close(db: *mut sqlite3) -> c_int;
+        pub fn sqlite3_exec(
+            db: *mut sqlite3,
+            sql: *const c_char,
+            callback: Option<ExecCallback>,
+            arg: *mut c_void,
+            errmsg: *mut *mut c_char,
+        ) -> c_int;
+        pub fn sqlite3_prepare_v2(
+            db: *mut sqlite3,
+            sql: *const c_char,
+            bytes: c_int,
+            stmt: *mut *mut sqlite3_stmt,
+            tail: *mut *const c_char,
+        ) -> c_int;
+        pub fn sqlite3_step(stmt: *mut sqlite3_stmt) -> c_int;
+        pub fn sqlite3_column_text(stmt: *mut sqlite3_stmt, column: c_int) -> *const c_uchar;
+        pub fn sqlite3_finalize(stmt: *mut sqlite3_stmt) -> c_int;
+        pub fn sqlite3_errmsg(db: *mut sqlite3) -> *const c_char;
+    }
+}
+
+fn main() {
+    let args: Vec<String> = std::env::args().skip(1).collect();
+    let (write, dir, prefix) = match args.as_slice() {
+        [command, dir] if command == "write" => (true, dir, "native"),
+        [command, dir, prefix] if command == "read" => (false, dir, prefix.as_str()),
+        _ => panic!("usage: interop write DIR | interop read DIR PREFIX"),
+    };
+    let marker = if write {
+        // Files of an earlier run must never pass for this one's.
+        let _ = std::fs::remove_dir_all(dir);
+        std::fs::create_dir_all(dir).unwrap();
+        exchange::NATIVE
+    } else {
+        exchange::WASM
+    };
+    for (cipher, pragmas) in exchange::CIPHERS {
+        let name = format!("{dir}/{prefix}-{cipher}.db");
+        if write {
+            exchange::write(&name, pragmas, marker);
+        }
+        let bytes = std::fs::read(&name).unwrap_or_else(|e| panic!("{name}: {e}"));
+        exchange::check(&name, &bytes, pragmas, marker);
+    }
+}

smoke/src/exchange.rs

@@ -0,0 +1,157 @@
+//! The encrypted files native and Wasm SQLite3MC hand each other, and the checks both sides run on them.
+//! The native `interop` binary and the Wasm tests include this file, each next to its own `ffi` module.
+
+use super::ffi;
+use std::ffi::{c_int, CStr, CString};
+
+const KEY: &str = "PRAGMA key = 'correct horse battery staple'";
+const WRONG_KEY: &str = "PRAGMA key = 'wrong horse battery staple'";
+/// Start of every row the native build writes.
+pub const NATIVE: &str = "written natively";
+/// Start of every row the Wasm build writes.
+pub const WASM: &str = "written by Wasm";
+/// Enough rows of about 100 bytes to fill several pages.
+const ROWS: u32 = 400;
+const SQLCIPHER: &str = "PRAGMA cipher = 'sqlcipher'";
+
+/// Every cipher SQLite3MC ships, by file name, as the pragmas that select it before `PRAGMA key`.
+/// A reader that skips the last pragma of a cipher must fail, which proves the pragmas took effect.
+pub const CIPHERS: &[(&str, &[&str])] = &[
+    ("chacha20", &[]),
+    ("aes128cbc", &["PRAGMA cipher = 'aes128cbc'"]),
+    ("aes256cbc", &["PRAGMA cipher = 'aes256cbc'"]),
+    ("sqlcipher1", &[SQLCIPHER, "PRAGMA legacy = 1"]),
+    ("sqlcipher2", &[SQLCIPHER, "PRAGMA legacy = 2"]),
+    ("sqlcipher3", &[SQLCIPHER, "PRAGMA legacy = 3"]),
+    ("sqlcipher4", &[SQLCIPHER, "PRAGMA legacy = 4"]),
+    ("rc4", &["PRAGMA cipher = 'rc4'"]),
+    ("ascon128", &["PRAGMA cipher = 'ascon128'"]),
+    ("aegis", &["PRAGMA cipher = 'aegis'"]),
+];
+
+/// A connection to `name` in the side's default VFS, which fails the test on any SQLite error.
+pub struct Db(*mut ffi::sqlite3, String);
+
+impl Db {
+    /// Opens `name` read-write, creating it if missing, and runs `pragmas`.
+    ///
+    /// # Panics
+    ///
+    /// If opening or a pragma fails.
+    #[must_use]
+    pub fn open(name: &str, pragmas: &[&str]) -> Self {
+        let c_name = CString::new(name).unwrap();
+        let mut handle = std::ptr::null_mut();
+        let flags = ffi::SQLITE_OPEN_READWRITE | ffi::SQLITE_OPEN_CREATE;
+        // `c_name` outlives the call, and SQLite copies it.
+        let rc =
+            unsafe { ffi::sqlite3_open_v2(c_name.as_ptr(), &mut handle, flags, std::ptr::null()) };
+        let db = Self(handle, name.to_owned());
+        assert_eq!(rc, ffi::SQLITE_OK, "{name}: {}", db.error());
+        for pragma in pragmas {
+            db.exec(pragma);
+        }
+        db
+    }
+
+    /// Runs `sql` and returns SQLite's result code.
+    fn status(&self, sql: &str) -> c_int {
+        let sql = CString::new(sql).unwrap();
+        // `sql` outlives the call, and no callback or error pointer is passed.
+        unsafe {
+            let (arg, errmsg) = (std::ptr::null_mut(), std::ptr::null_mut());
+            ffi::sqlite3_exec(self.0, sql.as_ptr(), None, arg, errmsg)
+        }
+    }
+
+    fn exec(&self, sql: &str) {
+        let rc = self.status(sql);
+        assert_eq!(rc, ffi::SQLITE_OK, "{}: {sql}: {}", self.1, self.error());
+    }
+
+    /// First column of the first row of `sql`, as text.
+    ///
+    /// # Panics
+    ///
+    /// If it fails or returns no text.
+    #[must_use]
+    pub fn text(&self, sql: &str) -> String {
+        let c_sql = CString::new(sql).unwrap();
+        let mut stmt = std::ptr::null_mut();
+        // `c_sql` outlives the call, which reads it up to its NUL.
+        let rc = unsafe {
+            ffi::sqlite3_prepare_v2(self.0, c_sql.as_ptr(), -1, &mut stmt, std::ptr::null_mut())
+        };
+        assert_eq!(rc, ffi::SQLITE_OK, "{}: {sql}: {}", self.1, self.error());
+        // `stmt` was just prepared, and the text is copied before it is finalized.
+        let text = unsafe {
+            let value = if ffi::sqlite3_step(stmt) == ffi::SQLITE_ROW {
+                ffi::sqlite3_column_text(stmt, 0)
+            } else {
+                std::ptr::null()
+            };
+            let text = (!value.is_null())
+                .then(|| CStr::from_ptr(value.cast()).to_string_lossy().into_owned());
+            let error = self.error();
+            ffi::sqlite3_finalize(stmt);
+            text.ok_or(error)
+        };
+        text.unwrap_or_else(|error| panic!("{}: {sql}: {error}", self.1))
+    }
+
+    fn error(&self) -> String {
+        // SQLite returns a NUL-terminated message for any handle, even a null one.
+        unsafe { CStr::from_ptr(ffi::sqlite3_errmsg(self.0)) }
+            .to_string_lossy()
+            .into_owned()
+    }
+}
+
+impl Drop for Db {
+    fn drop(&mut self) {
+        // The handle came from `sqlite3_open_v2`, and every statement on it is finalized.
+        unsafe { ffi::sqlite3_close(self.0) };
+    }
+}
+
+/// Writes `name` in the format `pragmas` select, with rows starting with `marker`.
+pub fn write(name: &str, pragmas: &[&str], marker: &str) {
+    Db::open(name, &[pragmas, &[KEY]].concat()).exec(&format!(
+        "CREATE TABLE t(v TEXT);
+         WITH RECURSIVE n(i) AS (SELECT 1 UNION ALL SELECT i + 1 FROM n WHERE i < {ROWS})
+         INSERT INTO t SELECT '{marker} ' || i || ' ' || hex(zeroblob(40)) FROM n;"
+    ));
+}
+
+/// Asserts `bytes`, the file `name` holding rows that start with `marker`, is encrypted on every page,
+/// and that it opens with its key and pragmas, but not with a wrong key or without its last pragma.
+///
+/// # Panics
+///
+/// If any of that does not hold.
+pub fn check(name: &str, bytes: &[u8], pragmas: &[&str], marker: &str) {
+    assert!(bytes.len() > 16 * 1024, "{name} is {} bytes", bytes.len());
+    for plain in [
+        b"SQLite format 3".as_slice(),
+        b"CREATE TABLE",
+        marker.as_bytes(),
+    ] {
+        let leak = bytes.windows(plain.len()).any(|w| w == plain);
+        assert!(!leak, "{name} holds {:?}", String::from_utf8_lossy(plain));
+    }
+    let db = Db::open(name, &[pragmas, &[KEY]].concat());
+    assert_eq!(db.text("PRAGMA quick_check"), "ok", "{name}");
+    let count = db.text(&format!("SELECT count(*) FROM t WHERE v LIKE '{marker} %'"));
+    assert_eq!(count, ROWS.to_string(), "{name}");
+    let probe = "SELECT count(*) FROM sqlite_schema";
+    let rc = Db::open(name, &[pragmas, &[WRONG_KEY]].concat()).status(probe);
+    assert_eq!(rc, ffi::SQLITE_NOTADB, "{name} opened with a wrong key");
+    if let Some((_, fewer)) = pragmas.split_last() {
+        let rc = Db::open(name, &[fewer, &[KEY]].concat()).status(probe);
+        assert_eq!(
+            rc,
+            ffi::SQLITE_NOTADB,
+            "{name} opened without its last pragma"
+        );
+    }
+}