From 5d89839363a6a93b457ad7bf71a7b1108e88720a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=B2=20Boschi?= Date: Fri, 24 Apr 2026 11:44:21 +0200 Subject: [PATCH 1/2] fix: address Windows, NixOS, and localhost startup issues (#9 #10 #11) - Windows (#9): extract theseus-rs ZIP bundle via the zip crate (was failing with "invalid gzip header"), add Windows to the pgvector platform map so pg0-windows-x86_64.exe ships with pgvector, and pick the right .exe-suffixed binary names. - NixOS (#11): pin timezone/log_timezone to UTC so PostgreSQL never reads /usr/share/zoneinfo at startup (the upstream binaries are compiled against Debian's tzdata path, which does not exist on NixOS). - Localhost (#10): use 127.0.0.1 in all generated connection URIs and client calls, avoiding DNS/host-resolution variance for localhost. Other: - drop pgbouncer from versions.env (never referenced in build.rs or main.rs, so it was dead config). - CI: add build-windows + sdk-tests-windows jobs that mirror the release-cli.yml compile step (same toolchain targets, same cargo invocation, same artifact path) and run the full pytest suite on windows-latest. Skip the SIGKILL crash-recovery test on Windows since signal.SIGKILL does not exist there. --- .github/workflows/ci.yml | 58 +++++++++++++++ Cargo.lock | 20 ++++- Cargo.toml | 6 ++ README.md | 2 +- build.rs | 11 +-- sdk/node/README.md | 2 +- sdk/python/README.md | 6 +- sdk/python/tests/test_pg0.py | 5 ++ src/main.rs | 138 ++++++++++++++++++++++++----------- versions.env | 5 -- 10 files changed, 191 insertions(+), 62 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b146cad..fceed9f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -25,6 +25,28 @@ jobs: name: pg0-macos path: target/release/pg0 + # Mirrors the windows-x86_64 build step in release-cli.yml so CI exercises + # the exact same compile path that produces the shipped Windows executable. + build-windows: + name: Build CLI (Windows) + runs-on: windows-latest + steps: + - uses: actions/checkout@v4 + + - name: Install Rust + uses: dtolnay/rust-toolchain@stable + with: + targets: x86_64-pc-windows-msvc + + - name: Build + run: cargo build --release --target x86_64-pc-windows-msvc + + - name: Upload CLI artifact + uses: actions/upload-artifact@v4 + with: + name: pg0-windows + path: target/x86_64-pc-windows-msvc/release/pg0.exe + sdk-tests: name: SDK Tests (macOS) needs: build @@ -56,6 +78,42 @@ jobs: uv pip install --system -e ".[dev]" pytest tests/ -v + sdk-tests-windows: + name: SDK Tests (Windows) + needs: build-windows + runs-on: windows-latest + steps: + - uses: actions/checkout@v4 + + - name: Download CLI + uses: actions/download-artifact@v4 + with: + name: pg0-windows + path: pg0-bin + + - name: Put CLI on PATH + shell: pwsh + run: | + $dest = "$env:USERPROFILE\pg0-bin" + New-Item -ItemType Directory -Force -Path $dest | Out-Null + Copy-Item pg0-bin\pg0.exe $dest\pg0.exe + Add-Content $env:GITHUB_PATH $dest + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.11" + + - name: Install uv + uses: astral-sh/setup-uv@v4 + + - name: Run Python SDK tests + working-directory: sdk/python + shell: pwsh + run: | + uv pip install --system -e ".[dev]" + pytest tests/ -v + # Docker tests - one job per platform, runs both CLI and Python SDK tests # Note: ARM64 tests are skipped because QEMU emulation is too slow for PostgreSQL setup docker-tests: diff --git a/Cargo.lock b/Cargo.lock index 1394cad..6321795 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1188,6 +1188,7 @@ dependencies = [ "thiserror 1.0.69", "tracing", "tracing-subscriber", + "zip 2.4.2", ] [[package]] @@ -1228,7 +1229,7 @@ dependencies = [ "thiserror 2.0.17", "tracing", "url", - "zip", + "zip 4.6.1", ] [[package]] @@ -2929,6 +2930,23 @@ dependencies = [ "syn", ] +[[package]] +name = "zip" +version = "2.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fabe6324e908f85a1c52063ce7aa26b68dcb7eb6dbc83a2d148403c9bc3eba50" +dependencies = [ + "arbitrary", + "crc32fast", + "crossbeam-utils", + "displaydoc", + "flate2", + "indexmap", + "memchr", + "thiserror 2.0.17", + "zopfli", +] + [[package]] name = "zip" version = "4.6.1" diff --git a/Cargo.toml b/Cargo.toml index 2cd1633..c9ebc92 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -23,6 +23,12 @@ tracing-subscriber = { version = "0.3", features = ["env-filter"] } flate2 = "1" tar = "0.4" +# ZIP extraction for the Windows PostgreSQL bundle, which theseus-rs ships as +# .zip (unlike every other platform's tar.gz). Only pulled in on Windows +# targets to avoid bloating other platforms' binaries. +[target.'cfg(windows)'.dependencies] +zip = { version = "2", default-features = false, features = ["deflate"] } + [profile.release] strip = true lto = true diff --git a/README.md b/README.md index 181aab8..2088f65 100644 --- a/README.md +++ b/README.md @@ -73,7 +73,7 @@ from pg0 import Pg0 # Start PostgreSQL pg = Pg0() pg.start() -print(pg.uri) # postgresql://postgres:postgres@localhost:5432/postgres +print(pg.uri) # postgresql://postgres:postgres@127.0.0.1:5432/postgres # Or use context manager with Pg0() as pg: diff --git a/build.rs b/build.rs index 17df9b7..72bac1d 100644 --- a/build.rs +++ b/build.rs @@ -111,16 +111,7 @@ fn bundle_pgvector(pg_version: &str, pgvector_tag: &str, pgvector_repo: &str, ou "x86_64-unknown-linux-musl" => "x86_64-unknown-linux-gnu", // musl uses gnu pgvector "aarch64-unknown-linux-gnu" => "aarch64-unknown-linux-gnu", "aarch64-unknown-linux-musl" => "aarch64-unknown-linux-gnu", // musl uses gnu pgvector - "x86_64-pc-windows-msvc" => { - eprintln!("Warning: pgvector not available for Windows, skipping bundle"); - let marker = out_dir.join("pgvector_bundle.tar.gz"); - fs::write(&marker, b"").expect("Failed to create empty pgvector marker"); - println!( - "cargo:rustc-env=PGVECTOR_BUNDLE_PATH={}", - marker.display() - ); - return; - } + "x86_64-pc-windows-msvc" => "x86_64-pc-windows-msvc", _ => { eprintln!( "Warning: Unknown target {}, pgvector will not be bundled", diff --git a/sdk/node/README.md b/sdk/node/README.md index 3d9792a..ae8946d 100644 --- a/sdk/node/README.md +++ b/sdk/node/README.md @@ -18,7 +18,7 @@ import { Pg0 } from "@vectorize-io/pg0"; // Basic usage const pg = new Pg0(); await pg.start(); -console.log(await pg.getUri()); // postgresql://postgres:postgres@localhost:5432/postgres +console.log(await pg.getUri()); // postgresql://postgres:postgres@127.0.0.1:5432/postgres await pg.execute("CREATE EXTENSION IF NOT EXISTS vector"); await pg.stop(); diff --git a/sdk/python/README.md b/sdk/python/README.md index 383ccf5..c6a6b69 100644 --- a/sdk/python/README.md +++ b/sdk/python/README.md @@ -17,7 +17,7 @@ from pg0 import Pg0 # Basic usage with Pg0() as pg: - print(pg.uri) # postgresql://postgres:postgres@localhost:5432/postgres + print(pg.uri) # postgresql://postgres:postgres@127.0.0.1:5432/postgres pg.execute("CREATE EXTENSION IF NOT EXISTS vector") pg.execute("SELECT version()") @@ -69,11 +69,11 @@ pg = Pg0() pg.start() # Using the uri property -print(pg.uri) # postgresql://postgres:postgres@localhost:5432/postgres +print(pg.uri) # postgresql://postgres:postgres@127.0.0.1:5432/postgres # Or using info() info = pg.info() -print(info.uri) # postgresql://postgres:postgres@localhost:5432/postgres +print(info.uri) # postgresql://postgres:postgres@127.0.0.1:5432/postgres print(info.port) # 5432 print(info.username) # postgres print(info.database) # postgres diff --git a/sdk/python/tests/test_pg0.py b/sdk/python/tests/test_pg0.py index fa405c8..bf4ad40 100644 --- a/sdk/python/tests/test_pg0.py +++ b/sdk/python/tests/test_pg0.py @@ -2,6 +2,7 @@ import os import signal +import sys import time import pytest @@ -151,6 +152,10 @@ def test_port_conflict_error(self, clean_instance): pg2.stop() pg0.drop(f"{TEST_NAME}-2") + @pytest.mark.skipif( + sys.platform == "win32", + reason="signal.SIGKILL does not exist on Windows; crash-recovery behavior is exercised by the Unix matrix.", + ) def test_data_survives_crash(self, clean_instance): """Test that data is preserved after an unclean shutdown (SIGKILL). diff --git a/src/main.rs b/src/main.rs index dd28d3f..3197bd1 100644 --- a/src/main.rs +++ b/src/main.rs @@ -360,62 +360,108 @@ fn read_latest_pg_log(data_dir: &PathBuf) -> Option { } } -/// Extract the bundled PostgreSQL to the installation directory -/// Returns the path to the version-specific directory (e.g., ~/.pg0/installation/18.1.0) -fn extract_bundled_postgresql(installation_dir: &PathBuf, pg_version: &str) -> Result { - let version_dir = installation_dir.join(pg_version); - - // Check if already extracted - let bin_dir = version_dir.join("bin"); - if bin_dir.exists() && bin_dir.join("postgres").exists() { - tracing::debug!("PostgreSQL already extracted at {}", version_dir.display()); - return Ok(version_dir); - } - - if POSTGRESQL_BUNDLE.is_empty() { - return Err(CliError::Other( - "PostgreSQL bundle is empty - this binary was not built with BUNDLE_POSTGRESQL=true".to_string() - )); - } - - println!("Extracting bundled PostgreSQL {}...", pg_version); - fs::create_dir_all(&version_dir)?; - - // Extract the tar.gz bundle - // The archive contains paths like "postgresql-18.1.0-aarch64-apple-darwin/bin/postgres" - // We need to extract to version_dir, stripping the first path component - let decoder = GzDecoder::new(POSTGRESQL_BUNDLE); +/// Name of the main PostgreSQL server binary for the current target platform. +/// theseus-rs bundles `postgres.exe` on Windows and `postgres` everywhere else. +#[cfg(windows)] +const POSTGRES_BINARY: &str = "postgres.exe"; +#[cfg(not(windows))] +const POSTGRES_BINARY: &str = "postgres"; + +/// Extract the embedded bundle into `version_dir`, stripping the top-level +/// directory entry (e.g. "postgresql-18.1.0-/"). theseus-rs publishes +/// the Windows bundle as a ZIP and every other platform as tar.gz. +#[cfg(not(windows))] +fn extract_postgresql_archive(bundle: &[u8], version_dir: &std::path::Path) -> Result<(), CliError> { + let decoder = GzDecoder::new(bundle); let mut archive = Archive::new(decoder); for entry in archive.entries()? { let mut entry = entry?; let path = entry.path()?; - // Strip the first component (e.g., "postgresql-18.1.0-aarch64-apple-darwin") let stripped_path: PathBuf = path.components().skip(1).collect(); if stripped_path.as_os_str().is_empty() { - continue; // Skip the root directory entry + continue; } let dest_path = version_dir.join(&stripped_path); - - // Create parent directories if needed if let Some(parent) = dest_path.parent() { fs::create_dir_all(parent)?; } - // Extract the entry if entry.header().entry_type().is_dir() { fs::create_dir_all(&dest_path)?; } else { entry.unpack(&dest_path)?; } } + Ok(()) +} + +#[cfg(windows)] +fn extract_postgresql_archive(bundle: &[u8], version_dir: &std::path::Path) -> Result<(), CliError> { + use std::io::Cursor; + let reader = Cursor::new(bundle); + let mut archive = zip::ZipArchive::new(reader) + .map_err(|e| CliError::Other(format!("Failed to read PostgreSQL ZIP archive: {}", e)))?; + + for i in 0..archive.len() { + let mut entry = archive + .by_index(i) + .map_err(|e| CliError::Other(format!("Failed to read ZIP entry {}: {}", i, e)))?; + + let entry_path = match entry.enclosed_name() { + Some(p) => p.to_path_buf(), + None => continue, // Skip unsafe / absolute / traversal-containing names + }; + + let stripped_path: PathBuf = entry_path.components().skip(1).collect(); + if stripped_path.as_os_str().is_empty() { + continue; + } + + let dest_path = version_dir.join(&stripped_path); + if entry.is_dir() { + fs::create_dir_all(&dest_path)?; + } else { + if let Some(parent) = dest_path.parent() { + fs::create_dir_all(parent)?; + } + let mut out = fs::File::create(&dest_path)?; + std::io::copy(&mut entry, &mut out)?; + } + } + Ok(()) +} + +/// Extract the bundled PostgreSQL to the installation directory +/// Returns the path to the version-specific directory (e.g., ~/.pg0/installation/18.1.0) +fn extract_bundled_postgresql(installation_dir: &PathBuf, pg_version: &str) -> Result { + let version_dir = installation_dir.join(pg_version); + + // Check if already extracted + let bin_dir = version_dir.join("bin"); + if bin_dir.exists() && bin_dir.join(POSTGRES_BINARY).exists() { + tracing::debug!("PostgreSQL already extracted at {}", version_dir.display()); + return Ok(version_dir); + } + + if POSTGRESQL_BUNDLE.is_empty() { + return Err(CliError::Other( + "PostgreSQL bundle is empty - this binary was not built with BUNDLE_POSTGRESQL=true".to_string() + )); + } + + println!("Extracting bundled PostgreSQL {}...", pg_version); + fs::create_dir_all(&version_dir)?; + + extract_postgresql_archive(POSTGRESQL_BUNDLE, &version_dir)?; // Verify extraction - if !bin_dir.join("postgres").exists() { + if !bin_dir.join(POSTGRES_BINARY).exists() { return Err(CliError::Other(format!( - "PostgreSQL extraction failed - postgres binary not found at {}", + "PostgreSQL extraction failed - {} not found at {}", + POSTGRES_BINARY, bin_dir.display() ))); } @@ -639,6 +685,13 @@ fn start( configuration.insert("log_rotation_age".to_string(), "1d".to_string()); configuration.insert("log_rotation_size".to_string(), "100MB".to_string()); + // Pin timezone to UTC so PostgreSQL never reads the tzdata directory at startup. + // The theseus-rs binaries are compiled with --with-system-tzdata=/usr/share/zoneinfo, + // which doesn't exist on NixOS (tzdata lives at /etc/zoneinfo) and causes a FATAL + // "could not find a suitable time zone abbreviations file" on server start. + configuration.insert("timezone".to_string(), "UTC".to_string()); + configuration.insert("log_timezone".to_string(), "UTC".to_string()); + // Parse and apply custom config options (these override defaults) for cfg in &config { if let Some((key, value)) = cfg.split_once('=') { @@ -696,7 +749,7 @@ fn start( username, username, password.replace('\'', "''") ); let status = std::process::Command::new(&psql_path) - .arg(&format!("postgresql://postgres:{}@localhost:{}/postgres", password, port)) + .arg(&format!("postgresql://postgres:{}@127.0.0.1:{}/postgres", password, port)) .arg("-c") .arg(&create_user_sql) .status()?; @@ -720,7 +773,7 @@ fn start( let psql_path = find_psql_binary(&installation_dir)?; let grant_sql = format!("GRANT ALL PRIVILEGES ON DATABASE \"{}\" TO \"{}\";", database, username); let _ = std::process::Command::new(&psql_path) - .arg(&format!("postgresql://postgres:{}@localhost:{}/postgres", password, port)) + .arg(&format!("postgresql://postgres:{}@127.0.0.1:{}/postgres", password, port)) .arg("-c") .arg(&grant_sql) .status(); @@ -754,7 +807,7 @@ fn start( println!(" Data dir: {}", data_dir.display()); println!(); println!( - "Connection URI: postgresql://{}:{}@localhost:{}/{}", + "Connection URI: postgresql://{}:{}@127.0.0.1:{}/{}", username, password, port, database ); println!(); @@ -910,7 +963,7 @@ fn info(name: String, output_format: OutputFormat) -> Result<(), CliError> { let running = is_process_running(info.pid); if running { let uri = format!( - "postgresql://{}:{}@localhost:{}/{}", + "postgresql://{}:{}@127.0.0.1:{}/{}", info.username, info.password, info.port, info.database ); InfoOutput { @@ -989,10 +1042,12 @@ fn info(name: String, output_format: OutputFormat) -> Result<(), CliError> { } fn find_psql_binary(installation_dir: &PathBuf) -> Result { + let psql_name = if cfg!(windows) { "psql.exe" } else { "psql" }; + // Look for psql in installation_dir/*/bin/psql (version subdirectory) if let Ok(entries) = fs::read_dir(installation_dir) { for entry in entries.flatten() { - let psql_path = entry.path().join("bin").join("psql"); + let psql_path = entry.path().join("bin").join(psql_name); if psql_path.exists() { return Ok(psql_path); } @@ -1000,7 +1055,7 @@ fn find_psql_binary(installation_dir: &PathBuf) -> Result { } // Fallback: try direct path (in case structure changes) - let direct_path = installation_dir.join("bin").join("psql"); + let direct_path = installation_dir.join("bin").join(psql_name); if direct_path.exists() { return Ok(direct_path); } @@ -1008,7 +1063,8 @@ fn find_psql_binary(installation_dir: &PathBuf) -> Result { Err(CliError::Io(std::io::Error::new( std::io::ErrorKind::NotFound, format!( - "psql not found in {}", + "{} not found in {}", + psql_name, installation_dir.display() ), ))) @@ -1026,7 +1082,7 @@ fn psql(name: String, args: Vec) -> Result<(), CliError> { // Build connection URI let uri = format!( - "postgresql://{}:{}@localhost:{}/{}", + "postgresql://{}:{}@127.0.0.1:{}/{}", info.username, info.password, info.port, info.database ); @@ -1211,7 +1267,7 @@ fn list(output_format: OutputFormat) -> Result<(), CliError> { let running = is_process_running(info.pid); let output = if running { let uri = format!( - "postgresql://{}:{}@localhost:{}/{}", + "postgresql://{}:{}@127.0.0.1:{}/{}", info.username, info.password, info.port, info.database ); InfoOutput { diff --git a/versions.env b/versions.env index f8a1cd7..9f83058 100644 --- a/versions.env +++ b/versions.env @@ -2,8 +2,3 @@ PG_VERSION=18.1.0 PGVECTOR_VERSION=0.8.1 PGVECTOR_COMPILED_TAG=v0.18.237 PGVECTOR_COMPILED_REPO=nicoloboschi/pgvector_compiled - -# PgBouncer for connection pooling -PGBOUNCER_VERSION=1.23.1 -PGBOUNCER_COMPILED_TAG=v1.23.1 -PGBOUNCER_COMPILED_REPO=nicoloboschi/pgbouncer_compiled From f79363ae936daf43cf47a43156b380ddd15a98d5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=B2=20Boschi?= Date: Fri, 24 Apr 2026 12:25:01 +0200 Subject: [PATCH 2/2] fix(sdk): avoid Windows pipe deadlock when pg0 spawns PostgreSQL MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit On Windows, `pg0 start` spawns PostgreSQL which inherits pg0's stdio handles. pg0 exits promptly (PG is intentionally detached via std::mem::forget), but PostgreSQL keeps writing to the inherited handles, so Python's `subprocess.run(capture_output=True)` never sees pipe EOF and hangs indefinitely. Unix is unaffected — `pg_ctl` closes inherited FDs before handing off to postgres. Route stdio through real temporary files on Windows. subprocess.call only waits on the process exit code, not on file handles held by grandchildren, so pg0 returns as soon as it exits regardless of what PostgreSQL does with the inherited file handles afterwards. Also add a 20-minute timeout-minutes to the sdk-tests-windows CI job so future hangs fail fast instead of consuming 6h of runner time. --- .github/workflows/ci.yml | 1 + sdk/python/pg0/__init__.py | 35 ++++++++++++++++++++++++++++++----- 2 files changed, 31 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index fceed9f..81de8a6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -82,6 +82,7 @@ jobs: name: SDK Tests (Windows) needs: build-windows runs-on: windows-latest + timeout-minutes: 20 steps: - uses: actions/checkout@v4 diff --git a/sdk/python/pg0/__init__.py b/sdk/python/pg0/__init__.py index 5d14fd4..8428efb 100644 --- a/sdk/python/pg0/__init__.py +++ b/sdk/python/pg0/__init__.py @@ -21,6 +21,7 @@ import os import shutil import subprocess +import tempfile import sys from dataclasses import dataclass from pathlib import Path @@ -130,11 +131,35 @@ def _run_pg0(*args: str, check: bool = True) -> subprocess.CompletedProcess: """Run a pg0 command.""" pg0_path = _find_pg0() try: - result = subprocess.run( - [pg0_path, *args], - capture_output=True, - text=True, - ) + if sys.platform == "win32": + # On Windows, `pg0 start` spawns PostgreSQL which inherits pg0's + # stdio handles. pg0 exits but PostgreSQL keeps writing to those + # handles, so Python's capture_output=True pipes never see EOF + # and subprocess.run hangs forever. Route stdio through real + # files instead — subprocess.run only waits on the process exit + # code, not on file handles held by grandchildren. + with tempfile.TemporaryFile() as out_f, tempfile.TemporaryFile() as err_f: + rc = subprocess.call( + [pg0_path, *args], + stdout=out_f, + stderr=err_f, + ) + out_f.seek(0) + err_f.seek(0) + stdout = out_f.read().decode("utf-8", errors="replace") + stderr = err_f.read().decode("utf-8", errors="replace") + result = subprocess.CompletedProcess( + args=[pg0_path, *args], + returncode=rc, + stdout=stdout, + stderr=stderr, + ) + else: + result = subprocess.run( + [pg0_path, *args], + capture_output=True, + text=True, + ) if check and result.returncode != 0: stderr = result.stderr.strip() if "already running" in stderr.lower():