From ecd038fae7ef61ec4aa1485db6146a2c5128d26a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=B2=20Boschi?= Date: Wed, 8 Apr 2026 12:26:14 +0200 Subject: [PATCH 1/3] fix: detect missing system shared libraries on Linux On some Linux distros (e.g. Arch Linux), system shared libraries like libxml2 may be missing. The OS can't execute the postgres binary, causing initdb to report a misleading "postgres not found" error. After extraction, run ldd on the postgres binary (Linux only) and report any missing .so dependencies with actionable install guidance. If ldd is unavailable, the check is silently skipped. Adds a Docker integration test that removes libxml2 and verifies the new error message is surfaced correctly. Ref: vectorize-io/hindsight#919 --- .github/workflows/ci.yml | 4 + docker-tests/run_all_tests.sh | 1 + docker-tests/test_missing_libs.sh | 123 ++++++++++++++++++++++++++++++ src/main.rs | 43 +++++++++++ 4 files changed, 171 insertions(+) create mode 100755 docker-tests/test_missing_libs.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a03b609..36264cd 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -71,6 +71,9 @@ jobs: - platform: alpine-amd64 cli_script: docker-tests/test_alpine_amd64.sh python_script: docker-tests/python/test_alpine_amd64.sh + - platform: missing-libs-debian-amd64 + cli_script: docker-tests/test_missing_libs.sh + python_script: "" steps: - uses: actions/checkout@v4 @@ -81,6 +84,7 @@ jobs: bash ${{ matrix.cli_script }} - name: Run Python SDK Docker test + if: matrix.python_script != '' run: | chmod +x ${{ matrix.python_script }} bash ${{ matrix.python_script }} diff --git a/docker-tests/run_all_tests.sh b/docker-tests/run_all_tests.sh index 410232b..1be1b88 100755 --- a/docker-tests/run_all_tests.sh +++ b/docker-tests/run_all_tests.sh @@ -50,6 +50,7 @@ run_test "Debian AMD64" "$DIR/test_debian_amd64.sh" run_test "Debian ARM64" "$DIR/test_debian_arm64.sh" run_test "Alpine AMD64" "$DIR/test_alpine_amd64.sh" run_test "Alpine ARM64" "$DIR/test_alpine_arm64.sh" +run_test "Missing Libs Detection (Debian AMD64)" "$DIR/test_missing_libs.sh" # Print summary echo "" diff --git a/docker-tests/test_missing_libs.sh b/docker-tests/test_missing_libs.sh new file mode 100755 index 0000000..783f382 --- /dev/null +++ b/docker-tests/test_missing_libs.sh @@ -0,0 +1,123 @@ +#!/bin/bash +set -e + +echo "=============================================" +echo "Testing pg0 missing shared library detection" +echo "Image: python:3.11-slim" +echo "Platform: linux/amd64" +echo "=============================================" + +# Get the script directory to find install.sh +SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +INSTALL_SCRIPT="$SCRIPT_DIR/../install.sh" + +# Check if PG0_BINARY_PATH is set (local binary to test) +VOLUME_ARGS="" +BINARY_ENV="" +if [ -n "${PG0_BINARY_PATH:-}" ]; then + echo "Using local binary: $PG0_BINARY_PATH" + VOLUME_ARGS="-v $PG0_BINARY_PATH:/tmp/pg0-binary:ro" + BINARY_ENV="-e PG0_BINARY_URL=file:///tmp/pg0-binary" +fi + +docker run --rm --platform=linux/amd64 \ + $BINARY_ENV \ + -v "$INSTALL_SCRIPT:/tmp/install.sh:ro" \ + $VOLUME_ARGS \ + python:3.11-slim bash -c ' +set -e + +echo "=== System Info ===" +uname -m +cat /etc/os-release | grep PRETTY_NAME + +echo "" +echo "=== Installing dependencies ===" +apt-get update -qq +apt-get install -y curl libxml2 libssl3 libgssapi-krb5-2 sudo procps 2>&1 | grep -v "^Get:" || true +apt-get install -y libicu72 2>/dev/null || apt-get install -y libicu74 2>/dev/null || apt-get install -y libicu* 2>&1 | head -5 + +echo "" +echo "=== Creating non-root user ===" +useradd -m -s /bin/bash pguser +echo "pguser ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers + +echo "" +echo "=== Copying local install.sh ===" +cp /tmp/install.sh /usr/local/bin/install.sh +chmod 755 /usr/local/bin/install.sh + +echo "" +echo "=== Phase 1: Install pg0 and do initial extraction ===" +su - pguser << EOF +set -e +export PG0_BINARY_URL="${PG0_BINARY_URL}" + +echo "=== Installing pg0 ===" +bash /usr/local/bin/install.sh +export PATH="\$HOME/.local/bin:\$PATH" + +echo "" +echo "=== Starting PostgreSQL (initial extraction) ===" +pg0 start +sleep 3 + +echo "" +echo "=== Stopping PostgreSQL ===" +pg0 stop +sleep 1 + +echo "" +echo "=== Removing extracted installation to force re-extraction ===" +rm -rf ~/.pg0/installation +echo "Installation directory cleared." +EOF + +echo "" +echo "=== Phase 2: Remove libxml2 to simulate missing library ===" +apt-get remove -y libxml2 2>&1 | tail -3 + +echo "" +echo "=== Phase 3: Verify pg0 detects missing libraries ===" +su - pguser << EOF +set -e +export PATH="\$HOME/.local/bin:\$PATH" + +echo "=== Starting pg0 (should fail with missing library error) ===" +OUTPUT=\$(pg0 start 2>&1 || true) +EXIT_CODE=\${PIPESTATUS[0]:-\$?} +echo "\$OUTPUT" + +echo "" +echo "=== Checking error message ===" + +if echo "\$OUTPUT" | grep -q "missing required system libraries"; then + echo "PASS: Found 'missing required system libraries' message" +else + echo "FAIL: Missing expected error message about shared libraries" + exit 1 +fi + +if echo "\$OUTPUT" | grep -q "libxml2"; then + echo "PASS: Found 'libxml2' in the missing library list" +else + echo "FAIL: Expected libxml2 to be listed as missing" + exit 1 +fi + +if echo "\$OUTPUT" | grep -q "Install the missing libraries"; then + echo "PASS: Found install guidance message" +else + echo "FAIL: Missing install guidance" + exit 1 +fi + +echo "" +echo "=============================================" +echo "ALL CHECKS PASSED - Missing libs detected" +echo "=============================================" +EOF +' + +echo "" +echo "Test completed successfully!" diff --git a/src/main.rs b/src/main.rs index 35e4a7a..c4134df 100644 --- a/src/main.rs +++ b/src/main.rs @@ -452,10 +452,53 @@ fn extract_bundled_postgresql(installation_dir: &PathBuf, pg_version: &str) -> R } } + // Check for missing shared libraries on Linux + #[cfg(target_os = "linux")] + check_shared_libraries(&bin_dir)?; + println!("PostgreSQL {} extracted successfully.", pg_version); Ok(version_dir) } +/// Check that the postgres binary can find all required shared libraries. +/// Only called on Linux. If ldd is unavailable, silently skips the check. +#[cfg(target_os = "linux")] +fn check_shared_libraries(bin_dir: &Path) -> Result<(), CliError> { + let postgres_path = bin_dir.join("postgres"); + let output = match std::process::Command::new("ldd") + .arg(&postgres_path) + .output() + { + Ok(output) => output, + Err(e) => { + tracing::debug!("Could not run ldd to check shared libraries: {}", e); + return Ok(()); + } + }; + + let stdout = String::from_utf8_lossy(&output.stdout); + let missing: Vec<&str> = stdout + .lines() + .filter(|line| line.contains("not found")) + .map(|line| line.trim()) + .collect(); + + if missing.is_empty() { + return Ok(()); + } + + let missing_list = missing.join("\n "); + Err(CliError::Other(format!( + "The bundled PostgreSQL binary is missing required system libraries:\n \ + {}\n\n\ + Install the missing libraries using your system package manager. For example:\n \ + Arch Linux: sudo pacman -S \n \ + Ubuntu/Debian: sudo apt install \n \ + Fedora/RHEL: sudo dnf install ", + missing_list + ))) +} + /// Install pgvector extension files into the PostgreSQL installation fn install_pgvector(installation_dir: &PathBuf, pg_version: &str) -> Result<(), CliError> { let pg_major = pg_version.split('.').next().unwrap_or("16"); From c43374825657ecb8840cbe8b2cd9abc6f420882c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=B2=20Boschi?= Date: Wed, 8 Apr 2026 12:34:33 +0200 Subject: [PATCH 2/3] fix: build Linux binary for missing-libs test and fix heredoc issues - Add Rust build step in CI for the missing-libs test so it uses the PR binary (with shared lib detection) instead of the released one - Rewrite test script to use a mounted script file instead of nested heredocs to avoid bash quoting issues - Require PG0_BINARY_PATH to be set (test needs the PR binary) --- .github/workflows/ci.yml | 10 +++++ docker-tests/test_missing_libs.sh | 73 +++++++++++++++---------------- 2 files changed, 46 insertions(+), 37 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 36264cd..b146cad 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -78,6 +78,16 @@ jobs: steps: - uses: actions/checkout@v4 + - name: Install Rust + if: matrix.platform == 'missing-libs-debian-amd64' + uses: dtolnay/rust-toolchain@stable + + - name: Build Linux binary + if: matrix.platform == 'missing-libs-debian-amd64' + run: | + cargo build --release + echo "PG0_BINARY_PATH=$(pwd)/target/release/pg0" >> $GITHUB_ENV + - name: Run CLI Docker test run: | chmod +x ${{ matrix.cli_script }} diff --git a/docker-tests/test_missing_libs.sh b/docker-tests/test_missing_libs.sh index 783f382..2dd9a41 100755 --- a/docker-tests/test_missing_libs.sh +++ b/docker-tests/test_missing_libs.sh @@ -11,20 +11,20 @@ echo "=============================================" SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" INSTALL_SCRIPT="$SCRIPT_DIR/../install.sh" -# Check if PG0_BINARY_PATH is set (local binary to test) -VOLUME_ARGS="" -BINARY_ENV="" -if [ -n "${PG0_BINARY_PATH:-}" ]; then - echo "Using local binary: $PG0_BINARY_PATH" - VOLUME_ARGS="-v $PG0_BINARY_PATH:/tmp/pg0-binary:ro" - BINARY_ENV="-e PG0_BINARY_URL=file:///tmp/pg0-binary" +# PG0_BINARY_PATH is required - this test must use a binary built from source +# (with the shared library detection code), not the released binary +if [ -z "${PG0_BINARY_PATH:-}" ]; then + echo "ERROR: PG0_BINARY_PATH must be set to a Linux binary built from this branch" + exit 1 fi -docker run --rm --platform=linux/amd64 \ - $BINARY_ENV \ - -v "$INSTALL_SCRIPT:/tmp/install.sh:ro" \ - $VOLUME_ARGS \ - python:3.11-slim bash -c ' +echo "Using local binary: $PG0_BINARY_PATH" + +# Create a temporary test script to run inside the container +# This avoids nested heredoc quoting issues +TEMP_SCRIPT=$(mktemp) +cat > "$TEMP_SCRIPT" << 'INNERSCRIPT' +#!/bin/bash set -e echo "=== System Info ===" @@ -43,35 +43,28 @@ useradd -m -s /bin/bash pguser echo "pguser ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers echo "" -echo "=== Copying local install.sh ===" -cp /tmp/install.sh /usr/local/bin/install.sh -chmod 755 /usr/local/bin/install.sh +echo "=== Installing pg0 from local binary ===" +cp /tmp/pg0-binary /usr/local/bin/pg0 +chmod 755 /usr/local/bin/pg0 echo "" -echo "=== Phase 1: Install pg0 and do initial extraction ===" -su - pguser << EOF +echo "=== Phase 1: Initial extraction with all deps present ===" +su -s /bin/bash - pguser -c ' set -e -export PG0_BINARY_URL="${PG0_BINARY_URL}" - -echo "=== Installing pg0 ===" -bash /usr/local/bin/install.sh -export PATH="\$HOME/.local/bin:\$PATH" +export PATH="/usr/local/bin:$PATH" -echo "" echo "=== Starting PostgreSQL (initial extraction) ===" pg0 start sleep 3 -echo "" echo "=== Stopping PostgreSQL ===" pg0 stop sleep 1 -echo "" echo "=== Removing extracted installation to force re-extraction ===" rm -rf ~/.pg0/installation echo "Installation directory cleared." -EOF +' echo "" echo "=== Phase 2: Remove libxml2 to simulate missing library ===" @@ -79,33 +72,32 @@ apt-get remove -y libxml2 2>&1 | tail -3 echo "" echo "=== Phase 3: Verify pg0 detects missing libraries ===" -su - pguser << EOF +su -s /bin/bash - pguser -c ' set -e -export PATH="\$HOME/.local/bin:\$PATH" +export PATH="/usr/local/bin:$PATH" echo "=== Starting pg0 (should fail with missing library error) ===" -OUTPUT=\$(pg0 start 2>&1 || true) -EXIT_CODE=\${PIPESTATUS[0]:-\$?} -echo "\$OUTPUT" +OUTPUT=$(pg0 start 2>&1 || true) +echo "$OUTPUT" echo "" echo "=== Checking error message ===" -if echo "\$OUTPUT" | grep -q "missing required system libraries"; then - echo "PASS: Found 'missing required system libraries' message" +if echo "$OUTPUT" | grep -q "missing required system libraries"; then + echo "PASS: Found missing required system libraries message" else echo "FAIL: Missing expected error message about shared libraries" exit 1 fi -if echo "\$OUTPUT" | grep -q "libxml2"; then - echo "PASS: Found 'libxml2' in the missing library list" +if echo "$OUTPUT" | grep -qi "libxml2"; then + echo "PASS: Found libxml2 in the missing library list" else echo "FAIL: Expected libxml2 to be listed as missing" exit 1 fi -if echo "\$OUTPUT" | grep -q "Install the missing libraries"; then +if echo "$OUTPUT" | grep -q "Install the missing libraries"; then echo "PASS: Found install guidance message" else echo "FAIL: Missing install guidance" @@ -116,8 +108,15 @@ echo "" echo "=============================================" echo "ALL CHECKS PASSED - Missing libs detected" echo "=============================================" -EOF ' +INNERSCRIPT + +docker run --rm --platform=linux/amd64 \ + -v "$PG0_BINARY_PATH:/tmp/pg0-binary:ro" \ + -v "$TEMP_SCRIPT:/tmp/test_script.sh:ro" \ + python:3.11-slim bash /tmp/test_script.sh + +rm -f "$TEMP_SCRIPT" echo "" echo "Test completed successfully!" From 1a29214bffaf035c689a6c6f91f0edf48ca77014 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=B2=20Boschi?= Date: Wed, 8 Apr 2026 12:41:33 +0200 Subject: [PATCH 3/3] fix: use fully qualified Path type in Linux-only function The check_shared_libraries function uses &Path but it's gated with #[cfg(target_os = "linux")]. Use std::path::Path inline to avoid unused import warnings on non-Linux platforms. --- src/main.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main.rs b/src/main.rs index c4134df..dd28d3f 100644 --- a/src/main.rs +++ b/src/main.rs @@ -463,7 +463,7 @@ fn extract_bundled_postgresql(installation_dir: &PathBuf, pg_version: &str) -> R /// Check that the postgres binary can find all required shared libraries. /// Only called on Linux. If ldd is unavailable, silently skips the check. #[cfg(target_os = "linux")] -fn check_shared_libraries(bin_dir: &Path) -> Result<(), CliError> { +fn check_shared_libraries(bin_dir: &std::path::Path) -> Result<(), CliError> { let postgres_path = bin_dir.join("postgres"); let output = match std::process::Command::new("ldd") .arg(&postgres_path)