diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 3ec60c8..f44cde7 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -46,15 +46,19 @@ jobs: - name: Initialize datadir run: | mkdir -p $PWD/var - ./build/runtime_output_directory/mysqld --initialize-insecure --datadir=$PWD/var/mysqld.1 + ./build/runtime_output_directory/mysqld --initialize-insecure \ + --datadir=$PWD/var/mysqld.1 \ + --log-error=$PWD/var/mysqld.log - name: Start MySQL server run: | + mkdir -p $PWD/lib/veb ./build/runtime_output_directory/mysqld \ --datadir=$PWD/var/mysqld.1 \ --socket=$PWD/var/mysql.sock \ --port=3306 \ --log-error=$PWD/var/mysqld.log \ + --veb-dir=$PWD/lib/veb \ &>/dev/null & echo "Waiting for MySQL to start..." for i in {1..30}; do @@ -67,24 +71,25 @@ jobs: - name: Build extension run: | - cd extension - mkdir build && cd build - cmake .. -DVillageSQL_BUILD_DIR=$PWD/../villagesql-server/build + cd ../extension + mkdir -p build && cd build + cmake .. -DVillageSQL_BUILD_DIR=$GITHUB_WORKSPACE/villagesql-server/build make -j$(nproc) - name: Install extension run: | - mysql -S $PWD/var/mysql.sock -u root \ - -e "INSTALL EXTENSION $PWD/extension/build/prometheus_exporter.veb" + cp "$GITHUB_WORKSPACE/extension/build/prometheus_exporter.veb" "$PWD/lib/veb/" + mysql -S "$PWD/var/mysql.sock" -u root -e "INSTALL EXTENSION prometheus_exporter" - name: Run MTR tests run: | - cd build/mysql-test - perl mysql-test-run.pl \ - --suite=$PWD/../extension/mysql-test \ + cp -r "$GITHUB_WORKSPACE/extension/mysql-test" "$PWD/mysql-test/suite/villagesql/prometheus_exporter" + cd $PWD + ./build/mysql-test/mysql-test-run.pl \ + --do-suite=villagesql/prometheus_exporter \ --parallel=auto \ --tmpdir=$PWD/var \ - --socket=$PWD/../var/mysql.sock \ + --socket=$PWD/var/mysql.sock \ --report-features - name: Upload test logs on failure @@ -93,5 +98,5 @@ jobs: with: name: test-logs path: | - build/mysql-test/var/log/*.log - villagesql-server/var/mysqld.log \ No newline at end of file + $PWD/mysql-test/var/log/*.log + $PWD/var/mysqld.log \ No newline at end of file diff --git a/docker/ci-validation/Dockerfile b/docker/ci-validation/Dockerfile new file mode 100644 index 0000000..4779612 --- /dev/null +++ b/docker/ci-validation/Dockerfile @@ -0,0 +1,37 @@ +# Copyright (c) 2026 VillageSQL Contributors +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, see . + +# Dockerfile for CI validation +# Extends proxysql-ci-base with needed tools for CI simulation + +FROM proxysql-ci-base:latest + +RUN apt-get update && \ + apt-get install -y --no-install-recommends \ + cmake \ + g++ \ + make \ + libtirpc-dev \ + libcurl4-openssl-dev \ + netcat-openbsd \ + libdbd-mysql-perl \ + libjson-perl \ + bison && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* || echo "Some packages may have failed" + +WORKDIR /workspace + +CMD ["/bin/bash"] \ No newline at end of file diff --git a/docker/ci-validation/local-ci.sh b/docker/ci-validation/local-ci.sh new file mode 100644 index 0000000..244e6d2 --- /dev/null +++ b/docker/ci-validation/local-ci.sh @@ -0,0 +1,145 @@ +#!/bin/bash +# Local CI simulation script for testing CI workflow changes +# Runs the same steps as .github/workflows/test.yml but locally in Docker + +set -e + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +EXTENSION_DIR="$(dirname "$SCRIPT_DIR")" +VILLAGESQL_SOURCE_DIR="${VILLAGESQL_SOURCE_DIR:-/data/rene/villagesql-server}" +VILLAGESQL_BUILD_DIR="${VILLAGESQL_BUILD_DIR:-/data/rene/build}" +USE_LOCAL_BUILD="${USE_LOCAL_BUILD:-true}" + +CONTAINER_NAME="vsql-ci-local-$$" +DOCKER_IMAGE="vsql-ci-validation:latest" + +log() { + echo "[local-ci] $(date '+%H:%M:%S') - $1" +} + +cleanup() { + log "Cleaning up container..." + docker rm -f "$CONTAINER_NAME" 2>/dev/null || true +} +trap cleanup EXIT + +build_docker_image() { + log "Building Docker image..." + docker build -t "$DOCKER_IMAGE" "$EXTENSION_DIR/docker/ci-validation" 2>&1 | tail -5 +} + +run_ci_step() { + local step_name="$1" + local cmd="$2" + log "STEP: $step_name" + echo "---" + eval "$cmd" + echo "---" +} + +# Build Docker image if needed +if ! docker image inspect "$DOCKER_IMAGE" &>/dev/null; then + build_docker_image +fi + +# Create test workspace +WORKSPACE_DIR="/tmp/vsql-ci-workspace-$$" +rm -rf "$WORKSPACE_DIR" +mkdir -p "$WORKSPACE_DIR" + +log "Starting CI simulation..." +log "VillageSQL source: $VILLAGESQL_SOURCE_DIR" +log "VillageSQL build: $VILLAGESQL_BUILD_DIR" +log "Extension: $EXTENSION_DIR" + +# Start container with volume mounts +log "Starting container..." +docker run -d --name "$CONTAINER_NAME" \ + -v "$VILLAGESQL_SOURCE_DIR:/villagesql-source:ro" \ + -v "$VILLAGESQL_BUILD_DIR:/villagesql-build:ro" \ + -v "$EXTENSION_DIR:/extension:ro" \ + -v "$WORKSPACE_DIR:/workspace" \ + -w /workspace \ + "$DOCKER_IMAGE" \ + sleep infinity >/dev/null + +# Copy VillageSQL to workspace if not using local build, or symlink +if [ "$USE_LOCAL_BUILD" = "true" ]; then + log "Using local VillageSQL build (skipping build step)" + # Symlink build into workspace structure + docker exec "$CONTAINER_NAME" bash -c "mkdir -p /workspace/villagesql-server && ln -s /villagesql-build /workspace/villagesql-server/build" +else + log "Copying VillageSQL source..." + docker exec "$CONTAINER_NAME" bash -c "cp -r /villagesql-source /workspace/villagesql-server" +fi + +# Create sibling extension directory +docker exec "$CONTAINER_NAME" bash -c "ln -s /extension /workspace/extension" + +# Step 1: Install dependencies (already in Docker image, but run anyway for consistency) +run_ci_step "Install dependencies" ' +docker exec '"$CONTAINER_NAME"' bash -c "sudo apt-get update -qq && sudo apt-get install -y -qq cmake gcc make libtirpc-dev libcurl4-openssl-dev netcat-openbsd 2>/dev/null || true" +' + +# Step 2: Configure (if not using local build) +if [ "$USE_LOCAL_BUILD" != "true" ]; then + run_ci_step "Configure VillageSQL" ' + docker exec '"$CONTAINER_NAME"' bash -c "cd /workspace/villagesql-server && mkdir -p build && cd build && cmake .. -DWITH_DEBUG=1 -DWITH_UNIT_TESTS=ON -DWITH_ROUTER=OFF" + ' + + # Step 3: Build + run_ci_step "Build VillageSQL" ' + docker exec '"$CONTAINER_NAME"' bash -c "cd /workspace/villagesql-server/build && make -j\$(nproc)" + ' +fi + +# Step 4: Initialize datadir +run_ci_step "Initialize datadir" ' +docker exec '"$CONTAINER_NAME"' bash -c "cd /workspace/villagesql-server && mkdir -p var && ./build/runtime_output_directory/mysqld --initialize-insecure --datadir=var/mysqld.1 --log-error=var/mysqld.log" +' + +# Step 5: Start MySQL server +run_ci_step "Start MySQL server" ' +docker exec '"$CONTAINER_NAME"' bash -c "cd /workspace/villagesql-server && ./build/runtime_output_directory/mysqld --datadir=var/mysqld.1 --socket=var/mysql.sock --port=3306 --log-error=var/mysqld.log &>/dev/null &" && sleep 2 && docker exec '"$CONTAINER_NAME"' bash -c "for i in {1..30}; do mysql -S /workspace/villagesql-server/var/mysql.sock -u root -e \"SELECT 1\" 2>/dev/null && echo \"MySQL ready\" && break || sleep 1; done" +' + +# Step 6: Build extension +run_ci_step "Build extension" ' +docker exec '"$CONTAINER_NAME"' bash -c "cd /workspace/extension && mkdir -p build && cd build && cmake .. -DVillageSQL_BUILD_DIR=/workspace/villagesql-server/build && make -j\$(nproc)" +' + +# Step 7: Install extension +run_ci_step "Install extension" ' +docker exec '"$CONTAINER_NAME"' bash -c "mysql -S /workspace/villagesql-server/var/mysql.sock -u root -e \"INSTALL EXTENSION /workspace/extension/build/prometheus_exporter.veb\"" +' + +# Step 8: Run MTR tests +log "Running MTR tests..." +TEST_OUTPUT="/tmp/vsql-ci-test-output-$$" +mkdir -p "$TEST_OUTPUT" + +run_ci_step "Run MTR tests" ' +docker exec '"$CONTAINER_NAME"' bash -c "cd /workspace/villagesql-server/build/mysql-test && perl mysql-test-run.pl --suite=/workspace/extension/mysql-test --parallel=auto --tmpdir=/workspace/villagesql-server/var --socket=/workspace/villagesql-server/var/mysql.sock --report-features 2>&1" | tee "'"$TEST_OUTPUT"'/mtr_output.txt" +' + +# Copy test logs +log "Copying test artifacts..." +docker cp "$CONTAINER_NAME:/workspace/villagesql-server/build/mysql-test/var/log" "$TEST_OUTPUT/" 2>/dev/null || true +docker cp "$CONTAINER_NAME:/workspace/villagesql-server/var/mysqld.log" "$TEST_OUTPUT/" 2>/dev/null || true + +# Check test results +PASSED=$(grep -c "mysql-test-run: .* \[ pass \]" "$TEST_OUTPUT/mtr_output.txt" 2>/dev/null || echo "0") +FAILED=$(grep -c "mysql-test-run: .* \[ fail \]" "$TEST_OUTPUT/mtr_output.txt" 2>/dev/null || echo "0") + +log "============================================" +log "TEST RESULTS: $PASSED passed, $FAILED failed" +log "Full output: $TEST_OUTPUT/mtr_output.txt" +log "============================================" + +if [ "$FAILED" -gt "0" ]; then + log "TESTS FAILED - CI workflow needs fixes" + exit 1 +else + log "ALL TESTS PASSED - CI workflow is valid" + exit 0 +fi \ No newline at end of file diff --git a/local-ci.sh b/local-ci.sh new file mode 100755 index 0000000..c34189e --- /dev/null +++ b/local-ci.sh @@ -0,0 +1,100 @@ +#!/bin/bash +# Local CI simulation - runs on host directly +# Mirrors .github/workflows/test.yml steps locally for fast iteration + +set -e + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +EXTENSION_DIR="$SCRIPT_DIR" +VILLAGESQL_SOURCE_DIR="${VILLAGESQL_SOURCE_DIR:-/data/rene/villagesql-server}" +VILLAGESQL_BUILD_DIR="${VILLAGESQL_BUILD_DIR:-/data/rene/build}" +USE_LOCAL_BUILD="${USE_LOCAL_BUILD:-true}" + +WORKSPACE="/tmp/vsql-ci-local-$$" +VAR_DIR="$WORKSPACE/var" + +log() { + echo "[local-ci] $(date '+%H:%M:%S') - $1" +} + +cleanup() { + log "Cleaning up..." + pkill -f "mysqld.*vsql-ci.*" 2>/dev/null || true + rm -rf "$WORKSPACE" 2>/dev/null || true +} +trap cleanup EXIT + +mkdir -p "$WORKSPACE" +mkdir -p "$VAR_DIR" + +export LD_LIBRARY_PATH="$VILLAGESQL_BUILD_DIR/library_output_directory:$VILLAGESQL_BUILD_DIR/plugin_output_directory:$LD_LIBRARY_PATH" + +log "Starting local CI simulation..." +log "VillageSQL source: $VILLAGESQL_SOURCE_DIR" +log "VillageSQL build: $VILLAGESQL_BUILD_DIR" +log "Extension: $EXTENSION_DIR" + +log "Initializing datadir..." +cd "$VILLAGESQL_SOURCE_DIR" +"$VILLAGESQL_BUILD_DIR/runtime_output_directory/mysqld" \ + --initialize-insecure \ + --datadir="$VAR_DIR/mysqld.1" \ + --log-error="$VAR_DIR/mysqld.log" 2>&1 | tail -5 + +log "Starting MySQL..." +"$VILLAGESQL_BUILD_DIR/runtime_output_directory/mysqld" \ + --datadir="$VAR_DIR/mysqld.1" \ + --socket="$VAR_DIR/mysql.sock" \ + --port=3306 \ + --log-error="$VAR_DIR/mysqld.log" &>/dev/null & + +log "Waiting for MySQL to start..." +for i in {1..30}; do + if "$VILLAGESQL_BUILD_DIR/runtime_output_directory/mysql" -S "$VAR_DIR/mysql.sock" -u root -e "SELECT 1" 2>/dev/null; then + log "MySQL is ready" + break + fi + sleep 1 +done + +log "Building extension..." +cd "$EXTENSION_DIR" +rm -rf build +mkdir -p build +cd build +cmake .. -DVillageSQL_BUILD_DIR="$VILLAGESQL_BUILD_DIR" 2>&1 | tail -3 +make -j$(nproc) 2>&1 | tail -3 + +log "Installing extension..." +"$VILLAGESQL_BUILD_DIR/runtime_output_directory/mysql" -S "$VAR_DIR/mysql.sock" -u root \ + -e "INSTALL EXTENSION $EXTENSION_DIR/build/prometheus_exporter.veb" 2>&1 + +log "Running MTR tests..." +cd "$VILLAGESQL_BUILD_DIR/mysql-test" +TEST_OUTPUT="/tmp/vsql-ci-test-output-$$" +mkdir -p "$TEST_OUTPUT" + +perl mysql-test-run.pl \ + --suite="$EXTENSION_DIR/mysql-test" \ + --parallel=auto \ + --tmpdir="$VAR_DIR" \ + --socket="$VAR_DIR/mysql.sock" \ + --report-features 2>&1 | tee "$TEST_OUTPUT/mtr_output.txt" + +cp "$VAR_DIR/mysqld.log" "$TEST_OUTPUT/" 2>/dev/null || true + +PASSED=$(grep -c "mysql-test-run: .* \[ pass \]" "$TEST_OUTPUT/mtr_output.txt" 2>/dev/null || echo "0") +FAILED=$(grep -c "mysql-test-run: .* \[ fail \]" "$TEST_OUTPUT/mtr_output.txt" 2>/dev/null || echo "0") + +log "============================================" +log "TEST RESULTS: $PASSED passed, $FAILED failed" +log "Full output: $TEST_OUTPUT/mtr_output.txt" +log "============================================" + +if [ "$FAILED" -gt "0" ]; then + log "TESTS FAILED - CI workflow needs fixes" + exit 1 +else + log "ALL TESTS PASSED - CI workflow is valid" + exit 0 +fi \ No newline at end of file diff --git a/scripts/local-ci.sh b/scripts/local-ci.sh new file mode 100755 index 0000000..935751c --- /dev/null +++ b/scripts/local-ci.sh @@ -0,0 +1,108 @@ +#!/bin/bash +# Local CI simulation - runs on host directly (no Docker overhead) +# Mirrors .github/workflows/test.yml steps locally for fast iteration + +set -e + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +EXTENSION_DIR="$SCRIPT_DIR" +VILLAGESQL_SOURCE_DIR="${VILLAGESQL_SOURCE_DIR:-/data/rene/villagesql-server}" +VILLAGESQL_BUILD_DIR="${VILLAGESQL_BUILD_DIR:-/data/rene/build}" +USE_LOCAL_BUILD="${USE_LOCAL_BUILD:-true}" + +WORKSPACE="/tmp/vsql-ci-local-$$" +VAR_DIR="$WORKSPACE/var" + +log() { + echo "[local-ci] $(date '+%H:%M:%S') - $1" +} + +cleanup() { + log "Cleaning up..." + pkill -f "mysqld.*vsql-ci.*" 2>/dev/null || true + rm -rf "$WORKSPACE" 2>/dev/null || true +} +trap cleanup EXIT + +mkdir -p "$WORKSPACE" +mkdir -p "$VAR_DIR" + +# Library path for VillageSQL +export LD_LIBRARY_PATH="$VILLAGESQL_BUILD_DIR/library_output_directory:$VILLAGESQL_BUILD_DIR/plugin_output_directory:$LD_LIBRARY_PATH" + +log "Starting local CI simulation..." +log "VillageSQL source: $VILLAGESQL_SOURCE_DIR" +log "VillageSQL build: $VILLAGESQL_BUILD_DIR" +log "Extension: $EXTENSION_DIR" + +# Step: Initialize datadir +log "Initializing datadir..." +cd "$VILLAGESQL_SOURCE_DIR" +"$VILLAGESQL_BUILD_DIR/runtime_output_directory/mysqld" \ + --initialize-insecure \ + --datadir="$VAR_DIR/mysqld.1" \ + --log-error="$VAR_DIR/mysqld.log" 2>&1 | tail -5 + +# Step: Start MySQL +log "Starting MySQL..." +"$VILLAGESQL_BUILD_DIR/runtime_output_directory/mysqld" \ + --datadir="$VAR_DIR/mysqld.1" \ + --socket="$VAR_DIR/mysql.sock" \ + --port=3306 \ + --log-error="$VAR_DIR/mysqld.log" &>/dev/null & + +log "Waiting for MySQL to start..." +for i in {1..30}; do + if "$VILLAGESQL_BUILD_DIR/runtime_output_directory/mysql" -S "$VAR_DIR/mysql.sock" -u root -e "SELECT 1" 2>/dev/null; then + log "MySQL is ready" + break + fi + sleep 1 +done + +# Step: Build extension +log "Building extension..." +cd "$EXTENSION_DIR" +rm -rf build +mkdir -p build +cd build +cmake .. -DVillageSQL_BUILD_DIR="$VILLAGESQL_BUILD_DIR" 2>&1 | tail -3 +make -j$(nproc) 2>&1 | tail -3 + +# Step: Install extension +log "Installing extension..." +"$VILLAGESQL_BUILD_DIR/runtime_output_directory/mysql" -S "$VAR_DIR/mysql.sock" -u root \ + -e "INSTALL EXTENSION $EXTENSION_DIR/build/prometheus_exporter.veb" 2>&1 + +# Step: Run MTR tests +log "Running MTR tests..." +cd "$VILLAGESQL_BUILD_DIR/mysql-test" +TEST_OUTPUT="/tmp/vsql-ci-test-output-$$" +mkdir -p "$TEST_OUTPUT" + +perl mysql-test-run.pl \ + --suite="$EXTENSION_DIR/mysql-test" \ + --parallel=auto \ + --tmpdir="$VAR_DIR" \ + --socket="$VAR_DIR/mysql.sock" \ + --report-features 2>&1 | tee "$TEST_OUTPUT/mtr_output.txt" + +# Copy logs +cp "$VAR_DIR/mysqld.log" "$TEST_OUTPUT/" 2>/dev/null || true + +# Check results +PASSED=$(grep -c "mysql-test-run: .* \[ pass \]" "$TEST_OUTPUT/mtr_output.txt" 2>/dev/null || echo "0") +FAILED=$(grep -c "mysql-test-run: .* \[ fail \]" "$TEST_OUTPUT/mtr_output.txt" 2>/dev/null || echo "0") + +log "============================================" +log "TEST RESULTS: $PASSED passed, $FAILED failed" +log "Full output: $TEST_OUTPUT/mtr_output.txt" +log "============================================" + +if [ "$FAILED" -gt "0" ]; then + log "TESTS FAILED - CI workflow needs fixes" + exit 1 +else + log "ALL TESTS PASSED - CI workflow is valid" + exit 0 +fi \ No newline at end of file