From f34a5d109cc05bdfff37d93f6be10ae74d619c83 Mon Sep 17 00:00:00 2001 From: Kybxd <627940450@qq.com> Date: Thu, 28 May 2026 15:50:04 +0800 Subject: [PATCH 01/66] ci: migrate protobuf install to vcpkg in CI --- .github/workflows/testing-cpp.yml | 102 ++++++++++++++++--------- .gitignore | 6 ++ test/cpp-tableau-loader/CMakeLists.txt | 32 +++++--- 3 files changed, 96 insertions(+), 44 deletions(-) diff --git a/.github/workflows/testing-cpp.yml b/.github/workflows/testing-cpp.yml index 80151d9..343975c 100644 --- a/.github/workflows/testing-cpp.yml +++ b/.github/workflows/testing-cpp.yml @@ -12,27 +12,38 @@ on: permissions: contents: read +env: + VCPKG_COMMIT: dc8d75cfc3281b8e2a4ed8ee4163c891190df932 + jobs: test: strategy: + fail-fast: false matrix: os: [ubuntu-latest, windows-latest] - protobuf-version: ["32.0", "3.19.3"] + config: + - label: modern + protobuf-version: "6.33.4" + - label: legacy-v3 + protobuf-version: "3.21.12" include: - os: ubuntu-latest - init_script: bash init.sh + triplet: x64-linux - os: windows-latest - init_script: cmd /c init.bat + triplet: x64-windows-static - name: test (${{ matrix.os }}, protobuf ${{ matrix.protobuf-version }}) + name: test (${{ matrix.os }}, ${{ matrix.config.label }}) runs-on: ${{ matrix.os }} - timeout-minutes: 20 + timeout-minutes: 45 + + env: + VCPKG_INSTALLED_DIR: ${{ github.workspace }}/vcpkg_installed + VCPKG_DEFAULT_TRIPLET: ${{ matrix.triplet }} + VCPKG_BINARY_SOURCES: "clear;x-gha,readwrite" steps: - name: Checkout Code uses: actions/checkout@v6 - with: - submodules: true - name: Install Go uses: actions/setup-go@v6 @@ -41,7 +52,7 @@ jobs: cache-dependency-path: go.sum cache: true - - name: Install dependencies (Ubuntu) + - name: Install Ninja (Ubuntu) if: runner.os == 'Linux' run: sudo apt-get update && sudo apt-get install -y ninja-build @@ -49,34 +60,50 @@ jobs: if: runner.os == 'Windows' uses: ilammy/msvc-dev-cmd@v1 - - name: Cache protobuf install - id: cache-protobuf - uses: actions/cache@v4 + - name: Export GitHub Actions cache env + uses: actions/github-script@v7 with: - path: | - third_party/_submodules/protobuf/.build/_install - key: protobuf-${{ matrix.os }}-${{ matrix.protobuf-version }}-${{ hashFiles('init.sh', 'init.bat', '.gitmodules') }} - restore-keys: | - protobuf-${{ matrix.os }}-${{ matrix.protobuf-version }}- - - - name: Init submodules and build protobuf - run: ${{ matrix.init_script }} - env: - PROTOBUF_REF: v${{ matrix.protobuf-version }} - - - name: Install Protoc - if: "!startsWith(matrix.protobuf-version, '3.')" - uses: arduino/setup-protoc@v3 - with: - version: ${{ matrix.protobuf-version }} - repo-token: ${{ secrets.GITHUB_TOKEN }} + script: | + core.exportVariable('ACTIONS_CACHE_URL', process.env.ACTIONS_CACHE_URL || ''); + core.exportVariable('ACTIONS_RUNTIME_TOKEN', process.env.ACTIONS_RUNTIME_TOKEN || ''); - - name: Install Protoc (legacy) - if: startsWith(matrix.protobuf-version, '3.') - uses: arduino/setup-protoc@v1 + - name: Render vcpkg.json + working-directory: test/cpp-tableau-loader + shell: bash + run: | + cat > vcpkg.json <> "$GITHUB_PATH" + + - name: Add vcpkg-installed protoc to PATH (Windows) + if: runner.os == 'Windows' + shell: pwsh + run: Add-Content -Path $env:GITHUB_PATH -Value "$env:VCPKG_INSTALLED_DIR\${{ matrix.triplet }}\tools\protobuf" + + - name: Verify protoc + shell: bash + run: | + which protoc + protoc --version - name: Install Buf uses: bufbuild/buf-action@v1 @@ -91,7 +118,14 @@ jobs: - name: CMake Configure working-directory: test/cpp-tableau-loader - run: cmake -S . -B build -G Ninja -DCMAKE_BUILD_TYPE=Debug -DCMAKE_CXX_STANDARD=17 + run: > + cmake -S . -B build -G Ninja + -DCMAKE_BUILD_TYPE=Debug + -DCMAKE_CXX_STANDARD=17 + -DCMAKE_TOOLCHAIN_FILE=${{ env.VCPKG_ROOT }}/scripts/buildsystems/vcpkg.cmake + -DVCPKG_TARGET_TRIPLET=${{ matrix.triplet }} + -DVCPKG_INSTALLED_DIR=${{ env.VCPKG_INSTALLED_DIR }} + -DVCPKG_MANIFEST_INSTALL=OFF - name: CMake Build working-directory: test/cpp-tableau-loader diff --git a/.gitignore b/.gitignore index 09abcd0..64218d1 100644 --- a/.gitignore +++ b/.gitignore @@ -50,3 +50,9 @@ test/csharp-tableau-loader/protoconf # C# Dev Kit language service cache (VS Code) *.lscache +# vcpkg manifest + install tree generated by CI (see .github/workflows/testing-cpp.yml). +# The manifest is rendered on the fly so the repo does not pin a vcpkg flow on +# local builds; the install tree is a build artifact. +test/cpp-tableau-loader/vcpkg.json +vcpkg_installed/ + diff --git a/test/cpp-tableau-loader/CMakeLists.txt b/test/cpp-tableau-loader/CMakeLists.txt index af39f0a..1e0f6dd 100644 --- a/test/cpp-tableau-loader/CMakeLists.txt +++ b/test/cpp-tableau-loader/CMakeLists.txt @@ -32,21 +32,33 @@ else() endif() # google protobuf -# Try to find protobuf from local submodule first, then fallback to system. -# Protobuf's install.cmake generates config files under: -# - Linux: build/lib64/cmake/protobuf/ (via CMAKE_INSTALL_LIBDIR) -# - Windows: build/cmake/ (MSVC uses "cmake" directly) -# Setting CMAKE_PREFIX_PATH to the build dir allows CMake to search both layouts. +# +# Resolution order (first match wins): +# 1. User-provided -DProtobuf_ROOT=... or -DCMAKE_PREFIX_PATH=... — for users +# bringing their own protobuf install (vcpkg / conan / Homebrew / custom). +# 2. Local submodule build at third_party/_submodules/protobuf/.build/_install +# — populated by running ./init.sh (or init.bat on Windows) at the repo root. +# This is OPTIONAL; loader does not require it. +# 3. System-wide protobuf discoverable by find_package (e.g. apt's +# libprotobuf-dev, system-installed CMake config). +# +# Whichever protobuf is picked, it MUST match the protoc that generated +# tableau.pb.{h,cc}: protobuf v22+ enforces a strict gencode/runtime version +# check via PROTOBUF_VERSION in the generated headers. Install matching +# protoc + libprotobuf yourself, or run ./init.sh with PROTOBUF_REF=vX.Y.Z to +# build a matching submodule copy. set(LOCAL_PROTOBUF_INSTALL_DIR "${PROJECT_SOURCE_DIR}/../../third_party/_submodules/protobuf/.build/_install") -set(LOCAL_PROTOBUF_SRC_DIR "${PROJECT_SOURCE_DIR}/../../third_party/_submodules/protobuf/src") -if(EXISTS "${LOCAL_PROTOBUF_INSTALL_DIR}" AND EXISTS "${LOCAL_PROTOBUF_SRC_DIR}") - message(STATUS "Found local protobuf submodule, using it preferentially.") - list(PREPEND CMAKE_PREFIX_PATH "${LOCAL_PROTOBUF_INSTALL_DIR}") +if(EXISTS "${LOCAL_PROTOBUF_INSTALL_DIR}") + message(STATUS "Found local protobuf submodule install at ${LOCAL_PROTOBUF_INSTALL_DIR}; " + "appending to CMAKE_PREFIX_PATH (user-provided paths still take priority).") + list(APPEND CMAKE_PREFIX_PATH "${LOCAL_PROTOBUF_INSTALL_DIR}") endif() # Use CONFIG mode explicitly to pick up protobuf's own protobuf-config.cmake # instead of CMake's built-in FindProtobuf.cmake module, which may not handle -# the local build directory layout correctly. +# every layout correctly (especially the submodule build dir on Linux vs. +# Windows). CONFIG also gives us the modern protobuf::libprotobuf imported +# target with proper INTERFACE_INCLUDE_DIRECTORIES. find_package(Protobuf CONFIG REQUIRED) message(STATUS "Using protobuf ${Protobuf_VERSION}") From 6cdd153ce0be9042b5b8da64444ab2d934b9510d Mon Sep 17 00:00:00 2001 From: Kybxd <627940450@qq.com> Date: Thu, 28 May 2026 16:56:11 +0800 Subject: [PATCH 02/66] feat: replace protobuf submodule with vcpkg Switch from building the protobuf submodule via init.sh/init.bat to installing protobuf through vcpkg. This removes the bundled protobuf build, simplifying the setup process and significantly reducing initial bootstrap time. Update prepare.bat to install vcpkg and protobuf (x64-windows-static), expose the vcpkg-built protoc on PATH, and export VCPKG_ROOT. Revise README.md instructions to document vcpkg (recommended), system package, Homebrew, and source-based protobuf installation paths. Adjust gitignore rules for the vcpkg manifest and install tree accordingly. --- .gitignore | 9 +- .gitmodules | 4 - README.md | 126 +++++++++++++-------- init.bat | 151 ------------------------- init.sh | 131 --------------------- prepare.bat | 136 +++++++++++++++++++++- test/cpp-tableau-loader/CMakeLists.txt | 59 +++++----- third_party/_submodules/protobuf | 1 - 8 files changed, 246 insertions(+), 371 deletions(-) delete mode 100644 .gitmodules delete mode 100644 init.bat delete mode 100755 init.sh delete mode 160000 third_party/_submodules/protobuf diff --git a/.gitignore b/.gitignore index 64218d1..089eedd 100644 --- a/.gitignore +++ b/.gitignore @@ -50,9 +50,10 @@ test/csharp-tableau-loader/protoconf # C# Dev Kit language service cache (VS Code) *.lscache -# vcpkg manifest + install tree generated by CI (see .github/workflows/testing-cpp.yml). -# The manifest is rendered on the fly so the repo does not pin a vcpkg flow on -# local builds; the install tree is a build artifact. -test/cpp-tableau-loader/vcpkg.json +# vcpkg install tree (build artifact, never checked in). vcpkg_installed/ +# vcpkg manifest is rendered on the fly by .github/workflows/testing-cpp.yml. +# Local users who want to commit a manifest can `git add -f vcpkg.json`. +test/cpp-tableau-loader/vcpkg.json + diff --git a/.gitmodules b/.gitmodules deleted file mode 100644 index 5f23743..0000000 --- a/.gitmodules +++ /dev/null @@ -1,4 +0,0 @@ -[submodule "third_party/_submodules/protobuf"] - path = third_party/_submodules/protobuf - url = https://github.com/protocolbuffers/protobuf - ignore = dirty diff --git a/README.md b/README.md index c6b88d4..3c8ce09 100644 --- a/README.md +++ b/README.md @@ -7,38 +7,73 @@ The official config loader for [Tableau](https://github.com/tableauio/tableau). > TODO: [devcontainer](https://code.visualstudio.com/docs/devcontainers/containers) - C++ standard: at least C++17 -- Prepare and init: - - macOS or Linux: `bash init.sh` - - Windows: - 1. Run `prepare.bat` **as Administrator** to automatically install all build dependencies ([Chocolatey](https://chocolatey.org/), [CMake](https://github.com/Kitware/CMake/releases), [Ninja](https://ninja-build.org/), MSVC build tools, and [buf](https://buf.build/)), configure `PATH`, and initialize the MSVC compiler environment: - ```bat - .\prepare.bat - ``` - > ⚠️ **Admin required:** This script uses Chocolatey and MSI installers that write to system-protected directories (`C:\ProgramData`, `C:\Program Files`). Right-click Command Prompt → **Run as administrator**, then execute the script. - > - > Preview what the script would do without making any changes: - > ```bat - > .\prepare.bat --dry-run - > ``` - 2. Run `init.bat` to initialize submodules and build protobuf: - ```bat - .\init.bat - ``` - > **Note:** The **installation** part of `prepare.bat` only runs once per machine — it detects already-installed tools (Chocolatey, Ninja, CMake, MSVC Build Tools, buf) and skips them, so no manual installation is required. - > - > However, the MSVC compiler environment (`cl.exe` on `PATH`, plus `INCLUDE` / `LIB` / `LIBPATH` / `WindowsSdkDir` / `VCToolsInstallDir`) is exported to the **current cmd session only** — `vcvarsall.bat` does not (and should not) write these into the persistent user `PATH`. You therefore need to re-run `.\prepare.bat` in **every new cmd window** before invoking `init.bat` or building the loader. Subsequent runs are near-instant since no installation work is repeated. - -> **Fast path (idempotent re-runs):** Building protobuf takes 5–15 minutes. To make repeated runs cheap, both `init.sh` and `init.bat` short-circuit and exit immediately when `third_party/_submodules/protobuf/.build/_install` already contains a valid `protobuf-config.cmake` (the marker that the previous build finished). This means: -> - Re-running `init.sh` / `init.bat` after a successful first run is a no-op (a second or two). -> - CI workflows cache `.build/_install` (see `.github/workflows/testing-cpp.yml`) and the fast path then turns the "build protobuf" step into a near-instant cache restore. -> - To force a clean rebuild (e.g. after changing protobuf flags or switching `PROTOBUF_REF` to a version whose previously-installed artefacts are still around), set `FORCE_REBUILD_PROTOBUF=1`: -> ```sh -> FORCE_REBUILD_PROTOBUF=1 bash init.sh # macOS / Linux -> ``` -> ```bat -> set FORCE_REBUILD_PROTOBUF=1 && .\init.bat :: Windows (cmd) -> ``` -> Or simply delete `third_party/_submodules/protobuf/.build/` before rerunning. +- A working **`protoc` + `libprotobuf`** toolchain on your machine. The same + protobuf release **must** provide both: protobuf v22+ enforces a strict + gencode/runtime version check via `PROTOBUF_VERSION` in the generated + headers, so a mismatched `protoc` and `libprotobuf` will fail to link. + +### Install protobuf + +Pick whichever channel fits your platform; loader does not bundle protobuf. + +- **vcpkg (recommended, cross-platform):** + ```sh + git clone https://github.com/microsoft/vcpkg.git ~/vcpkg + ~/vcpkg/bootstrap-vcpkg.sh # macOS / Linux + # .\vcpkg\bootstrap-vcpkg.bat # Windows + ~/vcpkg/vcpkg install protobuf # Linux: x64-linux + # ~/vcpkg/vcpkg install protobuf:x64-osx # macOS + # .\vcpkg\vcpkg install protobuf:x64-windows-static # Windows (matches loader's static CRT) + ``` + Pin to the legacy v3 line if you need it: append `--x-version=3.21.12`. + Then point CMake at vcpkg with `-DCMAKE_TOOLCHAIN_FILE=/scripts/buildsystems/vcpkg.cmake`. + +- **Linux (system package):** + ```sh + sudo apt-get install -y protobuf-compiler libprotobuf-dev # Debian / Ubuntu + sudo dnf install -y protobuf-compiler protobuf-devel # Fedora / RHEL 8+ / CentOS Stream / Rocky / Alma + sudo yum install -y epel-release \ + && sudo yum install -y protobuf-compiler protobuf-devel # CentOS 7 (via EPEL) + ``` + > Distro packages can lag well behind upstream (e.g. CentOS 7 ships protobuf 2.5; RHEL/Rocky 8 ships 3.x). If you need protobuf v22+ (or any specific version), prefer **vcpkg** above or **build from source**. + +- **macOS (Homebrew):** + ```sh + brew install protobuf + ``` + +- **From source:** see [Protocol Buffers C++ Installation](https://github.com/protocolbuffers/protobuf/tree/master/src). + After installing, point CMake at it with `-DCMAKE_PREFIX_PATH=/path/to/protobuf-install` + (or `-DProtobuf_ROOT=...`). + +### Windows: bootstrap the rest of the toolchain + +Run `prepare.bat` **as Administrator** to install everything you need on a +fresh Windows machine: [Chocolatey](https://chocolatey.org/), +[CMake](https://github.com/Kitware/CMake/releases), +[Ninja](https://ninja-build.org/), MSVC build tools, [buf](https://buf.build/), +**vcpkg**, and `protobuf:x64-windows-static`. It also activates the MSVC +compiler environment for the current cmd session. + +```bat +.\prepare.bat +``` + +> ⚠️ **Admin required:** This script uses Chocolatey and MSI installers that write to system-protected directories (`C:\ProgramData`, `C:\Program Files`). Right-click Command Prompt → **Run as administrator**, then execute the script. +> +> Preview what the script would do without making any changes: +> ```bat +> .\prepare.bat --dry-run +> ``` +> +> Override the protobuf vcpkg port version (e.g. for the legacy v3 ABI): +> ```bat +> set PROTOBUF_VCPKG_VERSION=3.21.12 && .\prepare.bat +> ``` + +> **Note:** The **installation** part of `prepare.bat` only runs once per machine — it detects already-installed tools (Chocolatey, Ninja, CMake, MSVC Build Tools, buf, vcpkg, protobuf) and skips them, so no manual installation is required. +> +> However, the MSVC compiler environment (`cl.exe` on `PATH`, plus `INCLUDE` / `LIB` / `LIBPATH` / `WindowsSdkDir` / `VCToolsInstallDir`) is exported to the **current cmd session only** — `vcvarsall.bat` does not (and should not) write these into the persistent user `PATH`. You therefore need to re-run `.\prepare.bat` in **every new cmd window** before building the loader. Subsequent runs are near-instant since no installation work is repeated. ### References @@ -47,6 +82,7 @@ The official config loader for [Tableau](https://github.com/tableauio/tableau). - [Ninja](https://ninja-build.org/) - [Visual Studio 2022](https://visualstudio.microsoft.com/downloads/) - [Use the Microsoft C++ Build Tools from the command line](https://learn.microsoft.com/en-us/cpp/build/building-on-the-command-line?view=msvc-170) +- [vcpkg](https://github.com/microsoft/vcpkg) - [buf CLI](https://buf.build/docs/cli/) ## C++ @@ -54,11 +90,16 @@ The official config loader for [Tableau](https://github.com/tableauio/tableau). ### Dev at Linux - Change dir: `cd test/cpp-tableau-loader` -- Generate protoconf: `PATH=../../third_party/_submodules/protobuf/.build/_install/bin:$PATH buf generate ..` -- CMake: +- Generate protoconf: `buf generate ..` (assumes `protoc` is on `PATH`; if you installed via vcpkg, prepend `/installed/x64-linux/tools/protobuf` to `PATH`) +- CMake (system protobuf): - C++17: `cmake -S . -B build -DCMAKE_BUILD_TYPE=Debug` - C++20: `cmake -S . -B build -DCMAKE_BUILD_TYPE=Debug -DCMAKE_CXX_STANDARD=20` - clang: `cmake -S . -B build -DCMAKE_BUILD_TYPE=Debug -DCMAKE_CXX_COMPILER=clang++` +- CMake (vcpkg-provided protobuf): + ```sh + cmake -S . -B build -DCMAKE_BUILD_TYPE=Debug \ + -DCMAKE_TOOLCHAIN_FILE=/scripts/buildsystems/vcpkg.cmake + ``` - Build: `cmake --build build --parallel` - Test: `ctest --test-dir build --output-on-failure` @@ -66,16 +107,14 @@ The official config loader for [Tableau](https://github.com/tableauio/tableau). > **Important:** CMake with Ninja requires MSVC environment variables (`cl.exe`, `INCLUDE`, `LIB`, etc.) to be active. Run `.\prepare.bat` from the **loader** root in the **same cmd session** (use **cmd**, not PowerShell — `prepare.bat` exports vars via `endlocal & set ...` which only works for a cmd parent process) before switching to the test directory. Opening a new terminal window will lose these variables. > -> **Build type:** The protobuf submodule is built as **Debug** (`/MTd`) by `init.bat`. To avoid LNK2038 `_ITERATOR_DEBUG_LEVEL` / `RuntimeLibrary` CRT-mismatch errors, the loader must also be built as Debug. `CMakeLists.txt` does not set a default, so always pass `-DCMAKE_BUILD_TYPE=Debug` explicitly — also required for multi-config generators (Visual Studio default = Debug, but stay explicit to match the cached protobuf). +> **Build type:** vcpkg's `x64-windows-static` triplet (and our `prepare.bat`) builds protobuf as **Debug** with the static CRT (`/MTd`). To avoid LNK2038 `_ITERATOR_DEBUG_LEVEL` / `RuntimeLibrary` CRT-mismatch errors, the loader must also be built as Debug. `CMakeLists.txt` does not set a default, so always pass `-DCMAKE_BUILD_TYPE=Debug` explicitly — also required for multi-config generators (Visual Studio default = Debug, but stay explicit to match the protobuf you installed). - Initialize MSVC environment (from loader root): `.\prepare.bat` - Change dir: `cd test\cpp-tableau-loader`, or change directory with Drive, e.g.: `cd /D D:\GitHub\loader\test\cpp-tableau-loader` -- Generate protoconf: - - cmd: `cmd /C "set PATH=..\..\third_party\_submodules\protobuf\.build\_install\bin;%PATH% && buf generate .."` - - PowerShell: `$env:PATH = "..\..\third_party\_submodules\protobuf\.build\_install\bin;" + $env:PATH; buf generate ..` -- CMake: - - C++17: `cmake -S . -B build -G Ninja -DCMAKE_BUILD_TYPE=Debug` - - C++20: `cmake -S . -B build -G Ninja -DCMAKE_BUILD_TYPE=Debug -DCMAKE_CXX_STANDARD=20` +- Generate protoconf: `buf generate ..` (the `prepare.bat` step above already puts the vcpkg-built `protoc.exe` on `PATH`) +- CMake (vcpkg-provided protobuf): + - C++17: `cmake -S . -B build -G Ninja -DCMAKE_BUILD_TYPE=Debug -DCMAKE_TOOLCHAIN_FILE=%VCPKG_ROOT%\scripts\buildsystems\vcpkg.cmake -DVCPKG_TARGET_TRIPLET=x64-windows-static` + - C++20: append `-DCMAKE_CXX_STANDARD=20` - Build: `cmake --build build --parallel` - Test: `ctest --test-dir build --output-on-failure` @@ -108,10 +147,7 @@ The official config loader for [Tableau](https://github.com/tableauio/tableau). - Install: **dotnet-sdk-8.0** - Change dir: `cd test/csharp-tableau-loader` -- Generate protoconf: - - macOS / Linux: `PATH=../../third_party/_submodules/protobuf/.build/_install/bin:$PATH buf generate ..` - - Windows (cmd): `cmd /C "set PATH=..\..\third_party\_submodules\protobuf\.build\_install\bin;%PATH% && buf generate .."` - - Windows (PowerShell): `$env:PATH = "..\..\third_party\_submodules\protobuf\.build\_install\bin;" + $env:PATH; buf generate ..` +- Generate protoconf: `buf generate ..` (requires `protoc` on `PATH`; install protobuf as described in [Install protobuf](#install-protobuf)) - Test: `dotnet test` > **Note:** Tests are written with [xUnit](https://xunit.net/). diff --git a/init.bat b/init.bat deleted file mode 100644 index 6d43624..0000000 --- a/init.bat +++ /dev/null @@ -1,151 +0,0 @@ -@echo off -setlocal enabledelayedexpansion - -REM Initialize build environment (installs choco/ninja/MSVC if needed, sets up PATH) -call "%~dp0prepare.bat" -if errorlevel 1 ( - echo [ERROR] prepare.bat failed. Aborting. - exit /b 1 -) - -for /f "delims=" %%i in ('git rev-parse --show-toplevel') do set repoRoot=%%i -cd /d "%repoRoot%" - -REM Initialize only the protobuf submodule (non-recursive). Protobuf's own -REM nested submodules (third_party/googletest, third_party/benchmark on v3.x) -REM are only needed when protobuf_BUILD_TESTS=ON / benchmarks are enabled, and -REM modern protobuf (v4+/v21+) has dropped git submodules entirely in favor of -REM CMake FetchContent. Skipping --recursive saves clone time and CI bandwidth. -git submodule update --init third_party/_submodules/protobuf - -REM Build and install the C++ Protocol Buffer runtime and the Protocol Buffer compiler (protoc) -cd third_party\_submodules\protobuf - -REM If PROTOBUF_REF is set, switch submodule to the specified ref -if not "%PROTOBUF_REF%"=="" ( - echo Switching protobuf submodule to %PROTOBUF_REF%... - git fetch --tags - git checkout %PROTOBUF_REF% -) - -REM --------------------------------------------------------------------------- -REM Detect protobuf major version and build the cmake command line. We compute -REM both up-front (before the fast-path check) so the signature comparison -REM below can include the exact cmake invocation we would run. -REM - protobuf v3.x : CMakeLists.txt is in cmake/ subdirectory -REM - protobuf v4+ : CMakeLists.txt is in root directory -REM --------------------------------------------------------------------------- -for /f "tokens=*" %%v in ('git describe --tags --abbrev^=0 2^>nul') do set PROTOBUF_VERSION=%%v -if not defined PROTOBUF_VERSION set PROTOBUF_VERSION=unknown -echo Detected protobuf version: %PROTOBUF_VERSION% - -REM Extract major version number from tag (e.g., v3.19.3 -> 3, v32.0 -> 32) -set "VER_STR=%PROTOBUF_VERSION:~1%" -for /f "tokens=1 delims=." %%a in ("%VER_STR%") do set MAJOR_VERSION=%%a - -if %MAJOR_VERSION% LEQ 3 ( - REM Legacy protobuf (v3.x): CMakeLists.txt is in cmake/ subdirectory - set "PROTOBUF_BUILD_VARIANT=legacy" - set "CMAKE_SRC=cmake" - set "CMAKE_FLAGS=-DCMAKE_BUILD_TYPE=Debug -DCMAKE_CXX_STANDARD=17 -DCMAKE_POLICY_VERSION_MINIMUM=3.5 -Dprotobuf_BUILD_TESTS=OFF -Dprotobuf_WITH_ZLIB=OFF -Dprotobuf_BUILD_SHARED_LIBS=OFF" -) else ( - REM Modern protobuf (v4+/v21+/v32+): CMakeLists.txt is in root directory - REM Refer: https://github.com/protocolbuffers/protobuf/blob/v32.0/cmake/README.md#cmake-configuration - REM - protobuf_MSVC_STATIC_RUNTIME defaults to ON, which uses static CRT (/MTd for Debug). - REM Our project's CMakeLists.txt also sets static CRT to match. - REM - protobuf_WITH_ZLIB=OFF: disable ZLIB dependency to avoid ZLIB::ZLIB link requirement - REM in protobuf's exported CMake targets, which simplifies cross-platform builds. - REM - protobuf_BUILD_SHARED_LIBS=OFF: build static libraries explicitly. - set "PROTOBUF_BUILD_VARIANT=modern" - set "CMAKE_SRC=." - set "CMAKE_FLAGS=-DCMAKE_BUILD_TYPE=Debug -DCMAKE_CXX_STANDARD=17 -DCMAKE_POLICY_VERSION_MINIMUM=3.5 -Dprotobuf_BUILD_TESTS=OFF -Dprotobuf_WITH_ZLIB=OFF -Dprotobuf_BUILD_SHARED_LIBS=OFF -Dutf8_range_ENABLE_INSTALL=ON" -) - -REM Build a stable, multi-line signature describing the inputs that determine -REM the contents of .build\_install. Any change to these values must -REM invalidate the fast-path. Adding new compile-time inputs? Append a line. -set "SIG_FILE=.build\_install\.build_signature" -set "SIG_LINE_1=schema=1" -set "SIG_LINE_2=version=!PROTOBUF_VERSION!" -set "SIG_LINE_3=variant=!PROTOBUF_BUILD_VARIANT!" -set "SIG_LINE_4=cmake_args=-S !CMAKE_SRC! -B .build -G Ninja !CMAKE_FLAGS!" - -REM --------------------------------------------------------------------------- -REM Fast path: if a previous build's signature file matches the one we're -REM about to use, skip the (very long) protobuf compile entirely. -REM Set FORCE_REBUILD_PROTOBUF=1 to bypass this short-circuit unconditionally. -REM --------------------------------------------------------------------------- -if not "%FORCE_REBUILD_PROTOBUF%"=="" goto :no_fast_path -if not exist "!SIG_FILE!" goto :no_fast_path - -REM Read the file 4 lines at a time and compare with the expected signature. -set "ACTUAL_LINE_1=" -set "ACTUAL_LINE_2=" -set "ACTUAL_LINE_3=" -set "ACTUAL_LINE_4=" -set "_LINE_NO=0" -for /f "usebackq delims=" %%L in ("!SIG_FILE!") do ( - set /a _LINE_NO+=1 - set "ACTUAL_LINE_!_LINE_NO!=%%L" -) - -if not "!ACTUAL_LINE_1!"=="!SIG_LINE_1!" goto :sig_mismatch -if not "!ACTUAL_LINE_2!"=="!SIG_LINE_2!" goto :sig_mismatch -if not "!ACTUAL_LINE_3!"=="!SIG_LINE_3!" goto :sig_mismatch -if not "!ACTUAL_LINE_4!"=="!SIG_LINE_4!" goto :sig_mismatch - -echo [INFO] Build signature matches; reusing existing protobuf install at .build\_install. -echo [INFO] Set FORCE_REBUILD_PROTOBUF=1 to force a clean rebuild. -goto :eof - -:sig_mismatch -echo [INFO] Build signature mismatch; rebuilding protobuf. -echo [INFO] actual: -echo [INFO] !ACTUAL_LINE_1! -echo [INFO] !ACTUAL_LINE_2! -echo [INFO] !ACTUAL_LINE_3! -echo [INFO] !ACTUAL_LINE_4! -echo [INFO] expected: -echo [INFO] !SIG_LINE_1! -echo [INFO] !SIG_LINE_2! -echo [INFO] !SIG_LINE_3! -echo [INFO] !SIG_LINE_4! - -:no_fast_path -REM Wipe any stale install dir so we don't leave half-overwritten files behind -REM when cmake flags change (e.g. Debug -> Release puts artifacts in different -REM places, an in-place re-install would mix old and new). -if exist .build rmdir /s /q .build - -REM --------------------------------------------------------------------------- -REM Configure -REM --------------------------------------------------------------------------- -if "!PROTOBUF_BUILD_VARIANT!"=="legacy" ( - echo Using legacy cmake\ subdirectory for protobuf %PROTOBUF_VERSION% -) else ( - echo Using root CMakeLists.txt for protobuf %PROTOBUF_VERSION% -) -cmake -S !CMAKE_SRC! -B .build -G Ninja !CMAKE_FLAGS! -if errorlevel 1 exit /b 1 - -REM Compile the code -cmake --build .build --parallel -if errorlevel 1 exit /b 1 - -REM Install into .build/_install so that protobuf-config.cmake (along with -REM absl and utf8_range configs) is generated for find_package(Protobuf CONFIG) -REM used by downstream CMakeLists.txt. -REM NOTE: .build/ is already in protobuf's .gitignore, so _install stays clean. -cmake --install .build --prefix .build\_install -if errorlevel 1 exit /b 1 - -REM Persist the signature so the next run can fast-path skip when nothing changed. -> "!SIG_FILE!" ( - echo !SIG_LINE_1! - echo !SIG_LINE_2! - echo !SIG_LINE_3! - echo !SIG_LINE_4! -) -echo [INFO] Wrote build signature to !SIG_FILE! - -endlocal diff --git a/init.sh b/init.sh deleted file mode 100755 index d5b2510..0000000 --- a/init.sh +++ /dev/null @@ -1,131 +0,0 @@ -#!/bin/bash -# set -eux -set -e -set -o pipefail - -cd "$(git rev-parse --show-toplevel)" - -# Initialize only the protobuf submodule (non-recursive). Protobuf's own nested -# submodules (third_party/googletest, third_party/benchmark on v3.x) are only -# needed when protobuf_BUILD_TESTS=ON / benchmarks are enabled, and modern -# protobuf (v4+/v21+) has dropped git submodules entirely in favor of CMake -# FetchContent. Skipping --recursive saves clone time and CI bandwidth. -git submodule update --init third_party/_submodules/protobuf - -# prerequisites -# On Ubuntu/Debian, you can install them with: -# sudo apt-get install autoconf automake libtool curl make g++ unzip - -# Build and install the C++ Protocol Buffer runtime and the Protocol Buffer compiler (protoc) -cd third_party/_submodules/protobuf - -# If PROTOBUF_REF is set, switch submodule to the specified ref -if [ -n "${PROTOBUF_REF:-}" ]; then - echo "Switching protobuf submodule to ${PROTOBUF_REF}..." - git fetch --tags - git checkout "${PROTOBUF_REF}" -fi - -# ----------------------------------------------------------------------------- -# Detect protobuf major version and build the cmake command line. We compute -# both up-front (before the fast-path check) so the signature comparison below -# can include the exact cmake invocation we would run. -# - protobuf v3.x : CMakeLists.txt is in cmake/ subdirectory -# - protobuf v4+ : CMakeLists.txt is in root directory -# ----------------------------------------------------------------------------- -PROTOBUF_VERSION=$(git describe --tags --abbrev=0 2>/dev/null || echo "unknown") -echo "Detected protobuf version: ${PROTOBUF_VERSION}" - -# Extract major version number from tag (e.g., v3.19.3 -> 3, v32.0 -> 32) -MAJOR_VERSION=$(echo "${PROTOBUF_VERSION}" | sed 's/^v//' | cut -d. -f1) - -if [ "${MAJOR_VERSION}" -le 3 ] 2>/dev/null; then - # Legacy protobuf (v3.x): CMakeLists.txt is in cmake/ subdirectory - PROTOBUF_BUILD_VARIANT="legacy" - CMAKE_ARGS=( - -S cmake - -B .build - -G Ninja - -DCMAKE_BUILD_TYPE=Debug - -DCMAKE_CXX_STANDARD=17 - -Dprotobuf_BUILD_TESTS=OFF - -Dprotobuf_WITH_ZLIB=OFF - -Dprotobuf_BUILD_SHARED_LIBS=OFF - ) -else - # Modern protobuf (v4+/v21+/v32+): CMakeLists.txt is in root directory - # Refer: https://github.com/protocolbuffers/protobuf/blob/v32.0/cmake/README.md#cmake-configuration - # - protobuf_WITH_ZLIB=OFF: disable ZLIB dependency to avoid ZLIB::ZLIB link requirement - # in protobuf's exported CMake targets, which simplifies cross-platform builds. - # - protobuf_BUILD_SHARED_LIBS=OFF: build static libraries explicitly. - PROTOBUF_BUILD_VARIANT="modern" - CMAKE_ARGS=( - -S . - -B .build - -G Ninja - -DCMAKE_BUILD_TYPE=Debug - -DCMAKE_CXX_STANDARD=17 - -Dprotobuf_BUILD_TESTS=OFF - -Dprotobuf_WITH_ZLIB=OFF - -Dprotobuf_BUILD_SHARED_LIBS=OFF - -Dutf8_range_ENABLE_INSTALL=ON - ) -fi - -# Build a stable, multi-line signature describing the inputs that determine -# the contents of .build/_install. Any change to these values must invalidate -# the fast-path. Adding new compile-time inputs? Append a line here. -SIG_FILE=".build/_install/.build_signature" -EXPECTED_SIGNATURE=$(printf '%s\n' \ - "schema=1" \ - "version=${PROTOBUF_VERSION}" \ - "variant=${PROTOBUF_BUILD_VARIANT}" \ - "cmake_args=${CMAKE_ARGS[*]}") - -# ----------------------------------------------------------------------------- -# Fast path: if a previous build's _install dir is present AND its embedded -# signature matches the one we're about to use, skip the (very long) protobuf -# compile entirely. -# Set FORCE_REBUILD_PROTOBUF=1 to bypass this short-circuit unconditionally. -# ----------------------------------------------------------------------------- -if [ -z "${FORCE_REBUILD_PROTOBUF:-}" ] && [ -f "${SIG_FILE}" ]; then - ACTUAL_SIGNATURE=$(cat "${SIG_FILE}") - if [ "${ACTUAL_SIGNATURE}" = "${EXPECTED_SIGNATURE}" ]; then - echo "[INFO] Build signature matches; reusing existing protobuf install at .build/_install." - echo "[INFO] Set FORCE_REBUILD_PROTOBUF=1 to force a clean rebuild." - exit 0 - fi - echo "[INFO] Build signature mismatch; rebuilding protobuf." - echo "[INFO] actual:" - printf '%s\n' "${ACTUAL_SIGNATURE}" | sed 's/^/[INFO] /' - echo "[INFO] expected:" - printf '%s\n' "${EXPECTED_SIGNATURE}" | sed 's/^/[INFO] /' -fi - -# Wipe any stale install dir so we don't leave half-overwritten files behind -# when cmake flags change (e.g. Debug -> Release puts artifacts in different -# places, an in-place re-install would mix old and new). -rm -rf .build 2>/dev/null || true - -# ----------------------------------------------------------------------------- -# Configure -# ----------------------------------------------------------------------------- -if [ "${PROTOBUF_BUILD_VARIANT}" = "legacy" ]; then - echo "Using legacy cmake/ subdirectory for protobuf ${PROTOBUF_VERSION}" -else - echo "Using root CMakeLists.txt for protobuf ${PROTOBUF_VERSION}" -fi -cmake "${CMAKE_ARGS[@]}" - -# Compile the code -cmake --build .build --parallel - -# Install into .build/_install so that protobuf-config.cmake (along with -# absl and utf8_range configs) is generated for find_package(Protobuf CONFIG) -# used by downstream CMakeLists.txt. -# NOTE: .build/ is already in protobuf's .gitignore, so _install stays clean. -cmake --install .build --prefix .build/_install - -# Persist the signature so the next run can fast-path skip when nothing changed. -printf '%s\n' "${EXPECTED_SIGNATURE}" >"${SIG_FILE}" -echo "[INFO] Wrote build signature to ${SIG_FILE}" diff --git a/prepare.bat b/prepare.bat index 8f3f023..cfc80e2 100644 --- a/prepare.bat +++ b/prepare.bat @@ -1,6 +1,27 @@ @echo off setlocal enabledelayedexpansion +REM =========================================================================== +REM prepare.bat — bootstrap a Windows build environment for the C++ loader. +REM +REM Installs (only if missing): Chocolatey, Ninja, CMake 3.31.8, MSVC Build +REM Tools (Visual Studio 2022 Build Tools), buf CLI, and vcpkg. +REM +REM Then installs `protobuf` (and friends) into vcpkg using the static-CRT +REM triplet x64-windows-static, so that downstream cmake builds can pick it +REM up via -DCMAKE_TOOLCHAIN_FILE=%VCPKG_ROOT%\scripts\buildsystems\vcpkg.cmake. +REM +REM Override `protobuf` to a specific vcpkg port version with PROTOBUF_VCPKG_VERSION: +REM set PROTOBUF_VCPKG_VERSION=3.21.12 && .\prepare.bat +REM (Default is whatever the vcpkg `master` baseline ships, currently the 6.x +REM line. Use 3.21.12 if you need the legacy v3 ABI.) +REM +REM This script is idempotent: re-running it on a machine that already has +REM everything installed is a no-op (a few seconds of probing). Only the MSVC +REM environment variables are re-exported each time, since vcvarsall.bat sets +REM cmd-session-local state that does not persist. +REM =========================================================================== + REM ----------------------------------------------------------------------- REM Parse arguments REM --dry-run : print what would be done, but do not install anything @@ -285,7 +306,118 @@ if "%BUF_FOUND%"=="0" ( echo [INFO] buf.exe already in PATH. ) +REM ----------------------------------------------------------------------- +REM Step 5: Ensure vcpkg is installed and `protobuf` is provisioned +REM +REM Resolution order for the vcpkg install location: +REM 1. Existing %VCPKG_ROOT% if it points at a usable bootstrap. +REM 2. Existing %VCPKG_INSTALLATION_ROOT% (set on GitHub-hosted runners). +REM 3. Fresh clone into %USERPROFILE%\vcpkg. +REM +REM We then run `vcpkg install protobuf:x64-windows-static` so that the +REM static-CRT libprotobuf + protoc match the loader build (CMakeLists.txt +REM forces /MT[d] via CMAKE_MSVC_RUNTIME_LIBRARY). +REM +REM Override the protobuf port version (e.g. for the legacy v3 line) with: +REM set PROTOBUF_VCPKG_VERSION=3.21.12 && .\prepare.bat +REM ----------------------------------------------------------------------- +set "VCPKG_TRIPLET=x64-windows-static" +set "VCPKG_EXE=" + +REM Honor pre-existing VCPKG_ROOT / VCPKG_INSTALLATION_ROOT if they look valid. +if "%SIMULATE_CLEAN%"=="0" ( + if defined VCPKG_ROOT ( + if exist "%VCPKG_ROOT%\vcpkg.exe" set "VCPKG_EXE=%VCPKG_ROOT%\vcpkg.exe" + ) + if not defined VCPKG_EXE ( + if defined VCPKG_INSTALLATION_ROOT ( + if exist "%VCPKG_INSTALLATION_ROOT%\vcpkg.exe" ( + set "VCPKG_ROOT=%VCPKG_INSTALLATION_ROOT%" + set "VCPKG_EXE=%VCPKG_INSTALLATION_ROOT%\vcpkg.exe" + ) + ) + ) + if not defined VCPKG_EXE ( + if exist "%USERPROFILE%\vcpkg\vcpkg.exe" ( + set "VCPKG_ROOT=%USERPROFILE%\vcpkg" + set "VCPKG_EXE=%USERPROFILE%\vcpkg\vcpkg.exe" + ) + ) +) + +if not defined VCPKG_EXE ( + echo [INFO] vcpkg not found. Installing into %USERPROFILE%\vcpkg ... + set "VCPKG_ROOT=%USERPROFILE%\vcpkg" + if "%DRY_RUN%"=="0" ( + if not exist "!VCPKG_ROOT!" ( + git clone --depth 1 https://github.com/microsoft/vcpkg.git "!VCPKG_ROOT!" + if errorlevel 1 ( + echo [ERROR] Failed to clone vcpkg. + exit /b 1 + ) + ) + call "!VCPKG_ROOT!\bootstrap-vcpkg.bat" -disableMetrics + if errorlevel 1 ( + echo [ERROR] Failed to bootstrap vcpkg. + exit /b 1 + ) + ) else ( + echo [DRY-RUN] Would run: git clone https://github.com/microsoft/vcpkg.git "!VCPKG_ROOT!" + echo [DRY-RUN] Would run: "!VCPKG_ROOT!\bootstrap-vcpkg.bat" -disableMetrics + ) + set "VCPKG_EXE=!VCPKG_ROOT!\vcpkg.exe" + REM Persist VCPKG_ROOT and PATH to user environment + if "%DRY_RUN%"=="0" ( + setx VCPKG_ROOT "!VCPKG_ROOT!" + for /f "usebackq tokens=2*" %%a in (`reg query "HKCU\Environment" /v PATH 2^>nul`) do set "USR_PATH=%%b" + echo !USR_PATH! | findstr /i /c:"!VCPKG_ROOT!" >nul 2>&1 + if errorlevel 1 ( + setx PATH "!VCPKG_ROOT!;!USR_PATH!" + echo [INFO] vcpkg path added to user PATH permanently. + ) + ) else ( + echo [DRY-RUN] Would run: setx VCPKG_ROOT "!VCPKG_ROOT!" + echo [DRY-RUN] Would run: setx PATH "!VCPKG_ROOT!;..." + ) + set "PATH=!VCPKG_ROOT!;%PATH%" + echo [INFO] vcpkg installed at !VCPKG_ROOT!. +) else ( + echo [INFO] vcpkg already available at !VCPKG_ROOT!. +) + +REM Install protobuf into vcpkg (idempotent: vcpkg detects already-installed +REM packages and skips them). If PROTOBUF_VCPKG_VERSION is set, pass --version. +if "%DRY_RUN%"=="0" ( + if defined PROTOBUF_VCPKG_VERSION ( + echo [INFO] Installing protobuf %PROTOBUF_VCPKG_VERSION% into vcpkg ^(triplet !VCPKG_TRIPLET!^)... + "!VCPKG_EXE!" install "protobuf:!VCPKG_TRIPLET!" --x-version=%PROTOBUF_VCPKG_VERSION% + ) else ( + echo [INFO] Installing protobuf into vcpkg ^(triplet !VCPKG_TRIPLET!^)... + "!VCPKG_EXE!" install "protobuf:!VCPKG_TRIPLET!" + ) + if errorlevel 1 ( + echo [ERROR] vcpkg failed to install protobuf. + exit /b 1 + ) +) else ( + if defined PROTOBUF_VCPKG_VERSION ( + echo [DRY-RUN] Would run: "!VCPKG_EXE!" install "protobuf:!VCPKG_TRIPLET!" --x-version=%PROTOBUF_VCPKG_VERSION% + ) else ( + echo [DRY-RUN] Would run: "!VCPKG_EXE!" install "protobuf:!VCPKG_TRIPLET!" + ) +) + +REM Expose vcpkg-installed protoc on PATH so `buf generate` finds it. +set "PROTOC_TOOLS_DIR=!VCPKG_ROOT!\installed\!VCPKG_TRIPLET!\tools\protobuf" +if exist "!PROTOC_TOOLS_DIR!\protoc.exe" ( + set "PATH=!PROTOC_TOOLS_DIR!;%PATH%" + echo [INFO] vcpkg protoc on PATH: !PROTOC_TOOLS_DIR! +) + echo [INFO] Build environment ready. -REM Export PATH and key MSVC vars back to the caller's environment -endlocal & set "PATH=%PATH%" & set "INCLUDE=%INCLUDE%" & set "LIB=%LIB%" & set "LIBPATH=%LIBPATH%" & set "WindowsSdkDir=%WindowsSdkDir%" & set "VCToolsInstallDir=%VCToolsInstallDir%" +REM Export PATH and key MSVC vars back to the caller's environment. +REM Also export VCPKG_ROOT so subsequent `cmake -DCMAKE_TOOLCHAIN_FILE=%VCPKG_ROOT%\...` +REM invocations resolve in this same cmd session even before the persisted +REM setx value takes effect in newly-spawned processes. +endlocal & set "PATH=%PATH%" & set "INCLUDE=%INCLUDE%" & set "LIB=%LIB%" & set "LIBPATH=%LIBPATH%" & set "WindowsSdkDir=%WindowsSdkDir%" & set "VCToolsInstallDir=%VCToolsInstallDir%" & set "VCPKG_ROOT=%VCPKG_ROOT%" diff --git a/test/cpp-tableau-loader/CMakeLists.txt b/test/cpp-tableau-loader/CMakeLists.txt index 1e0f6dd..7eaa566 100644 --- a/test/cpp-tableau-loader/CMakeLists.txt +++ b/test/cpp-tableau-loader/CMakeLists.txt @@ -33,49 +33,42 @@ endif() # google protobuf # -# Resolution order (first match wins): -# 1. User-provided -DProtobuf_ROOT=... or -DCMAKE_PREFIX_PATH=... — for users -# bringing their own protobuf install (vcpkg / conan / Homebrew / custom). -# 2. Local submodule build at third_party/_submodules/protobuf/.build/_install -# — populated by running ./init.sh (or init.bat on Windows) at the repo root. -# This is OPTIONAL; loader does not require it. -# 3. System-wide protobuf discoverable by find_package (e.g. apt's -# libprotobuf-dev, system-installed CMake config). +# This project does NOT vendor or build protobuf. Bring your own toolchain +# (matching protoc + libprotobuf, since protobuf v22+ enforces a strict +# gencode/runtime version check via PROTOBUF_VERSION in the generated headers). # -# Whichever protobuf is picked, it MUST match the protoc that generated -# tableau.pb.{h,cc}: protobuf v22+ enforces a strict gencode/runtime version -# check via PROTOBUF_VERSION in the generated headers. Install matching -# protoc + libprotobuf yourself, or run ./init.sh with PROTOBUF_REF=vX.Y.Z to -# build a matching submodule copy. -set(LOCAL_PROTOBUF_INSTALL_DIR "${PROJECT_SOURCE_DIR}/../../third_party/_submodules/protobuf/.build/_install") -if(EXISTS "${LOCAL_PROTOBUF_INSTALL_DIR}") - message(STATUS "Found local protobuf submodule install at ${LOCAL_PROTOBUF_INSTALL_DIR}; " - "appending to CMAKE_PREFIX_PATH (user-provided paths still take priority).") - list(APPEND CMAKE_PREFIX_PATH "${LOCAL_PROTOBUF_INSTALL_DIR}") -endif() - -# Use CONFIG mode explicitly to pick up protobuf's own protobuf-config.cmake -# instead of CMake's built-in FindProtobuf.cmake module, which may not handle -# every layout correctly (especially the submodule build dir on Linux vs. -# Windows). CONFIG also gives us the modern protobuf::libprotobuf imported -# target with proper INTERFACE_INCLUDE_DIRECTORIES. +# Common ways to point CMake at your protobuf install: +# - vcpkg (recommended, cross-platform): +# cmake -S . -B build \ +# -DCMAKE_TOOLCHAIN_FILE=/scripts/buildsystems/vcpkg.cmake +# - System package (apt's libprotobuf-dev, Homebrew's protobuf, ...): no +# extra flag needed; find_package picks it up from the default prefixes. +# - Custom install prefix: +# cmake -S . -B build -DCMAKE_PREFIX_PATH=/path/to/protobuf-install +# (or -DProtobuf_ROOT=/path/to/protobuf-install) +# +# CONFIG mode is used explicitly so we get protobuf's own protobuf-config.cmake +# (and the modern protobuf::libprotobuf imported target with proper +# INTERFACE_INCLUDE_DIRECTORIES) instead of CMake's bundled FindProtobuf.cmake. find_package(Protobuf CONFIG REQUIRED) message(STATUS "Using protobuf ${Protobuf_VERSION}") # GoogleTest via FetchContent, pinned to a stable release. Using FetchContent -# (instead of add_subdirectory on protobuf's bundled googletest) gives a -# consistent test framework regardless of which protobuf version is in use. +# (instead of relying on a system gtest) gives a consistent test framework +# regardless of which protobuf version is in use. # # gtest_force_shared_crt: # OFF -> let CMAKE_MSVC_RUNTIME_LIBRARY decide (we set it to MultiThreaded[Debug] # above, i.e. static CRT /MT or /MTd). # ON -> force googletest to use the DYNAMIC CRT (/MD or /MDd). -# Our protobuf submodule is built with protobuf_MSVC_STATIC_RUNTIME=ON (static -# CRT), and our loader_lib also targets static CRT. Forcing gtest to a different -# CRT would mix two C runtimes inside loader.exe; the link may still succeed but -# global-destructor sequencing at process exit can hit an Access Violation -# (gtest's STL objects holding handles allocated by a different heap). Keep gtest -# on the same static CRT as everything else. +# On Windows we expect protobuf to come from vcpkg's x64-windows-static triplet +# (or any other build that uses the static CRT) so that loader_lib, gtest and +# libprotobuf all share one CRT. Mixing CRTs inside loader.exe might still link +# but global-destructor sequencing at process exit can hit an Access Violation +# (gtest's STL objects holding handles allocated by a different heap). If you +# bring a libprotobuf built against the dynamic CRT, also pass +# -DCMAKE_MSVC_RUNTIME_LIBRARY=MultiThreaded$<$:Debug>DLL and +# flip gtest_force_shared_crt to ON to keep everything in sync. include(FetchContent) set(gtest_force_shared_crt OFF CACHE BOOL "" FORCE) FetchContent_Declare( diff --git a/third_party/_submodules/protobuf b/third_party/_submodules/protobuf deleted file mode 160000 index cc7b1b5..0000000 --- a/third_party/_submodules/protobuf +++ /dev/null @@ -1 +0,0 @@ -Subproject commit cc7b1b53234cd7a8f50d90ac3933b240dcf4cd97 From 0fe37c1d46edb24635b8e527b364a2b1b079876a Mon Sep 17 00:00:00 2001 From: Kybxd <627940450@qq.com> Date: Thu, 28 May 2026 17:10:15 +0800 Subject: [PATCH 03/66] fix: export ACTIONS_RESULTS_URL in CI workflow --- .github/workflows/testing-cpp.yml | 3 +- prepare.bat | 869 +++++++++++++++--------------- 2 files changed, 448 insertions(+), 424 deletions(-) diff --git a/.github/workflows/testing-cpp.yml b/.github/workflows/testing-cpp.yml index 343975c..4d8bbc3 100644 --- a/.github/workflows/testing-cpp.yml +++ b/.github/workflows/testing-cpp.yml @@ -64,7 +64,8 @@ jobs: uses: actions/github-script@v7 with: script: | - core.exportVariable('ACTIONS_CACHE_URL', process.env.ACTIONS_CACHE_URL || ''); + core.exportVariable('ACTIONS_CACHE_URL', process.env.ACTIONS_CACHE_URL || ''); + core.exportVariable('ACTIONS_RESULTS_URL', process.env.ACTIONS_RESULTS_URL || ''); core.exportVariable('ACTIONS_RUNTIME_TOKEN', process.env.ACTIONS_RUNTIME_TOKEN || ''); - name: Render vcpkg.json diff --git a/prepare.bat b/prepare.bat index cfc80e2..32d0e83 100644 --- a/prepare.bat +++ b/prepare.bat @@ -1,423 +1,446 @@ -@echo off -setlocal enabledelayedexpansion - -REM =========================================================================== -REM prepare.bat — bootstrap a Windows build environment for the C++ loader. -REM -REM Installs (only if missing): Chocolatey, Ninja, CMake 3.31.8, MSVC Build -REM Tools (Visual Studio 2022 Build Tools), buf CLI, and vcpkg. -REM -REM Then installs `protobuf` (and friends) into vcpkg using the static-CRT -REM triplet x64-windows-static, so that downstream cmake builds can pick it -REM up via -DCMAKE_TOOLCHAIN_FILE=%VCPKG_ROOT%\scripts\buildsystems\vcpkg.cmake. -REM -REM Override `protobuf` to a specific vcpkg port version with PROTOBUF_VCPKG_VERSION: -REM set PROTOBUF_VCPKG_VERSION=3.21.12 && .\prepare.bat -REM (Default is whatever the vcpkg `master` baseline ships, currently the 6.x -REM line. Use 3.21.12 if you need the legacy v3 ABI.) -REM -REM This script is idempotent: re-running it on a machine that already has -REM everything installed is a no-op (a few seconds of probing). Only the MSVC -REM environment variables are re-exported each time, since vcvarsall.bat sets -REM cmd-session-local state that does not persist. -REM =========================================================================== - -REM ----------------------------------------------------------------------- -REM Parse arguments -REM --dry-run : print what would be done, but do not install anything -REM --simulate-clean : pretend nothing is installed (implies --dry-run) -REM ----------------------------------------------------------------------- -set "DRY_RUN=0" -set "SIMULATE_CLEAN=0" -for %%A in (%*) do ( - if /i "%%A"=="--dry-run" set "DRY_RUN=1" - if /i "%%A"=="--simulate-clean" set "DRY_RUN=1" & set "SIMULATE_CLEAN=1" -) -if "%DRY_RUN%"=="1" echo [DRY-RUN] No changes will be made to the system. -if "%SIMULATE_CLEAN%"=="1" echo [DRY-RUN] Simulating a clean machine (all tools treated as not installed). - -echo [INFO] Preparing build environment... - -REM ----------------------------------------------------------------------- -REM Step 0: Ensure Chocolatey is installed -REM ----------------------------------------------------------------------- -set "CHOCO_EXE=" -set "CHOCO_BASE=" -if "%SIMULATE_CLEAN%"=="0" ( - REM Try env var first, then fall back to registry (HKCU then HKLM) - if defined ChocolateyInstall set "CHOCO_BASE=%ChocolateyInstall%" - if not defined CHOCO_BASE ( - for /f "usebackq tokens=2*" %%a in (`reg query "HKCU\Environment" /v ChocolateyInstall 2^>nul`) do set "CHOCO_BASE=%%b" - ) - if not defined CHOCO_BASE ( - for /f "usebackq tokens=2*" %%a in (`reg query "HKLM\SYSTEM\CurrentControlSet\Control\Session Manager\Environment" /v ChocolateyInstall 2^>nul`) do set "CHOCO_BASE=%%b" - ) - if not defined CHOCO_BASE set "CHOCO_BASE=%ALLUSERSPROFILE%\chocolatey" - if exist "!CHOCO_BASE!\bin\choco.exe" set "CHOCO_EXE=!CHOCO_BASE!\bin\choco.exe" - if exist "!CHOCO_BASE!\redirects\choco.exe" set "CHOCO_EXE=!CHOCO_BASE!\redirects\choco.exe" - if exist "!CHOCO_BASE!\tools\choco.exe" set "CHOCO_EXE=!CHOCO_BASE!\tools\choco.exe" -) -if not defined CHOCO_EXE ( - echo [INFO] Chocolatey not found. Installing Chocolatey... - if "%DRY_RUN%"=="0" ( - powershell -NoProfile -ExecutionPolicy Bypass -Command ^ - "[System.Net.ServicePointManager]::SecurityProtocol = [System.Net.ServicePointManager]::SecurityProtocol -bor 3072; iex ((New-Object System.Net.WebClient).DownloadString('https://community.chocolatey.org/install.ps1'))" - if errorlevel 1 ( - echo [ERROR] Failed to install Chocolatey. - exit /b 1 - ) - ) else ( - echo [DRY-RUN] Would run: powershell ... install Chocolatey - ) - REM Add Chocolatey to current session PATH - set "PATH=%ALLUSERSPROFILE%\chocolatey\bin;%PATH%" - REM Persist Chocolatey bin to user PATH permanently - if "%DRY_RUN%"=="0" ( - for /f "usebackq tokens=2*" %%a in (`reg query "HKCU\Environment" /v PATH 2^>nul`) do set "USR_PATH=%%b" - echo !USR_PATH! | findstr /i /c:"%ALLUSERSPROFILE%\chocolatey\bin" >nul 2>&1 - if errorlevel 1 ( - setx PATH "%ALLUSERSPROFILE%\chocolatey\bin;!USR_PATH!" - echo [INFO] Chocolatey bin added to user PATH permanently. - ) - ) else ( - echo [DRY-RUN] Would run: setx PATH "%%ALLUSERSPROFILE%%\chocolatey\bin;..." - ) - echo [INFO] Chocolatey installed successfully. -) else ( - echo [INFO] Chocolatey already installed. -) - -REM Refresh ChocolateyInstall var if it was just installed (also read from registry) -if not defined ChocolateyInstall ( - for /f "usebackq tokens=2*" %%a in (`reg query "HKCU\Environment" /v ChocolateyInstall 2^>nul`) do set "ChocolateyInstall=%%b" -) -if not defined ChocolateyInstall set "ChocolateyInstall=%ALLUSERSPROFILE%\chocolatey" -if "%SIMULATE_CLEAN%"=="0" ( - set "PATH=%ChocolateyInstall%\bin;%ChocolateyInstall%\lib\ninja\tools;%PATH%" -) - -REM ----------------------------------------------------------------------- -REM Step 1: Ensure Ninja is installed via Chocolatey -REM ----------------------------------------------------------------------- -set "NINJA_FOUND=0" -if "%SIMULATE_CLEAN%"=="0" ( - where ninja.exe >nul 2>&1 - if not errorlevel 1 set "NINJA_FOUND=1" -) -if "%NINJA_FOUND%"=="0" ( - echo [INFO] ninja.exe not found. Installing via choco... - if "%DRY_RUN%"=="0" ( - choco install ninja -y --no-progress - if errorlevel 1 ( - echo [ERROR] Failed to install ninja. - exit /b 1 - ) - ) else ( - echo [DRY-RUN] Would run: choco install ninja -y --no-progress - ) - REM Add ninja to current session PATH - if defined ChocolateyInstall ( - set "NINJA_PATH=!ChocolateyInstall!\lib\ninja\tools" - ) else ( - set "NINJA_PATH=%ALLUSERSPROFILE%\chocolatey\lib\ninja\tools" - ) - set "PATH=!NINJA_PATH!;%PATH%" - REM Persist ninja path to user PATH permanently - if "%DRY_RUN%"=="0" ( - for /f "usebackq tokens=2*" %%a in (`reg query "HKCU\Environment" /v PATH 2^>nul`) do set "USR_PATH=%%b" - echo !USR_PATH! | findstr /i /c:"ninja\tools" >nul 2>&1 - if errorlevel 1 ( - setx PATH "!NINJA_PATH!;!USR_PATH!" - echo [INFO] ninja path added to user PATH permanently. - ) - ) else ( - echo [DRY-RUN] Would run: setx PATH "!NINJA_PATH!;..." - ) - echo [INFO] ninja installed successfully. -) else ( - echo [INFO] ninja.exe already in PATH. -) - -REM ----------------------------------------------------------------------- -REM Step 2: Ensure CMake 3.31.8 is installed -REM Try Chocolatey first; fall back to direct MSI download. -REM ----------------------------------------------------------------------- -set "CMAKE_FOUND=0" -if "%SIMULATE_CLEAN%"=="0" ( - where cmake.exe >nul 2>&1 - if not errorlevel 1 set "CMAKE_FOUND=1" -) -if "%CMAKE_FOUND%"=="0" ( - echo [INFO] cmake.exe not found. Installing CMake 3.31.8... - if "%DRY_RUN%"=="0" ( - set "CMAKE_INSTALLED=0" - REM --- Attempt 1: Chocolatey --- - choco install cmake --version=3.31.8 --installargs "'ADD_CMAKE_TO_PATH=System'" -y --no-progress >nul 2>&1 && set "CMAKE_INSTALLED=1" - if "!CMAKE_INSTALLED!"=="0" ( - echo [WARN] choco install cmake failed. Falling back to direct MSI download... - set "CMAKE_MSI=%TEMP%\cmake-3.31.8-windows-x86_64.msi" - powershell -NoProfile -Command "(New-Object Net.WebClient).DownloadFile('https://github.com/Kitware/CMake/releases/download/v3.31.8/cmake-3.31.8-windows-x86_64.msi','!CMAKE_MSI!')" - if not exist "!CMAKE_MSI!" ( - echo [ERROR] Failed to download CMake MSI. - exit /b 1 - ) - msiexec /i "!CMAKE_MSI!" ADD_CMAKE_TO_PATH=System /quiet /norestart - if errorlevel 1 ( - echo [ERROR] Failed to install CMake from MSI. - exit /b 1 - ) - del /q "!CMAKE_MSI!" 2>nul - ) - ) else ( - echo [DRY-RUN] Would run: choco install cmake --version=3.31.8 ... (or fallback to MSI download) - ) - REM Add cmake to current session PATH - set "CMAKE_PATH=C:\Program Files\CMake\bin" - set "PATH=!CMAKE_PATH!;%PATH%" - REM Persist cmake path to user PATH permanently - if "%DRY_RUN%"=="0" ( - for /f "usebackq tokens=2*" %%a in (`reg query "HKCU\Environment" /v PATH 2^>nul`) do set "USR_PATH=%%b" - echo !USR_PATH! | findstr /i /c:"CMake\bin" >nul 2>&1 - if errorlevel 1 ( - setx PATH "!CMAKE_PATH!;!USR_PATH!" - echo [INFO] cmake path added to user PATH permanently. - ) - ) else ( - echo [DRY-RUN] Would run: setx PATH "!CMAKE_PATH!;..." - ) - echo [INFO] cmake installed successfully. -) else ( - echo [INFO] cmake.exe already in PATH. -) - -REM ----------------------------------------------------------------------- -REM Step 3: Ensure MSVC compiler (cl.exe) is available, then activate its -REM environment for this cmd session via vcvarsall.bat. The CI -REM workflow uses ilammy/msvc-dev-cmd@v1 to do the same thing. -REM ----------------------------------------------------------------------- -set "CL_FOUND=0" -if "%SIMULATE_CLEAN%"=="0" ( - where cl.exe >nul 2>&1 - if not errorlevel 1 set "CL_FOUND=1" -) -set "SKIP_MSVC=0" -if "%CL_FOUND%"=="0" ( - echo [INFO] cl.exe not found. Searching for existing VS installation... - set "VSWHERE=" - if "%SIMULATE_CLEAN%"=="0" ( - for %%d in ("%ProgramFiles(x86)%" "%ProgramFiles%") do ( - if not defined VSWHERE ( - if exist "%%~d\Microsoft Visual Studio\Installer\vswhere.exe" ( - set "VSWHERE=%%~d\Microsoft Visual Studio\Installer\vswhere.exe" - ) - ) - ) - ) - if not defined VSWHERE ( - echo [INFO] Visual Studio not found. Installing via choco... - if "%DRY_RUN%"=="0" ( - choco install visualstudio2022buildtools --package-parameters "--add Microsoft.VisualStudio.Workload.VCTools --includeRecommended --passive --locale en-US" -y - if errorlevel 1 ( - echo [ERROR] Failed to install Visual Studio Build Tools. - exit /b 1 - ) - echo [INFO] Visual Studio Build Tools installed successfully. - REM Re-search vswhere after installation - for %%d in ("%ProgramFiles(x86)%" "%ProgramFiles%") do ( - if not defined VSWHERE ( - if exist "%%~d\Microsoft Visual Studio\Installer\vswhere.exe" ( - set "VSWHERE=%%~d\Microsoft Visual Studio\Installer\vswhere.exe" - ) - ) - ) - ) else ( - echo [DRY-RUN] Would run: choco install visualstudio2022buildtools ... - echo [DRY-RUN] Would search vswhere.exe after installation. - set "SKIP_MSVC=1" - ) - ) - if "!SKIP_MSVC!"=="0" ( - if not defined VSWHERE ( - echo [ERROR] vswhere.exe still not found after installation. Please restart and retry. - exit /b 1 - ) - set "VCVARSALL=" - for /f "usebackq delims=" %%p in (`"!VSWHERE!" -latest -products * -requires Microsoft.VisualStudio.Component.VC.Tools.x86.x64 -property installationPath`) do ( - set "VCVARSALL=%%p\VC\Auxiliary\Build\vcvarsall.bat" - ) - if not defined VCVARSALL ( - echo [ERROR] No VS installation with C++ tools detected. - exit /b 1 - ) - if not exist "!VCVARSALL!" ( - echo [ERROR] vcvarsall.bat not found at: !VCVARSALL! - exit /b 1 - ) - echo [INFO] Initializing MSVC environment from: !VCVARSALL! - call "!VCVARSALL!" x64 - ) -) else ( - echo [INFO] cl.exe already in PATH, skipping MSVC environment setup. -) - -REM ----------------------------------------------------------------------- -REM Step 4: Ensure buf CLI is installed -REM The CI workflow uses bufbuild/buf-action@v1 (also pinned to -REM BUF_VERSION below) to do the same thing. -REM buf is a single self-contained .exe; install it under -REM %LOCALAPPDATA%\buf\bin\buf.exe to avoid requiring admin rights. -REM ----------------------------------------------------------------------- -set "BUF_VERSION=1.67.0" -set "BUF_FOUND=0" -if "%SIMULATE_CLEAN%"=="0" ( - where buf.exe >nul 2>&1 - if not errorlevel 1 set "BUF_FOUND=1" -) -if "%BUF_FOUND%"=="0" ( - echo [INFO] buf.exe not found. Installing buf %BUF_VERSION%... - set "BUF_DIR=%LOCALAPPDATA%\buf\bin" - set "BUF_EXE=!BUF_DIR!\buf.exe" - set "BUF_URL=https://github.com/bufbuild/buf/releases/download/v%BUF_VERSION%/buf-Windows-x86_64.exe" - if "%DRY_RUN%"=="0" ( - if not exist "!BUF_DIR!" mkdir "!BUF_DIR!" - powershell -NoProfile -Command "(New-Object Net.WebClient).DownloadFile('!BUF_URL!','!BUF_EXE!')" - if not exist "!BUF_EXE!" ( - echo [ERROR] Failed to download buf from !BUF_URL!. - exit /b 1 - ) - ) else ( - echo [DRY-RUN] Would run: download !BUF_URL! to !BUF_EXE! - ) - REM Add buf to current session PATH - set "PATH=!BUF_DIR!;%PATH%" - REM Persist buf path to user PATH permanently - if "%DRY_RUN%"=="0" ( - for /f "usebackq tokens=2*" %%a in (`reg query "HKCU\Environment" /v PATH 2^>nul`) do set "USR_PATH=%%b" - echo !USR_PATH! | findstr /i /c:"buf\bin" >nul 2>&1 - if errorlevel 1 ( - setx PATH "!BUF_DIR!;!USR_PATH!" - echo [INFO] buf path added to user PATH permanently. - ) - ) else ( - echo [DRY-RUN] Would run: setx PATH "!BUF_DIR!;..." - ) - echo [INFO] buf installed successfully. -) else ( - echo [INFO] buf.exe already in PATH. -) - -REM ----------------------------------------------------------------------- -REM Step 5: Ensure vcpkg is installed and `protobuf` is provisioned -REM -REM Resolution order for the vcpkg install location: -REM 1. Existing %VCPKG_ROOT% if it points at a usable bootstrap. -REM 2. Existing %VCPKG_INSTALLATION_ROOT% (set on GitHub-hosted runners). -REM 3. Fresh clone into %USERPROFILE%\vcpkg. -REM -REM We then run `vcpkg install protobuf:x64-windows-static` so that the -REM static-CRT libprotobuf + protoc match the loader build (CMakeLists.txt -REM forces /MT[d] via CMAKE_MSVC_RUNTIME_LIBRARY). -REM -REM Override the protobuf port version (e.g. for the legacy v3 line) with: -REM set PROTOBUF_VCPKG_VERSION=3.21.12 && .\prepare.bat -REM ----------------------------------------------------------------------- -set "VCPKG_TRIPLET=x64-windows-static" -set "VCPKG_EXE=" - -REM Honor pre-existing VCPKG_ROOT / VCPKG_INSTALLATION_ROOT if they look valid. -if "%SIMULATE_CLEAN%"=="0" ( - if defined VCPKG_ROOT ( - if exist "%VCPKG_ROOT%\vcpkg.exe" set "VCPKG_EXE=%VCPKG_ROOT%\vcpkg.exe" - ) - if not defined VCPKG_EXE ( - if defined VCPKG_INSTALLATION_ROOT ( - if exist "%VCPKG_INSTALLATION_ROOT%\vcpkg.exe" ( - set "VCPKG_ROOT=%VCPKG_INSTALLATION_ROOT%" - set "VCPKG_EXE=%VCPKG_INSTALLATION_ROOT%\vcpkg.exe" - ) - ) - ) - if not defined VCPKG_EXE ( - if exist "%USERPROFILE%\vcpkg\vcpkg.exe" ( - set "VCPKG_ROOT=%USERPROFILE%\vcpkg" - set "VCPKG_EXE=%USERPROFILE%\vcpkg\vcpkg.exe" - ) - ) -) - -if not defined VCPKG_EXE ( - echo [INFO] vcpkg not found. Installing into %USERPROFILE%\vcpkg ... - set "VCPKG_ROOT=%USERPROFILE%\vcpkg" - if "%DRY_RUN%"=="0" ( - if not exist "!VCPKG_ROOT!" ( - git clone --depth 1 https://github.com/microsoft/vcpkg.git "!VCPKG_ROOT!" - if errorlevel 1 ( - echo [ERROR] Failed to clone vcpkg. - exit /b 1 - ) - ) - call "!VCPKG_ROOT!\bootstrap-vcpkg.bat" -disableMetrics - if errorlevel 1 ( - echo [ERROR] Failed to bootstrap vcpkg. - exit /b 1 - ) - ) else ( - echo [DRY-RUN] Would run: git clone https://github.com/microsoft/vcpkg.git "!VCPKG_ROOT!" - echo [DRY-RUN] Would run: "!VCPKG_ROOT!\bootstrap-vcpkg.bat" -disableMetrics - ) - set "VCPKG_EXE=!VCPKG_ROOT!\vcpkg.exe" - REM Persist VCPKG_ROOT and PATH to user environment - if "%DRY_RUN%"=="0" ( - setx VCPKG_ROOT "!VCPKG_ROOT!" - for /f "usebackq tokens=2*" %%a in (`reg query "HKCU\Environment" /v PATH 2^>nul`) do set "USR_PATH=%%b" - echo !USR_PATH! | findstr /i /c:"!VCPKG_ROOT!" >nul 2>&1 - if errorlevel 1 ( - setx PATH "!VCPKG_ROOT!;!USR_PATH!" - echo [INFO] vcpkg path added to user PATH permanently. - ) - ) else ( - echo [DRY-RUN] Would run: setx VCPKG_ROOT "!VCPKG_ROOT!" - echo [DRY-RUN] Would run: setx PATH "!VCPKG_ROOT!;..." - ) - set "PATH=!VCPKG_ROOT!;%PATH%" - echo [INFO] vcpkg installed at !VCPKG_ROOT!. -) else ( - echo [INFO] vcpkg already available at !VCPKG_ROOT!. -) - -REM Install protobuf into vcpkg (idempotent: vcpkg detects already-installed -REM packages and skips them). If PROTOBUF_VCPKG_VERSION is set, pass --version. -if "%DRY_RUN%"=="0" ( - if defined PROTOBUF_VCPKG_VERSION ( - echo [INFO] Installing protobuf %PROTOBUF_VCPKG_VERSION% into vcpkg ^(triplet !VCPKG_TRIPLET!^)... - "!VCPKG_EXE!" install "protobuf:!VCPKG_TRIPLET!" --x-version=%PROTOBUF_VCPKG_VERSION% - ) else ( - echo [INFO] Installing protobuf into vcpkg ^(triplet !VCPKG_TRIPLET!^)... - "!VCPKG_EXE!" install "protobuf:!VCPKG_TRIPLET!" - ) - if errorlevel 1 ( - echo [ERROR] vcpkg failed to install protobuf. - exit /b 1 - ) -) else ( - if defined PROTOBUF_VCPKG_VERSION ( - echo [DRY-RUN] Would run: "!VCPKG_EXE!" install "protobuf:!VCPKG_TRIPLET!" --x-version=%PROTOBUF_VCPKG_VERSION% - ) else ( - echo [DRY-RUN] Would run: "!VCPKG_EXE!" install "protobuf:!VCPKG_TRIPLET!" - ) -) - -REM Expose vcpkg-installed protoc on PATH so `buf generate` finds it. -set "PROTOC_TOOLS_DIR=!VCPKG_ROOT!\installed\!VCPKG_TRIPLET!\tools\protobuf" -if exist "!PROTOC_TOOLS_DIR!\protoc.exe" ( - set "PATH=!PROTOC_TOOLS_DIR!;%PATH%" - echo [INFO] vcpkg protoc on PATH: !PROTOC_TOOLS_DIR! -) - -echo [INFO] Build environment ready. - -REM Export PATH and key MSVC vars back to the caller's environment. -REM Also export VCPKG_ROOT so subsequent `cmake -DCMAKE_TOOLCHAIN_FILE=%VCPKG_ROOT%\...` -REM invocations resolve in this same cmd session even before the persisted -REM setx value takes effect in newly-spawned processes. -endlocal & set "PATH=%PATH%" & set "INCLUDE=%INCLUDE%" & set "LIB=%LIB%" & set "LIBPATH=%LIBPATH%" & set "WindowsSdkDir=%WindowsSdkDir%" & set "VCToolsInstallDir=%VCToolsInstallDir%" & set "VCPKG_ROOT=%VCPKG_ROOT%" +@echo off +setlocal enabledelayedexpansion + +REM =========================================================================== +REM prepare.bat — bootstrap a Windows build environment for the C++ loader. +REM +REM Installs (only if missing): Chocolatey, Ninja, CMake 3.31.8, MSVC Build +REM Tools (Visual Studio 2022 Build Tools), buf CLI, and vcpkg. +REM +REM Then installs `protobuf` (and friends) into vcpkg using the static-CRT +REM triplet x64-windows-static, so that downstream cmake builds can pick it +REM up via -DCMAKE_TOOLCHAIN_FILE=%VCPKG_ROOT%\scripts\buildsystems\vcpkg.cmake. +REM +REM Override `protobuf` to a specific vcpkg port version with PROTOBUF_VCPKG_VERSION: +REM set PROTOBUF_VCPKG_VERSION=3.21.12 && .\prepare.bat +REM (Default is whatever the vcpkg `master` baseline ships, currently the 6.x +REM line. Use 3.21.12 if you need the legacy v3 ABI.) +REM +REM This script is idempotent: re-running it on a machine that already has +REM everything installed is a no-op (a few seconds of probing). Only the MSVC +REM environment variables are re-exported each time, since vcvarsall.bat sets +REM cmd-session-local state that does not persist. +REM =========================================================================== + +REM ----------------------------------------------------------------------- +REM Parse arguments +REM --dry-run : print what would be done, but do not install anything +REM --simulate-clean : pretend nothing is installed (implies --dry-run) +REM ----------------------------------------------------------------------- +set "DRY_RUN=0" +set "SIMULATE_CLEAN=0" +for %%A in (%*) do ( + if /i "%%A"=="--dry-run" set "DRY_RUN=1" + if /i "%%A"=="--simulate-clean" set "DRY_RUN=1" & set "SIMULATE_CLEAN=1" +) +if "%DRY_RUN%"=="1" echo [DRY-RUN] No changes will be made to the system. +if "%SIMULATE_CLEAN%"=="1" echo [DRY-RUN] Simulating a clean machine (all tools treated as not installed). + +echo [INFO] Preparing build environment... + +REM ----------------------------------------------------------------------- +REM Step 0: Ensure Chocolatey is installed +REM ----------------------------------------------------------------------- +set "CHOCO_EXE=" +set "CHOCO_BASE=" +if "%SIMULATE_CLEAN%"=="0" ( + REM Try env var first, then fall back to registry (HKCU then HKLM) + if defined ChocolateyInstall set "CHOCO_BASE=%ChocolateyInstall%" + if not defined CHOCO_BASE ( + for /f "usebackq tokens=2*" %%a in (`reg query "HKCU\Environment" /v ChocolateyInstall 2^>nul`) do set "CHOCO_BASE=%%b" + ) + if not defined CHOCO_BASE ( + for /f "usebackq tokens=2*" %%a in (`reg query "HKLM\SYSTEM\CurrentControlSet\Control\Session Manager\Environment" /v ChocolateyInstall 2^>nul`) do set "CHOCO_BASE=%%b" + ) + if not defined CHOCO_BASE set "CHOCO_BASE=%ALLUSERSPROFILE%\chocolatey" + if exist "!CHOCO_BASE!\bin\choco.exe" set "CHOCO_EXE=!CHOCO_BASE!\bin\choco.exe" + if exist "!CHOCO_BASE!\redirects\choco.exe" set "CHOCO_EXE=!CHOCO_BASE!\redirects\choco.exe" + if exist "!CHOCO_BASE!\tools\choco.exe" set "CHOCO_EXE=!CHOCO_BASE!\tools\choco.exe" +) +if not defined CHOCO_EXE ( + echo [INFO] Chocolatey not found. Installing Chocolatey... + if "%DRY_RUN%"=="0" ( + powershell -NoProfile -ExecutionPolicy Bypass -Command ^ + "[System.Net.ServicePointManager]::SecurityProtocol = [System.Net.ServicePointManager]::SecurityProtocol -bor 3072; iex ((New-Object System.Net.WebClient).DownloadString('https://community.chocolatey.org/install.ps1'))" + if errorlevel 1 ( + echo [ERROR] Failed to install Chocolatey. + exit /b 1 + ) + ) else ( + echo [DRY-RUN] Would run: powershell ... install Chocolatey + ) + REM Add Chocolatey to current session PATH + set "PATH=%ALLUSERSPROFILE%\chocolatey\bin;%PATH%" + REM Persist Chocolatey bin to user PATH permanently + if "%DRY_RUN%"=="0" ( + for /f "usebackq tokens=2*" %%a in (`reg query "HKCU\Environment" /v PATH 2^>nul`) do set "USR_PATH=%%b" + echo !USR_PATH! | findstr /i /c:"%ALLUSERSPROFILE%\chocolatey\bin" >nul 2>&1 + if errorlevel 1 ( + setx PATH "%ALLUSERSPROFILE%\chocolatey\bin;!USR_PATH!" + echo [INFO] Chocolatey bin added to user PATH permanently. + ) + ) else ( + echo [DRY-RUN] Would run: setx PATH "%%ALLUSERSPROFILE%%\chocolatey\bin;..." + ) + echo [INFO] Chocolatey installed successfully. +) else ( + echo [INFO] Chocolatey already installed. +) + +REM Refresh ChocolateyInstall var if it was just installed (also read from registry) +if not defined ChocolateyInstall ( + for /f "usebackq tokens=2*" %%a in (`reg query "HKCU\Environment" /v ChocolateyInstall 2^>nul`) do set "ChocolateyInstall=%%b" +) +if not defined ChocolateyInstall set "ChocolateyInstall=%ALLUSERSPROFILE%\chocolatey" +if "%SIMULATE_CLEAN%"=="0" ( + set "PATH=%ChocolateyInstall%\bin;%ChocolateyInstall%\lib\ninja\tools;%PATH%" +) + +REM ----------------------------------------------------------------------- +REM Step 1: Ensure Ninja is installed via Chocolatey +REM ----------------------------------------------------------------------- +set "NINJA_FOUND=0" +if "%SIMULATE_CLEAN%"=="0" ( + where ninja.exe >nul 2>&1 + if not errorlevel 1 set "NINJA_FOUND=1" +) +if "%NINJA_FOUND%"=="0" ( + echo [INFO] ninja.exe not found. Installing via choco... + if "%DRY_RUN%"=="0" ( + choco install ninja -y --no-progress + if errorlevel 1 ( + echo [ERROR] Failed to install ninja. + exit /b 1 + ) + ) else ( + echo [DRY-RUN] Would run: choco install ninja -y --no-progress + ) + REM Add ninja to current session PATH + if defined ChocolateyInstall ( + set "NINJA_PATH=!ChocolateyInstall!\lib\ninja\tools" + ) else ( + set "NINJA_PATH=%ALLUSERSPROFILE%\chocolatey\lib\ninja\tools" + ) + set "PATH=!NINJA_PATH!;%PATH%" + REM Persist ninja path to user PATH permanently + if "%DRY_RUN%"=="0" ( + for /f "usebackq tokens=2*" %%a in (`reg query "HKCU\Environment" /v PATH 2^>nul`) do set "USR_PATH=%%b" + echo !USR_PATH! | findstr /i /c:"ninja\tools" >nul 2>&1 + if errorlevel 1 ( + setx PATH "!NINJA_PATH!;!USR_PATH!" + echo [INFO] ninja path added to user PATH permanently. + ) + ) else ( + echo [DRY-RUN] Would run: setx PATH "!NINJA_PATH!;..." + ) + echo [INFO] ninja installed successfully. +) else ( + echo [INFO] ninja.exe already in PATH. +) + +REM ----------------------------------------------------------------------- +REM Step 2: Ensure CMake 3.31.8 is installed +REM Try Chocolatey first; fall back to direct MSI download. +REM ----------------------------------------------------------------------- +set "CMAKE_FOUND=0" +if "%SIMULATE_CLEAN%"=="0" ( + where cmake.exe >nul 2>&1 + if not errorlevel 1 set "CMAKE_FOUND=1" +) +if "%CMAKE_FOUND%"=="0" ( + echo [INFO] cmake.exe not found. Installing CMake 3.31.8... + if "%DRY_RUN%"=="0" ( + set "CMAKE_INSTALLED=0" + REM --- Attempt 1: Chocolatey --- + choco install cmake --version=3.31.8 --installargs "'ADD_CMAKE_TO_PATH=System'" -y --no-progress >nul 2>&1 && set "CMAKE_INSTALLED=1" + if "!CMAKE_INSTALLED!"=="0" ( + echo [WARN] choco install cmake failed. Falling back to direct MSI download... + set "CMAKE_MSI=%TEMP%\cmake-3.31.8-windows-x86_64.msi" + powershell -NoProfile -Command "(New-Object Net.WebClient).DownloadFile('https://github.com/Kitware/CMake/releases/download/v3.31.8/cmake-3.31.8-windows-x86_64.msi','!CMAKE_MSI!')" + if not exist "!CMAKE_MSI!" ( + echo [ERROR] Failed to download CMake MSI. + exit /b 1 + ) + msiexec /i "!CMAKE_MSI!" ADD_CMAKE_TO_PATH=System /quiet /norestart + if errorlevel 1 ( + echo [ERROR] Failed to install CMake from MSI. + exit /b 1 + ) + del /q "!CMAKE_MSI!" 2>nul + ) + ) else ( + echo [DRY-RUN] Would run: choco install cmake --version=3.31.8 ... (or fallback to MSI download) + ) + REM Add cmake to current session PATH + set "CMAKE_PATH=C:\Program Files\CMake\bin" + set "PATH=!CMAKE_PATH!;%PATH%" + REM Persist cmake path to user PATH permanently + if "%DRY_RUN%"=="0" ( + for /f "usebackq tokens=2*" %%a in (`reg query "HKCU\Environment" /v PATH 2^>nul`) do set "USR_PATH=%%b" + echo !USR_PATH! | findstr /i /c:"CMake\bin" >nul 2>&1 + if errorlevel 1 ( + setx PATH "!CMAKE_PATH!;!USR_PATH!" + echo [INFO] cmake path added to user PATH permanently. + ) + ) else ( + echo [DRY-RUN] Would run: setx PATH "!CMAKE_PATH!;..." + ) + echo [INFO] cmake installed successfully. +) else ( + echo [INFO] cmake.exe already in PATH. +) + +REM ----------------------------------------------------------------------- +REM Step 3: Ensure MSVC compiler (cl.exe) is available, then activate its +REM environment for this cmd session via vcvarsall.bat. The CI +REM workflow uses ilammy/msvc-dev-cmd@v1 to do the same thing. +REM ----------------------------------------------------------------------- +set "CL_FOUND=0" +if "%SIMULATE_CLEAN%"=="0" ( + where cl.exe >nul 2>&1 + if not errorlevel 1 set "CL_FOUND=1" +) +set "SKIP_MSVC=0" +if "%CL_FOUND%"=="0" ( + echo [INFO] cl.exe not found. Searching for existing VS installation... + set "VSWHERE=" + if "%SIMULATE_CLEAN%"=="0" ( + for %%d in ("%ProgramFiles(x86)%" "%ProgramFiles%") do ( + if not defined VSWHERE ( + if exist "%%~d\Microsoft Visual Studio\Installer\vswhere.exe" ( + set "VSWHERE=%%~d\Microsoft Visual Studio\Installer\vswhere.exe" + ) + ) + ) + ) + if not defined VSWHERE ( + echo [INFO] Visual Studio not found. Installing via choco... + if "%DRY_RUN%"=="0" ( + choco install visualstudio2022buildtools --package-parameters "--add Microsoft.VisualStudio.Workload.VCTools --includeRecommended --passive --locale en-US" -y + if errorlevel 1 ( + echo [ERROR] Failed to install Visual Studio Build Tools. + exit /b 1 + ) + echo [INFO] Visual Studio Build Tools installed successfully. + REM Re-search vswhere after installation + for %%d in ("%ProgramFiles(x86)%" "%ProgramFiles%") do ( + if not defined VSWHERE ( + if exist "%%~d\Microsoft Visual Studio\Installer\vswhere.exe" ( + set "VSWHERE=%%~d\Microsoft Visual Studio\Installer\vswhere.exe" + ) + ) + ) + ) else ( + echo [DRY-RUN] Would run: choco install visualstudio2022buildtools ... + echo [DRY-RUN] Would search vswhere.exe after installation. + set "SKIP_MSVC=1" + ) + ) + if "!SKIP_MSVC!"=="0" ( + if not defined VSWHERE ( + echo [ERROR] vswhere.exe still not found after installation. Please restart and retry. + exit /b 1 + ) + set "VCVARSALL=" + for /f "usebackq delims=" %%p in (`"!VSWHERE!" -latest -products * -requires Microsoft.VisualStudio.Component.VC.Tools.x86.x64 -property installationPath`) do ( + set "VCVARSALL=%%p\VC\Auxiliary\Build\vcvarsall.bat" + ) + if not defined VCVARSALL ( + echo [ERROR] No VS installation with C++ tools detected. + exit /b 1 + ) + if not exist "!VCVARSALL!" ( + echo [ERROR] vcvarsall.bat not found at: !VCVARSALL! + exit /b 1 + ) + echo [INFO] Initializing MSVC environment from: !VCVARSALL! + call "!VCVARSALL!" x64 + ) +) else ( + echo [INFO] cl.exe already in PATH, skipping MSVC environment setup. +) + +REM ----------------------------------------------------------------------- +REM Step 4: Ensure buf CLI is installed +REM The CI workflow uses bufbuild/buf-action@v1 (also pinned to +REM BUF_VERSION below) to do the same thing. +REM buf is a single self-contained .exe; install it under +REM %LOCALAPPDATA%\buf\bin\buf.exe to avoid requiring admin rights. +REM ----------------------------------------------------------------------- +set "BUF_VERSION=1.67.0" +set "BUF_FOUND=0" +if "%SIMULATE_CLEAN%"=="0" ( + where buf.exe >nul 2>&1 + if not errorlevel 1 set "BUF_FOUND=1" +) +if "%BUF_FOUND%"=="0" ( + echo [INFO] buf.exe not found. Installing buf %BUF_VERSION%... + set "BUF_DIR=%LOCALAPPDATA%\buf\bin" + set "BUF_EXE=!BUF_DIR!\buf.exe" + set "BUF_URL=https://github.com/bufbuild/buf/releases/download/v%BUF_VERSION%/buf-Windows-x86_64.exe" + if "%DRY_RUN%"=="0" ( + if not exist "!BUF_DIR!" mkdir "!BUF_DIR!" + powershell -NoProfile -Command "(New-Object Net.WebClient).DownloadFile('!BUF_URL!','!BUF_EXE!')" + if not exist "!BUF_EXE!" ( + echo [ERROR] Failed to download buf from !BUF_URL!. + exit /b 1 + ) + ) else ( + echo [DRY-RUN] Would run: download !BUF_URL! to !BUF_EXE! + ) + REM Add buf to current session PATH + set "PATH=!BUF_DIR!;%PATH%" + REM Persist buf path to user PATH permanently + if "%DRY_RUN%"=="0" ( + for /f "usebackq tokens=2*" %%a in (`reg query "HKCU\Environment" /v PATH 2^>nul`) do set "USR_PATH=%%b" + echo !USR_PATH! | findstr /i /c:"buf\bin" >nul 2>&1 + if errorlevel 1 ( + setx PATH "!BUF_DIR!;!USR_PATH!" + echo [INFO] buf path added to user PATH permanently. + ) + ) else ( + echo [DRY-RUN] Would run: setx PATH "!BUF_DIR!;..." + ) + echo [INFO] buf installed successfully. +) else ( + echo [INFO] buf.exe already in PATH. +) + +REM ----------------------------------------------------------------------- +REM Step 5: Ensure vcpkg is installed and `protobuf` is provisioned +REM +REM Resolution order for the vcpkg install location: +REM 1. Existing %VCPKG_ROOT% if it points at a usable classic-mode bootstrap. +REM 2. Existing %VCPKG_INSTALLATION_ROOT% (set on GitHub-hosted runners). +REM 3. Existing %USERPROFILE%\vcpkg (a previous run of this script). +REM 4. Fresh clone into %USERPROFILE%\vcpkg. +REM +REM A "usable" vcpkg root must contain BOTH vcpkg.exe AND bootstrap-vcpkg.bat. +REM This deliberately rejects the manifest-only vcpkg shipped under +REM C:\Program Files\Microsoft Visual Studio\2022\\VC\vcpkg +REM which has no bootstrap script and refuses classic-mode `vcpkg install +REM :` with: "Could not locate a manifest (vcpkg.json) above +REM the current working directory. This vcpkg distribution does not have a +REM classic mode instance." +REM +REM We then run `vcpkg install protobuf:x64-windows-static` so that the +REM static-CRT libprotobuf + protoc match the loader build (CMakeLists.txt +REM forces /MT[d] via CMAKE_MSVC_RUNTIME_LIBRARY). +REM +REM Override the protobuf port version (e.g. for the legacy v3 line) with: +REM set PROTOBUF_VCPKG_VERSION=3.21.12 && .\prepare.bat +REM ----------------------------------------------------------------------- +set "VCPKG_TRIPLET=x64-windows-static" +set "VCPKG_EXE=" + +REM Honor pre-existing VCPKG_ROOT / VCPKG_INSTALLATION_ROOT only if they +REM point at a classic-mode-capable vcpkg (i.e. bootstrap-vcpkg.bat is present). +if "%SIMULATE_CLEAN%"=="0" ( + if defined VCPKG_ROOT ( + if exist "%VCPKG_ROOT%\vcpkg.exe" ( + if exist "%VCPKG_ROOT%\bootstrap-vcpkg.bat" ( + set "VCPKG_EXE=%VCPKG_ROOT%\vcpkg.exe" + ) else ( + echo [WARN] %VCPKG_ROOT% looks like a manifest-only vcpkg ^(no bootstrap-vcpkg.bat^); ignoring. + set "VCPKG_ROOT=" + ) + ) + ) + if not defined VCPKG_EXE ( + if defined VCPKG_INSTALLATION_ROOT ( + if exist "%VCPKG_INSTALLATION_ROOT%\vcpkg.exe" ( + if exist "%VCPKG_INSTALLATION_ROOT%\bootstrap-vcpkg.bat" ( + set "VCPKG_ROOT=%VCPKG_INSTALLATION_ROOT%" + set "VCPKG_EXE=%VCPKG_INSTALLATION_ROOT%\vcpkg.exe" + ) else ( + echo [WARN] %VCPKG_INSTALLATION_ROOT% looks like a manifest-only vcpkg; ignoring. + ) + ) + ) + ) + if not defined VCPKG_EXE ( + if exist "%USERPROFILE%\vcpkg\vcpkg.exe" ( + if exist "%USERPROFILE%\vcpkg\bootstrap-vcpkg.bat" ( + set "VCPKG_ROOT=%USERPROFILE%\vcpkg" + set "VCPKG_EXE=%USERPROFILE%\vcpkg\vcpkg.exe" + ) + ) + ) +) + +if not defined VCPKG_EXE ( + echo [INFO] vcpkg not found. Installing into %USERPROFILE%\vcpkg ... + set "VCPKG_ROOT=%USERPROFILE%\vcpkg" + if "%DRY_RUN%"=="0" ( + if not exist "!VCPKG_ROOT!" ( + git clone --depth 1 https://github.com/microsoft/vcpkg.git "!VCPKG_ROOT!" + if errorlevel 1 ( + echo [ERROR] Failed to clone vcpkg. + exit /b 1 + ) + ) + call "!VCPKG_ROOT!\bootstrap-vcpkg.bat" -disableMetrics + if errorlevel 1 ( + echo [ERROR] Failed to bootstrap vcpkg. + exit /b 1 + ) + ) else ( + echo [DRY-RUN] Would run: git clone https://github.com/microsoft/vcpkg.git "!VCPKG_ROOT!" + echo [DRY-RUN] Would run: "!VCPKG_ROOT!\bootstrap-vcpkg.bat" -disableMetrics + ) + set "VCPKG_EXE=!VCPKG_ROOT!\vcpkg.exe" + REM Persist VCPKG_ROOT and PATH to user environment + if "%DRY_RUN%"=="0" ( + setx VCPKG_ROOT "!VCPKG_ROOT!" + for /f "usebackq tokens=2*" %%a in (`reg query "HKCU\Environment" /v PATH 2^>nul`) do set "USR_PATH=%%b" + echo !USR_PATH! | findstr /i /c:"!VCPKG_ROOT!" >nul 2>&1 + if errorlevel 1 ( + setx PATH "!VCPKG_ROOT!;!USR_PATH!" + echo [INFO] vcpkg path added to user PATH permanently. + ) + ) else ( + echo [DRY-RUN] Would run: setx VCPKG_ROOT "!VCPKG_ROOT!" + echo [DRY-RUN] Would run: setx PATH "!VCPKG_ROOT!;..." + ) + set "PATH=!VCPKG_ROOT!;%PATH%" + echo [INFO] vcpkg installed at !VCPKG_ROOT!. +) else ( + echo [INFO] vcpkg already available at !VCPKG_ROOT!. +) + +REM Install protobuf into vcpkg (idempotent: vcpkg detects already-installed +REM packages and skips them). If PROTOBUF_VCPKG_VERSION is set, pass --version. +if "%DRY_RUN%"=="0" ( + if defined PROTOBUF_VCPKG_VERSION ( + echo [INFO] Installing protobuf %PROTOBUF_VCPKG_VERSION% into vcpkg ^(triplet !VCPKG_TRIPLET!^)... + "!VCPKG_EXE!" install "protobuf:!VCPKG_TRIPLET!" --x-version=%PROTOBUF_VCPKG_VERSION% + ) else ( + echo [INFO] Installing protobuf into vcpkg ^(triplet !VCPKG_TRIPLET!^)... + "!VCPKG_EXE!" install "protobuf:!VCPKG_TRIPLET!" + ) + if errorlevel 1 ( + echo [ERROR] vcpkg failed to install protobuf. + exit /b 1 + ) +) else ( + if defined PROTOBUF_VCPKG_VERSION ( + echo [DRY-RUN] Would run: "!VCPKG_EXE!" install "protobuf:!VCPKG_TRIPLET!" --x-version=%PROTOBUF_VCPKG_VERSION% + ) else ( + echo [DRY-RUN] Would run: "!VCPKG_EXE!" install "protobuf:!VCPKG_TRIPLET!" + ) +) + +REM Expose vcpkg-installed protoc on PATH so `buf generate` finds it. +set "PROTOC_TOOLS_DIR=!VCPKG_ROOT!\installed\!VCPKG_TRIPLET!\tools\protobuf" +if exist "!PROTOC_TOOLS_DIR!\protoc.exe" ( + set "PATH=!PROTOC_TOOLS_DIR!;%PATH%" + echo [INFO] vcpkg protoc on PATH: !PROTOC_TOOLS_DIR! +) + +echo [INFO] Build environment ready. + +REM Export PATH and key MSVC vars back to the caller's environment. +REM Also export VCPKG_ROOT so subsequent `cmake -DCMAKE_TOOLCHAIN_FILE=%VCPKG_ROOT%\...` +REM invocations resolve in this same cmd session even before the persisted +REM setx value takes effect in newly-spawned processes. +endlocal & set "PATH=%PATH%" & set "INCLUDE=%INCLUDE%" & set "LIB=%LIB%" & set "LIBPATH=%LIBPATH%" & set "WindowsSdkDir=%WindowsSdkDir%" & set "VCToolsInstallDir=%VCToolsInstallDir%" & set "VCPKG_ROOT=%VCPKG_ROOT%" From 632b81f8e360ca2ab8e98c3284dbd0086a198819 Mon Sep 17 00:00:00 2001 From: Kybxd <627940450@qq.com> Date: Thu, 28 May 2026 19:15:03 +0800 Subject: [PATCH 04/66] ci: replace vcpkg binary cache with actions/cache --- .github/workflows/testing-cpp.yml | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/.github/workflows/testing-cpp.yml b/.github/workflows/testing-cpp.yml index 4d8bbc3..dfe3105 100644 --- a/.github/workflows/testing-cpp.yml +++ b/.github/workflows/testing-cpp.yml @@ -39,7 +39,6 @@ jobs: env: VCPKG_INSTALLED_DIR: ${{ github.workspace }}/vcpkg_installed VCPKG_DEFAULT_TRIPLET: ${{ matrix.triplet }} - VCPKG_BINARY_SOURCES: "clear;x-gha,readwrite" steps: - name: Checkout Code @@ -60,14 +59,6 @@ jobs: if: runner.os == 'Windows' uses: ilammy/msvc-dev-cmd@v1 - - name: Export GitHub Actions cache env - uses: actions/github-script@v7 - with: - script: | - core.exportVariable('ACTIONS_CACHE_URL', process.env.ACTIONS_CACHE_URL || ''); - core.exportVariable('ACTIONS_RESULTS_URL', process.env.ACTIONS_RESULTS_URL || ''); - core.exportVariable('ACTIONS_RUNTIME_TOKEN', process.env.ACTIONS_RUNTIME_TOKEN || ''); - - name: Render vcpkg.json working-directory: test/cpp-tableau-loader shell: bash @@ -84,6 +75,13 @@ jobs: } EOF + - name: Cache vcpkg_installed + id: cache-vcpkg-installed + uses: actions/cache@v4 + with: + path: ${{ env.VCPKG_INSTALLED_DIR }} + key: vcpkg-installed-${{ runner.os }}-${{ matrix.triplet }}-${{ env.VCPKG_COMMIT }}-${{ hashFiles('test/cpp-tableau-loader/vcpkg.json') }} + - name: Setup vcpkg & install protobuf uses: lukka/run-vcpkg@v11 with: From 0021723b20d8e2b492b6c775daeccd69f59cf6b4 Mon Sep 17 00:00:00 2001 From: Kybxd <627940450@qq.com> Date: Thu, 28 May 2026 19:46:07 +0800 Subject: [PATCH 05/66] ci: update protobuf versions and restructure matrix config --- .github/workflows/testing-csharp.yml | 17 +++++++++++------ .github/workflows/testing-go.yml | 4 ++-- 2 files changed, 13 insertions(+), 8 deletions(-) diff --git a/.github/workflows/testing-csharp.yml b/.github/workflows/testing-csharp.yml index 06b19f5..6d5f21a 100644 --- a/.github/workflows/testing-csharp.yml +++ b/.github/workflows/testing-csharp.yml @@ -15,11 +15,16 @@ permissions: jobs: test: strategy: + fail-fast: false matrix: os: [ubuntu-latest, windows-latest] - protobuf-version: ["32.0", "3.19.3"] + config: + - label: modern + protobuf-version: "6.33.4" + - label: legacy-v3 + protobuf-version: "3.21.12" - name: test (${{ matrix.os }}, protobuf ${{ matrix.protobuf-version }}) + name: test (${{ matrix.os }}, ${{ matrix.config.label }}) runs-on: ${{ matrix.os }} timeout-minutes: 10 @@ -42,17 +47,17 @@ jobs: dotnet-version: "8.0.x" - name: Install Protoc - if: "!startsWith(matrix.protobuf-version, '3.')" + if: matrix.config.label != 'legacy-v3' uses: arduino/setup-protoc@v3 with: - version: ${{ matrix.protobuf-version }} + version: ${{ matrix.config.protobuf-version }} repo-token: ${{ secrets.GITHUB_TOKEN }} - name: Install Protoc (legacy) - if: startsWith(matrix.protobuf-version, '3.') + if: matrix.config.label == 'legacy-v3' uses: arduino/setup-protoc@v1 with: - version: ${{ matrix.protobuf-version }} + version: ${{ matrix.config.protobuf-version }} repo-token: ${{ secrets.GITHUB_TOKEN }} - name: Install Buf diff --git a/.github/workflows/testing-go.yml b/.github/workflows/testing-go.yml index d7e430d..2c064e3 100644 --- a/.github/workflows/testing-go.yml +++ b/.github/workflows/testing-go.yml @@ -15,11 +15,11 @@ permissions: jobs: test: strategy: + fail-fast: false matrix: os: [ubuntu-latest, windows-latest] - protobuf-version: ["32.0", "3.19.3"] - name: test (${{ matrix.os }}, protobuf ${{ matrix.protobuf-version }}) + name: test (${{ matrix.os }}) runs-on: ${{ matrix.os }} timeout-minutes: 10 From e2d9243f94e2131e3fe743d6878b1f59e0301a9c Mon Sep 17 00:00:00 2001 From: Kybxd <627940450@qq.com> Date: Thu, 28 May 2026 19:53:46 +0800 Subject: [PATCH 06/66] ci: update protoc setup to use v3 for all configs --- .github/workflows/testing-csharp.yml | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/.github/workflows/testing-csharp.yml b/.github/workflows/testing-csharp.yml index 6d5f21a..ff956db 100644 --- a/.github/workflows/testing-csharp.yml +++ b/.github/workflows/testing-csharp.yml @@ -20,7 +20,7 @@ jobs: os: [ubuntu-latest, windows-latest] config: - label: modern - protobuf-version: "6.33.4" + protobuf-version: "33.4" - label: legacy-v3 protobuf-version: "3.21.12" @@ -47,19 +47,11 @@ jobs: dotnet-version: "8.0.x" - name: Install Protoc - if: matrix.config.label != 'legacy-v3' uses: arduino/setup-protoc@v3 with: version: ${{ matrix.config.protobuf-version }} repo-token: ${{ secrets.GITHUB_TOKEN }} - - name: Install Protoc (legacy) - if: matrix.config.label == 'legacy-v3' - uses: arduino/setup-protoc@v1 - with: - version: ${{ matrix.config.protobuf-version }} - repo-token: ${{ secrets.GITHUB_TOKEN }} - - name: Install Buf uses: bufbuild/buf-action@v1 with: From f7f7f14bd50677def21b435ab99e188d518eaa14 Mon Sep 17 00:00:00 2001 From: Kybxd <627940450@qq.com> Date: Thu, 28 May 2026 19:56:19 +0800 Subject: [PATCH 07/66] ci: use setup-protoc v1 for legacy-v3 builds --- .github/workflows/testing-csharp.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.github/workflows/testing-csharp.yml b/.github/workflows/testing-csharp.yml index ff956db..8e3fd61 100644 --- a/.github/workflows/testing-csharp.yml +++ b/.github/workflows/testing-csharp.yml @@ -47,11 +47,19 @@ jobs: dotnet-version: "8.0.x" - name: Install Protoc + if: matrix.config.label != 'legacy-v3' uses: arduino/setup-protoc@v3 with: version: ${{ matrix.config.protobuf-version }} repo-token: ${{ secrets.GITHUB_TOKEN }} + - name: Install Protoc (legacy) + if: matrix.config.label == 'legacy-v3' + uses: arduino/setup-protoc@v1 + with: + version: ${{ matrix.config.protobuf-version }} + repo-token: ${{ secrets.GITHUB_TOKEN }} + - name: Install Buf uses: bufbuild/buf-action@v1 with: From 8061774c33e03b75a00ffcdf4c2438c28bb8e174 Mon Sep 17 00:00:00 2001 From: Kybxd <627940450@qq.com> Date: Thu, 28 May 2026 19:59:19 +0800 Subject: [PATCH 08/66] ci: update legacy protobuf version and unify setup-protoc --- .github/workflows/testing-csharp.yml | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/.github/workflows/testing-csharp.yml b/.github/workflows/testing-csharp.yml index 8e3fd61..0cf146b 100644 --- a/.github/workflows/testing-csharp.yml +++ b/.github/workflows/testing-csharp.yml @@ -22,7 +22,7 @@ jobs: - label: modern protobuf-version: "33.4" - label: legacy-v3 - protobuf-version: "3.21.12" + protobuf-version: "21.12" name: test (${{ matrix.os }}, ${{ matrix.config.label }}) runs-on: ${{ matrix.os }} @@ -47,19 +47,11 @@ jobs: dotnet-version: "8.0.x" - name: Install Protoc - if: matrix.config.label != 'legacy-v3' uses: arduino/setup-protoc@v3 with: version: ${{ matrix.config.protobuf-version }} repo-token: ${{ secrets.GITHUB_TOKEN }} - - name: Install Protoc (legacy) - if: matrix.config.label == 'legacy-v3' - uses: arduino/setup-protoc@v1 - with: - version: ${{ matrix.config.protobuf-version }} - repo-token: ${{ secrets.GITHUB_TOKEN }} - - name: Install Buf uses: bufbuild/buf-action@v1 with: From 145574e8547874b1d0b6563c9b25520205fe1203 Mon Sep 17 00:00:00 2001 From: Kybxd <627940450@qq.com> Date: Thu, 28 May 2026 20:27:20 +0800 Subject: [PATCH 09/66] docs: warn against RHEL protobuf packages --- README.md | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 3c8ce09..58bc486 100644 --- a/README.md +++ b/README.md @@ -26,16 +26,20 @@ Pick whichever channel fits your platform; loader does not bundle protobuf. # .\vcpkg\vcpkg install protobuf:x64-windows-static # Windows (matches loader's static CRT) ``` Pin to the legacy v3 line if you need it: append `--x-version=3.21.12`. - Then point CMake at vcpkg with `-DCMAKE_TOOLCHAIN_FILE=/scripts/buildsystems/vcpkg.cmake`. + + Then put `protoc` on `PATH` (so `buf generate` works) and pass + `-DCMAKE_TOOLCHAIN_FILE=/scripts/buildsystems/vcpkg.cmake` to + CMake. See [Dev at Linux](#dev-at-linux) / [Dev at Windows](#dev-at-windows) + for the exact commands. - **Linux (system package):** ```sh sudo apt-get install -y protobuf-compiler libprotobuf-dev # Debian / Ubuntu - sudo dnf install -y protobuf-compiler protobuf-devel # Fedora / RHEL 8+ / CentOS Stream / Rocky / Alma - sudo yum install -y epel-release \ - && sudo yum install -y protobuf-compiler protobuf-devel # CentOS 7 (via EPEL) ``` - > Distro packages can lag well behind upstream (e.g. CentOS 7 ships protobuf 2.5; RHEL/Rocky 8 ships 3.x). If you need protobuf v22+ (or any specific version), prefer **vcpkg** above or **build from source**. + > **Avoid `dnf` / `yum` on RHEL-family distros.** The `protobuf-devel` + > shipped by Fedora / RHEL / TencentOS repos is typically stuck on + > protobuf **3.5.x**, which is far behind what loader expects and predates + > the v22 / Abseil split. Use vcpkg or build from source instead. - **macOS (Homebrew):** ```sh From 47deb0c1f2e83e505c9dbc40b571b8fb6015709a Mon Sep 17 00:00:00 2001 From: wenchy Date: Fri, 29 May 2026 21:27:45 +0800 Subject: [PATCH 10/66] fix: address PR review feedback for vcpkg migration - prepare.bat: switch to vcpkg manifest mode when PROTOBUF_VCPKG_VERSION is set. Classic-mode `--x-version` is silently a no-op, so the previous pin attempt would lie to users. Render a vcpkg.json under %LOCALAPPDATA%\loader\vcpkg-manifest\ with builtin-baseline + overrides, install via `vcpkg install --x-install-root=...`, and post-assert that the resolved port version matches the request. Pin the vcpkg checkout itself to VCPKG_BASELINE_COMMIT (mirroring testing-cpp.yml's VCPKG_COMMIT) so classic mode is also reproducible. Drop `--depth 1` so the commit pin is reachable. Export VCPKG_INSTALLED_DIR for downstream cmake. - prepare.bat: add a `where cl.exe` preflight at the top of Step 5 so an unactivated MSVC environment fails fast with an actionable message instead of cryptic vcpkg compiler-detection errors. - testing-cpp.yml: replace deprecated `version-string` with `version` in the rendered vcpkg.json. - testing-csharp.yml: document inline that `protobuf-version` is the protoc release tag (e.g. 33.4) and explain how it maps to the C++ libprotobuf SemVer used in testing-cpp.yml. - README.md: add a "Migrating from the bundled-protobuf layout" callout with the submodule-deinit recipe so existing checkouts know how to clean up. Replace the misleading classic-mode `--x-version` snippet with a working manifest-mode example. Document the extra cmake flags (-DVCPKG_INSTALLED_DIR / -DVCPKG_MANIFEST_INSTALL=OFF) required when PROTOBUF_VCPKG_VERSION is used. - .gitignore: ignore .claude/settings.local.json. Co-Authored-By: Claude Opus 4.7 (1M context) --- .github/workflows/testing-cpp.yml | 2 +- .github/workflows/testing-csharp.yml | 8 ++ .gitignore | 2 + README.md | 44 +++++++- prepare.bat | 153 +++++++++++++++++++++++---- 5 files changed, 184 insertions(+), 25 deletions(-) diff --git a/.github/workflows/testing-cpp.yml b/.github/workflows/testing-cpp.yml index dfe3105..41a8d27 100644 --- a/.github/workflows/testing-cpp.yml +++ b/.github/workflows/testing-cpp.yml @@ -66,7 +66,7 @@ jobs: cat > vcpkg.json < **Migrating from the bundled-protobuf layout?** Loader used to vendor +> protobuf as a git submodule under `third_party/_submodules/protobuf` plus +> an `init.sh` / `init.bat` build pipeline. Both are gone. If you've checked +> the repo out before this change, clean up the orphan worktree and submodule +> metadata before building: +> +> ```sh +> git submodule deinit -f third_party/_submodules/protobuf +> rm -rf third_party/_submodules/protobuf .git/modules/third_party/_submodules/protobuf +> ``` +> +> Then install protobuf via one of the channels documented in +> [Install protobuf](#install-protobuf). + ### Install protobuf Pick whichever channel fits your platform; loader does not bundle protobuf. @@ -25,7 +39,24 @@ Pick whichever channel fits your platform; loader does not bundle protobuf. # ~/vcpkg/vcpkg install protobuf:x64-osx # macOS # .\vcpkg\vcpkg install protobuf:x64-windows-static # Windows (matches loader's static CRT) ``` - Pin to the legacy v3 line if you need it: append `--x-version=3.21.12`. + This installs whatever protobuf version the vcpkg checkout's baseline ships + (currently the 6.x line). To pin a specific version, use vcpkg **manifest + mode**: drop a `vcpkg.json` in your build directory with a `builtin-baseline` + + `overrides`, e.g. + + ```json + { + "name": "loader-build", + "version": "0.1.0", + "dependencies": ["protobuf"], + "overrides": [{ "name": "protobuf", "version": "3.21.12" }], + "builtin-baseline": "" + } + ``` + + > **Note:** classic-mode `vcpkg install --x-version=...` is silently a no-op; + > version pinning only works in manifest mode. See + > `.github/workflows/testing-cpp.yml` for the exact pattern CI uses. Then put `protoc` on `PATH` (so `buf generate` works) and pass `-DCMAKE_TOOLCHAIN_FILE=/scripts/buildsystems/vcpkg.cmake` to @@ -74,6 +105,13 @@ compiler environment for the current cmd session. > ```bat > set PROTOBUF_VCPKG_VERSION=3.21.12 && .\prepare.bat > ``` +> Setting this switches the script to vcpkg **manifest mode** — the only mode +> in which the version pin actually takes effect. The install root moves from +> `%VCPKG_ROOT%\installed\x64-windows-static\` to a manifest dir under +> `%LOCALAPPDATA%\loader\vcpkg-manifest\vcpkg_installed\`, and `prepare.bat` +> exports its path as `%VCPKG_INSTALLED_DIR%`. Your downstream CMake invocation +> must then add `-DVCPKG_INSTALLED_DIR=%VCPKG_INSTALLED_DIR%` and +> `-DVCPKG_MANIFEST_INSTALL=OFF` (see [Dev at Windows](#dev-at-windows)). > **Note:** The **installation** part of `prepare.bat` only runs once per machine — it detects already-installed tools (Chocolatey, Ninja, CMake, MSVC Build Tools, buf, vcpkg, protobuf) and skips them, so no manual installation is required. > @@ -116,9 +154,11 @@ compiler environment for the current cmd session. - Initialize MSVC environment (from loader root): `.\prepare.bat` - Change dir: `cd test\cpp-tableau-loader`, or change directory with Drive, e.g.: `cd /D D:\GitHub\loader\test\cpp-tableau-loader` - Generate protoconf: `buf generate ..` (the `prepare.bat` step above already puts the vcpkg-built `protoc.exe` on `PATH`) -- CMake (vcpkg-provided protobuf): +- CMake (vcpkg-provided protobuf, classic mode — default): - C++17: `cmake -S . -B build -G Ninja -DCMAKE_BUILD_TYPE=Debug -DCMAKE_TOOLCHAIN_FILE=%VCPKG_ROOT%\scripts\buildsystems\vcpkg.cmake -DVCPKG_TARGET_TRIPLET=x64-windows-static` - C++20: append `-DCMAKE_CXX_STANDARD=20` +- CMake (vcpkg manifest mode — only when you ran `prepare.bat` with `PROTOBUF_VCPKG_VERSION` set): + - C++17: `cmake -S . -B build -G Ninja -DCMAKE_BUILD_TYPE=Debug -DCMAKE_TOOLCHAIN_FILE=%VCPKG_ROOT%\scripts\buildsystems\vcpkg.cmake -DVCPKG_TARGET_TRIPLET=x64-windows-static -DVCPKG_INSTALLED_DIR="%VCPKG_INSTALLED_DIR%" -DVCPKG_MANIFEST_INSTALL=OFF` - Build: `cmake --build build --parallel` - Test: `ctest --test-dir build --output-on-failure` diff --git a/prepare.bat b/prepare.bat index 32d0e83..def5017 100644 --- a/prepare.bat +++ b/prepare.bat @@ -11,10 +11,22 @@ REM Then installs `protobuf` (and friends) into vcpkg using the static-CRT REM triplet x64-windows-static, so that downstream cmake builds can pick it REM up via -DCMAKE_TOOLCHAIN_FILE=%VCPKG_ROOT%\scripts\buildsystems\vcpkg.cmake. REM +REM The vcpkg checkout is pinned to VCPKG_BASELINE_COMMIT below for +REM reproducibility — the same commit testing-cpp.yml uses in CI. +REM REM Override `protobuf` to a specific vcpkg port version with PROTOBUF_VCPKG_VERSION: REM set PROTOBUF_VCPKG_VERSION=3.21.12 && .\prepare.bat -REM (Default is whatever the vcpkg `master` baseline ships, currently the 6.x -REM line. Use 3.21.12 if you need the legacy v3 ABI.) +REM +REM When PROTOBUF_VCPKG_VERSION is set we install in vcpkg MANIFEST mode (a +REM rendered vcpkg.json under %LOCALAPPDATA%\loader\vcpkg-manifest\), which +REM is the only mode where `--x-version` / `overrides` actually pin the port +REM version. In that case the install root is %VCPKG_INSTALLED_DIR%, and +REM downstream cmake invocations must add: +REM -DVCPKG_INSTALLED_DIR=%VCPKG_INSTALLED_DIR% -DVCPKG_MANIFEST_INSTALL=OFF +REM (matching the CI flow in .github/workflows/testing-cpp.yml). +REM +REM When PROTOBUF_VCPKG_VERSION is NOT set we install in vcpkg CLASSIC mode +REM (no manifest), and downstream cmake works with just the toolchain file. REM REM This script is idempotent: re-running it on a machine that already has REM everything installed is a no-op (a few seconds of probing). Only the MSVC @@ -323,13 +335,39 @@ REM :` with: "Could not locate a manifest (vcpkg.json) above REM the current working directory. This vcpkg distribution does not have a REM classic mode instance." REM +REM When we install vcpkg ourselves we pin it to VCPKG_BASELINE_COMMIT so +REM the protobuf port version is reproducible. When we adopt a pre-existing +REM user-managed vcpkg, we leave its checkout state alone (the user owns it). +REM REM We then run `vcpkg install protobuf:x64-windows-static` so that the REM static-CRT libprotobuf + protoc match the loader build (CMakeLists.txt REM forces /MT[d] via CMAKE_MSVC_RUNTIME_LIBRARY). REM REM Override the protobuf port version (e.g. for the legacy v3 line) with: REM set PROTOBUF_VCPKG_VERSION=3.21.12 && .\prepare.bat +REM In that case we switch to manifest mode (see header comment above). REM ----------------------------------------------------------------------- +REM Pre-flight: vcpkg compiles protobuf from source via MSVC, so cl.exe must +REM resolve here. If Step 3 didn't activate the MSVC environment (e.g. choco +REM just installed VS Build Tools and the registration hasn't fully landed +REM in this shell), fail fast with an actionable message rather than letting +REM vcpkg emit cryptic compiler-detection errors deep into the install. +where cl.exe >nul 2>&1 +if errorlevel 1 ( + if "%DRY_RUN%"=="0" ( + echo [ERROR] cl.exe not on PATH; the MSVC environment is not active in this shell. + echo [ERROR] Open a new "Developer Command Prompt for VS 2022" or rerun this script + echo [ERROR] in a fresh cmd window so vcvarsall.bat can take effect, then retry. + exit /b 1 + ) else ( + echo [DRY-RUN] [WARN] cl.exe not on PATH; would error out before vcpkg install. + ) +) + +REM Pin both the vcpkg checkout and the manifest's builtin-baseline to the +REM same commit testing-cpp.yml uses. Bumping vcpkg? Bump both this value +REM and VCPKG_COMMIT in .github/workflows/testing-cpp.yml in lockstep. +set "VCPKG_BASELINE_COMMIT=dc8d75cfc3281b8e2a4ed8ee4163c891190df932" set "VCPKG_TRIPLET=x64-windows-static" set "VCPKG_EXE=" @@ -373,12 +411,22 @@ if not defined VCPKG_EXE ( set "VCPKG_ROOT=%USERPROFILE%\vcpkg" if "%DRY_RUN%"=="0" ( if not exist "!VCPKG_ROOT!" ( - git clone --depth 1 https://github.com/microsoft/vcpkg.git "!VCPKG_ROOT!" + REM Full clone (no --depth 1) so we can `git checkout` an arbitrary + REM commit below for reproducibility. + git clone https://github.com/microsoft/vcpkg.git "!VCPKG_ROOT!" if errorlevel 1 ( echo [ERROR] Failed to clone vcpkg. exit /b 1 ) ) + REM Pin the checkout so port versions are reproducible. Safe to run + REM repeatedly: a no-op when we're already on VCPKG_BASELINE_COMMIT. + git -C "!VCPKG_ROOT!" fetch --quiet origin %VCPKG_BASELINE_COMMIT% + git -C "!VCPKG_ROOT!" checkout --quiet %VCPKG_BASELINE_COMMIT% + if errorlevel 1 ( + echo [ERROR] Failed to checkout vcpkg @ %VCPKG_BASELINE_COMMIT%. + exit /b 1 + ) call "!VCPKG_ROOT!\bootstrap-vcpkg.bat" -disableMetrics if errorlevel 1 ( echo [ERROR] Failed to bootstrap vcpkg. @@ -386,6 +434,7 @@ if not defined VCPKG_EXE ( ) ) else ( echo [DRY-RUN] Would run: git clone https://github.com/microsoft/vcpkg.git "!VCPKG_ROOT!" + echo [DRY-RUN] Would run: git -C "!VCPKG_ROOT!" checkout %VCPKG_BASELINE_COMMIT% echo [DRY-RUN] Would run: "!VCPKG_ROOT!\bootstrap-vcpkg.bat" -disableMetrics ) set "VCPKG_EXE=!VCPKG_ROOT!\vcpkg.exe" @@ -403,44 +452,104 @@ if not defined VCPKG_EXE ( echo [DRY-RUN] Would run: setx PATH "!VCPKG_ROOT!;..." ) set "PATH=!VCPKG_ROOT!;%PATH%" - echo [INFO] vcpkg installed at !VCPKG_ROOT!. + echo [INFO] vcpkg installed at !VCPKG_ROOT! ^(pinned to %VCPKG_BASELINE_COMMIT%^). ) else ( - echo [INFO] vcpkg already available at !VCPKG_ROOT!. + echo [INFO] vcpkg already available at !VCPKG_ROOT! ^(user-managed; not re-pinning^). ) -REM Install protobuf into vcpkg (idempotent: vcpkg detects already-installed -REM packages and skips them). If PROTOBUF_VCPKG_VERSION is set, pass --version. -if "%DRY_RUN%"=="0" ( - if defined PROTOBUF_VCPKG_VERSION ( - echo [INFO] Installing protobuf %PROTOBUF_VCPKG_VERSION% into vcpkg ^(triplet !VCPKG_TRIPLET!^)... - "!VCPKG_EXE!" install "protobuf:!VCPKG_TRIPLET!" --x-version=%PROTOBUF_VCPKG_VERSION% +REM Install protobuf into vcpkg. +REM +REM Branching on PROTOBUF_VCPKG_VERSION: +REM - unset: CLASSIC mode. Installs into %VCPKG_ROOT%\installed\\, +REM auto-discovered by the vcpkg cmake toolchain — no extra cmake +REM flags needed downstream. Whatever protobuf the pinned vcpkg +REM checkout (VCPKG_BASELINE_COMMIT) ships is what you get. +REM - set: MANIFEST mode. Renders a vcpkg.json under +REM %LOCALAPPDATA%\loader\vcpkg-manifest\ with builtin-baseline + +REM an override pinning protobuf to the requested version. This +REM is the only mode in which `--x-version` / `overrides` actually +REM take effect; classic-mode `--x-version` is silently a no-op. +REM Install root: \vcpkg_installed\\. +REM +REM Both modes are idempotent: vcpkg detects already-installed packages and +REM skips them. +if defined PROTOBUF_VCPKG_VERSION ( + set "VCPKG_MANIFEST_DIR=%LOCALAPPDATA%\loader\vcpkg-manifest" + set "VCPKG_INSTALLED_DIR=!VCPKG_MANIFEST_DIR!\vcpkg_installed" + if "%DRY_RUN%"=="0" ( + if not exist "!VCPKG_MANIFEST_DIR!" mkdir "!VCPKG_MANIFEST_DIR!" + REM Render vcpkg.json. The `>` redirection truncates on first line and + REM `>>` appends the rest, mirroring a here-doc. + > "!VCPKG_MANIFEST_DIR!\vcpkg.json" echo { + >>"!VCPKG_MANIFEST_DIR!\vcpkg.json" echo "name": "loader-prepare", + >>"!VCPKG_MANIFEST_DIR!\vcpkg.json" echo "version": "0.1.0", + >>"!VCPKG_MANIFEST_DIR!\vcpkg.json" echo "dependencies": ["protobuf"], + >>"!VCPKG_MANIFEST_DIR!\vcpkg.json" echo "overrides": [{ "name": "protobuf", "version": "%PROTOBUF_VCPKG_VERSION%" }], + >>"!VCPKG_MANIFEST_DIR!\vcpkg.json" echo "builtin-baseline": "%VCPKG_BASELINE_COMMIT%" + >>"!VCPKG_MANIFEST_DIR!\vcpkg.json" echo } + echo [INFO] Installing protobuf %PROTOBUF_VCPKG_VERSION% into vcpkg ^(manifest mode, triplet !VCPKG_TRIPLET!^)... + pushd "!VCPKG_MANIFEST_DIR!" + "!VCPKG_EXE!" install --triplet=!VCPKG_TRIPLET! --x-install-root="!VCPKG_INSTALLED_DIR!" + set "VCPKG_INSTALL_RC=!ERRORLEVEL!" + popd + if not "!VCPKG_INSTALL_RC!"=="0" ( + echo [ERROR] vcpkg failed to install protobuf %PROTOBUF_VCPKG_VERSION%. + exit /b 1 + ) + REM Sanity check: assert the resolved version actually starts with + REM the requested one. vcpkg port versions can have a `#N` port-revision + REM suffix, so we match by prefix rather than equality. This is the + REM safety net that catches future regressions in vcpkg's manifest + REM resolution silently producing the wrong version. + set "VCPKG_INFO_FILE=!VCPKG_INSTALLED_DIR!\vcpkg\info\.unused" + for /f "delims=" %%f in ('dir /b /a-d "!VCPKG_INSTALLED_DIR!\vcpkg\info\protobuf_*_!VCPKG_TRIPLET!.list" 2^>nul') do ( + set "VCPKG_INFO_FILE=%%f" + ) + echo !VCPKG_INFO_FILE! | findstr /c:"protobuf_%PROTOBUF_VCPKG_VERSION%" >nul 2>&1 + if errorlevel 1 ( + echo [ERROR] Installed protobuf does not match requested version %PROTOBUF_VCPKG_VERSION%. + echo [ERROR] vcpkg installed file marker: !VCPKG_INFO_FILE! + echo [ERROR] This usually means VCPKG_BASELINE_COMMIT is too old to know about that + echo [ERROR] version. Bump the commit at the top of Step 5 and retry. + exit /b 1 + ) ) else ( - echo [INFO] Installing protobuf into vcpkg ^(triplet !VCPKG_TRIPLET!^)... - "!VCPKG_EXE!" install "protobuf:!VCPKG_TRIPLET!" - ) - if errorlevel 1 ( - echo [ERROR] vcpkg failed to install protobuf. - exit /b 1 + echo [DRY-RUN] Would render: !VCPKG_MANIFEST_DIR!\vcpkg.json ^(protobuf %PROTOBUF_VCPKG_VERSION%, baseline %VCPKG_BASELINE_COMMIT%^) + echo [DRY-RUN] Would run: pushd "!VCPKG_MANIFEST_DIR!" ^&^& "!VCPKG_EXE!" install --triplet=!VCPKG_TRIPLET! --x-install-root="!VCPKG_INSTALLED_DIR!" ) + set "PROTOC_TOOLS_DIR=!VCPKG_INSTALLED_DIR!\!VCPKG_TRIPLET!\tools\protobuf" ) else ( - if defined PROTOBUF_VCPKG_VERSION ( - echo [DRY-RUN] Would run: "!VCPKG_EXE!" install "protobuf:!VCPKG_TRIPLET!" --x-version=%PROTOBUF_VCPKG_VERSION% + if "%DRY_RUN%"=="0" ( + echo [INFO] Installing protobuf into vcpkg ^(classic mode, triplet !VCPKG_TRIPLET!^)... + "!VCPKG_EXE!" install "protobuf:!VCPKG_TRIPLET!" + if errorlevel 1 ( + echo [ERROR] vcpkg failed to install protobuf. + exit /b 1 + ) ) else ( echo [DRY-RUN] Would run: "!VCPKG_EXE!" install "protobuf:!VCPKG_TRIPLET!" ) + set "VCPKG_INSTALLED_DIR=" + set "PROTOC_TOOLS_DIR=!VCPKG_ROOT!\installed\!VCPKG_TRIPLET!\tools\protobuf" ) REM Expose vcpkg-installed protoc on PATH so `buf generate` finds it. -set "PROTOC_TOOLS_DIR=!VCPKG_ROOT!\installed\!VCPKG_TRIPLET!\tools\protobuf" if exist "!PROTOC_TOOLS_DIR!\protoc.exe" ( set "PATH=!PROTOC_TOOLS_DIR!;%PATH%" echo [INFO] vcpkg protoc on PATH: !PROTOC_TOOLS_DIR! ) +if defined VCPKG_INSTALLED_DIR ( + echo [INFO] Manifest-mode install root: !VCPKG_INSTALLED_DIR! + echo [INFO] When invoking cmake, also pass: + echo [INFO] -DVCPKG_INSTALLED_DIR="!VCPKG_INSTALLED_DIR!" -DVCPKG_MANIFEST_INSTALL=OFF +) + echo [INFO] Build environment ready. REM Export PATH and key MSVC vars back to the caller's environment. REM Also export VCPKG_ROOT so subsequent `cmake -DCMAKE_TOOLCHAIN_FILE=%VCPKG_ROOT%\...` REM invocations resolve in this same cmd session even before the persisted -REM setx value takes effect in newly-spawned processes. -endlocal & set "PATH=%PATH%" & set "INCLUDE=%INCLUDE%" & set "LIB=%LIB%" & set "LIBPATH=%LIBPATH%" & set "WindowsSdkDir=%WindowsSdkDir%" & set "VCToolsInstallDir=%VCToolsInstallDir%" & set "VCPKG_ROOT=%VCPKG_ROOT%" +REM setx value takes effect in newly-spawned processes. VCPKG_INSTALLED_DIR +REM is set only in manifest mode (PROTOBUF_VCPKG_VERSION pinned). +endlocal & set "PATH=%PATH%" & set "INCLUDE=%INCLUDE%" & set "LIB=%LIB%" & set "LIBPATH=%LIBPATH%" & set "WindowsSdkDir=%WindowsSdkDir%" & set "VCToolsInstallDir=%VCToolsInstallDir%" & set "VCPKG_ROOT=%VCPKG_ROOT%" & set "VCPKG_INSTALLED_DIR=%VCPKG_INSTALLED_DIR%" From 60885c3f09dfcd0658935894e69744160c216b0e Mon Sep 17 00:00:00 2001 From: wenchy Date: Fri, 29 May 2026 22:24:19 +0800 Subject: [PATCH 11/66] docs: design spec for devcontainer Captures the brainstormed design for adding a Dev Container under .devcontainer/ to give contributors a one-command, reproducible multi-language toolchain (C++17 + Go 1.24 + .NET 8 + Node 20 + buf 1.67.0 + protobuf 6.33.4 via vcpkg, all pinned to CI's exact versions). Key decisions captured: - Single all-in-one Ubuntu 24.04 image (covers all four languages) - Dockerfile in repo, build on-demand (no ghcr.io publish in v1) - Multi-arch native via TARGETARCH (amd64 + arm64; no QEMU on Apple Silicon) - Pinnable protobuf version via LOADER_PROTOBUF_VERSION host env var, flowing through devcontainer.json build args into vcpkg manifest mode - Devcontainer is recommended path; prepare.bat / per-language manual setup stays as fallback for contributors who can't run Docker - CI keeps lukka/run-vcpkg directly (devcontainer is not used in CI) Implementation plan to follow via writing-plans. Co-Authored-By: Claude Opus 4.7 (1M context) --- .../specs/2026-05-29-devcontainer-design.md | 200 ++++++++++++++++++ 1 file changed, 200 insertions(+) create mode 100644 docs/superpowers/specs/2026-05-29-devcontainer-design.md diff --git a/docs/superpowers/specs/2026-05-29-devcontainer-design.md b/docs/superpowers/specs/2026-05-29-devcontainer-design.md new file mode 100644 index 0000000..923095f --- /dev/null +++ b/docs/superpowers/specs/2026-05-29-devcontainer-design.md @@ -0,0 +1,200 @@ +# Dev Container for tableauio/loader — Design + +**Status:** Approved through brainstorming. Implementation plan to follow via `writing-plans`. +**Date:** 2026-05-29 +**Scope:** Add a Dev Container that pins the full multi-language toolchain +(C++17, Go 1.24, .NET 8, Node 20, buf 1.67.0, protobuf 6.33.4 via vcpkg) so +contributors on any host OS get a reproducible build environment that mirrors +CI exactly. + +## Goals + +1. **One-command setup** on any host (Windows, macOS, Linux) — "Reopen in Container" replaces the existing per-OS, per-language manual setup as the *recommended* path. +2. **Reproducibility** — protobuf, buf, Go, .NET, Node, and the vcpkg checkout are pinned to the exact versions / commit SHA used by CI (`testing-cpp.yml`, `testing-go.yml`, `testing-csharp.yml`). +3. **Multi-arch native** — Apple Silicon contributors build natively as arm64 (no QEMU emulation); amd64 contributors build natively as amd64. One Dockerfile, no buildx publish step required. +4. **Pinnable protobuf version** — daily dev runs against modern (6.33.4) by default; legacy-v3 (3.21.12) is reachable by setting one host env var, with no second Dockerfile. +5. **Daily commands stay unchanged** — every shell snippet currently in README's per-language sections (`buf generate ..`, `cmake -S . -B build -DCMAKE_BUILD_TYPE=Debug`, `go test ./...`, `dotnet test`) works inside the container without flag tweaks. + +## Non-goals + +- **Prebuilt-and-pushed image on ghcr.io.** Out of scope for v1; revisit only if first-run latency (~25 min vcpkg compile) becomes a real complaint. Adding it later is a CI-only change with no Dockerfile churn. +- **CI running inside the devcontainer.** CI keeps `lukka/run-vcpkg` for cached vcpkg installs; rebuilding the devcontainer image per matrix entry would be strictly slower with no reproducibility win. +- **Unity-side C# workflow.** Unity Editor doesn't run in a Linux container; the container covers `.NET 8 + xUnit` (which is what `test/csharp-tableau-loader/` exercises) only. +- **Replacing `prepare.bat` or per-language manual setup.** Both stay as fallback paths for contributors who can't run Docker (corp policy, restricted machines). + +## File layout + +Three new files, one directory; nothing existing moves. + +``` +.devcontainer/ +├── Dockerfile # ~95 lines, single stage, multi-arch, multi-language +├── devcontainer.json # ~30 lines +└── README.md # 1-pager: prerequisites, how to enter, host caveats +``` + +## Architecture + +### Image base + +`mcr.microsoft.com/devcontainers/cpp:1-ubuntu-24.04` — Microsoft's officially-maintained Dev Container base image. Provides: + +- gcc-13, glibc 2.39, cmake, ninja, git, sudo +- Non-root `vscode` user (uid/gid 1000) with passwordless sudo +- Multi-arch: pulls amd64 or arm64 automatically based on host +- Dev Containers shell hooks (`postCreateCommand` etc.) + +### Toolchain layers (Dockerfile, ordered for cache friendliness) + +Each layer is a single `RUN` block, version-pinned, named in a comment block. Order goes least-likely-to-change to most-likely-to-change so editing the latter doesn't invalidate the former. + +1. **Architecture detection.** `ARG TARGETARCH` (BuildKit auto-populates from build host). One `RUN` writes the resolved arch-dependent values to `/opt/buildargs.env` for downstream layers: + + | TARGETARCH | GO_ARCH | BUF_ARCH | VCPKG_TRIPLET | + |---|---|---|---| + | amd64 | amd64 | x86_64 | x64-linux | + | arm64 | arm64 | aarch64 | arm64-linux | + + Unknown arches fail the build with a clear message. + +2. **Go 1.24.0** — download official tarball for `${GO_ARCH}` to `/usr/local/go`. PATH is exposed via `ENV PATH=/usr/local/go/bin:/home/vscode/go/bin:${PATH}` (not `/etc/profile.d/`) so that non-interactive shells like the `postCreateCommand` and any `RUN` in downstream Dockerfiles see Go without sourcing profile. `/home/vscode/go/bin` is included so `go install`-placed binaries land on PATH automatically. + +3. **buf 1.67.0** — single binary download `buf-Linux-${BUF_ARCH}` to `/usr/local/bin/buf`. + +4. **vcpkg + protobuf via manifest mode** *(the heavy layer; ~25 min on first build).* + - Pinned commit: `VCPKG_BASELINE_COMMIT=dc8d75cfc3281b8e2a4ed8ee4163c891190df932` (lock-step with `prepare.bat` and `testing-cpp.yml`'s `VCPKG_COMMIT`). + - Pinned port: `PROTOBUF_VERSION=6.33.4` (a `Dockerfile ARG`, override-friendly — see "Pinnable protobuf version" below). + - Cloned into `/opt/vcpkg`, checked out to the pinned commit, bootstrapped with `-disableMetrics`. + - A small `vcpkg.json` is rendered into `/opt/vcpkg-manifest/` carrying `dependencies: ["protobuf"]`, `overrides: [{name: protobuf, version: ${PROTOBUF_VERSION}}]`, `builtin-baseline: ${VCPKG_BASELINE_COMMIT}`. + - `vcpkg install --triplet=${VCPKG_TRIPLET} --x-install-root=/opt/vcpkg-manifest/vcpkg_installed` runs from that directory. + - **Post-install assertion:** the same `dir-bin/findstr` pattern from `prepare.bat`, ported to bash — fail the build if the resolved port version doesn't have `${PROTOBUF_VERSION}` as a prefix. This is the safety net against future vcpkg-resolution regressions silently producing the wrong version. + - `ln -s /opt/vcpkg-manifest/vcpkg_installed/${VCPKG_TRIPLET} /opt/vcpkg/active` so the architecture-dependent path collapses behind a stable symlink. + - `ln -s /opt/vcpkg/active/tools/protobuf/protoc /usr/local/bin/protoc` so `buf generate` finds protoc with no PATH dance. + +5. **.NET SDK 8.0 + Node.js 20 LTS** — Microsoft's `packages-microsoft-prod.deb` and NodeSource's `setup_20.x` apt repos; `apt-get install -y dotnet-sdk-8.0 nodejs`; clean `/var/lib/apt/lists` to keep the layer trim. + +6. **Final environment.** + ```dockerfile + ENV CMAKE_PREFIX_PATH=/opt/vcpkg/active + ENV VCPKG_ROOT=/opt/vcpkg + ``` + Stable paths, no triplet leakage. + +### `devcontainer.json` + +```jsonc +{ + "name": "tableauio/loader", + "build": { + "dockerfile": "Dockerfile", + "args": { + "PROTOBUF_VERSION": "${localEnv:LOADER_PROTOBUF_VERSION:6.33.4}" + } + }, + + // Keep Go's module cache across container rebuilds. + "mounts": [ + "source=loader-go-mod,target=/home/vscode/go,type=volume" + ], + + "remoteUser": "vscode", + "workspaceFolder": "/workspaces/loader", + + "customizations": { + "vscode": { + "extensions": [ + "golang.go", + "ms-vscode.cmake-tools", + "ms-vscode.cpptools", + "ms-dotnettools.csharp", + "bufbuild.vscode-buf", + "zxh404.vscode-proto3" + ], + "settings": { + "go.toolsManagement.autoUpdate": false, + "cmake.configureOnOpen": false + } + } + }, + + "postCreateCommand": "printf 'tableauio/loader devcontainer ready.\\n go: %s\\n buf: %s\\n protoc: %s\\n dotnet: %s\\n node: %s\\n' \"$(go version | cut -d' ' -f3)\" \"$(buf --version)\" \"$(protoc --version)\" \"$(dotnet --version)\" \"$(node --version)\"" +} +``` + +Three intentional choices: + +1. **`${localEnv:LOADER_PROTOBUF_VERSION:6.33.4}`** — host env var picked up at container-build time. Workflow: `LOADER_PROTOBUF_VERSION=3.21.12 code .` → Reopen in Container → legacy-v3 image. No second devcontainer.json. +2. **Named volume `loader-go-mod` for `~/go`** — Go module cache persists across rebuilds. Workspace itself uses VS Code's default bind-mount (edits sync to host). +3. **`go.toolsManagement.autoUpdate: false`, `cmake.configureOnOpen: false`** — stops both extensions from auto-running their setup actions on first open, which would fight manual `cmake -S . -B build` invocations and trigger a 2-minute background `gopls` install. + +The `postCreateCommand` is pure echo — it prints the five tool versions so the contributor immediately knows the container is healthy. No installs, no conditionals. + +## Integration with existing flows + +### README change (additive, surgical) + +A new subsection at the top of `Prerequisites`, **above** "Install protobuf": + +> ### Recommended: Dev Container (any host OS) +> +> The fastest way to get a reproducible build environment is to open the +> repo in VS Code and choose **Reopen in Container**. The devcontainer +> under `.devcontainer/` has everything pinned to the exact versions CI +> uses (Go 1.24, buf 1.67.0, protobuf 6.33.4 via vcpkg, .NET 8.0, +> Node 20). First container build is one-time ~25 minutes (vcpkg +> compiles protobuf from source); subsequent reopens are instant. +> +> After the container starts you can skip the per-language setup below +> and jump straight to **C++** / **Go** / **C#** / **TypeScript**. +> +> Requirements: Docker Desktop (Windows + macOS) or Docker Engine (Linux), +> and the VS Code "Dev Containers" extension. See `.devcontainer/README.md` +> for the longer how-to. + +The existing `Windows: bootstrap…` block and per-language `Install protobuf` block both stay as written. Each gains a one-line lead-in: *"If you can't or don't want to use the devcontainer (corp Docker policy, etc.), follow the steps below."* + +### Container-side env so daily commands stay flag-free + +The Dockerfile's final `ENV CMAKE_PREFIX_PATH=/opt/vcpkg/active` is the *only* mechanism needed to make the existing "Dev at Linux → CMake (system protobuf)" recipe work in the container — `find_package(Protobuf CONFIG)` resolves to vcpkg's pinned protobuf without any toolchain-file flag. The contributor types the same commands they would on a host with system-installed protobuf; they happen to land on vcpkg's pinned 6.33.4. None of the four `buf.gen.yaml` files change. + +### Things explicitly NOT changed by this design + +- `prepare.bat` (already correct; stays as Windows-host fallback) +- Any `buf.gen.yaml` +- `test/cpp-tableau-loader/CMakeLists.txt` +- `.github/workflows/*.yml` (CI keeps `lukka/run-vcpkg` directly) + +## Verification matrix + +| Host | Container arch | First-run cost | Daily-cmd cost | Notes | +|---|---|---|---|---| +| Linux x86 | amd64 native | ~25 min build | bind-mount IO native | reference path | +| macOS Apple Silicon | **arm64 native** | ~25 min build | bind-mount IO native | no Rosetta tax | +| macOS Intel | amd64 native | ~25 min build | bind-mount IO native | | +| Windows + WSL2 | amd64 native | ~25 min build | bind-mount IO good if workspace is under WSL2 (`\\wsl.localhost\Ubuntu\…`), poor under `/mnt/c/...` | flagged in `.devcontainer/README.md` | + +**Acceptance gates:** + +1. `docker build .devcontainer/` succeeds clean on amd64 and arm64 hosts (gate 1: image actually builds). +2. Container start runs `postCreateCommand` and prints all five tool versions (gate 2: toolchain is wired correctly). +3. Inside the container, all four E2E paths from the README run green: + - `cd test/go-tableau-loader && buf generate .. && go test ./...` + - `cd test/cpp-tableau-loader && buf generate .. && cmake -S . -B build -DCMAKE_BUILD_TYPE=Debug && cmake --build build && ctest --test-dir build --output-on-failure` + - `cd test/csharp-tableau-loader && buf generate .. && dotnet test` + - `cd _lab/ts && npm install && npm run generate && npm run test` *(stretch — TS lab isn't in CI)* +4. `LOADER_PROTOBUF_VERSION=3.21.12 code .` → Reopen in Container → all C++ test paths still green (gate 3: protobuf version pinning works end-to-end). + +## Trade-offs and explicit deferrals + +- **First-run latency.** ~25 min on cold build (vcpkg compiles protobuf from source). Consequences: every change to the protobuf-installation `RUN` invalidates that layer for everyone who pulls the change. Mitigated by ordering it late in the layer chain (Go/buf changes don't trigger it). If pain becomes acute, prebuild-on-ghcr.io is the escape hatch. +- **No multi-arch publishing.** The Dockerfile is arch-agnostic, but `docker build` only builds the host arch. Apple Silicon contributors get arm64 natively because Docker Desktop's BuildKit picks `linux/arm64`. We do **not** run `docker buildx build --platform linux/amd64,linux/arm64 --push` anywhere. If we ever publish to ghcr.io, that's the moment to add buildx. +- **Modern protobuf default.** Daily dev runs against 6.33.4. Legacy-v3 contributors must rebuild the container with `LOADER_PROTOBUF_VERSION=3.21.12`. Acceptable because (a) CI catches legacy-v3 regressions automatically and (b) the container rebuild is incremental — only the vcpkg layer reruns. +- **Named volume for `~/go`.** Persists the Go module cache across rebuilds (~30s saved per first `go test` post-rebuild). If a contributor wants pure isolation, they `docker volume rm loader-go-mod`. + +## Implementation outline (for the writing-plans step that follows) + +1. Add `.devcontainer/Dockerfile` (multi-arch, manifest-mode vcpkg, version assertion). +2. Add `.devcontainer/devcontainer.json` (build args, named volume, extensions, postCreate banner). +3. Add `.devcontainer/README.md` (Docker prereqs, host-OS caveats, the `LOADER_PROTOBUF_VERSION` knob). +4. Update repo-root `README.md` Prerequisites section: add the new "Recommended: Dev Container" subsection; lead the existing Windows / per-language blocks with the "If you can't / don't want to use the devcontainer…" line. +5. Verification: `docker build` locally for amd64; container start produces the banner; all four E2E test commands green; `LOADER_PROTOBUF_VERSION=3.21.12` rebuild produces a working legacy-v3 image. From e8d4d10d2b565baeb4fe2d31e87f129ed688b482 Mon Sep 17 00:00:00 2001 From: wenchy Date: Fri, 29 May 2026 22:32:41 +0800 Subject: [PATCH 12/66] docs: implementation plan for devcontainer 12-task plan implementing the design at docs/superpowers/specs/2026-05-29-devcontainer-design.md. Each task is a single Dockerfile layer or config file with build/verify/commit steps and concrete expected outputs. Refs: docs/superpowers/specs/2026-05-29-devcontainer-design.md Co-Authored-By: Claude Opus 4.7 (1M context) --- .../plans/2026-05-29-devcontainer.md | 1047 +++++++++++++++++ 1 file changed, 1047 insertions(+) create mode 100644 docs/superpowers/plans/2026-05-29-devcontainer.md diff --git a/docs/superpowers/plans/2026-05-29-devcontainer.md b/docs/superpowers/plans/2026-05-29-devcontainer.md new file mode 100644 index 0000000..0188240 --- /dev/null +++ b/docs/superpowers/plans/2026-05-29-devcontainer.md @@ -0,0 +1,1047 @@ +# Dev Container Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Add a Dev Container under `.devcontainer/` so contributors on any host (Windows/macOS/Linux) get a one-command, reproducible build environment that mirrors CI's exact toolchain (C++17, Go 1.24, .NET 8, Node 20, buf 1.67.0, protobuf 6.33.4 via vcpkg). + +**Architecture:** Single-stage `Dockerfile` based on `mcr.microsoft.com/devcontainers/cpp:1-ubuntu-24.04`, layered Go → buf → vcpkg/protobuf → .NET/Node. Multi-arch via BuildKit `TARGETARCH` (amd64 + arm64 native, no QEMU). Protobuf version pinnable via the `LOADER_PROTOBUF_VERSION` host env var, flowing through `devcontainer.json` build args into a vcpkg manifest-mode install with a post-install version assertion. + +**Tech Stack:** Docker (BuildKit), `mcr.microsoft.com/devcontainers/cpp:1-ubuntu-24.04`, vcpkg manifest mode, VS Code Dev Containers spec. + +**Spec:** [`docs/superpowers/specs/2026-05-29-devcontainer-design.md`](./../specs/2026-05-29-devcontainer-design.md) + +--- + +## Files Created / Modified + +| Path | Action | Purpose | +|---|---|---| +| `.devcontainer/Dockerfile` | Create | Multi-arch, multi-language toolchain image | +| `.devcontainer/devcontainer.json` | Create | VS Code config: build args, named volume, extensions, banner | +| `.devcontainer/README.md` | Create | Host prerequisites, how-to, host-OS caveats | +| `README.md` | Modify | Add "Recommended: Dev Container" subsection at top of Prerequisites; add "If you can't / don't want to use the devcontainer…" lead-in to existing Windows + per-language blocks | + +CI workflows (`.github/workflows/*.yml`), `prepare.bat`, `buf.gen.yaml` files, and `CMakeLists.txt` are **not** touched. + +--- + +## Conventions for this plan + +- Each Dockerfile change is **one logical layer**, built and verified before the next is added. The "test" for a layer is `docker build` + a `docker run --rm` smoke check that the binary on PATH reports the expected version. +- Build target image tag: `loader-devcontainer:dev` (overwritten each build). +- Build context is `.devcontainer/` so all `docker build` commands use `docker build -t loader-devcontainer:dev .devcontainer/`. +- Smoke checks use `docker run --rm loader-devcontainer:dev `. +- After Task 5 the image takes ~25 minutes to rebuild from scratch on first run because vcpkg compiles protobuf from source. Subsequent builds reuse layers and only re-run the layer you changed. + +--- + +### Task 1: Stub Dockerfile with the base image only + +**Files:** +- Create: `.devcontainer/Dockerfile` + +- [ ] **Step 1: Create the file** + +`.devcontainer/Dockerfile`: + +```dockerfile +# syntax=docker/dockerfile:1.7 +# tableauio/loader devcontainer +# +# Single-stage, multi-arch (amd64 + arm64) image bringing the full +# C++/Go/.NET/Node toolchain plus protobuf 6.33.4 (via vcpkg) at the +# exact versions CI uses. See docs/superpowers/specs/2026-05-29-devcontainer-design.md. + +FROM mcr.microsoft.com/devcontainers/cpp:1-ubuntu-24.04 +``` + +- [ ] **Step 2: Build the base layer** + +```sh +docker build -t loader-devcontainer:dev .devcontainer/ +``` + +Expected: build completes in <1 min on a warm Docker, ending with +`=> => naming to docker.io/library/loader-devcontainer:dev`. + +- [ ] **Step 3: Smoke-check the base image runs and gives us the vscode user** + +```sh +docker run --rm loader-devcontainer:dev id +``` + +Expected output: `uid=0(root) gid=0(root) groups=0(root)` (RUN context defaults to root; the `vscode` user is set later via `devcontainer.json`'s `remoteUser`). Confirm with: + +```sh +docker run --rm loader-devcontainer:dev id vscode +``` + +Expected: `uid=1000(vscode) gid=1000(vscode) groups=1000(vscode),...` — confirming the base image ships the `vscode` user. + +- [ ] **Step 4: Commit** + +```sh +git add .devcontainer/Dockerfile +git commit -m "$(cat <<'EOF' +feat(devcontainer): add base Dockerfile (Ubuntu 24.04 cpp image) + +Bootstrap the devcontainer image with Microsoft's multi-arch +mcr.microsoft.com/devcontainers/cpp:1-ubuntu-24.04 base. Subsequent +commits layer Go, buf, vcpkg/protobuf, .NET, and Node on top. + +Refs: docs/superpowers/specs/2026-05-29-devcontainer-design.md +EOF +)" +``` + +--- + +### Task 2: Add architecture-detection layer + +Resolves `TARGETARCH` (BuildKit auto-populates it from the host) into the per-arch values that downstream layers need: Go's tarball arch, buf's release-asset arch, and the vcpkg triplet. Writes them to `/opt/buildargs.env` so later `RUN` commands can `source` them — Dockerfile `ARG`s don't persist across `RUN` blocks the way ENV does, but a shell-readable file does. + +**Files:** +- Modify: `.devcontainer/Dockerfile` + +- [ ] **Step 1: Append the architecture detection block** + +Append to `.devcontainer/Dockerfile`: + +```dockerfile +# --------------------------------------------------------------------------- +# Architecture detection. BuildKit auto-populates TARGETARCH; we resolve it +# into per-arch download-name fragments (Go's tarball, buf's release asset, +# vcpkg triplet) and persist them to /opt/buildargs.env so later RUN layers +# can `source` them — Dockerfile ARGs don't survive across RUN boundaries. +# --------------------------------------------------------------------------- +ARG TARGETARCH +RUN < /opt/buildargs.env +EOF +``` + +- [ ] **Step 2: Build** + +```sh +docker build -t loader-devcontainer:dev .devcontainer/ +``` + +Expected: success in <1 min. The architecture-detection layer is just a `case` + `printf`. + +- [ ] **Step 3: Smoke-check the resolved values** + +```sh +docker run --rm loader-devcontainer:dev cat /opt/buildargs.env +``` + +Expected (on an amd64 host): +``` +GO_ARCH=amd64 +BUF_ARCH=x86_64 +VCPKG_TRIPLET=x64-linux +``` + +(arm64 host would print `GO_ARCH=arm64`, `BUF_ARCH=aarch64`, `VCPKG_TRIPLET=arm64-linux`.) + +- [ ] **Step 4: Commit** + +```sh +git add .devcontainer/Dockerfile +git commit -m "$(cat <<'EOF' +feat(devcontainer): add architecture detection + +Resolves TARGETARCH (amd64 or arm64) into per-arch values +(Go tarball arch, buf release-asset arch, vcpkg triplet) and +writes them to /opt/buildargs.env for downstream RUN layers +to source. Unknown arches fail the build. +EOF +)" +``` + +--- + +### Task 3: Add Go 1.24.0 + +**Files:** +- Modify: `.devcontainer/Dockerfile` + +- [ ] **Step 1: Append the Go layer** + +Append to `.devcontainer/Dockerfile`: + +```dockerfile +# --------------------------------------------------------------------------- +# Go 1.24.0 — official tarball into /usr/local/go. +# +# PATH is set via ENV (not /etc/profile.d/) so non-interactive shells like +# the postCreateCommand and downstream RUNs see Go without sourcing profile. +# /home/vscode/go/bin lands `go install`-placed binaries on PATH automatically. +# --------------------------------------------------------------------------- +ARG GO_VERSION=1.24.0 +RUN < /opt/vcpkg-manifest/vcpkg.json </dev/null | head -n1) +case "$(basename "${INFO_FILE:-/missing}" 2>/dev/null)" in + protobuf_${PROTOBUF_VERSION}*) + ;; + *) + echo "ERROR: installed protobuf does not match requested version ${PROTOBUF_VERSION}." + echo " vcpkg installed-file marker: ${INFO_FILE:-}" + echo " Bump VCPKG_BASELINE_COMMIT (in this Dockerfile, prepare.bat," + echo " and testing-cpp.yml) to a commit that knows about the requested version." + exit 1 + ;; +esac + +# 5. Stable symlinks so ENV CMAKE_PREFIX_PATH (last layer) doesn't have to +# care about the underlying triplet. +ln -s /opt/vcpkg-manifest/vcpkg_installed/${VCPKG_TRIPLET} /opt/vcpkg/active +ln -s /opt/vcpkg/active/tools/protobuf/protoc /usr/local/bin/protoc +EOF +``` + +- [ ] **Step 2: Build (this is the slow one, ~25 minutes on cold cache)** + +```sh +docker build -t loader-devcontainer:dev .devcontainer/ +``` + +Expected: success after ~25 minutes on first run; subsequent builds reuse the layer instantly. The build log includes lines like `protobuf:x64-linux@6.33.4 -- Building`, then a long CMake/ninja compile, then `Total install time:`. + +- [ ] **Step 3: Smoke-check protoc** + +```sh +docker run --rm loader-devcontainer:dev protoc --version +``` + +Expected: `libprotoc 33.4` (the protoc binary reports the umbrella protoc release tag, which is `33.4` for the libprotobuf C++ 6.33.4 line — same mapping `testing-csharp.yml` documents). + +- [ ] **Step 4: Smoke-check the symlinks** + +```sh +docker run --rm loader-devcontainer:dev sh -c 'readlink -f /usr/local/bin/protoc; readlink -f /opt/vcpkg/active' +``` + +Expected (on amd64): +``` +/opt/vcpkg-manifest/vcpkg_installed/x64-linux/tools/protobuf/protoc +/opt/vcpkg-manifest/vcpkg_installed/x64-linux +``` + +- [ ] **Step 5: Smoke-check the version-assertion marker file** + +```sh +docker run --rm loader-devcontainer:dev sh -c 'ls /opt/vcpkg-manifest/vcpkg_installed/vcpkg/info/protobuf_*.list' +``` + +Expected: a single file named like `protobuf_6.33.4_x64-linux.list` (or with a `#N` port-revision suffix — the assertion uses prefix matching so either passes). + +- [ ] **Step 6: Commit** + +```sh +git add .devcontainer/Dockerfile +git commit -m "$(cat <<'EOF' +feat(devcontainer): add vcpkg + protobuf via manifest mode + +Pin vcpkg to commit dc8d75c…df932 (lock-step with prepare.bat and +testing-cpp.yml's VCPKG_COMMIT). Render a minimal vcpkg.json manifest +with the protobuf override + builtin-baseline, install via +manifest mode (the only mode where the version pin actually takes +effect), and post-assert that the resolved port version starts with +the requested PROTOBUF_VERSION. Default is 6.33.4; legacy v3 reachable +via --build-arg PROTOBUF_VERSION=3.21.12. + +Symlink /opt/vcpkg/active → installed/ and /usr/local/bin/protoc +→ active/tools/protobuf/protoc so downstream ENV/PATH stays +arch-independent. +EOF +)" +``` + +--- + +### Task 6: Verify the protobuf version-pinning path (no Dockerfile change) + +This task is verification-only: a deliberate counterexample build that proves the `LOADER_PROTOBUF_VERSION` knob works end-to-end. No commit produced. + +- [ ] **Step 1: Build with protobuf 3.21.12 (legacy v3 line)** + +```sh +docker build \ + --build-arg PROTOBUF_VERSION=3.21.12 \ + -t loader-devcontainer:legacy-v3 .devcontainer/ +``` + +Expected: vcpkg layer rebuilds (~10 min on cache hit for everything before that layer; the protobuf compile itself takes the whole time). Final assertion succeeds because the resolved port matches `3.21.12`. + +- [ ] **Step 2: Smoke-check protoc reports the legacy version** + +```sh +docker run --rm loader-devcontainer:legacy-v3 protoc --version +``` + +Expected: `libprotoc 3.21.12` (the legacy-v3 line still uses `3.x.y` in `protoc --version`; the umbrella tag is `v21.12`). + +- [ ] **Step 3: Verify the assertion fires when given an impossible version** + +```sh +docker build \ + --build-arg PROTOBUF_VERSION=999.0.0 \ + -t loader-devcontainer:never .devcontainer/ 2>&1 | tail -20 +``` + +Expected: build fails. The vcpkg manifest install errors out (or the assertion catches it), and the last lines include either `error: while looking up version 999.0.0` (vcpkg-side rejection) or `ERROR: installed protobuf does not match requested version 999.0.0` (our assertion). Either failure is acceptable; both prove the silent-wrong-version regression is impossible. + +- [ ] **Step 4: Drop the temporary tags** + +```sh +docker rmi loader-devcontainer:legacy-v3 loader-devcontainer:never 2>/dev/null || true +``` + +(No commit — this task is purely verification of behaviour established in Task 5.) + +--- + +### Task 7: Add .NET SDK 8.0 + Node.js 20 LTS + +**Files:** +- Modify: `.devcontainer/Dockerfile` + +- [ ] **Step 1: Append the .NET + Node layer** + +Append to `.devcontainer/Dockerfile`: + +```dockerfile +# --------------------------------------------------------------------------- +# .NET SDK 8.0 + Node.js 20 LTS — apt-based installs from the official +# Microsoft and NodeSource repositories. apt-get clean + rm /var/lib/apt/lists +# at the end keeps the layer small. +# --------------------------------------------------------------------------- +RUN <` (latest 8.0.x at apt-cache time; e.g. `8.0.404`). + +- [ ] **Step 4: Smoke-check Node** + +```sh +docker run --rm loader-devcontainer:dev node --version +``` + +Expected: `v20..` (latest v20 LTS at NodeSource setup time). + +- [ ] **Step 5: Commit** + +```sh +git add .devcontainer/Dockerfile +git commit -m "$(cat <<'EOF' +feat(devcontainer): add .NET SDK 8.0 and Node.js 20 LTS + +Microsoft apt repo for dotnet-sdk-8.0 (matches testing-csharp.yml's +target). NodeSource setup_20.x for Node 20 LTS (covers the +experimental _lab/ts/ workflow; not currently in CI). Both clean +their apt caches to keep the layer small. +EOF +)" +``` + +--- + +### Task 8: Final ENV (CMAKE_PREFIX_PATH) + +This is the single line that lets every existing per-language daily command from the README work inside the container without flag tweaks. `find_package(Protobuf CONFIG)` resolves to vcpkg's pinned protobuf because `CMAKE_PREFIX_PATH` includes `/opt/vcpkg/active`. + +**Files:** +- Modify: `.devcontainer/Dockerfile` + +- [ ] **Step 1: Append the final ENV block** + +Append to `.devcontainer/Dockerfile`: + +```dockerfile +# --------------------------------------------------------------------------- +# Final environment: with CMAKE_PREFIX_PATH pointing at vcpkg's installed +# tree, find_package(Protobuf CONFIG) resolves automatically — the existing +# README's "Dev at Linux → CMake (system protobuf)" recipe works as written +# inside the container. +# VCPKG_ROOT is set above (Task 5) for users who want to invoke the toolchain +# file explicitly: -DCMAKE_TOOLCHAIN_FILE=$VCPKG_ROOT/scripts/buildsystems/vcpkg.cmake. +# --------------------------------------------------------------------------- +ENV CMAKE_PREFIX_PATH=/opt/vcpkg/active +``` + +- [ ] **Step 2: Build** + +```sh +docker build -t loader-devcontainer:dev .devcontainer/ +``` + +Expected: success in <30 s (only the ENV layer changes). + +- [ ] **Step 3: Smoke-check the env** + +```sh +docker run --rm loader-devcontainer:dev sh -c 'echo "VCPKG_ROOT=$VCPKG_ROOT"; echo "CMAKE_PREFIX_PATH=$CMAKE_PREFIX_PATH"' +``` + +Expected: +``` +VCPKG_ROOT=/opt/vcpkg +CMAKE_PREFIX_PATH=/opt/vcpkg/active +``` + +- [ ] **Step 4: Commit** + +```sh +git add .devcontainer/Dockerfile +git commit -m "$(cat <<'EOF' +feat(devcontainer): finalize CMAKE_PREFIX_PATH + +CMAKE_PREFIX_PATH=/opt/vcpkg/active lets the existing README cmake +recipe (-DCMAKE_BUILD_TYPE=Debug, no toolchain file) resolve protobuf +inside the container without any flag changes from the host workflow. +EOF +)" +``` + +--- + +### Task 9: Add `devcontainer.json` + +**Files:** +- Create: `.devcontainer/devcontainer.json` + +- [ ] **Step 1: Create the file** + +`.devcontainer/devcontainer.json`: + +```jsonc +{ + // tableauio/loader Dev Container. + // See docs/superpowers/specs/2026-05-29-devcontainer-design.md for the design. + "name": "tableauio/loader", + + // Build args wire host env to Dockerfile ARGs: + // LOADER_PROTOBUF_VERSION on the host -> PROTOBUF_VERSION inside. + // Default 6.33.4 (CI's modern matrix entry). To rebuild against legacy v3: + // LOADER_PROTOBUF_VERSION=3.21.12 code . # then Reopen in Container. + "build": { + "dockerfile": "Dockerfile", + "args": { + "PROTOBUF_VERSION": "${localEnv:LOADER_PROTOBUF_VERSION:6.33.4}" + } + }, + + // Persist the Go module cache across container rebuilds. Workspace itself + // uses VS Code's default bind-mount so edits sync to the host. + "mounts": [ + "source=loader-go-mod,target=/home/vscode/go,type=volume" + ], + + "remoteUser": "vscode", + "workspaceFolder": "/workspaces/loader", + + "customizations": { + "vscode": { + "extensions": [ + "golang.go", + "ms-vscode.cmake-tools", + "ms-vscode.cpptools", + "ms-dotnettools.csharp", + "bufbuild.vscode-buf", + "zxh404.vscode-proto3" + ], + "settings": { + // Don't auto-install gopls and friends on first open — let the user + // do it explicitly from the Go extension's command palette. + "go.toolsManagement.autoUpdate": false, + // Don't auto-cmake-configure on workspace open; we run cmake manually + // per the existing README recipes. + "cmake.configureOnOpen": false + } + } + }, + + // One-line ready banner so the developer knows the container is healthy. + // Pure echo — no installs, no version-pinning at runtime, no surprises. + "postCreateCommand": "printf 'tableauio/loader devcontainer ready.\\n go: %s\\n buf: %s\\n protoc: %s\\n dotnet: %s\\n node: %s\\n' \"$(go version | cut -d' ' -f3)\" \"$(buf --version)\" \"$(protoc --version)\" \"$(dotnet --version)\" \"$(node --version)\"" +} +``` + +- [ ] **Step 2: Validate the JSON parses (with jsonc comments stripped)** + +```sh +python3 - <<'PY' +import json, re, pathlib +src = pathlib.Path('.devcontainer/devcontainer.json').read_text() +# Strip // line comments (devcontainer.json is jsonc). +stripped = re.sub(r'(^|[^:])//.*$', r'\1', src, flags=re.MULTILINE) +parsed = json.loads(stripped) +assert parsed['name'] == 'tableauio/loader' +assert parsed['build']['dockerfile'] == 'Dockerfile' +assert parsed['build']['args']['PROTOBUF_VERSION'] == '${localEnv:LOADER_PROTOBUF_VERSION:6.33.4}' +print('devcontainer.json OK') +PY +``` + +Expected: `devcontainer.json OK`. + +- [ ] **Step 3: Commit** + +```sh +git add .devcontainer/devcontainer.json +git commit -m "$(cat <<'EOF' +feat(devcontainer): add devcontainer.json + +Wires the Dockerfile under build.args, mounts a named volume for the +Go module cache, declares the VS Code extension set, and prints a +one-line ready-banner via postCreateCommand. PROTOBUF_VERSION flows +from the host LOADER_PROTOBUF_VERSION env var (default 6.33.4) so +contributors can rebuild against the legacy v3 line via: + LOADER_PROTOBUF_VERSION=3.21.12 code . +EOF +)" +``` + +--- + +### Task 10: Add `.devcontainer/README.md` + +**Files:** +- Create: `.devcontainer/README.md` + +- [ ] **Step 1: Create the file** + +`.devcontainer/README.md`: + +````markdown +# Dev Container + +The recommended way to develop on `tableauio/loader`. One container, all +four target languages (C++17, Go 1.24, .NET 8, Node 20) plus protobuf +6.33.4 via vcpkg, pinned to the exact toolchain CI uses. + +## Prerequisites + +- Docker Desktop (Windows / macOS) or Docker Engine (Linux) +- VS Code with the [Dev Containers extension](https://marketplace.visualstudio.com/items?itemName=ms-vscode-remote.remote-containers) + +## Open the container + +```sh +code . # in the repo root +``` + +In VS Code, run **Dev Containers: Reopen in Container** from the command +palette. First build is one-time ~25 minutes (vcpkg compiles protobuf +6.33.4 from source); subsequent reopens are near-instant. + +When the container is ready, the integrated terminal prints a banner with +five toolchain versions. After that, every command from the per-language +sections of the repo root [`README.md`](../README.md) works as written — +no PATH dance, no extra cmake flags. + +## Pin a different protobuf version + +Daily dev runs against protobuf 6.33.4 (CI's "modern" matrix entry). To +rebuild against the legacy v3 line: + +```sh +LOADER_PROTOBUF_VERSION=3.21.12 code . +``` + +…then **Dev Containers: Reopen in Container** (or **Rebuild Container** +if the container is already running). The vcpkg layer rebuilds with the +manifest pinning protobuf 3.21.12; everything else is reused from the +cache. + +## Host-OS caveats + +- **Windows.** WSL2 backend required. **Check the workspace out under + WSL2** (e.g. `\\wsl.localhost\Ubuntu\home\\loader`) — not under + `/mnt/c/...` — for good bind-mount performance. Files under `/mnt/c/` + work but file-watching and large `cmake --build` operations are 5–10× + slower. + +- **Apple Silicon.** Docker builds the container natively as arm64. No + Rosetta or QEMU emulation. Confirm with `docker info | grep Architecture` + → expect `linux/arm64`. + +- **Linux (native Docker Engine).** No special configuration. + +## Architecture + +Single-stage Dockerfile based on +`mcr.microsoft.com/devcontainers/cpp:1-ubuntu-24.04`, with these layers: + +1. Architecture detection (`TARGETARCH` → Go arch, buf arch, vcpkg triplet) +2. Go 1.24.0 (official tarball, multi-arch) +3. buf 1.67.0 (single-binary release, multi-arch) +4. vcpkg pinned to `dc8d75c…df932`, protobuf installed via vcpkg manifest + mode and asserted against the requested version +5. .NET SDK 8.0 (Microsoft apt repo) +6. Node.js 20 LTS (NodeSource apt repo) +7. `ENV CMAKE_PREFIX_PATH=/opt/vcpkg/active` so `find_package(Protobuf CONFIG)` + resolves automatically + +The architecture choice is detected from BuildKit's `TARGETARCH` and fed +into Go / buf / vcpkg triplet selection. Docker auto-selects the host +arch on build. + +## Falling back + +If you can't run Docker (corp policy, restricted machines, etc.) the +existing manual setup paths in the [repo README](../README.md) — Windows +`prepare.bat`, per-language `Install protobuf` instructions — still work. +The devcontainer is the recommended path; the rest is the supported +fallback. +```` + +- [ ] **Step 2: Verify the file renders as expected** + +```sh +ls -la .devcontainer/README.md && wc -l .devcontainer/README.md +``` + +Expected: file exists, ~70 lines. + +- [ ] **Step 3: Commit** + +```sh +git add .devcontainer/README.md +git commit -m "$(cat <<'EOF' +docs(devcontainer): add .devcontainer/README.md + +One-pager covering prerequisites (Docker Desktop / Engine + VS Code +Dev Containers extension), how to open the container, the +LOADER_PROTOBUF_VERSION knob, host-OS caveats (WSL2 workspace +location, Apple Silicon native arm64), and the layered Dockerfile +architecture. Points users at prepare.bat / per-language manual setup +as the explicit fallback path. +EOF +)" +``` + +--- + +### Task 11: Update repo root `README.md` + +Adds the "Recommended: Dev Container" subsection at the top of `Prerequisites` and prefixes the existing Windows + per-language blocks with a short opt-out lead-in. + +**Files:** +- Modify: `README.md` + +- [ ] **Step 1: Add the "Recommended: Dev Container" subsection** + +Open `README.md` and find the Prerequisites bullet list that ends with `…will fail to link.`. Immediately after the existing migration callout (the `> **Migrating from the bundled-protobuf layout?**` block), insert this new subsection — i.e. right before `### Install protobuf`: + +```markdown +### Recommended: Dev Container (any host OS) + +The fastest way to get a reproducible build environment is to open the +repo in VS Code and choose **Reopen in Container**. The devcontainer +under [`.devcontainer/`](./.devcontainer/) has everything pinned to the +exact versions CI uses (Go 1.24, buf 1.67.0, protobuf 6.33.4 via vcpkg, +.NET 8.0, Node 20). First container build is one-time ~25 minutes (vcpkg +compiles protobuf from source); subsequent reopens are near-instant. + +After the container starts you can skip the per-language setup below and +jump straight to **[C++](#c)** / **[Go](#go)** / **[C#](#c-1)** / +**[TypeScript](#typescript)**. + +Requirements: Docker Desktop (Windows + macOS) or Docker Engine (Linux), +and the VS Code "Dev Containers" extension. See +[`.devcontainer/README.md`](./.devcontainer/README.md) for the longer +how-to. +``` + +- [ ] **Step 2: Add an opt-out lead-in to the `Install protobuf` section** + +In `README.md`, find the line that currently reads: + +```markdown +### Install protobuf + +Pick whichever channel fits your platform; loader does not bundle protobuf. +``` + +Replace it with: + +```markdown +### Install protobuf + +> **Skip this section if you're using the [devcontainer](#recommended-dev-container-any-host-os).** +> The instructions below cover the manual fallback for hosts where +> Docker isn't available. + +Pick whichever channel fits your platform; loader does not bundle protobuf. +``` + +- [ ] **Step 3: Add an opt-out lead-in to the Windows bootstrap section** + +Find: + +```markdown +### Windows: bootstrap the rest of the toolchain + +Run `prepare.bat` **as Administrator** to install everything you need on a +fresh Windows machine: [Chocolatey](https://chocolatey.org/), +``` + +Replace with: + +```markdown +### Windows: bootstrap the rest of the toolchain + +> **Skip this section if you're using the [devcontainer](#recommended-dev-container-any-host-os).** +> `prepare.bat` is the manual fallback for Windows hosts that can't run +> Docker. + +Run `prepare.bat` **as Administrator** to install everything you need on a +fresh Windows machine: [Chocolatey](https://chocolatey.org/), +``` + +- [ ] **Step 4: Verify the README still renders sanely** + +```sh +grep -nE '^#{1,4} ' README.md | head -30 +``` + +Expected: shows the heading skeleton, with the new `### Recommended: Dev Container (any host OS)` heading appearing between the Prerequisites bullets and `### Install protobuf`. + +- [ ] **Step 5: Commit** + +```sh +git add README.md +git commit -m "$(cat <<'EOF' +docs: recommend the devcontainer in repo README + +Add a new "Recommended: Dev Container (any host OS)" subsection at the +top of Prerequisites pointing contributors at .devcontainer/. Add a +"Skip this section if you're using the devcontainer" lead-in to the +existing "Install protobuf" and "Windows: bootstrap" blocks so the +manual paths are clearly the fallback, not the primary route. +EOF +)" +``` + +--- + +### Task 12: End-to-end integration check (verification only, no commit) + +Final smoke test: bring the container up exactly the way a contributor would, and run the four E2E test commands from the README to confirm the toolchain inside the container actually exercises the repo. No commit produced. + +- [ ] **Step 1: Build the final container** + +```sh +docker build -t loader-devcontainer:dev .devcontainer/ +``` + +Expected: success. If Tasks 1–10 were committed individually, this should be all-cache-hits and finish in <10 s. + +- [ ] **Step 2: Run the postCreate banner manually** + +```sh +docker run --rm loader-devcontainer:dev sh -c " +printf 'tableauio/loader devcontainer ready.\n go: %s\n buf: %s\n protoc: %s\n dotnet: %s\n node: %s\n' \"\$(go version | cut -d' ' -f3)\" \"\$(buf --version)\" \"\$(protoc --version)\" \"\$(dotnet --version)\" \"\$(node --version)\" +" +``` + +Expected output (versions may vary slightly): +``` +tableauio/loader devcontainer ready. + go: go1.24.0 + buf: 1.67.0 + protoc: libprotoc 33.4 + dotnet: 8.0.404 + node: v20.18.0 +``` + +- [ ] **Step 3: Run the Go E2E inside the container** + +```sh +docker run --rm -v "$(pwd):/workspaces/loader" -w /workspaces/loader/test/go-tableau-loader \ + loader-devcontainer:dev sh -c "buf generate .. && go test ./..." +``` + +Expected: `buf generate` regenerates the Go protoconf and loader stubs, then `go test ./...` reports `ok` for `test/go-tableau-loader`, `internal/index`, `pkg/treemap`, `pkg/udiff`, etc. + +- [ ] **Step 4: Run the C++ E2E inside the container** + +```sh +docker run --rm -v "$(pwd):/workspaces/loader" -w /workspaces/loader/test/cpp-tableau-loader \ + loader-devcontainer:dev sh -c " + buf generate .. && + cmake -S . -B build -G Ninja -DCMAKE_BUILD_TYPE=Debug && + cmake --build build --parallel && + ctest --test-dir build --output-on-failure + " +``` + +Expected: `buf generate` regenerates `*.pb.*` and `*.pc.*`, cmake configure picks up `protobuf::libprotobuf` via `CMAKE_PREFIX_PATH`, build succeeds, ctest reports all tests passed. + +- [ ] **Step 5: Run the C# E2E inside the container** + +```sh +docker run --rm -v "$(pwd):/workspaces/loader" -w /workspaces/loader/test/csharp-tableau-loader \ + loader-devcontainer:dev sh -c "buf generate .. && dotnet test --nologo --logger 'console;verbosity=normal'" +``` + +Expected: protobuf C# stubs regenerated, dotnet builds the project, xUnit reports passed. + +- [ ] **Step 6: Confirm the named volume mount works (interactive)** + +```sh +docker volume create loader-go-mod >/dev/null +docker run --rm -v "$(pwd):/workspaces/loader" \ + -v loader-go-mod:/home/vscode/go \ + -w /workspaces/loader/test/go-tableau-loader \ + loader-devcontainer:dev go test ./... +``` + +Run the command twice. The second run should be noticeably faster (Go's module cache is warm in `/home/vscode/go/pkg/mod`). + +Cleanup: + +```sh +docker volume rm loader-go-mod +``` + +- [ ] **Step 7: Push the branch (no commit; everything is already committed in earlier tasks)** + +```sh +git push origin HEAD +``` + +Expected: branch is pushed to remote with all 10 task commits visible in `git log`. + +--- + +## Self-Review + +**Spec coverage** — every Goals item in the spec maps to a task: + +| Spec goal | Implementing task(s) | +|---|---| +| One-command setup on any host (Reopen in Container) | Tasks 9 (devcontainer.json) + 10 (.devcontainer/README.md) + 11 (repo README) | +| Reproducibility — pinned versions matching CI | Tasks 3 (Go 1.24), 4 (buf 1.67.0), 5 (vcpkg + protobuf 6.33.4 with assertion), 7 (.NET 8.0, Node 20) | +| Multi-arch native (amd64 + arm64) | Tasks 2 (TARGETARCH detection), 3, 4, 5 (per-arch downloads + triplet) | +| Pinnable protobuf version via LOADER_PROTOBUF_VERSION | Tasks 5 (Dockerfile ARG + manifest mode) + 9 (devcontainer.json `${localEnv:...}`) + 6 (verification) | +| Daily commands stay unchanged (CMAKE_PREFIX_PATH) | Task 8 | + +Non-goals (no ghcr.io publish, no CI inside container, no Unity, no replacing prepare.bat) — none of them produce a task, by design. + +**Placeholder scan** — none. Every step has runnable code/commands; commit messages are concrete; expected outputs are spelled out. + +**Type/path consistency** — `loader-devcontainer:dev` is the consistent image tag across all tasks; `.devcontainer/Dockerfile` and `.devcontainer/devcontainer.json` are referenced with consistent paths; `/opt/vcpkg/active`, `/opt/vcpkg-manifest/...`, `/opt/buildargs.env` all used identically across tasks. + +--- + +## Execution Handoff + +Plan complete and saved to `docs/superpowers/plans/2026-05-29-devcontainer.md`. Two execution options: + +**1. Subagent-Driven (recommended)** — I dispatch a fresh subagent per task, review between tasks, fast iteration with two-stage review. + +**2. Inline Execution** — Execute tasks in this session using executing-plans, batch execution with checkpoints. + +Which approach? From 40e4328543fa10f8f4c6308c23a940093a7f89fb Mon Sep 17 00:00:00 2001 From: wenchy Date: Fri, 29 May 2026 22:47:42 +0800 Subject: [PATCH 13/66] feat(devcontainer): add base Dockerfile (Ubuntu 24.04 cpp image) Bootstrap the devcontainer image with Microsoft's multi-arch mcr.microsoft.com/devcontainers/cpp:1-ubuntu-24.04 base. Subsequent commits layer Go, buf, vcpkg/protobuf, .NET, and Node on top. Refs: docs/superpowers/specs/2026-05-29-devcontainer-design.md Co-Authored-By: Claude Opus 4.7 (1M context) --- .devcontainer/Dockerfile | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 .devcontainer/Dockerfile diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile new file mode 100644 index 0000000..337f9d4 --- /dev/null +++ b/.devcontainer/Dockerfile @@ -0,0 +1,8 @@ +# syntax=docker/dockerfile:1.7 +# tableauio/loader devcontainer +# +# Single-stage, multi-arch (amd64 + arm64) image bringing the full +# C++/Go/.NET/Node toolchain plus protobuf 6.33.4 (via vcpkg) at the +# exact versions CI uses. See docs/superpowers/specs/2026-05-29-devcontainer-design.md. + +FROM mcr.microsoft.com/devcontainers/cpp:1-ubuntu-24.04 From 181863f677adae1406e22e8f704a054388240139 Mon Sep 17 00:00:00 2001 From: wenchy Date: Fri, 29 May 2026 22:56:56 +0800 Subject: [PATCH 14/66] feat(devcontainer): add architecture detection Resolves TARGETARCH (amd64 or arm64) into per-arch values (Go tarball arch, buf release-asset arch, vcpkg triplet) and writes them to /opt/buildargs.env for downstream RUN layers to source. Unknown arches fail the build. Co-Authored-By: Claude Opus 4.7 (1M context) --- .devcontainer/Dockerfile | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index 337f9d4..e1d7dc9 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -6,3 +6,22 @@ # exact versions CI uses. See docs/superpowers/specs/2026-05-29-devcontainer-design.md. FROM mcr.microsoft.com/devcontainers/cpp:1-ubuntu-24.04 + +# --------------------------------------------------------------------------- +# Architecture detection. BuildKit auto-populates TARGETARCH; we resolve it +# into per-arch download-name fragments (Go's tarball, buf's release asset, +# vcpkg triplet) and persist them to /opt/buildargs.env so later RUN layers +# can `source` them — Dockerfile ARGs don't survive across RUN boundaries. +# --------------------------------------------------------------------------- +ARG TARGETARCH +RUN < /opt/buildargs.env +EOF From 8a104df2eaf2c7fa9619f9e5e9d93d48a96e1c73 Mon Sep 17 00:00:00 2001 From: wenchy Date: Fri, 29 May 2026 23:04:20 +0800 Subject: [PATCH 15/66] feat(devcontainer): add Go 1.24.0 Install Go from the official multi-arch tarball into /usr/local/go. PATH is exposed via ENV (not /etc/profile.d) so non-interactive shells (postCreateCommand, downstream RUNs) see `go` without sourcing profile. Co-Authored-By: Claude Opus 4.7 (1M context) --- .devcontainer/Dockerfile | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index e1d7dc9..b2d2e8a 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -25,3 +25,19 @@ mkdir -p /opt printf 'GO_ARCH=%s\nBUF_ARCH=%s\nVCPKG_TRIPLET=%s\n' \ "${GO_ARCH}" "${BUF_ARCH}" "${TRIPLET}" > /opt/buildargs.env EOF + +# --------------------------------------------------------------------------- +# Go 1.24.0 — official tarball into /usr/local/go. +# +# PATH is set via ENV (not /etc/profile.d/) so non-interactive shells like +# the postCreateCommand and downstream RUNs see Go without sourcing profile. +# /home/vscode/go/bin lands `go install`-placed binaries on PATH automatically. +# --------------------------------------------------------------------------- +ARG GO_VERSION=1.24.0 +RUN < Date: Fri, 29 May 2026 23:06:39 +0800 Subject: [PATCH 16/66] feat(devcontainer): add buf 1.67.0 Single-binary release into /usr/local/bin/buf. Pinned to the same version testing-cpp.yml / testing-csharp.yml use. Co-Authored-By: Claude Opus 4.7 (1M context) --- .devcontainer/Dockerfile | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index b2d2e8a..bb00063 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -41,3 +41,15 @@ curl -fsSL "https://go.dev/dl/go${GO_VERSION}.linux-${GO_ARCH}.tar.gz" \ | tar -C /usr/local -xz EOF ENV PATH=/usr/local/go/bin:/home/vscode/go/bin:${PATH} + +# --------------------------------------------------------------------------- +# buf 1.67.0 — single-binary release on PATH. +# --------------------------------------------------------------------------- +ARG BUF_VERSION=1.67.0 +RUN < Date: Fri, 29 May 2026 23:14:49 +0800 Subject: [PATCH 17/66] feat(devcontainer): add vcpkg + protobuf via manifest mode MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Pin vcpkg to commit dc8d75c…df932 (lock-step with prepare.bat and testing-cpp.yml's VCPKG_COMMIT). Render a minimal vcpkg.json manifest with the protobuf override + builtin-baseline, install via manifest mode (the only mode where the version pin actually takes effect), and post-assert that the resolved port version starts with the requested PROTOBUF_VERSION. Default is 6.33.4; legacy v3 reachable via --build-arg PROTOBUF_VERSION=3.21.12. Symlink /opt/vcpkg/active → installed/ and /usr/local/bin/protoc → active/tools/protobuf/protoc so downstream ENV/PATH stays arch-independent. Co-Authored-By: Claude Opus 4.7 (1M context) --- .devcontainer/Dockerfile | 71 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 71 insertions(+) diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index bb00063..0648756 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -53,3 +53,74 @@ curl -fsSL -o /usr/local/bin/buf \ "https://github.com/bufbuild/buf/releases/download/v${BUF_VERSION}/buf-Linux-${BUF_ARCH}" chmod +x /usr/local/bin/buf EOF + +# --------------------------------------------------------------------------- +# vcpkg + protobuf via manifest mode. +# +# Pins: +# VCPKG_BASELINE_COMMIT — same commit testing-cpp.yml's VCPKG_COMMIT and +# prepare.bat's VCPKG_BASELINE_COMMIT use. Bumping vcpkg = bump all three. +# PROTOBUF_VERSION — defaults to the modern 6.33.4 line (CI's primary). +# Override at build time: +# docker build --build-arg PROTOBUF_VERSION=3.21.12 ... +# devcontainer.json wires this to the LOADER_PROTOBUF_VERSION host env var. +# +# Why manifest mode: classic-mode `vcpkg install --x-version=...` is silently +# a no-op; only manifest mode + overrides actually pin the port version. The +# post-install assertion catches the case where vcpkg's resolution still picks +# a different port revision than requested. +# --------------------------------------------------------------------------- +ARG VCPKG_BASELINE_COMMIT=dc8d75cfc3281b8e2a4ed8ee4163c891190df932 +ARG PROTOBUF_VERSION=6.33.4 +ENV VCPKG_ROOT=/opt/vcpkg + +RUN < /opt/vcpkg-manifest/vcpkg.json </dev/null | head -n1) +case "$(basename "${INFO_FILE:-/missing}" 2>/dev/null)" in + protobuf_${PROTOBUF_VERSION}*) + ;; + *) + echo "ERROR: installed protobuf does not match requested version ${PROTOBUF_VERSION}." + echo " vcpkg installed-file marker: ${INFO_FILE:-}" + echo " Bump VCPKG_BASELINE_COMMIT (in this Dockerfile, prepare.bat," + echo " and testing-cpp.yml) to a commit that knows about the requested version." + exit 1 + ;; +esac + +# 5. Stable symlinks so ENV CMAKE_PREFIX_PATH (last layer) doesn't have to +# care about the underlying triplet. +ln -s /opt/vcpkg-manifest/vcpkg_installed/${VCPKG_TRIPLET} /opt/vcpkg/active +ln -s /opt/vcpkg/active/tools/protobuf/protoc /usr/local/bin/protoc +EOF From 871a20d6ffb23639c0461446d555bfa2118a8fc9 Mon Sep 17 00:00:00 2001 From: wenchy Date: Fri, 29 May 2026 23:33:36 +0800 Subject: [PATCH 18/66] feat(devcontainer): add .NET SDK 8.0 and Node.js 20 LTS Microsoft apt repo for dotnet-sdk-8.0 (matches testing-csharp.yml's target). NodeSource setup_20.x for Node 20 LTS (covers the experimental _lab/ts/ workflow; not currently in CI). Both clean their apt caches to keep the layer small. Co-Authored-By: Claude Opus 4.7 (1M context) --- .devcontainer/Dockerfile | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index 0648756..39b8530 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -124,3 +124,22 @@ esac ln -s /opt/vcpkg-manifest/vcpkg_installed/${VCPKG_TRIPLET} /opt/vcpkg/active ln -s /opt/vcpkg/active/tools/protobuf/protoc /usr/local/bin/protoc EOF + +# --------------------------------------------------------------------------- +# .NET SDK 8.0 + Node.js 20 LTS — apt-based installs from the official +# Microsoft and NodeSource repositories. apt-get clean + rm /var/lib/apt/lists +# at the end keeps the layer small. +# --------------------------------------------------------------------------- +RUN < Date: Fri, 29 May 2026 23:35:37 +0800 Subject: [PATCH 19/66] feat(devcontainer): finalize CMAKE_PREFIX_PATH CMAKE_PREFIX_PATH=/opt/vcpkg/active lets the existing README cmake recipe (-DCMAKE_BUILD_TYPE=Debug, no toolchain file) resolve protobuf inside the container without any flag changes from the host workflow. Co-Authored-By: Claude Opus 4.7 (1M context) --- .devcontainer/Dockerfile | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index 39b8530..3ae9a05 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -143,3 +143,13 @@ apt-get install -y --no-install-recommends \ apt-get clean rm -rf /var/lib/apt/lists/* EOF + +# --------------------------------------------------------------------------- +# Final environment: with CMAKE_PREFIX_PATH pointing at vcpkg's installed +# tree, find_package(Protobuf CONFIG) resolves automatically — the existing +# README's "Dev at Linux → CMake (system protobuf)" recipe works as written +# inside the container. +# VCPKG_ROOT is set above (Task 5) for users who want to invoke the toolchain +# file explicitly: -DCMAKE_TOOLCHAIN_FILE=$VCPKG_ROOT/scripts/buildsystems/vcpkg.cmake. +# --------------------------------------------------------------------------- +ENV CMAKE_PREFIX_PATH=/opt/vcpkg/active From 04d2e09771ba1bb3efd1719223ef77bb327b5e9d Mon Sep 17 00:00:00 2001 From: wenchy Date: Fri, 29 May 2026 23:39:05 +0800 Subject: [PATCH 20/66] feat(devcontainer): add devcontainer.json Wires the Dockerfile under build.args, mounts a named volume for the Go module cache, declares the VS Code extension set, and prints a one-line ready-banner via postCreateCommand. PROTOBUF_VERSION flows from the host LOADER_PROTOBUF_VERSION env var (default 6.33.4) so contributors can rebuild against the legacy v3 line via: LOADER_PROTOBUF_VERSION=3.21.12 code . Co-Authored-By: Claude Opus 4.7 (1M context) --- .devcontainer/devcontainer.json | 50 +++++++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) create mode 100644 .devcontainer/devcontainer.json diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json new file mode 100644 index 0000000..8d17e90 --- /dev/null +++ b/.devcontainer/devcontainer.json @@ -0,0 +1,50 @@ +{ + // tableauio/loader Dev Container. + // See docs/superpowers/specs/2026-05-29-devcontainer-design.md for the design. + "name": "tableauio/loader", + + // Build args wire host env to Dockerfile ARGs: + // LOADER_PROTOBUF_VERSION on the host -> PROTOBUF_VERSION inside. + // Default 6.33.4 (CI's modern matrix entry). To rebuild against legacy v3: + // LOADER_PROTOBUF_VERSION=3.21.12 code . # then Reopen in Container. + "build": { + "dockerfile": "Dockerfile", + "args": { + "PROTOBUF_VERSION": "${localEnv:LOADER_PROTOBUF_VERSION:6.33.4}" + } + }, + + // Persist the Go module cache across container rebuilds. Workspace itself + // uses VS Code's default bind-mount so edits sync to the host. + "mounts": [ + "source=loader-go-mod,target=/home/vscode/go,type=volume" + ], + + "remoteUser": "vscode", + "workspaceFolder": "/workspaces/loader", + + "customizations": { + "vscode": { + "extensions": [ + "golang.go", + "ms-vscode.cmake-tools", + "ms-vscode.cpptools", + "ms-dotnettools.csharp", + "bufbuild.vscode-buf", + "zxh404.vscode-proto3" + ], + "settings": { + // Don't auto-install gopls and friends on first open — let the user + // do it explicitly from the Go extension's command palette. + "go.toolsManagement.autoUpdate": false, + // Don't auto-cmake-configure on workspace open; we run cmake manually + // per the existing README recipes. + "cmake.configureOnOpen": false + } + } + }, + + // One-line ready banner so the developer knows the container is healthy. + // Pure echo — no installs, no version-pinning at runtime, no surprises. + "postCreateCommand": "printf 'tableauio/loader devcontainer ready.\\n go: %s\\n buf: %s\\n protoc: %s\\n dotnet: %s\\n node: %s\\n' \"$(go version | cut -d' ' -f3)\" \"$(buf --version)\" \"$(protoc --version)\" \"$(dotnet --version)\" \"$(node --version)\"" +} From 841e7278f7322d71c796329bcb93db471c151f5b Mon Sep 17 00:00:00 2001 From: wenchy Date: Fri, 29 May 2026 23:47:43 +0800 Subject: [PATCH 21/66] docs(devcontainer): add .devcontainer/README.md One-pager covering prerequisites (Docker Desktop / Engine + VS Code Dev Containers extension), how to open the container, the LOADER_PROTOBUF_VERSION knob, host-OS caveats (WSL2 workspace location, Apple Silicon native arm64), and the layered Dockerfile architecture. Points users at prepare.bat / per-language manual setup as the explicit fallback path. Co-Authored-By: Claude Opus 4.7 (1M context) --- .devcontainer/README.md | 80 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 80 insertions(+) create mode 100644 .devcontainer/README.md diff --git a/.devcontainer/README.md b/.devcontainer/README.md new file mode 100644 index 0000000..0d1158b --- /dev/null +++ b/.devcontainer/README.md @@ -0,0 +1,80 @@ +# Dev Container + +The recommended way to develop on `tableauio/loader`. One container, all +four target languages (C++17, Go 1.24, .NET 8, Node 20) plus protobuf +6.33.4 via vcpkg, pinned to the exact toolchain CI uses. + +## Prerequisites + +- Docker Desktop (Windows / macOS) or Docker Engine (Linux) +- VS Code with the [Dev Containers extension](https://marketplace.visualstudio.com/items?itemName=ms-vscode-remote.remote-containers) + +## Open the container + +```sh +code . # in the repo root +``` + +In VS Code, run **Dev Containers: Reopen in Container** from the command +palette. First build is one-time ~25 minutes (vcpkg compiles protobuf +6.33.4 from source); subsequent reopens are near-instant. + +When the container is ready, the integrated terminal prints a banner with +five toolchain versions. After that, every command from the per-language +sections of the repo root [`README.md`](../README.md) works as written — +no PATH dance, no extra cmake flags. + +## Pin a different protobuf version + +Daily dev runs against protobuf 6.33.4 (CI's "modern" matrix entry). To +rebuild against the legacy v3 line: + +```sh +LOADER_PROTOBUF_VERSION=3.21.12 code . +``` + +…then **Dev Containers: Reopen in Container** (or **Rebuild Container** +if the container is already running). The vcpkg layer rebuilds with the +manifest pinning protobuf 3.21.12; everything else is reused from the +cache. + +## Host-OS caveats + +- **Windows.** WSL2 backend required. **Check the workspace out under + WSL2** (e.g. `\\wsl.localhost\Ubuntu\home\\loader`) — not under + `/mnt/c/...` — for good bind-mount performance. Files under `/mnt/c/` + work but file-watching and large `cmake --build` operations are 5–10× + slower. + +- **Apple Silicon.** Docker builds the container natively as arm64. No + Rosetta or QEMU emulation. Confirm with `docker info | grep Architecture` + → expect `linux/arm64`. + +- **Linux (native Docker Engine).** No special configuration. + +## Architecture + +Single-stage Dockerfile based on +`mcr.microsoft.com/devcontainers/cpp:1-ubuntu-24.04`, with these layers: + +1. Architecture detection (`TARGETARCH` → Go arch, buf arch, vcpkg triplet) +2. Go 1.24.0 (official tarball, multi-arch) +3. buf 1.67.0 (single-binary release, multi-arch) +4. vcpkg pinned to `dc8d75c…df932`, protobuf installed via vcpkg manifest + mode and asserted against the requested version +5. .NET SDK 8.0 (Microsoft apt repo) +6. Node.js 20 LTS (NodeSource apt repo) +7. `ENV CMAKE_PREFIX_PATH=/opt/vcpkg/active` so `find_package(Protobuf CONFIG)` + resolves automatically + +The architecture choice is detected from BuildKit's `TARGETARCH` and fed +into Go / buf / vcpkg triplet selection. Docker auto-selects the host +arch on build. + +## Falling back + +If you can't run Docker (corp policy, restricted machines, etc.) the +existing manual setup paths in the [repo README](../README.md) — Windows +`prepare.bat`, per-language `Install protobuf` instructions — still work. +The devcontainer is the recommended path; the rest is the supported +fallback. From 00bcb381d5923fb4e41074853522ad915f385102 Mon Sep 17 00:00:00 2001 From: wenchy Date: Fri, 29 May 2026 23:51:49 +0800 Subject: [PATCH 22/66] docs: recommend the devcontainer in repo README Add a new "Recommended: Dev Container (any host OS)" subsection at the top of Prerequisites pointing contributors at .devcontainer/. Add a "Skip this section if you're using the devcontainer" lead-in to the existing "Install protobuf" and "Windows: bootstrap" blocks so the manual paths are clearly the fallback, not the primary route. Co-Authored-By: Claude Opus 4.7 (1M context) --- README.md | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/README.md b/README.md index ff606c1..fec71bf 100644 --- a/README.md +++ b/README.md @@ -26,8 +26,30 @@ The official config loader for [Tableau](https://github.com/tableauio/tableau). > Then install protobuf via one of the channels documented in > [Install protobuf](#install-protobuf). +### Recommended: Dev Container (any host OS) + +The fastest way to get a reproducible build environment is to open the +repo in VS Code and choose **Reopen in Container**. The devcontainer +under [`.devcontainer/`](./.devcontainer/) has everything pinned to the +exact versions CI uses (Go 1.24, buf 1.67.0, protobuf 6.33.4 via vcpkg, +.NET 8.0, Node 20). First container build is one-time ~25 minutes (vcpkg +compiles protobuf from source); subsequent reopens are near-instant. + +After the container starts you can skip the per-language setup below and +jump straight to **[C++](#c)** / **[Go](#go)** / **[C#](#c-1)** / +**[TypeScript](#typescript)**. + +Requirements: Docker Desktop (Windows + macOS) or Docker Engine (Linux), +and the VS Code "Dev Containers" extension. See +[`.devcontainer/README.md`](./.devcontainer/README.md) for the longer +how-to. + ### Install protobuf +> **Skip this section if you're using the [devcontainer](#recommended-dev-container-any-host-os).** +> The instructions below cover the manual fallback for hosts where +> Docker isn't available. + Pick whichever channel fits your platform; loader does not bundle protobuf. - **vcpkg (recommended, cross-platform):** @@ -83,6 +105,10 @@ Pick whichever channel fits your platform; loader does not bundle protobuf. ### Windows: bootstrap the rest of the toolchain +> **Skip this section if you're using the [devcontainer](#recommended-dev-container-any-host-os).** +> `prepare.bat` is the manual fallback for Windows hosts that can't run +> Docker. + Run `prepare.bat` **as Administrator** to install everything you need on a fresh Windows machine: [Chocolatey](https://chocolatey.org/), [CMake](https://github.com/Kitware/CMake/releases), From 8a583ba11913dbe37e94db6a36389b317ed1531c Mon Sep 17 00:00:00 2001 From: wenchy Date: Sat, 30 May 2026 00:14:34 +0800 Subject: [PATCH 23/66] docs(devcontainer): document stale-codegen trap Contributors with an existing host checkout can hit confusing build errors when stale .pb.cc / .pb.cs files from a previous protoc-3.x session linger in the workspace (gitignored, so git pull doesn't remove them). The container's modern protoc 33.4 emits a different file layout, and the leftovers shadow what's freshly generated. Add a Troubleshooting section with the rm -rf recipe to wipe and retry. A fresh clone doesn't hit this. Co-Authored-By: Claude Opus 4.7 (1M context) --- .devcontainer/README.md | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/.devcontainer/README.md b/.devcontainer/README.md index 0d1158b..8a32fd5 100644 --- a/.devcontainer/README.md +++ b/.devcontainer/README.md @@ -71,6 +71,31 @@ The architecture choice is detected from BuildKit's `TARGETARCH` and fed into Go / buf / vcpkg triplet selection. Docker auto-selects the host arch on build. +## Troubleshooting + +### `buf generate` works but the C++ or C# build then fails with stale-codegen errors + +If you're hitting errors like + +- C++: `fatal error: google/protobuf/generated_message_table_driven.h: No such file or directory` +- C#: hundreds of `error CS0101: The namespace already contains a definition for ...` + +your host workspace probably has generated files from a *different* protobuf +version (left over from a previous host toolchain — gitignored, so `git pull` +doesn't remove them). They shadow what the container's `protoc` produces. + +Wipe and retry: + +```sh +rm -rf test/cpp-tableau-loader/src/tableau \ + test/cpp-tableau-loader/build \ + test/csharp-tableau-loader/protoconf/tableau \ + test/csharp-tableau-loader/{bin,obj} +``` + +Then re-run `buf generate ..` from the affected language's `test/-tableau-loader/` +directory. A fresh clone doesn't have these stale artefacts. + ## Falling back If you can't run Docker (corp policy, restricted machines, etc.) the From 5fdb0da8be0f025ac511fa4d8017f84aaadcd9bc Mon Sep 17 00:00:00 2001 From: wenchy Date: Sat, 30 May 2026 00:41:07 +0800 Subject: [PATCH 24/66] fix(devcontainer): correct stale-codegen rm paths; drop stale README TODO MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Troubleshooting recipe added in 8a583ba targeted directories that don't actually hold stale generated files: src/tableau is empty (real C++ stale files live at src/protoconf/tableau/), and protoconf/tableau doesn't exist (C# protoc emits flat .cs files into protoconf/). Fix the rm -rf paths so the recipe actually unblocks the failure mode it documents. Also drop the now-contradictory "> TODO: [devcontainer]" placeholder from README.md line 7 — the devcontainer is implemented and recommended in the section immediately below it. Co-Authored-By: Claude Opus 4.7 (1M context) --- .devcontainer/README.md | 4 ++-- README.md | 2 -- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/.devcontainer/README.md b/.devcontainer/README.md index 8a32fd5..aef0330 100644 --- a/.devcontainer/README.md +++ b/.devcontainer/README.md @@ -87,9 +87,9 @@ doesn't remove them). They shadow what the container's `protoc` produces. Wipe and retry: ```sh -rm -rf test/cpp-tableau-loader/src/tableau \ +rm -rf test/cpp-tableau-loader/src/protoconf/tableau \ test/cpp-tableau-loader/build \ - test/csharp-tableau-loader/protoconf/tableau \ + test/csharp-tableau-loader/protoconf \ test/csharp-tableau-loader/{bin,obj} ``` diff --git a/README.md b/README.md index fec71bf..fc6a3c3 100644 --- a/README.md +++ b/README.md @@ -4,8 +4,6 @@ The official config loader for [Tableau](https://github.com/tableauio/tableau). ## Prerequisites -> TODO: [devcontainer](https://code.visualstudio.com/docs/devcontainers/containers) - - C++ standard: at least C++17 - A working **`protoc` + `libprotobuf`** toolchain on your machine. The same protobuf release **must** provide both: protobuf v22+ enforces a strict From 4f106f1762092b7c7296314e2fa345755180e0f6 Mon Sep 17 00:00:00 2001 From: wenchy Date: Sat, 30 May 2026 00:51:07 +0800 Subject: [PATCH 25/66] feat: add CLAUDE.md --- CLAUDE.md | 135 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 135 insertions(+) create mode 100644 CLAUDE.md diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 0000000..a6ac1b5 --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,135 @@ +# CLAUDE.md + +This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository. + +## What this repo is + +`github.com/tableauio/loader` is the official config-loader generator for [Tableau](https://github.com/tableauio/tableau). It ships **three `protoc` plugins** written in Go that read protobuf files annotated with the `tableau.workbook` / `tableau.worksheet` / `tableau.field` extensions and emit strongly-typed loader code in three target languages: + +| Plugin (under `cmd/`) | Output language | Generated file extension | +| --- | --- | --- | +| `protoc-gen-go-tableau-loader` | Go | `*.pc.go` | +| `protoc-gen-cpp-tableau-loader` | C++17 | `*.pc.h` / `*.pc.cc` | +| `protoc-gen-csharp-tableau-loader` | C# (Unity 2022.3 LTS / .NET 8) | `*.pc.cs` | + +Generated code is opinionated: every worksheet message becomes a `Messager` with `Load`/`Store`/`Get*`/index/ordered-map accessors, all messagers register into a singleton-ish `Hub`, and the runtime delegates file IO + protobuf (un)marshaling to the upstream `github.com/tableauio/tableau` Go module's `format`/`load`/`store` packages. + +## Common commands + +All build and test work happens **per language** inside `test/-tableau-loader/`. The repo root only hosts the Go module and the plugin sources; running `go test ./...` from the root only exercises the small shared packages. + +### Dev Container (recommended for most contributors) + +`.devcontainer/` ships a single Ubuntu 24.04 image with the entire toolchain pinned to CI's exact versions: Go 1.24, buf 1.67.0, protobuf 6.33.4 (via vcpkg, manifest mode), .NET 8, Node 20. Open the repo in VS Code and run **Dev Containers: Reopen in Container** — first build is one-time ~25 min (vcpkg compiles protobuf from source); reopens after that are instant. Inside the container every command in the per-language sections below works as written, *and* the C++ section's `-DCMAKE_TOOLCHAIN_FILE=...` flag is unnecessary (the image sets `CMAKE_PREFIX_PATH=/opt/vcpkg/active`). The container is the recommended path on every host OS; `prepare.bat` and the per-language `Install protobuf` recipes in the repo README stay as the explicit fallback for hosts that can't run Docker. + +To rebuild against the legacy v3 protobuf line: `LOADER_PROTOBUF_VERSION=3.21.12 code .` then **Reopen in Container**. The Dockerfile ARG `PROTOBUF_VERSION` is wired to that host env var via `devcontainer.json` build args; vcpkg manifest mode pins the override and the post-install assertion fails the build if anything resolves to the wrong version. See `.devcontainer/README.md` for the longer how-to and host-OS caveats (notably: Windows users should check the workspace out under WSL2, not `/mnt/c/`, for usable bind-mount perf). + +The container is **not** used in CI. CI workflows still run `lukka/run-vcpkg` directly (faster cached vcpkg installs); the container exists for local-dev parity only. `VCPKG_BASELINE_COMMIT` in `.devcontainer/Dockerfile` is in lock-step with `prepare.bat`'s `VCPKG_BASELINE_COMMIT` and `testing-cpp.yml`'s `VCPKG_COMMIT` — bump all three together. + +### Plugin development (Go module at repo root) + +```sh +go vet ./... +go test ./... # tests live in internal/index, internal/loadutil, pkg/treemap, pkg/udiff +go test ./internal/index -run Test_ParseIndexDescriptor # single test +go build -o /tmp/p ./cmd/protoc-gen-go-tableau-loader # smoke-build a plugin +``` + +The plugins are always invoked through `buf generate` from a test directory — `buf.gen.yaml` runs them via `local: ["go", "run", "../../cmd/protoc-gen-go-tableau-loader"]`, so plugin changes take effect on the next `buf generate` without an explicit install step. + +### Go end-to-end (`test/go-tableau-loader/`) + +```sh +cd test/go-tableau-loader +buf generate .. # regenerate *.pb.go + protoconf/loader/*.pc.go +go test ./... # runs hub, index, and main test suites +go test -run Test_ActivityConf_OrderedMap ./... # single test +``` + +CI mirrors this with `go test -v -timeout 30m -race ./... -coverprofile=coverage.txt`. + +### C++ end-to-end (`test/cpp-tableau-loader/`) + +Requires a matching `protoc` + `libprotobuf` toolchain (protobuf v22+ enforces a strict gencode/runtime version check). Loader does **not** vendor protobuf — bring your own via vcpkg, system package, or source build. + +```sh +cd test/cpp-tableau-loader +buf generate .. # writes src/protoconf/*.pb.* and *.pc.* +cmake -S . -B build -DCMAKE_BUILD_TYPE=Debug \ + -DCMAKE_TOOLCHAIN_FILE=/scripts/buildsystems/vcpkg.cmake +cmake --build build --parallel +ctest --test-dir build --output-on-failure +ctest --test-dir build -R HubTest.Load --output-on-failure # single test +``` + +Windows additionally requires running `.\prepare.bat` from the repo root (in **cmd**, not PowerShell, **as Administrator** the first time) in every new shell — it installs the toolchain on first run and re-loads MSVC env vars (`cl.exe`, `INCLUDE`, `LIB`, `VCPKG_ROOT`, …) every run because `vcvarsall.bat` does not persist them. Use `-DCMAKE_BUILD_TYPE=Debug -DVCPKG_TARGET_TRIPLET=x64-windows-static` to match the static-CRT protobuf installed by `prepare.bat`; mismatched CRTs surface as `LNK2038 _ITERATOR_DEBUG_LEVEL` errors. GoogleTest is fetched via CMake `FetchContent` — no manual install. (None of this applies inside the Dev Container — Windows users on that path interact only with Docker Desktop + WSL2.) + +### C# end-to-end (`test/csharp-tableau-loader/`) + +```sh +cd test/csharp-tableau-loader +buf generate .. # writes protoconf/ and tableau/*.pc.cs +dotnet test # xUnit +dotnet test --filter "FullyQualifiedName~HubTest.Load" # single test +``` + +### TypeScript scratch (`_lab/ts/`) + +Experimental, not wired into CI. `npm install && npm run generate && npm run test`. + +## Big-picture architecture + +### Plugin pipeline (the part you'll modify most) + +Every plugin's `main.go` is the same shape: parse flags, set protogen feature bits (proto2 → editions 2024, FEATURE_PROTO3_OPTIONAL), iterate `gen.Files`, and for each file/message decide what to generate. The decision logic is centralized in `internal/options`: + +- `options.NeedGenFile(f)` — file-level gate: must have `(tableau.workbook)` set and at least one message with `(tableau.worksheet)`. +- `options.NeedGenOrderedMap` / `NeedGenIndex` / `NeedGenOrderedIndex` — message-level gates that *also* honour the `lang_options` map on `WorksheetOptions`, e.g. `lang_options: { key: "Index" value: "go" }` means "only generate the index accessors for Go". `internal/options/options.go` defines the language IDs (`cpp`, `go`, `cs`). + +Each plugin then splits work between two passes: + +1. **Per-message generation** (`messager.go` in each plugin) — emits one `*.pc.{go,h,cc,cs}` per `.proto`. Delegates ordered-map field/method emission to `cmd//orderedmap/` and index field/method emission to `cmd//indexes/`. The shared semantic model — what the index syntax means — lives in **`internal/index`**, not in each plugin: `ParseIndexDescriptor` walks the message tree and returns a `LevelMessage` linked-list describing what indices apply at which nesting level (map → list → map → list, etc.). Plugins consume this descriptor to emit language-specific code; do not duplicate this parsing per-language. +2. **Cross-message ("embed") generation** — emits `hub.pc.*`, `messager_container.pc.go`, `util.pc.*`, etc. Driven by `cmd//embed.go`, which `//go:embed`s the templates under `cmd//embed/templates/` (Go) or `cmd//embed/` (C++/C# also include verbatim `*.pc.{h,cc,cs}` files that are emitted unchanged). The list of "all messagers in source order" the templates iterate over comes from **`internal/xproto.ParseProtoFiles`**. + +The C++ plugin is the only one with sharding: `--shards=N` in `buf.gen.yaml` makes `xproto.ProtoFiles.SplitShards(N)` partition messagers across N `hub_shard*.pc.cc` files to parallelize the (very heavy) compile. It also has a tri-state `--mode` flag (`default` / `hub` / `messager`) for users who want to split protoconf generation across separate `buf generate` invocations. + +### Index syntax (`internal/index/index.go`) + +The `worksheet.index` / `worksheet.ordered_index` strings use a compact mini-language parsed by one regex. Knowing the shape helps when reading or writing tests in `test/proto/index_conf.proto`: + +``` +ID # single-column index on field "ID" +ID@Item # named "Item" +ID@Item # sort within group by SortedCol +(ID, Name)@Item # composite index +CountryItemAttrName # CamelCase concatenation of nested-field path Country.Item.Attr.Name +``` + +Multi-level nested maps/lists are flattened: `CountryItemAttrName` reaches into `country_list[].item_map[].attr_list[].name`. Generated indexes return leveled key structs whose names are derived in `helper.ParseLeveledMapPrefix` — a 3-level map with indexes only at the 2nd level produces fewer key structs than one with indexes at every level (compare `Fruit5Conf` vs `Fruit4Conf` in `test/proto/index_conf.proto`). + +### Hub and Messager runtime (Go) + +Hub state is held in `atomic.Pointer[MessagerContainer]` so `Load(...)` can swap the entire snapshot atomically while `Get*()` callers race-freely read the previous one. The container is generated (`messager_container.pc.go`) and exposes both a generic `GetMessager(name)` and a typed `Get()` per messager. `Hub.NewContext` / `FromContext` propagate the snapshot through `context.Context` so request handlers see a stable view. + +Mutability detection (`hub.WithMutableCheck`) is opt-in: enabling it flips `enableBackup()` on each messager so they retain the originally-loaded `proto.Message`, then a goroutine periodically `proto.Equal`s `originalMessage()` vs `Message()` and calls `OnMutate(name, original, current)` (default handler prints a unified diff via `pkg/udiff`). + +`processAfterLoad` (per-messager, runs as `Load` finishes) and `ProcessAfterLoadAll(hub *Hub)` (cross-messager, runs after all are loaded against a temporary hub) are the two extension points. `test/go-tableau-loader/customconf/custom_item_conf.go` shows how a hand-written messager registers via `tableau.Register(func() Messager { ... })` and uses `ProcessAfterLoadAll` to consume data from another messager — that's the canonical pattern for derived/computed configs. + +### Test data flow + +1. `test/proto/*.proto` — hand-written annotated protos (the source of truth for what generators are exercised). +2. `test/testdata/conf/*.json`, `patchconf/`, `patchconf2/`, `patchresult/` — JSON inputs the upstream `tableau` toolchain would produce from spreadsheets; loader tests read them at runtime. +3. Each `test/-tableau-loader/` runs `buf generate ..` to produce its language-specific stubs into a per-language output dir, then runs that language's native test runner against `test/testdata/`. + +Patch tests verify three semantics defined upstream (merge / replace / recursive_patch) — the loader's job is just to wire `patch_paths`/`patch_dirs` through `load.MessagerOptions`. + +## Versioning and releases + +Each plugin has its own `version` constant in its `main.go` and is released independently via tags shaped `cmd/protoc-gen-{go,cpp,csharp}-tableau-loader/`. The matching workflows in `.github/workflows/release-*.yml` build cross-platform binaries and attach them to the GitHub release. Bump the relevant `const version = "..."` in lockstep with the tag. + +## Style and conventions + +- C++ / proto: `clang-format` with the rules in `.clang-format` (Google base, 120-col). +- Go: standard `gofmt` / `go vet`; CI runs `go vet ./...` and `go test -race`. +- Generated files end in `.pc.` (the `extensions.PC` constant). Anything matching `*.pb.*` is `.gitignore`d — never check generated proto-runtime files in. +- Worksheet language gating: when adding a feature that only some target languages support, add it behind a `lang_options` check in `internal/options` rather than making the per-plugin `messager.go` know the rules. From 384612cfcd872bb2c6c33862535982b7705edc1a Mon Sep 17 00:00:00 2001 From: wenchy Date: Mon, 1 Jun 2026 10:46:16 +0800 Subject: [PATCH 26/66] chore: upgrade tableau to v0.16.0 --- go.mod | 43 +++++++++++++--------- go.sum | 114 +++++++++++++++++++++++++++++++++------------------------ 2 files changed, 91 insertions(+), 66 deletions(-) diff --git a/go.mod b/go.mod index 10bf1e8..d2625c9 100644 --- a/go.mod +++ b/go.mod @@ -5,34 +5,41 @@ go 1.24.0 require ( github.com/aymanbagabas/go-udiff v0.2.0 github.com/iancoleman/strcase v0.3.0 - github.com/stretchr/testify v1.10.0 - github.com/tableauio/tableau v0.15.1 + github.com/stretchr/testify v1.11.1 + github.com/tableauio/tableau v0.16.0 google.golang.org/protobuf v1.36.11 ) require ( - github.com/antchfx/xpath v0.0.0-20170515025933-1f3266e77307 // indirect - github.com/bufbuild/protocompile v0.10.0 // indirect + buf.build/gen/go/bufbuild/protovalidate/protocolbuffers/go v1.36.11-20260415201107-50325440f8f2.1 // indirect + buf.build/go/protovalidate v1.2.0 // indirect + cel.dev/expr v0.25.1 // indirect + github.com/antchfx/xpath v1.3.6 // indirect + github.com/antlr4-go/antlr/v4 v4.13.1 // indirect + github.com/bufbuild/protocompile v0.14.1 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/emirpasic/gods v1.18.1 // indirect + github.com/google/cel-go v0.28.0 // indirect github.com/mitchellh/go-wordwrap v1.0.1 // indirect - github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/protocolbuffers/txtpbfmt v0.0.0-20240820135758-21b1d9897dc7 // indirect - github.com/richardlehane/mscfb v1.0.4 // indirect - github.com/richardlehane/msoleps v1.0.4 // indirect + github.com/richardlehane/mscfb v1.0.6 // indirect + github.com/richardlehane/msoleps v1.0.6 // indirect github.com/subchen/go-xmldom v1.1.2 // indirect - github.com/valyala/fastjson v1.6.7 // indirect - github.com/xuri/efp v0.0.0-20240408161823-9ad904a10d6d // indirect - github.com/xuri/excelize/v2 v2.9.0 // indirect - github.com/xuri/nfp v0.0.0-20240318013403-ab9948c2c4a7 // indirect - go.uber.org/atomic v1.7.0 // indirect - go.uber.org/multierr v1.6.0 // indirect - go.uber.org/zap v1.24.0 // indirect - golang.org/x/crypto v0.28.0 // indirect - golang.org/x/net v0.30.0 // indirect - golang.org/x/sync v0.8.0 // indirect - golang.org/x/text v0.19.0 // indirect + github.com/tiendc/go-deepcopy v1.7.2 // indirect + github.com/valyala/fastjson v1.6.10 // indirect + github.com/xuri/efp v0.0.1 // indirect + github.com/xuri/excelize/v2 v2.10.1 // indirect + github.com/xuri/nfp v0.0.2-0.20250530014748-2ddeb826f9a9 // indirect + go.uber.org/multierr v1.10.0 // indirect + go.uber.org/zap v1.27.1 // indirect + golang.org/x/crypto v0.48.0 // indirect + golang.org/x/exp v0.0.0-20250813145105-42675adae3e6 // indirect + golang.org/x/net v0.50.0 // indirect + golang.org/x/sync v0.19.0 // indirect + golang.org/x/text v0.34.0 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20250811230008-5f3141c8851a // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20250811230008-5f3141c8851a // indirect gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index 08feefe..8a8f3c1 100644 --- a/go.sum +++ b/go.sum @@ -1,75 +1,93 @@ -github.com/antchfx/xpath v0.0.0-20170515025933-1f3266e77307 h1:C735MoY/X+UOx6SECmHk5pVOj51h839Ph13pEoY8UmU= +buf.build/gen/go/bufbuild/protovalidate/protocolbuffers/go v1.36.11-20260415201107-50325440f8f2.1 h1:s6hzCXtND/ICdGPTMGk7C+/BFlr2Jg5GyH0NKf4XGXg= +buf.build/gen/go/bufbuild/protovalidate/protocolbuffers/go v1.36.11-20260415201107-50325440f8f2.1/go.mod h1:tvtbpgaVXZX4g6Pn+AnzFycuRK3MOz5HJfEGeEllXYM= +buf.build/go/protovalidate v1.2.0 h1:DQVrUWkmGTBij+kOYv/x2LLxwcLaGKMdzShj1/6/3H0= +buf.build/go/protovalidate v1.2.0/go.mod h1:7rYiQEhqvAipoazpVNBBH2S2f8bjG4huMVy1V2Yofn4= +cel.dev/expr v0.25.1 h1:1KrZg61W6TWSxuNZ37Xy49ps13NUovb66QLprthtwi4= +cel.dev/expr v0.25.1/go.mod h1:hrXvqGP6G6gyx8UAHSHJ5RGk//1Oj5nXQ2NI02Nrsg4= github.com/antchfx/xpath v0.0.0-20170515025933-1f3266e77307/go.mod h1:Yee4kTMuNiPYJ7nSNorELQMr1J33uOpXDMByNYhvtNk= +github.com/antchfx/xpath v1.3.6 h1:s0y+ElRRtTQdfHP609qFu0+c6bglDv20pqOViQjjdPI= +github.com/antchfx/xpath v1.3.6/go.mod h1:i54GszH55fYfBmoZXapTHN8T8tkcHfRgLyVwwqzXNcs= +github.com/antlr4-go/antlr/v4 v4.13.1 h1:SqQKkuVZ+zWkMMNkjy5FZe5mr5WURWnlpmOuzYWrPrQ= +github.com/antlr4-go/antlr/v4 v4.13.1/go.mod h1:GKmUxMtwp6ZgGwZSva4eWPC5mS6vUAmOABFgjdkM7Nw= github.com/aymanbagabas/go-udiff v0.2.0 h1:TK0fH4MteXUDspT88n8CKzvK0X9O2xu9yQjWpi6yML8= github.com/aymanbagabas/go-udiff v0.2.0/go.mod h1:RE4Ex0qsGkTAJoQdQQCA0uG+nAzJO/pI/QwceO5fgrA= -github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= -github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= -github.com/bufbuild/protocompile v0.10.0 h1:+jW/wnLMLxaCEG8AX9lD0bQ5v9h1RUiMKOBOT5ll9dM= -github.com/bufbuild/protocompile v0.10.0/go.mod h1:G9qQIQo0xZ6Uyj6CMNz0saGmx2so+KONo8/KrELABiY= -github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/brianvoe/gofakeit/v6 v6.28.0 h1:Xib46XXuQfmlLS2EXRuJpqcw8St6qSZz75OUo0tgAW4= +github.com/brianvoe/gofakeit/v6 v6.28.0/go.mod h1:Xj58BMSnFqcn/fAQeSK+/PLtC5kSb7FJIq4JyGa8vEs= +github.com/bufbuild/protocompile v0.14.1 h1:iA73zAf/fyljNjQKwYzUHD6AD4R8KMasmwa/FBatYVw= +github.com/bufbuild/protocompile v0.14.1/go.mod h1:ppVdAIhbr2H8asPk6k4pY7t9zB1OU5DoEw9xY/FUi1c= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/emirpasic/gods v1.18.1 h1:FXtiHYKDGKCW2KzwZKx0iC0PQmdlorYgdFG9jPXJ1Bc= github.com/emirpasic/gods v1.18.1/go.mod h1:8tpGGwCnJ5H4r6BWwaV6OrWmMoPhUl5jm/FMNAnJvWQ= +github.com/google/cel-go v0.28.0 h1:KjSWstCpz/MN5t4a8gnGJNIYUsJRpdi/r97xWDphIQc= +github.com/google/cel-go v0.28.0/go.mod h1:X0bD6iVNR8pkROSOoHVdgTkzmRcosof7WQqCD6wcMc8= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/iancoleman/strcase v0.3.0 h1:nTXanmYxhfFAMjZL34Ov6gkzEsSJZ5DbhxWjvSASxEI= github.com/iancoleman/strcase v0.3.0/go.mod h1:iwCmte+B7n89clKwxIoIXy/HfoL7AsD47ZCWhYzw7ho= +github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= github.com/mitchellh/go-wordwrap v1.0.1 h1:TLuKupo69TCn6TQSyGxwI1EblZZEsQ0vMlAFQflz0v0= github.com/mitchellh/go-wordwrap v1.0.1/go.mod h1:R62XHJLzvMFRBbcrT7m7WgmE1eOyTSsCt+hzestvNj0= -github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 h1:RWengNIwukTxcDr9M+97sNutRR1RKhG96O6jWumTTnw= -github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826/go.mod h1:TaXosZuwdSHYgviHp1DAtfrULt5eUgsSMsZf+YrPgl8= -github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= -github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/protocolbuffers/txtpbfmt v0.0.0-20240820135758-21b1d9897dc7 h1:GkKZUEPNgwIk3LK4Er5vxnaNKk1pdjI3Oc6oTBwBsxQ= github.com/protocolbuffers/txtpbfmt v0.0.0-20240820135758-21b1d9897dc7/go.mod h1:jgxiZysxFPM+iWKwQwPR+y+Jvo54ARd4EisXxKYpB5c= -github.com/richardlehane/mscfb v1.0.4 h1:WULscsljNPConisD5hR0+OyZjwK46Pfyr6mPu5ZawpM= -github.com/richardlehane/mscfb v1.0.4/go.mod h1:YzVpcZg9czvAuhk9T+a3avCpcFPMUWm7gK3DypaEsUk= -github.com/richardlehane/msoleps v1.0.1/go.mod h1:BWev5JBpU9Ko2WAgmZEuiz4/u3ZYTKbjLycmwiWUfWg= -github.com/richardlehane/msoleps v1.0.4 h1:WuESlvhX3gH2IHcd8UqyCuFY5yiq/GR/yqaSM/9/g00= -github.com/richardlehane/msoleps v1.0.4/go.mod h1:BWev5JBpU9Ko2WAgmZEuiz4/u3ZYTKbjLycmwiWUfWg= -github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= -github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= -github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/richardlehane/mscfb v1.0.6 h1:eN3bvvZCp00bs7Zf52bxNwAx5lJDBK1tCuH19qq5aC8= +github.com/richardlehane/mscfb v1.0.6/go.mod h1:pe0+IUIc0AHh0+teNzBlJCtSyZdFOGgV4ZK9bsoV+Jo= +github.com/richardlehane/msoleps v1.0.6 h1:9BvkpjvD+iUBalUY4esMwv6uBkfOip/Lzvd93jvR9gg= +github.com/richardlehane/msoleps v1.0.6/go.mod h1:BWev5JBpU9Ko2WAgmZEuiz4/u3ZYTKbjLycmwiWUfWg= +github.com/rodaine/protogofakeit v0.1.1 h1:ZKouljuRM3A+TArppfBqnH8tGZHOwM/pjvtXe9DaXH8= +github.com/rodaine/protogofakeit v0.1.1/go.mod h1:pXn/AstBYMaSfc1/RqH3N82pBuxtWgejz1AlYpY1mI0= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= github.com/subchen/go-xmldom v1.1.2 h1:7evI2YqfYYOnuj+PBwyaOZZYjl3iWq35P6KfBUw9jeU= github.com/subchen/go-xmldom v1.1.2/go.mod h1:6Pg/HuX5/T4Jlj0IPJF1sRxKVoI/rrKP6LIMge9d5/8= -github.com/tableauio/tableau v0.15.1 h1:NbTxG7xft57s5SJUXVykNiKWE7/JPKIZsQR0S57hsuE= -github.com/tableauio/tableau v0.15.1/go.mod h1:TjQ0ZZMLQ4lBsFO+pMYdfxhlu68taqtpA4/BlAiqBbI= -github.com/valyala/fastjson v1.6.7 h1:ZE4tRy0CIkh+qDc5McjatheGX2czdn8slQjomexVpBM= -github.com/valyala/fastjson v1.6.7/go.mod h1:CLCAqky6SMuOcxStkYQvblddUtoRxhYMGLrsQns1aXY= -github.com/xuri/efp v0.0.0-20240408161823-9ad904a10d6d h1:llb0neMWDQe87IzJLS4Ci7psK/lVsjIS2otl+1WyRyY= -github.com/xuri/efp v0.0.0-20240408161823-9ad904a10d6d/go.mod h1:ybY/Jr0T0GTCnYjKqmdwxyxn2BQf2RcQIIvex5QldPI= -github.com/xuri/excelize/v2 v2.9.0 h1:1tgOaEq92IOEumR1/JfYS/eR0KHOCsRv/rYXXh6YJQE= -github.com/xuri/excelize/v2 v2.9.0/go.mod h1:uqey4QBZ9gdMeWApPLdhm9x+9o2lq4iVmjiLfBS5hdE= -github.com/xuri/nfp v0.0.0-20240318013403-ab9948c2c4a7 h1:hPVCafDV85blFTabnqKgNhDCkJX25eik94Si9cTER4A= -github.com/xuri/nfp v0.0.0-20240318013403-ab9948c2c4a7/go.mod h1:WwHg+CVyzlv/TX9xqBFXEZAuxOPxn2k1GNHwG41IIUQ= -go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw= -go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= -go.uber.org/goleak v1.1.11 h1:wy28qYRKZgnJTxGxvye5/wgWr1EKjmUDGYox5mGlRlI= -go.uber.org/goleak v1.1.11/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= -go.uber.org/multierr v1.6.0 h1:y6IPFStTAIT5Ytl7/XYmHvzXQ7S3g/IeZW9hyZ5thw4= -go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= -go.uber.org/zap v1.24.0 h1:FiJd5l1UOLj0wCgbSE0rwwXHzEdAZS6hiiSnxJN/D60= -go.uber.org/zap v1.24.0/go.mod h1:2kMP+WWQ8aoFoedH3T2sq6iJ2yDWpHbP0f6MQbS9Gkg= -golang.org/x/crypto v0.28.0 h1:GBDwsMXVQi34v5CCYUm2jkJvu4cbtru2U4TN2PSyQnw= -golang.org/x/crypto v0.28.0/go.mod h1:rmgy+3RHxRZMyY0jjAJShp2zgEdOqj2AO7U0pYmeQ7U= -golang.org/x/image v0.18.0 h1:jGzIakQa/ZXI1I0Fxvaa9W7yP25TqT6cHIHn+6CqvSQ= -golang.org/x/image v0.18.0/go.mod h1:4yyo5vMFQjVjUcVk4jEQcU9MGy/rulF5WvUILseCM2E= -golang.org/x/net v0.30.0 h1:AcW1SDZMkb8IpzCdQUaIq2sP4sZ4zw+55h6ynffypl4= -golang.org/x/net v0.30.0/go.mod h1:2wGyMJ5iFasEhkwi13ChkO/t1ECNC4X4eBKkVFyYFlU= -golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ= -golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= -golang.org/x/text v0.19.0 h1:kTxAhCbGbxhK0IwgSKiMO5awPoDQ0RpfiVYBfK860YM= -golang.org/x/text v0.19.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= +github.com/tableauio/tableau v0.16.0 h1:cUSPkTJIHTivTXnEsWH2iGawLeI4cNJnp8BGDkvKKuk= +github.com/tableauio/tableau v0.16.0/go.mod h1:TAlrDUuokQaodTyHAymVRRnqKV00qd5eCuPvl4W0GPg= +github.com/tiendc/go-deepcopy v1.7.2 h1:Ut2yYR7W9tWjTQitganoIue4UGxZwCcJy3orjrrIj44= +github.com/tiendc/go-deepcopy v1.7.2/go.mod h1:4bKjNC2r7boYOkD2IOuZpYjmlDdzjbpTRyCx+goBCJQ= +github.com/valyala/fastjson v1.6.10 h1:/yjJg8jaVQdYR3arGxPE2X5z89xrlhS0eGXdv+ADTh4= +github.com/valyala/fastjson v1.6.10/go.mod h1:e6FubmQouUNP73jtMLmcbxS6ydWIpOfhz34TSfO3JaE= +github.com/xuri/efp v0.0.1 h1:fws5Rv3myXyYni8uwj2qKjVaRP30PdjeYe2Y6FDsCL8= +github.com/xuri/efp v0.0.1/go.mod h1:ybY/Jr0T0GTCnYjKqmdwxyxn2BQf2RcQIIvex5QldPI= +github.com/xuri/excelize/v2 v2.10.1 h1:V62UlqopMqha3kOpnlHy2CcRVw1V8E63jFoWUmMzxN0= +github.com/xuri/excelize/v2 v2.10.1/go.mod h1:iG5tARpgaEeIhTqt3/fgXCGoBRt4hNXgCp3tfXKoOIc= +github.com/xuri/nfp v0.0.2-0.20250530014748-2ddeb826f9a9 h1:+C0TIdyyYmzadGaL/HBLbf3WdLgC29pgyhTjAT/0nuE= +github.com/xuri/nfp v0.0.2-0.20250530014748-2ddeb826f9a9/go.mod h1:WwHg+CVyzlv/TX9xqBFXEZAuxOPxn2k1GNHwG41IIUQ= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +go.uber.org/multierr v1.10.0 h1:S0h4aNzvfcFsC3dRF1jLoaov7oRaKqRGC/pUEJ2yvPQ= +go.uber.org/multierr v1.10.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +go.uber.org/zap v1.27.1 h1:08RqriUEv8+ArZRYSTXy1LeBScaMpVSTBhCeaZYfMYc= +go.uber.org/zap v1.27.1/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= +go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= +go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= +golang.org/x/crypto v0.48.0 h1:/VRzVqiRSggnhY7gNRxPauEQ5Drw9haKdM0jqfcCFts= +golang.org/x/crypto v0.48.0/go.mod h1:r0kV5h3qnFPlQnBSrULhlsRfryS2pmewsg+XfMgkVos= +golang.org/x/exp v0.0.0-20250813145105-42675adae3e6 h1:SbTAbRFnd5kjQXbczszQ0hdk3ctwYf3qBNH9jIsGclE= +golang.org/x/exp v0.0.0-20250813145105-42675adae3e6/go.mod h1:4QTo5u+SEIbbKW1RacMZq1YEfOBqeXa19JeshGi+zc4= +golang.org/x/image v0.25.0 h1:Y6uW6rH1y5y/LK1J8BPWZtr6yZ7hrsy6hFrXjgsc2fQ= +golang.org/x/image v0.25.0/go.mod h1:tCAmOEGthTtkalusGp1g3xa2gke8J6c2N565dTyl9Rs= +golang.org/x/net v0.50.0 h1:ucWh9eiCGyDR3vtzso0WMQinm2Dnt8cFMuQa9K33J60= +golang.org/x/net v0.50.0/go.mod h1:UgoSli3F/pBgdJBHCTc+tp3gmrU4XswgGRgtnwWTfyM= +golang.org/x/sync v0.19.0 h1:vV+1eWNmZ5geRlYjzm2adRgW2/mcpevXNg50YZtPCE4= +golang.org/x/sync v0.19.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= +golang.org/x/text v0.34.0 h1:oL/Qq0Kdaqxa1KbNeMKwQq0reLCCaFtqu2eNuSeNHbk= +golang.org/x/text v0.34.0/go.mod h1:homfLqTYRFyVYemLBFl5GgL/DWEiH5wcsQ5gSh1yziA= +google.golang.org/genproto/googleapis/api v0.0.0-20250811230008-5f3141c8851a h1:DMCgtIAIQGZqJXMVzJF4MV8BlWoJh2ZuFiRdAleyr58= +google.golang.org/genproto/googleapis/api v0.0.0-20250811230008-5f3141c8851a/go.mod h1:y2yVLIE/CSMCPXaHnSKXxu1spLPnglFLegmgdY23uuE= +google.golang.org/genproto/googleapis/rpc v0.0.0-20250811230008-5f3141c8851a h1:tPE/Kp+x9dMSwUm/uM0JKK0IfdiJkwAbSMSeZBXXJXc= +google.golang.org/genproto/googleapis/rpc v0.0.0-20250811230008-5f3141c8851a/go.mod h1:gw1tLEfykwDz2ET4a12jcXt4couGAm7IwsVaTy0Sflo= google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/natefinch/lumberjack.v2 v2.2.1 h1:bBRl1b0OH9s/DuPhuXpNl+VtCaJXFZ5/uEFST95x9zc= gopkg.in/natefinch/lumberjack.v2 v2.2.1/go.mod h1:YD8tP3GAjkrDg1eZH7EGmyESg/lsYskCTPBJVb9jqSc= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= From 6486e90b9308006f5e61cad6314850f36011f53a Mon Sep 17 00:00:00 2001 From: Kybxd <627940450@qq.com> Date: Mon, 1 Jun 2026 11:00:09 +0800 Subject: [PATCH 27/66] chore(devcontainer): fix GOPATH volume ownership and ignore dotnet runfile cache Docker creates named volumes as root:root by default, which leaves the Go module cache mounted at /home/vscode/go unwritable for the `vscode` user inside the devcontainer. Chown it to `vscode:vscode` in postCreateCommand so `go mod download` / `go build` work out of the box. Also ignore dotnet/runfile-discovery/, an auto-generated .NET SDK 10 file-based app discovery cache produced by the dotnet CLI / C# Dev Kit. --- .devcontainer/devcontainer.json | 5 ++++- .gitignore | 3 +++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 8d17e90..1d35928 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -46,5 +46,8 @@ // One-line ready banner so the developer knows the container is healthy. // Pure echo — no installs, no version-pinning at runtime, no surprises. - "postCreateCommand": "printf 'tableauio/loader devcontainer ready.\\n go: %s\\n buf: %s\\n protoc: %s\\n dotnet: %s\\n node: %s\\n' \"$(go version | cut -d' ' -f3)\" \"$(buf --version)\" \"$(protoc --version)\" \"$(dotnet --version)\" \"$(node --version)\"" + // Also fix ownership of the named volume mounted at /home/vscode/go: Docker + // creates named volumes as root:root by default, which makes the Go module + // cache (GOPATH/pkg) unwritable for the `vscode` user. + "postCreateCommand": "sudo chown -R vscode:vscode /home/vscode/go && printf 'tableauio/loader devcontainer ready.\\n go: %s\\n buf: %s\\n protoc: %s\\n dotnet: %s\\n node: %s\\n' \"$(go version | cut -d' ' -f3)\" \"$(buf --version)\" \"$(protoc --version)\" \"$(dotnet --version)\" \"$(node --version)\"" } diff --git a/.gitignore b/.gitignore index 6718c6b..b92e31d 100644 --- a/.gitignore +++ b/.gitignore @@ -59,3 +59,6 @@ test/cpp-tableau-loader/vcpkg.json .claude/settings.local.json +# .NET SDK 10 file-based app / runfile discovery cache (auto-generated by dotnet CLI / C# Dev Kit) +dotnet/runfile-discovery/ + From 5813f694906c2f86a475013653d64f66d8c51515 Mon Sep 17 00:00:00 2001 From: Kybxd <627940450@qq.com> Date: Mon, 1 Jun 2026 11:20:26 +0800 Subject: [PATCH 28/66] chore(deps): bump buf.build/tableauio/tableau from v0.15.1 to v0.16.0 Update test/buf.yaml dependency pin and refresh test/buf.lock with the new commit / digest. --- test/buf.lock | 4 ++-- test/buf.yaml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/test/buf.lock b/test/buf.lock index 13c53ed..5b5e434 100644 --- a/test/buf.lock +++ b/test/buf.lock @@ -2,5 +2,5 @@ version: v2 deps: - name: buf.build/tableauio/tableau - commit: 04b3a2c59b6644318d341d4f2220a578 - digest: b5:59bca610664e985502469393a4d47370da9d40d874312a76db4fe133fe61e27fc23b72d2ad45d67c3af76e6620ab1c86c3de0bd13d3517abf6e8b75a346b78fd + commit: 276d7564e86d4bcc80c493e28c5a3897 + digest: b5:507b516066e0e658f97a12e77021a399400baf6acd501c720e7650f711963eae8db5fedf2ae36bce32cfdcd42a81de9fe5c9ebbca7f57fcdae781a562b1a3d65 diff --git a/test/buf.yaml b/test/buf.yaml index d699560..a7a0495 100644 --- a/test/buf.yaml +++ b/test/buf.yaml @@ -1,5 +1,5 @@ version: v2 deps: - - buf.build/tableauio/tableau:v0.15.1 + - buf.build/tableauio/tableau:v0.16.0 modules: - path: proto From e073d462bb2a9b70704b540717e4f39772d9a817 Mon Sep 17 00:00:00 2001 From: wenchy Date: Mon, 1 Jun 2026 03:45:04 +0000 Subject: [PATCH 29/66] fix: permisions --- .devcontainer/devcontainer.json | 102 ++++++++++++++++---------------- 1 file changed, 50 insertions(+), 52 deletions(-) diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 1d35928..00fc00e 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -1,53 +1,51 @@ { - // tableauio/loader Dev Container. - // See docs/superpowers/specs/2026-05-29-devcontainer-design.md for the design. - "name": "tableauio/loader", - - // Build args wire host env to Dockerfile ARGs: - // LOADER_PROTOBUF_VERSION on the host -> PROTOBUF_VERSION inside. - // Default 6.33.4 (CI's modern matrix entry). To rebuild against legacy v3: - // LOADER_PROTOBUF_VERSION=3.21.12 code . # then Reopen in Container. - "build": { - "dockerfile": "Dockerfile", - "args": { - "PROTOBUF_VERSION": "${localEnv:LOADER_PROTOBUF_VERSION:6.33.4}" - } - }, - - // Persist the Go module cache across container rebuilds. Workspace itself - // uses VS Code's default bind-mount so edits sync to the host. - "mounts": [ - "source=loader-go-mod,target=/home/vscode/go,type=volume" - ], - - "remoteUser": "vscode", - "workspaceFolder": "/workspaces/loader", - - "customizations": { - "vscode": { - "extensions": [ - "golang.go", - "ms-vscode.cmake-tools", - "ms-vscode.cpptools", - "ms-dotnettools.csharp", - "bufbuild.vscode-buf", - "zxh404.vscode-proto3" - ], - "settings": { - // Don't auto-install gopls and friends on first open — let the user - // do it explicitly from the Go extension's command palette. - "go.toolsManagement.autoUpdate": false, - // Don't auto-cmake-configure on workspace open; we run cmake manually - // per the existing README recipes. - "cmake.configureOnOpen": false - } - } - }, - - // One-line ready banner so the developer knows the container is healthy. - // Pure echo — no installs, no version-pinning at runtime, no surprises. - // Also fix ownership of the named volume mounted at /home/vscode/go: Docker - // creates named volumes as root:root by default, which makes the Go module - // cache (GOPATH/pkg) unwritable for the `vscode` user. - "postCreateCommand": "sudo chown -R vscode:vscode /home/vscode/go && printf 'tableauio/loader devcontainer ready.\\n go: %s\\n buf: %s\\n protoc: %s\\n dotnet: %s\\n node: %s\\n' \"$(go version | cut -d' ' -f3)\" \"$(buf --version)\" \"$(protoc --version)\" \"$(dotnet --version)\" \"$(node --version)\"" -} + // tableauio/loader Dev Container. + // See docs/superpowers/specs/2026-05-29-devcontainer-design.md for the design. + "name": "tableauio/loader", + // Build args wire host env to Dockerfile ARGs: + // LOADER_PROTOBUF_VERSION on the host -> PROTOBUF_VERSION inside. + // Default 6.33.4 (CI's modern matrix entry). To rebuild against legacy v3: + // LOADER_PROTOBUF_VERSION=3.21.12 code . # then Reopen in Container. + "build": { + "dockerfile": "Dockerfile", + "args": { + "PROTOBUF_VERSION": "${localEnv:LOADER_PROTOBUF_VERSION:6.33.4}" + } + }, + // Persist the Go module cache across container rebuilds. Workspace itself + // uses VS Code's default bind-mount so edits sync to the host. + "mounts": [ + "source=loader-go-mod,target=/home/vscode/go,type=volume" + ], + "remoteUser": "vscode", + "workspaceFolder": "/workspaces/loader", + "customizations": { + "vscode": { + "extensions": [ + "golang.go", + "ms-vscode.cmake-tools", + "ms-vscode.cpptools", + "ms-dotnettools.csharp", + "bufbuild.vscode-buf", + "DrBlury.protobuf-vsc" + ], + "settings": { + // Don't auto-install gopls and friends on first open — let the user + // do it explicitly from the Go extension's command palette. + "go.toolsManagement.autoUpdate": false, + // Don't auto-cmake-configure on workspace open; we run cmake manually + // per the existing README recipes. + "cmake.configureOnOpen": false + } + } + }, + // One-line ready banner so the developer knows the container is healthy. + // Pure echo — no installs, no version-pinning at runtime, no surprises. + "postCreateCommand": "printf 'tableauio/loader devcontainer ready.\\n go: %s\\n buf: %s\\n protoc: %s\\n dotnet: %s\\n node: %s\\n' \"$(go version | cut -d' ' -f3)\" \"$(buf --version)\" \"$(protoc --version)\" \"$(dotnet --version)\" \"$(node --version)\"", + // The loader-go-mod named volume is mounted at /home/vscode/go (see "mounts" + // above). Docker creates named volumes root-owned by default, masking any + // ownership set in the Dockerfile. Re-chown on every start so `go install` + // (e.g. gopls auto-install from the Go extension) can write into GOPATH. + // Idempotent and ~50ms on an already-correct tree. + "postStartCommand": "sudo chown -R vscode:vscode /home/vscode/go" +} \ No newline at end of file From 3195ad51bbb9607f63eae93a44b6fb807e0324e9 Mon Sep 17 00:00:00 2001 From: wenchy Date: Mon, 1 Jun 2026 16:09:14 +0800 Subject: [PATCH 30/66] feat(devcontainer): add Windows-container variant + centralize toolchain pins Splits .devcontainer/ into linux/ (multi-arch, recommended) and windows/ (windows/amd64 only, native MSVC for Windows hosts that don't want WSL2), plus a macos/ docs path explaining why no macOS container exists. All toolchain versions (Go, buf, protobuf, vcpkg baseline, .NET, Node, CMake) move into a single source of truth at .devcontainer/shared/versions.env, consumed by both Dockerfiles, prepare.bat, and the CI workflows via a new ./.github/actions/load-versions composite action. Bumping any version is now a one-line change instead of touching 3-5 files. Adds two path-gated smoke workflows (devcontainer-{linux,windows}-smoke.yml) that build the devcontainer images on devcontainer-touching PRs so a versions.env typo or Dockerfile regression won't ship. Linux smoke runs on amd64 + arm64 to cover the multi-arch claim that testing-cpp.yml doesn't exercise. Co-Authored-By: Claude Opus 4.6 --- .devcontainer/{ => linux}/Dockerfile | 73 +++-- .devcontainer/{ => linux}/README.md | 59 ++-- .devcontainer/{ => linux}/devcontainer.json | 30 ++- .devcontainer/macos/README.md | 84 ++++++ .devcontainer/shared/README.md | 69 +++++ .devcontainer/shared/postcreate-banner.ps1 | 10 + .devcontainer/shared/postcreate-banner.sh | 13 + .devcontainer/shared/versions.env | 54 ++++ .devcontainer/windows/Dockerfile | 160 +++++++++++ .devcontainer/windows/README.md | 106 ++++++++ .devcontainer/windows/devcontainer.json | 60 +++++ .devcontainer/windows/vcpkg-install.ps1 | 98 +++++++ .github/actions/load-versions/action.yml | 33 +++ .../workflows/devcontainer-linux-smoke.yml | 104 +++++++ .../workflows/devcontainer-windows-smoke.yml | 86 ++++++ .github/workflows/testing-cpp.yml | 254 +++++++++--------- .github/workflows/testing-csharp.yml | 7 +- .github/workflows/testing-go.yml | 5 +- CLAUDE.md | 2 +- README.md | 38 ++- prepare.bat | 50 +++- 21 files changed, 1195 insertions(+), 200 deletions(-) rename .devcontainer/{ => linux}/Dockerfile (67%) rename .devcontainer/{ => linux}/README.md (57%) rename .devcontainer/{ => linux}/devcontainer.json (65%) create mode 100644 .devcontainer/macos/README.md create mode 100644 .devcontainer/shared/README.md create mode 100644 .devcontainer/shared/postcreate-banner.ps1 create mode 100644 .devcontainer/shared/postcreate-banner.sh create mode 100644 .devcontainer/shared/versions.env create mode 100644 .devcontainer/windows/Dockerfile create mode 100644 .devcontainer/windows/README.md create mode 100644 .devcontainer/windows/devcontainer.json create mode 100644 .devcontainer/windows/vcpkg-install.ps1 create mode 100644 .github/actions/load-versions/action.yml create mode 100644 .github/workflows/devcontainer-linux-smoke.yml create mode 100644 .github/workflows/devcontainer-windows-smoke.yml diff --git a/.devcontainer/Dockerfile b/.devcontainer/linux/Dockerfile similarity index 67% rename from .devcontainer/Dockerfile rename to .devcontainer/linux/Dockerfile index 3ae9a05..694e9c5 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/linux/Dockerfile @@ -1,12 +1,26 @@ # syntax=docker/dockerfile:1.7 -# tableauio/loader devcontainer +# tableauio/loader devcontainer (linux) # # Single-stage, multi-arch (amd64 + arm64) image bringing the full -# C++/Go/.NET/Node toolchain plus protobuf 6.33.4 (via vcpkg) at the -# exact versions CI uses. See docs/superpowers/specs/2026-05-29-devcontainer-design.md. +# C++/Go/.NET/Node toolchain plus protobuf at the exact versions CI uses. +# +# All version pins are read from ../shared/versions.env (the single source +# of truth shared with the Windows container, prepare.bat, and CI). To bump +# Go / buf / protobuf / .NET / Node / vcpkg-baseline, edit that file — not +# this one. +# +# Build context is .devcontainer/ (the parent of this directory), so that +# `COPY shared/...` resolves. devcontainer.json sets `"context": ".."`. FROM mcr.microsoft.com/devcontainers/cpp:1-ubuntu-24.04 +# --------------------------------------------------------------------------- +# Pull pinned versions from the shared file. We `COPY` it into a build-stage +# location and `source` it in every RUN that needs the values; ARG/ENV alone +# don't survive across RUN boundaries in BuildKit, so we re-source per layer. +# --------------------------------------------------------------------------- +COPY shared/versions.env /opt/versions.env + # --------------------------------------------------------------------------- # Architecture detection. BuildKit auto-populates TARGETARCH; we resolve it # into per-arch download-name fragments (Go's tarball, buf's release asset, @@ -27,15 +41,15 @@ printf 'GO_ARCH=%s\nBUF_ARCH=%s\nVCPKG_TRIPLET=%s\n' \ EOF # --------------------------------------------------------------------------- -# Go 1.24.0 — official tarball into /usr/local/go. +# Go — official tarball into /usr/local/go. # # PATH is set via ENV (not /etc/profile.d/) so non-interactive shells like # the postCreateCommand and downstream RUNs see Go without sourcing profile. # /home/vscode/go/bin lands `go install`-placed binaries on PATH automatically. # --------------------------------------------------------------------------- -ARG GO_VERSION=1.24.0 RUN </dev/null)" in *) echo "ERROR: installed protobuf does not match requested version ${PROTOBUF_VERSION}." echo " vcpkg installed-file marker: ${INFO_FILE:-}" - echo " Bump VCPKG_BASELINE_COMMIT (in this Dockerfile, prepare.bat," - echo " and testing-cpp.yml) to a commit that knows about the requested version." + echo " Bump VCPKG_BASELINE_COMMIT in .devcontainer/shared/versions.env" + echo " (and prepare.bat + testing-cpp.yml will pick it up automatically)" + echo " to a commit that knows about the requested version." exit 1 ;; esac @@ -126,19 +153,20 @@ ln -s /opt/vcpkg/active/tools/protobuf/protoc /usr/local/bin/protoc EOF # --------------------------------------------------------------------------- -# .NET SDK 8.0 + Node.js 20 LTS — apt-based installs from the official -# Microsoft and NodeSource repositories. apt-get clean + rm /var/lib/apt/lists -# at the end keeps the layer small. +# .NET SDK + Node.js — apt-based installs from the official Microsoft and +# NodeSource repositories. Versions are read from /opt/versions.env. +# apt-get clean + rm /var/lib/apt/lists at the end keeps the layer small. # --------------------------------------------------------------------------- RUN < PROTOBUF_VERSION inside. - // Default 6.33.4 (CI's modern matrix entry). To rebuild against legacy v3: + // Default falls through to versions.env's PROTOBUF_VERSION (CI's modern + // matrix entry). To rebuild against the legacy v3 line: // LOADER_PROTOBUF_VERSION=3.21.12 code . # then Reopen in Container. + // + // "context" is the parent .devcontainer/ directory so the Dockerfile's + // `COPY shared/...` lines resolve. "build": { "dockerfile": "Dockerfile", + "context": "..", "args": { - "PROTOBUF_VERSION": "${localEnv:LOADER_PROTOBUF_VERSION:6.33.4}" + "PROTOBUF_VERSION": "${localEnv:LOADER_PROTOBUF_VERSION:}" } }, // Persist the Go module cache across container rebuilds. Workspace itself @@ -39,13 +51,13 @@ } } }, - // One-line ready banner so the developer knows the container is healthy. - // Pure echo — no installs, no version-pinning at runtime, no surprises. - "postCreateCommand": "printf 'tableauio/loader devcontainer ready.\\n go: %s\\n buf: %s\\n protoc: %s\\n dotnet: %s\\n node: %s\\n' \"$(go version | cut -d' ' -f3)\" \"$(buf --version)\" \"$(protoc --version)\" \"$(dotnet --version)\" \"$(node --version)\"", + // Ready banner so the developer knows the container is healthy. + // Script is baked into the image at build time (see Dockerfile). + "postCreateCommand": "/usr/local/bin/loader-devcontainer-banner", // The loader-go-mod named volume is mounted at /home/vscode/go (see "mounts" // above). Docker creates named volumes root-owned by default, masking any // ownership set in the Dockerfile. Re-chown on every start so `go install` // (e.g. gopls auto-install from the Go extension) can write into GOPATH. // Idempotent and ~50ms on an already-correct tree. "postStartCommand": "sudo chown -R vscode:vscode /home/vscode/go" -} \ No newline at end of file +} diff --git a/.devcontainer/macos/README.md b/.devcontainer/macos/README.md new file mode 100644 index 0000000..7f234ad --- /dev/null +++ b/.devcontainer/macos/README.md @@ -0,0 +1,84 @@ +# Dev Container — macOS notes + +> **There is no macOS devcontainer.** Apple's macOS SLA prohibits +> virtualising macOS on non-Apple hardware, and Microsoft / Docker +> publish no `mcr.microsoft.com/macos/...` base image. This directory is +> documentation only — it intentionally has no `devcontainer.json` so +> the VS Code Dev Containers picker does not list it. + +## Recommended path on macOS + +Use the [Linux devcontainer](../linux/). Docker Desktop on macOS runs +Linux containers natively: + +| Host | Container arch the Linux Dockerfile builds | Native? | +| --- | --- | --- | +| Apple Silicon (M-series) | `linux/arm64` | yes | +| Intel Mac | `linux/amd64` | yes | + +Confirm with: + +```sh +docker info | grep Architecture +``` + +→ `aarch64` on Apple Silicon, `x86_64` on Intel. Neither uses Rosetta or +QEMU emulation. + +## If you want to build on macOS *without* a container + +The same toolchain versions live in +[`../shared/versions.env`](../shared/versions.env) — read them and +install via Homebrew. There is no automated script (yet) because the +Linux container is the recommended path; this is documented for parity +with Windows's `prepare.bat` fallback. + +```sh +# Read pinned versions +. .devcontainer/shared/versions.env + +# Toolchain via Homebrew +brew install \ + "go@${GO_VERSION%.*}" \ + "buf" \ + "protobuf" \ + "dotnet@${DOTNET_VERSION%.*}" \ + "node@${NODE_VERSION}" \ + "cmake" \ + "ninja" +``` + +> **Caveats** +> +> - Homebrew tracks the latest formula version, not `versions.env`'s +> exact pin. For protobuf in particular, this means you may end up +> with whichever 6.x or later release Homebrew currently ships, not +> `${PROTOBUF_VERSION}`. If the gencode/runtime version check bites, +> either install via vcpkg manifest mode (see the repo +> [README → Install protobuf](../../README.md#install-protobuf)) or +> switch to the Linux devcontainer. +> +> - `dotnet@8` and `node@20` are versioned brews; the unversioned `dotnet` +> / `node` formulae track their respective LTS lines and may drift. +> +> - Apple Silicon users: nothing in the loader requires Rosetta. If you +> see x86_64 Homebrew complaints, run `arch -arm64 brew ...`. + +## Why no macOS container? + +Three blocking reasons: + +1. **Apple licensing.** macOS may only run on Apple-branded hardware. + Container images are by design hardware-portable; the licence + forbids that. +2. **No base image.** Microsoft (the publisher of nearly every Windows / + Linux base image used by Docker) doesn't publish a macOS image, and + no third party does either. +3. **No runtime.** Even if a base image existed, `dockerd` on macOS runs + a Linux VM under the hood (LinuxKit on Intel, virtualization-framework + on Apple Silicon). It cannot run a macOS container. + +If your build genuinely needs Apple-specific toolchains (Xcode, +codesigning, `security` keychain), use a real macOS host — that's what +GitHub Actions' `macos-latest` runners are. We don't have one of those +in our matrix today, so this isn't a parity loss. diff --git a/.devcontainer/shared/README.md b/.devcontainer/shared/README.md new file mode 100644 index 0000000..f7b3558 --- /dev/null +++ b/.devcontainer/shared/README.md @@ -0,0 +1,69 @@ +# tableauio/loader — `.devcontainer/shared/` + +Files in this directory are consumed by **multiple** devcontainer paths +(`linux/`, `windows/`) and by host-side scripts (`prepare.bat`, CI +workflows). Edit them with cross-platform parsing in mind. + +## Files + +| File | Consumers | Format | +| --- | --- | --- | +| [`versions.env`](./versions.env) | All Dockerfiles, `prepare.bat`, every `.github/workflows/*.yml` | `KEY=VALUE`, one per line, no quotes, no `$VAR` expansion | +| [`postcreate-banner.sh`](./postcreate-banner.sh) | `linux/devcontainer.json` `postCreateCommand` | POSIX `sh` | +| [`postcreate-banner.ps1`](./postcreate-banner.ps1) | `windows/devcontainer.json` `postCreateCommand` | PowerShell | + +## `versions.env` parsing rules + +Every consumer needs to read this file with at most a one-liner. The format +is therefore extremely conservative: + +- **One assignment per line**, exactly `KEY=VALUE`. +- **No quotes**, no spaces around `=`, no inline comments after the value. +- **Comments start at column 0** with `#`. +- **Blank lines** are ignored. +- **No shell expansion** — values are bare literals. + +Quick parsers per language: + +```sh +# POSIX shell (Linux Dockerfile) +. .devcontainer/shared/versions.env +echo "$GO_VERSION" +``` + +```powershell +# PowerShell (Windows Dockerfile) +$v = Get-Content .devcontainer/shared/versions.env | + Where-Object { $_ -and $_ -notmatch '^\s*#' } | + ConvertFrom-StringData +$v.GO_VERSION +``` + +```cmd +:: Windows cmd (prepare.bat) +for /f "tokens=1,2 delims==" %%a in (.devcontainer\shared\versions.env) do ( + if not "%%a"=="" if not "%%a:~0,1%"=="#" set "%%a=%%b" +) +echo %GO_VERSION% +``` + +```yaml +# GitHub Actions (read once, export to $GITHUB_ENV) +- name: Read pinned versions + shell: bash + run: | + while IFS='=' read -r k v; do + [ -z "$k" ] && continue + [ "${k#\#}" != "$k" ] && continue + printf '%s=%s\n' "$k" "$v" >> "$GITHUB_ENV" + done < .devcontainer/shared/versions.env +``` + +## Lockstep rule + +`VCPKG_BASELINE_COMMIT` is the trickiest pin: it must know about every +`PROTOBUF_VERSION` value used anywhere (devcontainer default + every +`testing-cpp.yml` matrix entry). Bumping `PROTOBUF_VERSION` to a value +the current baseline doesn't know is caught at build time by the +post-install assertion in both Dockerfiles and `prepare.bat` — fail loud, +no silent wrong-version installs. diff --git a/.devcontainer/shared/postcreate-banner.ps1 b/.devcontainer/shared/postcreate-banner.ps1 new file mode 100644 index 0000000..8c98f51 --- /dev/null +++ b/.devcontainer/shared/postcreate-banner.ps1 @@ -0,0 +1,10 @@ +# Post-create banner for the Windows devcontainer. +# Same shape as ../shared/postcreate-banner.sh (Linux) — five lines, +# pure version queries, no installs. +$ErrorActionPreference = 'Stop' +Write-Host 'tableauio/loader devcontainer ready (windows).' +Write-Host (' go: {0}' -f ((go version) -split '\s+')[2]) +Write-Host (' buf: {0}' -f (buf --version 2>&1)) +Write-Host (' protoc: {0}' -f (protoc --version)) +Write-Host (' dotnet: {0}' -f (dotnet --version)) +Write-Host (' node: {0}' -f (node --version)) diff --git a/.devcontainer/shared/postcreate-banner.sh b/.devcontainer/shared/postcreate-banner.sh new file mode 100644 index 0000000..5aa8d40 --- /dev/null +++ b/.devcontainer/shared/postcreate-banner.sh @@ -0,0 +1,13 @@ +#!/bin/sh +# Post-create banner for the Linux devcontainer. +# Pure echo — no installs, no version-pinning at runtime, no surprises. +# Mirrors the same five-line summary the previous inline postCreateCommand +# emitted; extracted to a script so the Windows container can have a +# parallel postcreate-banner.ps1 with the same shape. +set -e +printf 'tableauio/loader devcontainer ready (linux).\n' +printf ' go: %s\n' "$(go version | cut -d' ' -f3)" +printf ' buf: %s\n' "$(buf --version 2>&1)" +printf ' protoc: %s\n' "$(protoc --version)" +printf ' dotnet: %s\n' "$(dotnet --version)" +printf ' node: %s\n' "$(node --version)" diff --git a/.devcontainer/shared/versions.env b/.devcontainer/shared/versions.env new file mode 100644 index 0000000..aed69a5 --- /dev/null +++ b/.devcontainer/shared/versions.env @@ -0,0 +1,54 @@ +# tableauio/loader — pinned toolchain versions. +# +# Single source of truth consumed by: +# - .devcontainer/linux/Dockerfile (sourced as a shell file) +# - .devcontainer/windows/Dockerfile (parsed line-by-line in PowerShell) +# - prepare.bat (parsed via `for /f` in cmd) +# - .github/workflows/*.yml (read by a "Read versions" step into $GITHUB_ENV) +# +# Format rules so every consumer can parse this file with a single regex: +# - One KEY=VALUE per line, no quotes, no spaces around `=`. +# - Comments start with `#` at column 0 only (not after a value). +# - Blank lines are ignored. +# - Values are bare strings — no shell expansion, no $VAR references. +# +# Bumping any of these is a one-line change. The matching post-install +# assertion in each Dockerfile / prepare.bat catches the case where the +# vcpkg baseline doesn't yet know about the requested PROTOBUF_VERSION. + +# Go SDK shipped in the devcontainer. Must be ≥ the `go` directive in go.mod; +# CI workflows resolve their Go version from go.mod via setup-go's +# go-version-file, so devcontainer is the only place Go is hard-pinned. +GO_VERSION=1.24.0 + +# buf CLI release. Pinned in devcontainers, prepare.bat, and all three +# testing-*.yml workflows. Bump everywhere together. +BUF_VERSION=1.67.0 + +# Default protobuf version (CI's "modern" matrix entry). devcontainer.json +# wires this to the LOADER_PROTOBUF_VERSION host env var so users can rebuild +# against the legacy v3 line (3.21.12) without editing this file. +PROTOBUF_VERSION=6.33.4 + +# vcpkg checkout commit. Same value used by: +# - .devcontainer/linux/Dockerfile (ARG VCPKG_BASELINE_COMMIT) +# - .devcontainer/windows/Dockerfile +# - prepare.bat (set VCPKG_BASELINE_COMMIT=...) +# - .github/workflows/testing-cpp.yml (env: VCPKG_COMMIT) +# This commit MUST know about every PROTOBUF_VERSION in the testing-cpp.yml +# matrix; bump it forward (never sideways) when adding a new protobuf entry. +VCPKG_BASELINE_COMMIT=dc8d75cfc3281b8e2a4ed8ee4163c891190df932 + +# .NET SDK major.minor. apt installs `dotnet-sdk-${DOTNET_VERSION}` on Linux, +# Chocolatey installs `dotnet-${DOTNET_VERSION}-sdk` on Windows. CI uses +# `${DOTNET_VERSION}.x` with actions/setup-dotnet. +DOTNET_VERSION=8.0 + +# Node.js LTS major. NodeSource apt repo is `setup_${NODE_VERSION}.x`, +# Chocolatey package is `nodejs-lts --version=${NODE_VERSION}.*`. +NODE_VERSION=20 + +# CMake version installed by prepare.bat (Linux devcontainer base image +# already ships a recent cmake; Windows container installs this exact one +# to match the native prepare.bat experience). +CMAKE_VERSION=3.31.8 diff --git a/.devcontainer/windows/Dockerfile b/.devcontainer/windows/Dockerfile new file mode 100644 index 0000000..fee3fc2 --- /dev/null +++ b/.devcontainer/windows/Dockerfile @@ -0,0 +1,160 @@ +# escape=` +# tableauio/loader devcontainer (windows) +# +# Windows-container variant. Runs ONLY on Windows hosts (Hyper-V or +# process isolation, Docker Desktop in Windows-containers mode). +# windows/amd64 ONLY — Microsoft does not publish arm64 Windows base +# images, so arm64 hosts must use the Linux container under ../linux/. +# +# All version pins are read from ../shared/versions.env (the single source +# of truth shared with the Linux container, prepare.bat, and CI). To bump +# Go / buf / protobuf / .NET / Node / vcpkg-baseline / cmake, edit that +# file — not this one. +# +# Build context is .devcontainer/ (the parent of this directory) so that +# `COPY shared\...` resolves. devcontainer.json sets `"context": ".."`. +# +# Why servercore (not nanoserver): +# vcpkg compiles protobuf from source via MSVC — that needs a real +# Win32 environment (PowerShell, Win32 APIs, CRT) that nanoserver does +# not provide. ltsc2022 matches GitHub Actions' windows-latest runner +# family, so the post-install assertion's resolved versions match CI. + +FROM mcr.microsoft.com/windows/servercore:ltsc2022 + +SHELL ["powershell", "-NoProfile", "-NoLogo", "-ExecutionPolicy", "Bypass", "-Command", "$ErrorActionPreference='Stop'; $ProgressPreference='SilentlyContinue';"] + +# --------------------------------------------------------------------------- +# Pull pinned versions from the shared file. PowerShell consumers parse it +# with `ConvertFrom-StringData` after stripping comment / blank lines. +# We render the parsed values into machine-wide environment variables so +# subsequent RUN layers (and the running container) see them as $env:KEY. +# --------------------------------------------------------------------------- +COPY shared\versions.env C:\loader\versions.env + +RUN $raw = Get-Content C:\loader\versions.env | ` + Where-Object { $_ -and $_ -notmatch '^\s*#' }; ` + $vars = $raw | ConvertFrom-StringData; ` + foreach ($k in $vars.Keys) { ` + [System.Environment]::SetEnvironmentVariable($k, $vars[$k], 'Machine'); ` + } + +# Re-import machine env into this build session so each RUN below sees +# $env:GO_VERSION etc. without re-parsing the file. +SHELL ["powershell", "-NoProfile", "-NoLogo", "-ExecutionPolicy", "Bypass", "-Command", "$ErrorActionPreference='Stop'; $ProgressPreference='SilentlyContinue'; foreach ($k in @('GO_VERSION','BUF_VERSION','PROTOBUF_VERSION','VCPKG_BASELINE_COMMIT','DOTNET_VERSION','NODE_VERSION','CMAKE_VERSION')) { Set-Item -Path \"env:$k\" -Value ([System.Environment]::GetEnvironmentVariable($k,'Machine')) };"] + +# --------------------------------------------------------------------------- +# Chocolatey — bootstrap the package manager that installs Git, CMake, +# .NET, Node, and the MSVC build tools. Same channel `prepare.bat` uses +# on a bare-metal Windows host. +# --------------------------------------------------------------------------- +RUN [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12; ` + Invoke-Expression ((New-Object System.Net.WebClient).DownloadString('https://community.chocolatey.org/install.ps1')) + +ENV ChocolateyInstall=C:\ProgramData\chocolatey +RUN $env:PATH = \"$env:ChocolateyInstall\bin;$env:PATH\"; ` + [System.Environment]::SetEnvironmentVariable('PATH', $env:PATH, 'Machine') + +# --------------------------------------------------------------------------- +# Git, CMake, Ninja — same versions/sources prepare.bat uses. +# Git is needed for `git clone` of vcpkg below. +# --------------------------------------------------------------------------- +RUN choco install -y --no-progress git ninja; ` + choco install -y --no-progress cmake --version=$env:CMAKE_VERSION --installargs '\"ADD_CMAKE_TO_PATH=System\"' + +# Refresh PATH so the new tools are visible in this build session. +RUN $env:PATH = [System.Environment]::GetEnvironmentVariable('PATH','Machine') + ';' + [System.Environment]::GetEnvironmentVariable('PATH','User'); ` + [System.Environment]::SetEnvironmentVariable('PATH', $env:PATH, 'Process') + +# --------------------------------------------------------------------------- +# Visual Studio 2022 Build Tools (VC++ workload). This is the heavy layer +# (~6 GB on disk). Matches the `prepare.bat` Step 3 install pattern. +# `--passive --wait --norestart` so Docker doesn't see a hung installer. +# --------------------------------------------------------------------------- +RUN choco install -y --no-progress visualstudio2022buildtools ` + --package-parameters '\"--add Microsoft.VisualStudio.Workload.VCTools --includeRecommended --passive --wait --norestart --locale en-US\"' + +# --------------------------------------------------------------------------- +# Go SDK — official MSI to C:\Go. Pinned to versions.env's GO_VERSION. +# --------------------------------------------------------------------------- +RUN $url = \"https://go.dev/dl/go${env:GO_VERSION}.windows-amd64.msi\"; ` + Invoke-WebRequest -Uri $url -OutFile C:\go.msi; ` + Start-Process msiexec.exe -ArgumentList '/i','C:\go.msi','/qn','/norestart' -Wait; ` + Remove-Item C:\go.msi +ENV GOPATH=C:\Users\ContainerUser\go +RUN [System.Environment]::SetEnvironmentVariable('PATH', ` + 'C:\Program Files\Go\bin;C:\Users\ContainerUser\go\bin;' + ` + [System.Environment]::GetEnvironmentVariable('PATH','Machine'), ` + 'Machine') + +# --------------------------------------------------------------------------- +# buf CLI — single-binary Windows release. Pinned to versions.env's +# BUF_VERSION. Mirrors prepare.bat Step 4. +# --------------------------------------------------------------------------- +RUN New-Item -ItemType Directory -Force -Path C:\Tools\buf | Out-Null; ` + $url = \"https://github.com/bufbuild/buf/releases/download/v${env:BUF_VERSION}/buf-Windows-x86_64.exe\"; ` + Invoke-WebRequest -Uri $url -OutFile C:\Tools\buf\buf.exe; ` + [System.Environment]::SetEnvironmentVariable('PATH', ` + 'C:\Tools\buf;' + [System.Environment]::GetEnvironmentVariable('PATH','Machine'), ` + 'Machine') + +# --------------------------------------------------------------------------- +# vcpkg + protobuf via manifest mode. +# +# Triplet: x64-windows-static. Same triplet prepare.bat uses, so a +# user moving between native (prepare.bat) and container builds avoids +# the LNK2038 _ITERATOR_DEBUG_LEVEL CRT-mismatch trap. +# +# Why manifest mode: classic-mode `vcpkg install --x-version=...` is +# silently a no-op; only manifest mode + overrides actually pin the port +# version. The post-install assertion catches the case where vcpkg's +# resolution still picks a different port revision than requested. +# --------------------------------------------------------------------------- +ARG PROTOBUF_VERSION= +ENV VCPKG_ROOT=C:\vcpkg ` + VCPKG_DEFAULT_TRIPLET=x64-windows-static + +RUN git clone https://github.com/microsoft/vcpkg.git $env:VCPKG_ROOT; ` + git -C $env:VCPKG_ROOT checkout $env:VCPKG_BASELINE_COMMIT; ` + & \"$env:VCPKG_ROOT\bootstrap-vcpkg.bat\" -disableMetrics + +# Render the manifest (PROTOBUF_VERSION build-arg overrides versions.env if set), +# then install via vcpkg, then assert the resolved version matches what we asked for. +# vcpkg compiles protobuf from source under MSVC, which needs the VS Build Tools +# environment active — load it from vsdevcmd.bat into the build session. +COPY windows\vcpkg-install.ps1 C:\loader\vcpkg-install.ps1 +RUN $effective = if ($env:PROTOBUF_VERSION) { $env:PROTOBUF_VERSION } else { ` + ([System.Environment]::GetEnvironmentVariable('PROTOBUF_VERSION','Machine')) }; ` + $env:PROTOBUF_VERSION = $effective; ` + & C:\loader\vcpkg-install.ps1 + +# Stable PATH entries so `protoc` and the vcpkg tools are visible. +RUN [System.Environment]::SetEnvironmentVariable('PATH', ` + 'C:\vcpkg;C:\vcpkg-manifest\vcpkg_installed\x64-windows-static\tools\protobuf;' + ` + [System.Environment]::GetEnvironmentVariable('PATH','Machine'), ` + 'Machine') +ENV CMAKE_PREFIX_PATH=C:\vcpkg-manifest\vcpkg_installed\x64-windows-static + +# --------------------------------------------------------------------------- +# .NET SDK + Node.js LTS. Versions from versions.env. Chocolatey is the +# install channel for both — same one prepare.bat uses for VS Build Tools. +# --------------------------------------------------------------------------- +RUN $dotnetPkg = \"dotnet-${env:DOTNET_VERSION}-sdk\"; ` + choco install -y --no-progress $dotnetPkg; ` + choco install -y --no-progress nodejs-lts --version=\"${env:NODE_VERSION}.0.0\" + +# --------------------------------------------------------------------------- +# Post-create banner script (parallel to the Linux variant). +# --------------------------------------------------------------------------- +COPY shared\postcreate-banner.ps1 C:\loader\postcreate-banner.ps1 + +# --------------------------------------------------------------------------- +# Workspace dir. Matches devcontainer.json's workspaceFolder. +# --------------------------------------------------------------------------- +WORKDIR C:\workspaces\loader + +# Drop back to a normal shell for the running container — the build-time +# version-import shell is no longer needed. +SHELL ["powershell", "-NoProfile", "-NoLogo", "-Command"] + +CMD ["powershell.exe", "-NoLogo"] diff --git a/.devcontainer/windows/README.md b/.devcontainer/windows/README.md new file mode 100644 index 0000000..ed5024d --- /dev/null +++ b/.devcontainer/windows/README.md @@ -0,0 +1,106 @@ +# Dev Container — Windows variant + +A **Windows-container** image (windows/amd64 only) for Windows hosts that +want a native MSVC build environment inside the container, without +WSL2. The Linux container under [`../linux/`](../linux/) is the +recommended path for almost everyone — use this variant only when you +specifically need Windows-native tooling. + +## When to use this variant + +| Goal | Use | +| --- | --- | +| Develop on Windows + WSL2 | `../linux/` | +| Develop on Windows without WSL2, native MSVC | **this** | +| Develop on macOS / Linux | `../linux/` | +| Develop on arm64 Windows (Surface Pro X) | `../linux/` (linux/arm64 native) | +| Match CI's `windows-latest` toolchain locally | **this** | + +## Prerequisites + +- **Windows 10/11 Pro or Enterprise.** Home edition cannot run Windows + containers (Hyper-V missing). +- **Docker Desktop in Windows-containers mode.** Right-click the tray + icon → *Switch to Windows containers*. This is a per-host toggle: + Linux and Windows containers cannot run simultaneously. +- **Disk space.** ~12 GB for MSVC Build Tools alone. Total image is + ~15 GB. +- **VS Code** with the Dev Containers extension. + +## Open the container + +```sh +code . # in the repo root +``` + +In VS Code, run **Dev Containers: Reopen in Container** from the command +palette. VS Code shows a picker; choose **tableauio/loader (windows)**. + +First build is one-time, ~45 minutes (vcpkg compiles protobuf from +source under MSVC). Subsequent reopens are near-instant. + +## Pin a different protobuf version + +```cmd +set LOADER_PROTOBUF_VERSION=3.21.12 +code . +``` + +…then **Dev Containers: Rebuild Container**. Same knob as the Linux +variant, same default sourced from +[`../shared/versions.env`](../shared/versions.env). + +## Architecture + +Single Dockerfile based on `mcr.microsoft.com/windows/servercore:ltsc2022`: + +1. Imports `versions.env` into machine-wide env so each `RUN` sees `$env:KEY`. +2. Chocolatey bootstrap. +3. Git, Ninja, CMake (CMake version pinned by `versions.env`). +4. Visual Studio 2022 Build Tools — VC++ workload. +5. Go SDK — official MSI, version from `versions.env`. +6. buf CLI — single-binary release, version from `versions.env`. +7. vcpkg (pinned to `VCPKG_BASELINE_COMMIT`) + protobuf via manifest mode + with the `x64-windows-static` triplet — same triplet `prepare.bat` + uses on bare metal, so a developer moving between native and + container builds avoids the LNK2038 `_ITERATOR_DEBUG_LEVEL` CRT-mismatch + trap. Post-install assertion catches version drift. +8. .NET SDK + Node.js LTS via Chocolatey. +9. `ENV CMAKE_PREFIX_PATH=C:\vcpkg-manifest\vcpkg_installed\x64-windows-static` + so `find_package(Protobuf CONFIG)` resolves automatically. + +## Why is the image so large? + +`visualstudio2022buildtools` with the VC++ workload is ~6 GB on disk; the +Windows base image is another ~5 GB. There is no nanoserver path because +vcpkg compiles protobuf from source under MSVC and that needs the full +Win32 environment. + +## Why isolation=hyperv? + +Process isolation requires the **host's** Windows build to be ≥ the +container image's Windows build. ltsc2022 is conservative enough to +work on most Windows 10 21H2+ and Windows 11 hosts under Hyper-V +isolation. If your host is Windows 11 22H2+ you can drop `runArgs` +or change to `--isolation=process` for slightly faster startup. + +## Falling back + +If the container build is too slow or too large for your machine, the +existing manual setup paths still work: + +- Native Windows: [`prepare.bat`](../../prepare.bat) at the repo root. + Same vcpkg baseline and triplet as this container. +- Linux container under WSL2: [`../linux/`](../linux/). Same toolchain + versions, much smaller image. + +## Limitations + +- **windows/amd64 only.** No arm64 Windows base image exists. Use the + Linux container on arm64 Windows hosts (it builds linux/arm64 natively + under Docker's Linux-container engine). +- **Windows containers don't run on macOS or Linux.** Don't try to build + this image on a non-Windows host; it won't work. +- **Heavier than the Linux variant.** Both build time and disk footprint + are roughly 2× the Linux container. If you don't specifically need + Windows-native tooling, use the Linux variant. diff --git a/.devcontainer/windows/devcontainer.json b/.devcontainer/windows/devcontainer.json new file mode 100644 index 0000000..b5bf621 --- /dev/null +++ b/.devcontainer/windows/devcontainer.json @@ -0,0 +1,60 @@ +{ + // tableauio/loader Dev Container — Windows variant. + // + // Windows-host-only. Runs windows/amd64 ONLY (Microsoft does not + // publish arm64 Windows base images). arm64 hosts and macOS hosts + // must use the Linux container under ../linux/ instead. + // + // Prerequisites: + // - Windows 10/11 Pro or Enterprise (Hyper-V required; Home edition + // cannot run Windows containers) + // - Docker Desktop in **Windows-containers mode** + // (right-click tray icon → "Switch to Windows containers") + // - Host Windows build ≥ ltsc2022 base image build, OR accept + // Hyper-V isolation (slower but compatible across builds) + // + // All version pins live in ../shared/versions.env. See the + // ../README.md for the longer how-to. + "name": "tableauio/loader (windows)", + "build": { + "dockerfile": "Dockerfile", + // Build context is the parent .devcontainer/ directory so that + // `COPY shared\...` and `COPY windows\...` resolve. + "context": "..", + "args": { + // Wire LOADER_PROTOBUF_VERSION host env to the Dockerfile's + // PROTOBUF_VERSION ARG. Empty default → Dockerfile falls back + // to versions.env's PROTOBUF_VERSION. + "PROTOBUF_VERSION": "${localEnv:LOADER_PROTOBUF_VERSION:}" + } + }, + // Process isolation needs host build ≥ ltsc2022; Hyper-V isolation is + // the safe default and works everywhere Windows containers run. + "runArgs": ["--isolation=hyperv"], + "remoteUser": "ContainerUser", + "workspaceFolder": "C:\\workspaces\\loader", + // No named-volume go-mod cache here. Windows-container named volumes + // have well-documented ACL footguns; accept that mod cache rebuilds + // each container rebuild. If that's painful, add a host bind-mount + // here pointing at a local directory. + "customizations": { + "vscode": { + "extensions": [ + "golang.go", + "ms-vscode.cmake-tools", + "ms-vscode.cpptools", + "ms-dotnettools.csharp", + "bufbuild.vscode-buf", + "DrBlury.protobuf-vsc" + ], + "settings": { + "go.toolsManagement.autoUpdate": false, + "cmake.configureOnOpen": false, + // PowerShell as default terminal inside the container. + "terminal.integrated.defaultProfile.windows": "PowerShell" + } + } + }, + // Ready banner so the developer knows the container is healthy. + "postCreateCommand": "powershell -NoProfile -ExecutionPolicy Bypass -File C:\\loader\\postcreate-banner.ps1" +} diff --git a/.devcontainer/windows/vcpkg-install.ps1 b/.devcontainer/windows/vcpkg-install.ps1 new file mode 100644 index 0000000..cb51110 --- /dev/null +++ b/.devcontainer/windows/vcpkg-install.ps1 @@ -0,0 +1,98 @@ +# tableauio/loader — Windows-container vcpkg + protobuf install. +# +# Called from windows/Dockerfile during image build. Mirrors the Linux +# Dockerfile's vcpkg manifest-mode install + post-install version +# assertion, in PowerShell + cmd. +# +# Inputs (env vars; set in the Dockerfile from versions.env): +# PROTOBUF_VERSION — protobuf vcpkg port version, e.g. 6.33.4 +# VCPKG_BASELINE_COMMIT — commit to pin in builtin-baseline +# VCPKG_ROOT — C:\vcpkg +# +# Side effects: +# - Renders C:\vcpkg-manifest\vcpkg.json +# - Runs `vcpkg install --triplet=x64-windows-static` +# - Asserts the resolved port version matches PROTOBUF_VERSION +# +# vcpkg compiles protobuf from source under MSVC — that needs the VS Build +# Tools env (cl.exe, INCLUDE, LIB) active in the running shell. We invoke +# vsdevcmd.bat through cmd.exe and capture the resulting environment, then +# replay it into PowerShell before invoking vcpkg. + +$ErrorActionPreference = 'Stop' +$ProgressPreference = 'SilentlyContinue' + +$triplet = 'x64-windows-static' +$manifestDir = 'C:\vcpkg-manifest' +$installRoot = Join-Path $manifestDir 'vcpkg_installed' +New-Item -ItemType Directory -Force -Path $manifestDir | Out-Null + +# 1. Render manifest. JSON-escape the substituted values just in case. +$pv = $env:PROTOBUF_VERSION +$bc = $env:VCPKG_BASELINE_COMMIT +if (-not $pv) { throw 'PROTOBUF_VERSION env var is empty.' } +if (-not $bc) { throw 'VCPKG_BASELINE_COMMIT env var is empty.' } + +$manifest = @{ + name = 'loader-devcontainer-windows' + version = '0.1.0' + dependencies = @('protobuf') + overrides = @(@{ name = 'protobuf'; version = $pv }) + 'builtin-baseline' = $bc +} | ConvertTo-Json -Depth 4 +Set-Content -Path (Join-Path $manifestDir 'vcpkg.json') -Value $manifest -Encoding ASCII + +# 2. Activate the VS 2022 Build Tools environment for this PS session. +# Mirrors what prepare.bat's `call "!VCVARSALL!" x64` does on bare metal. +$vswhere = 'C:\Program Files (x86)\Microsoft Visual Studio\Installer\vswhere.exe' +if (-not (Test-Path $vswhere)) { + throw "vswhere.exe not found at $vswhere; VS Build Tools install layer failed." +} +$installPath = & $vswhere -latest -products * ` + -requires Microsoft.VisualStudio.Component.VC.Tools.x86.x64 ` + -property installationPath +if (-not $installPath) { throw 'No VS install with C++ tools detected.' } +$vsdevcmd = Join-Path $installPath 'Common7\Tools\VsDevCmd.bat' +if (-not (Test-Path $vsdevcmd)) { throw "VsDevCmd.bat not found: $vsdevcmd" } + +# Capture the environment that vsdevcmd produces. +$envDump = & cmd.exe /s /c "`"$vsdevcmd`" -arch=amd64 -host_arch=amd64 && set" +foreach ($line in $envDump) { + if ($line -match '^([^=]+)=(.*)$') { + Set-Item -Path "env:$($Matches[1])" -Value $Matches[2] + } +} +if (-not (Get-Command cl.exe -ErrorAction SilentlyContinue)) { + throw 'cl.exe not on PATH after VsDevCmd activation; cannot proceed.' +} + +# 3. Manifest-mode install. +Push-Location $manifestDir +try { + & "$env:VCPKG_ROOT\vcpkg.exe" install ` + "--triplet=$triplet" ` + "--x-install-root=$installRoot" + if ($LASTEXITCODE -ne 0) { + throw "vcpkg install failed (exit code $LASTEXITCODE)" + } +} finally { + Pop-Location +} + +# 4. Post-install assertion — same shape as the Linux Dockerfile's case +# statement and prepare.bat's findstr check. +$infoDir = Join-Path $installRoot 'vcpkg\info' +$marker = Get-ChildItem -Path $infoDir -Filter "protobuf_*_${triplet}.list" ` + -ErrorAction SilentlyContinue | Select-Object -First 1 +if (-not $marker) { + throw "vcpkg installed-file marker not found under $infoDir" +} +if ($marker.Name -notlike "protobuf_${pv}*") { + Write-Error "Installed protobuf does not match requested version $pv." + Write-Error " vcpkg installed-file marker: $($marker.Name)" + Write-Error " Bump VCPKG_BASELINE_COMMIT in .devcontainer/shared/versions.env" + Write-Error " to a commit that knows about the requested version." + exit 1 +} + +Write-Host "vcpkg protobuf $pv installed under $installRoot ($triplet)." diff --git a/.github/actions/load-versions/action.yml b/.github/actions/load-versions/action.yml new file mode 100644 index 0000000..8d7f0bd --- /dev/null +++ b/.github/actions/load-versions/action.yml @@ -0,0 +1,33 @@ +name: Load pinned versions +description: > + Reads .devcontainer/shared/versions.env (the single source of truth shared + with the devcontainers and prepare.bat) and exports each KEY=VALUE pair to + $GITHUB_ENV so subsequent steps can reference them as ${{ env.KEY }}. + + Format rules of versions.env (kept simple so every consumer can parse it + with a builtin): + - One KEY=VALUE per line, no quotes, no spaces around `=`. + - Comments start with `#` at column 0. + - Blank lines are ignored. + +runs: + using: composite + steps: + - name: Export versions.env to $GITHUB_ENV + shell: bash + run: | + set -eu + file=.devcontainer/shared/versions.env + if [ ! -f "$file" ]; then + echo "::error::Missing $file; cannot resolve pinned tool versions." + exit 1 + fi + while IFS='=' read -r k v; do + [ -z "$k" ] && continue + case "$k" in \#*) continue ;; esac + printf '%s=%s\n' "$k" "$v" >> "$GITHUB_ENV" + done < "$file" + # Mirror VCPKG_BASELINE_COMMIT under VCPKG_COMMIT for backward + # compat with testing-cpp.yml's pre-existing variable name. + grep '^VCPKG_BASELINE_COMMIT=' "$file" \ + | sed 's/^VCPKG_BASELINE_COMMIT=/VCPKG_COMMIT=/' >> "$GITHUB_ENV" diff --git a/.github/workflows/devcontainer-linux-smoke.yml b/.github/workflows/devcontainer-linux-smoke.yml new file mode 100644 index 0000000..5fda640 --- /dev/null +++ b/.github/workflows/devcontainer-linux-smoke.yml @@ -0,0 +1,104 @@ +name: Devcontainer Linux Smoke + +# Builds the Linux-container variant of the devcontainer and runs a +# minimal smoke test inside it. Triggered only when files that affect +# the Linux image change, so we don't burn CI minutes on unrelated PRs. +# +# What this catches that testing-cpp.yml does not: +# 1. arm64 coverage. The Linux Dockerfile claims linux/amd64 + +# linux/arm64; testing-cpp.yml only exercises amd64. This workflow +# builds the image natively on ubuntu-24.04-arm runners. +# 2. shared/versions.env regressions. A typo in the shared file (stray +# space, quote, etc.) silently breaks the devcontainer build for +# anyone running `Reopen in Container`. The other testing-*.yml +# workflows now consume versions.env too, but only the keys they +# need; only this workflow exercises the full file in a real +# Dockerfile build. +# +# What this DOES NOT do: +# - Run the full C++ / C# test matrices. Those are testing-{cpp,csharp}.yml, +# which use lukka/run-vcpkg directly on the runner (much faster than +# building the devcontainer image first). The container is for +# local-dev parity, not the primary CI test path. + +on: + pull_request: + paths: + - '.devcontainer/linux/**' + - '.devcontainer/shared/**' + - '.github/workflows/devcontainer-linux-smoke.yml' + push: + branches: [master, main] + paths: + - '.devcontainer/linux/**' + - '.devcontainer/shared/**' + workflow_dispatch: + +permissions: + contents: read + +jobs: + build-and-smoke: + strategy: + fail-fast: false + matrix: + # amd64 is partially covered by testing-cpp.yml's ubuntu-latest + # job (vcpkg+protobuf, x64-linux). arm64 is the genuinely-new + # coverage this workflow adds: GitHub-hosted ubuntu-24.04-arm + # runners build the image natively, no QEMU emulation. + include: + - runner: ubuntu-latest + arch: amd64 + - runner: ubuntu-24.04-arm + arch: arm64 + name: build linux devcontainer + smoke (${{ matrix.arch }}) + runs-on: ${{ matrix.runner }} + timeout-minutes: 45 + + steps: + - name: Checkout + uses: actions/checkout@v6 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Build devcontainer image + working-directory: .devcontainer + run: | + # Context is .devcontainer/ so `COPY shared/...` resolves. + # --load puts the image into the local docker daemon so the + # subsequent `docker run` smoke steps can use it. + docker buildx build \ + --load \ + --file linux/Dockerfile \ + --tag loader-devcontainer-linux:smoke \ + . + + - name: Smoke — version banner + run: | + # Confirms the image runs at all and that all five toolchain + # binaries (go, buf, protoc, dotnet, node) resolve. + docker run --rm loader-devcontainer-linux:smoke \ + /usr/local/bin/loader-devcontainer-banner + + - name: Smoke — buf generate (Go) + run: | + # Bind-mount the checkout into the container's workspace dir + # and regenerate Go protoconf. Catches protoc/buf integration + # regressions that wouldn't show up in the banner. + docker run --rm \ + -v "${{ github.workspace }}:/workspaces/loader" \ + -w /workspaces/loader/test/go-tableau-loader \ + loader-devcontainer-linux:smoke \ + buf generate .. + + - name: Smoke — go vet (plugin packages only) + run: | + # Vet the plugin sources and shared internal packages. Skip + # ./test/... and ./internal/index — those depend on freshly + # generated *.pb.go that we don't produce in this smoke job. + docker run --rm \ + -v "${{ github.workspace }}:/workspaces/loader" \ + -w /workspaces/loader \ + loader-devcontainer-linux:smoke \ + go vet ./cmd/... ./pkg/... ./internal/options/... ./internal/loadutil/... ./internal/xproto/... diff --git a/.github/workflows/devcontainer-windows-smoke.yml b/.github/workflows/devcontainer-windows-smoke.yml new file mode 100644 index 0000000..4810c11 --- /dev/null +++ b/.github/workflows/devcontainer-windows-smoke.yml @@ -0,0 +1,86 @@ +name: Devcontainer Windows Smoke + +# Builds the Windows-container variant of the devcontainer and runs a +# minimal smoke test inside it. Triggered only when files that affect +# the Windows image change, so we don't burn windows-2022 CI minutes on +# unrelated PRs. The Linux container is dogfooded by humans (per +# CLAUDE.md); this job exists because a Windows-container regression is +# much harder for a human to notice locally. + +on: + pull_request: + paths: + - '.devcontainer/windows/**' + - '.devcontainer/shared/**' + - '.github/workflows/devcontainer-windows-smoke.yml' + push: + branches: [master, main] + paths: + - '.devcontainer/windows/**' + - '.devcontainer/shared/**' + workflow_dispatch: + +permissions: + contents: read + +jobs: + build-and-smoke: + name: build windows devcontainer + smoke + runs-on: windows-2022 + timeout-minutes: 90 + + steps: + - name: Checkout + uses: actions/checkout@v6 + + - name: Switch Docker to Windows containers + shell: pwsh + run: | + # GitHub-hosted windows-2022 runners ship Docker in Linux-containers + # mode by default. Flip to Windows containers; the runner has + # DockerCli.exe in PATH for exactly this reason. + & "$env:ProgramFiles\Docker\Docker\DockerCli.exe" -SwitchDaemon + docker info | Select-String -Pattern 'OSType|Server Version|Operating System' + + - name: Build devcontainer image + shell: pwsh + working-directory: .devcontainer + run: | + # Context is .devcontainer/ so `COPY shared\...` resolves. + # Tag mirrors the devcontainer.json name so it's recognisable + # in `docker images`. + docker build ` + --file windows\Dockerfile ` + --tag loader-devcontainer-windows:smoke ` + . + + - name: Smoke test — Go + shell: pwsh + run: | + # Run `go test ./...` from the repo's Go module inside the + # built image. The container's working dir is C:\workspaces\loader; + # bind-mount the checkout there. + docker run --rm ` + --isolation=hyperv ` + -v "${{ github.workspace }}:C:\workspaces\loader" ` + -w C:\workspaces\loader ` + loader-devcontainer-windows:smoke ` + powershell -NoProfile -Command "go vet ./... ; if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }" + + - name: Smoke test — buf generate (Go) + shell: pwsh + run: | + docker run --rm ` + --isolation=hyperv ` + -v "${{ github.workspace }}:C:\workspaces\loader" ` + -w C:\workspaces\loader\test\go-tableau-loader ` + loader-devcontainer-windows:smoke ` + powershell -NoProfile -Command "buf generate .. ; if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }" + + - name: Smoke test — version banner + shell: pwsh + run: | + docker run --rm ` + --isolation=hyperv ` + loader-devcontainer-windows:smoke ` + powershell -NoProfile -ExecutionPolicy Bypass -File C:\loader\postcreate-banner.ps1 diff --git a/.github/workflows/testing-cpp.yml b/.github/workflows/testing-cpp.yml index 41a8d27..7b86935 100644 --- a/.github/workflows/testing-cpp.yml +++ b/.github/workflows/testing-cpp.yml @@ -2,134 +2,134 @@ name: Testing C++ # Trigger on pushes, PRs (excluding documentation changes), and nightly. on: - push: - branches: [master, main] - pull_request: - schedule: - - cron: 0 0 * * * # daily at 00:00 - workflow_dispatch: + push: + branches: [master, main] + pull_request: + schedule: + - cron: 0 0 * * * # daily at 00:00 + workflow_dispatch: permissions: - contents: read - -env: - VCPKG_COMMIT: dc8d75cfc3281b8e2a4ed8ee4163c891190df932 + contents: read jobs: - test: - strategy: - fail-fast: false - matrix: - os: [ubuntu-latest, windows-latest] - config: - - label: modern - protobuf-version: "6.33.4" - - label: legacy-v3 - protobuf-version: "3.21.12" - include: - - os: ubuntu-latest - triplet: x64-linux - - os: windows-latest - triplet: x64-windows-static - - name: test (${{ matrix.os }}, ${{ matrix.config.label }}) - runs-on: ${{ matrix.os }} - timeout-minutes: 45 - - env: - VCPKG_INSTALLED_DIR: ${{ github.workspace }}/vcpkg_installed - VCPKG_DEFAULT_TRIPLET: ${{ matrix.triplet }} - - steps: - - name: Checkout Code - uses: actions/checkout@v6 - - - name: Install Go - uses: actions/setup-go@v6 - with: - go-version-file: go.mod - cache-dependency-path: go.sum - cache: true - - - name: Install Ninja (Ubuntu) - if: runner.os == 'Linux' - run: sudo apt-get update && sudo apt-get install -y ninja-build - - - name: Setup MSVC (Windows) - if: runner.os == 'Windows' - uses: ilammy/msvc-dev-cmd@v1 - - - name: Render vcpkg.json - working-directory: test/cpp-tableau-loader - shell: bash - run: | - cat > vcpkg.json <> "$GITHUB_PATH" - - - name: Add vcpkg-installed protoc to PATH (Windows) - if: runner.os == 'Windows' - shell: pwsh - run: Add-Content -Path $env:GITHUB_PATH -Value "$env:VCPKG_INSTALLED_DIR\${{ matrix.triplet }}\tools\protobuf" - - - name: Verify protoc - shell: bash - run: | - which protoc - protoc --version - - - name: Install Buf - uses: bufbuild/buf-action@v1 - with: - version: 1.67.0 - setup_only: true - github_token: ${{ secrets.GITHUB_TOKEN }} - - - name: Generate protoconf - working-directory: test/cpp-tableau-loader - run: buf generate .. - - - name: CMake Configure - working-directory: test/cpp-tableau-loader - run: > - cmake -S . -B build -G Ninja - -DCMAKE_BUILD_TYPE=Debug - -DCMAKE_CXX_STANDARD=17 - -DCMAKE_TOOLCHAIN_FILE=${{ env.VCPKG_ROOT }}/scripts/buildsystems/vcpkg.cmake - -DVCPKG_TARGET_TRIPLET=${{ matrix.triplet }} - -DVCPKG_INSTALLED_DIR=${{ env.VCPKG_INSTALLED_DIR }} - -DVCPKG_MANIFEST_INSTALL=OFF - - - name: CMake Build - working-directory: test/cpp-tableau-loader - run: cmake --build build --parallel - - - name: Run tests - working-directory: test/cpp-tableau-loader - run: ctest --test-dir build --output-on-failure + test: + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, windows-latest] + config: + - label: modern + protobuf-version: "6.33.4" + - label: legacy-v3 + protobuf-version: "3.21.12" + include: + - os: ubuntu-latest + triplet: x64-linux + - os: windows-latest + triplet: x64-windows-static + + name: test (${{ matrix.os }}, ${{ matrix.config.label }}) + runs-on: ${{ matrix.os }} + timeout-minutes: 45 + + env: + VCPKG_INSTALLED_DIR: ${{ github.workspace }}/vcpkg_installed + VCPKG_DEFAULT_TRIPLET: ${{ matrix.triplet }} + + steps: + - name: Checkout Code + uses: actions/checkout@v6 + + - name: Read pinned versions + uses: ./.github/actions/load-versions + + - name: Install Go + uses: actions/setup-go@v6 + with: + go-version-file: go.mod + cache-dependency-path: go.sum + cache: true + + - name: Install Ninja (Ubuntu) + if: runner.os == 'Linux' + run: sudo apt-get update && sudo apt-get install -y ninja-build + + - name: Setup MSVC (Windows) + if: runner.os == 'Windows' + uses: ilammy/msvc-dev-cmd@v1 + + - name: Render vcpkg.json + working-directory: test/cpp-tableau-loader + shell: bash + run: | + cat > vcpkg.json <> "$GITHUB_PATH" + + - name: Add vcpkg-installed protoc to PATH (Windows) + if: runner.os == 'Windows' + shell: pwsh + run: Add-Content -Path $env:GITHUB_PATH -Value "$env:VCPKG_INSTALLED_DIR\${{ matrix.triplet }}\tools\protobuf" + + - name: Verify protoc + shell: bash + run: | + which protoc + protoc --version + + - name: Install Buf + uses: bufbuild/buf-action@v1 + with: + version: ${{ env.BUF_VERSION }} + setup_only: true + github_token: ${{ secrets.GITHUB_TOKEN }} + + - name: Generate protoconf + working-directory: test/cpp-tableau-loader + run: buf generate .. + + - name: CMake Configure + working-directory: test/cpp-tableau-loader + run: > + cmake -S . -B build -G Ninja + -DCMAKE_BUILD_TYPE=Debug + -DCMAKE_CXX_STANDARD=17 + -DCMAKE_TOOLCHAIN_FILE=${{ env.VCPKG_ROOT }}/scripts/buildsystems/vcpkg.cmake + -DVCPKG_TARGET_TRIPLET=${{ matrix.triplet }} + -DVCPKG_INSTALLED_DIR=${{ env.VCPKG_INSTALLED_DIR }} + -DVCPKG_MANIFEST_INSTALL=OFF + + - name: CMake Build + working-directory: test/cpp-tableau-loader + run: cmake --build build --parallel + + - name: Run tests + working-directory: test/cpp-tableau-loader + run: ctest --test-dir build --output-on-failure diff --git a/.github/workflows/testing-csharp.yml b/.github/workflows/testing-csharp.yml index 5e3e736..da40e18 100644 --- a/.github/workflows/testing-csharp.yml +++ b/.github/workflows/testing-csharp.yml @@ -42,6 +42,9 @@ jobs: with: submodules: recursive + - name: Read pinned versions + uses: ./.github/actions/load-versions + - name: Install Go uses: actions/setup-go@v6 with: @@ -52,7 +55,7 @@ jobs: - name: Install .NET SDK uses: actions/setup-dotnet@v4 with: - dotnet-version: "8.0.x" + dotnet-version: "${{ env.DOTNET_VERSION }}.x" - name: Install Protoc uses: arduino/setup-protoc@v3 @@ -63,7 +66,7 @@ jobs: - name: Install Buf uses: bufbuild/buf-action@v1 with: - version: 1.67.0 + version: ${{ env.BUF_VERSION }} setup_only: true github_token: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/testing-go.yml b/.github/workflows/testing-go.yml index 2c064e3..258efa8 100644 --- a/.github/workflows/testing-go.yml +++ b/.github/workflows/testing-go.yml @@ -29,6 +29,9 @@ jobs: with: submodules: recursive + - name: Read pinned versions + uses: ./.github/actions/load-versions + - name: Install Go uses: actions/setup-go@v6 with: @@ -39,7 +42,7 @@ jobs: - name: Install Buf uses: bufbuild/buf-action@v1 with: - version: 1.67.0 + version: ${{ env.BUF_VERSION }} setup_only: true github_token: ${{ secrets.GITHUB_TOKEN }} diff --git a/CLAUDE.md b/CLAUDE.md index a6ac1b5..3457e10 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -24,7 +24,7 @@ All build and test work happens **per language** inside `test/-tableau-loa To rebuild against the legacy v3 protobuf line: `LOADER_PROTOBUF_VERSION=3.21.12 code .` then **Reopen in Container**. The Dockerfile ARG `PROTOBUF_VERSION` is wired to that host env var via `devcontainer.json` build args; vcpkg manifest mode pins the override and the post-install assertion fails the build if anything resolves to the wrong version. See `.devcontainer/README.md` for the longer how-to and host-OS caveats (notably: Windows users should check the workspace out under WSL2, not `/mnt/c/`, for usable bind-mount perf). -The container is **not** used in CI. CI workflows still run `lukka/run-vcpkg` directly (faster cached vcpkg installs); the container exists for local-dev parity only. `VCPKG_BASELINE_COMMIT` in `.devcontainer/Dockerfile` is in lock-step with `prepare.bat`'s `VCPKG_BASELINE_COMMIT` and `testing-cpp.yml`'s `VCPKG_COMMIT` — bump all three together. +CI's primary test path does **not** build the devcontainer images. The `testing-{cpp,go,csharp}.yml` workflows run `lukka/run-vcpkg` directly on GitHub-hosted runners (faster cached vcpkg installs). Two **smoke** workflows additionally build the devcontainer images themselves on devcontainer-touching PRs: `devcontainer-linux-smoke.yml` (amd64 + arm64, runs `buf generate` + `go vet` inside the image) and `devcontainer-windows-smoke.yml` (windows-2022, Windows-containers mode). They're path-gated to `.devcontainer/{linux,windows,shared}/**` so they don't burn CI minutes on unrelated PRs, but a `versions.env` typo or a Dockerfile regression won't ship. All toolchain versions (Go, buf, protobuf, vcpkg baseline commit, .NET, Node, CMake) are pinned in **`.devcontainer/shared/versions.env`** — the single source of truth consumed by `.devcontainer/linux/Dockerfile`, `.devcontainer/windows/Dockerfile`, `prepare.bat`, and every `.github/workflows/testing-*.yml` (via the **`./.github/actions/load-versions`** composite action that exports each `KEY=VALUE` to `$GITHUB_ENV`). Bump versions there; everything downstream picks them up automatically. The `.devcontainer/` directory has three subfolders: `linux/` (multi-arch, recommended), `windows/` (windows/amd64-only Windows container for native-MSVC dev), and `macos/` (docs only — Apple's licence forbids macOS containers; macOS users run the Linux container). ### Plugin development (Go module at repo root) diff --git a/README.md b/README.md index fc6a3c3..754b40d 100644 --- a/README.md +++ b/README.md @@ -27,20 +27,36 @@ The official config loader for [Tableau](https://github.com/tableauio/tableau). ### Recommended: Dev Container (any host OS) The fastest way to get a reproducible build environment is to open the -repo in VS Code and choose **Reopen in Container**. The devcontainer -under [`.devcontainer/`](./.devcontainer/) has everything pinned to the -exact versions CI uses (Go 1.24, buf 1.67.0, protobuf 6.33.4 via vcpkg, -.NET 8.0, Node 20). First container build is one-time ~25 minutes (vcpkg -compiles protobuf from source); subsequent reopens are near-instant. - -After the container starts you can skip the per-language setup below and -jump straight to **[C++](#c)** / **[Go](#go)** / **[C#](#c-1)** / +repo in VS Code and choose **Reopen in Container**. The +[`.devcontainer/`](./.devcontainer/) directory ships **two** container +variants plus a macOS docs path: + +- [`.devcontainer/linux/`](./.devcontainer/linux/) — recommended for + every host (Linux, macOS Intel + Apple Silicon, Windows + WSL2). + Multi-arch (linux/amd64 + linux/arm64). +- [`.devcontainer/windows/`](./.devcontainer/windows/) — + Windows-host-only (windows/amd64), for users who specifically need a + native MSVC environment inside the container without WSL2. +- [`.devcontainer/macos/`](./.devcontainer/macos/) — documentation only. + Apple's licence forbids macOS containers; macOS users use the Linux + variant. + +When you run **Dev Containers: Reopen in Container** and both `linux/` +and `windows/` are present, VS Code shows a picker — choose whichever +matches your host. All version pins (Go, buf, protobuf, vcpkg baseline, +.NET, Node, CMake) live in +[`.devcontainer/shared/versions.env`](./.devcontainer/shared/versions.env), +the single source of truth shared with `prepare.bat` and CI. First +container build is one-time ~25 min (Linux) / ~45 min (Windows); vcpkg +compiles protobuf from source. Subsequent reopens are near-instant. + +After the container starts you can skip the per-language setup below +and jump straight to **[C++](#c)** / **[Go](#go)** / **[C#](#c-1)** / **[TypeScript](#typescript)**. Requirements: Docker Desktop (Windows + macOS) or Docker Engine (Linux), -and the VS Code "Dev Containers" extension. See -[`.devcontainer/README.md`](./.devcontainer/README.md) for the longer -how-to. +and the VS Code "Dev Containers" extension. See the per-variant READMEs +linked above for the longer how-to. ### Install protobuf diff --git a/prepare.bat b/prepare.bat index def5017..4e460b5 100644 --- a/prepare.bat +++ b/prepare.bat @@ -4,7 +4,8 @@ setlocal enabledelayedexpansion REM =========================================================================== REM prepare.bat — bootstrap a Windows build environment for the C++ loader. REM -REM Installs (only if missing): Chocolatey, Ninja, CMake 3.31.8, MSVC Build +REM Installs (only if missing): Chocolatey, Ninja, CMake (version pinned in +REM .devcontainer/shared/versions.env), MSVC Build REM Tools (Visual Studio 2022 Build Tools), buf CLI, and vcpkg. REM REM Then installs `protobuf` (and friends) into vcpkg using the static-CRT @@ -50,6 +51,32 @@ if "%SIMULATE_CLEAN%"=="1" echo [DRY-RUN] Simulating a clean machine (all tools echo [INFO] Preparing build environment... +REM ----------------------------------------------------------------------- +REM Load pinned tool versions from .devcontainer/shared/versions.env. +REM +REM Single source of truth shared with the Linux/Windows devcontainers +REM and with the .github/workflows/*.yml CI workflows. Format is one +REM KEY=VALUE per line, no quotes, no $VAR expansion. See +REM .devcontainer/shared/README.md for the full format spec. +REM ----------------------------------------------------------------------- +set "VERSIONS_FILE=%~dp0.devcontainer\shared\versions.env" +if not exist "%VERSIONS_FILE%" ( + echo [ERROR] Missing %VERSIONS_FILE%; cannot resolve pinned tool versions. + exit /b 1 +) +for /f "usebackq tokens=1,2 delims==" %%a in ("%VERSIONS_FILE%") do ( + set "_KEY=%%a" + set "_VAL=%%b" + REM Skip blank lines and comment lines (start with #). + if defined _KEY if not "!_KEY:~0,1!"=="#" ( + set "!_KEY!=!_VAL!" + ) +) +set "_KEY=" +set "_VAL=" +echo [INFO] Pinned versions: cmake=%CMAKE_VERSION% buf=%BUF_VERSION% vcpkg-baseline=%VCPKG_BASELINE_COMMIT% + + REM ----------------------------------------------------------------------- REM Step 0: Ensure Chocolatey is installed REM ----------------------------------------------------------------------- @@ -151,7 +178,7 @@ if "%NINJA_FOUND%"=="0" ( ) REM ----------------------------------------------------------------------- -REM Step 2: Ensure CMake 3.31.8 is installed +REM Step 2: Ensure CMake (CMAKE_VERSION from versions.env) is installed REM Try Chocolatey first; fall back to direct MSI download. REM ----------------------------------------------------------------------- set "CMAKE_FOUND=0" @@ -160,15 +187,15 @@ if "%SIMULATE_CLEAN%"=="0" ( if not errorlevel 1 set "CMAKE_FOUND=1" ) if "%CMAKE_FOUND%"=="0" ( - echo [INFO] cmake.exe not found. Installing CMake 3.31.8... + echo [INFO] cmake.exe not found. Installing CMake %CMAKE_VERSION%... if "%DRY_RUN%"=="0" ( set "CMAKE_INSTALLED=0" REM --- Attempt 1: Chocolatey --- - choco install cmake --version=3.31.8 --installargs "'ADD_CMAKE_TO_PATH=System'" -y --no-progress >nul 2>&1 && set "CMAKE_INSTALLED=1" + choco install cmake --version=%CMAKE_VERSION% --installargs "'ADD_CMAKE_TO_PATH=System'" -y --no-progress >nul 2>&1 && set "CMAKE_INSTALLED=1" if "!CMAKE_INSTALLED!"=="0" ( echo [WARN] choco install cmake failed. Falling back to direct MSI download... - set "CMAKE_MSI=%TEMP%\cmake-3.31.8-windows-x86_64.msi" - powershell -NoProfile -Command "(New-Object Net.WebClient).DownloadFile('https://github.com/Kitware/CMake/releases/download/v3.31.8/cmake-3.31.8-windows-x86_64.msi','!CMAKE_MSI!')" + set "CMAKE_MSI=%TEMP%\cmake-%CMAKE_VERSION%-windows-x86_64.msi" + powershell -NoProfile -Command "(New-Object Net.WebClient).DownloadFile('https://github.com/Kitware/CMake/releases/download/v%CMAKE_VERSION%/cmake-%CMAKE_VERSION%-windows-x86_64.msi','!CMAKE_MSI!')" if not exist "!CMAKE_MSI!" ( echo [ERROR] Failed to download CMake MSI. exit /b 1 @@ -181,7 +208,7 @@ if "%CMAKE_FOUND%"=="0" ( del /q "!CMAKE_MSI!" 2>nul ) ) else ( - echo [DRY-RUN] Would run: choco install cmake --version=3.31.8 ... (or fallback to MSI download) + echo [DRY-RUN] Would run: choco install cmake --version=%CMAKE_VERSION% ... (or fallback to MSI download) ) REM Add cmake to current session PATH set "CMAKE_PATH=C:\Program Files\CMake\bin" @@ -278,8 +305,8 @@ REM The CI workflow uses bufbuild/buf-action@v1 (also pinned to REM BUF_VERSION below) to do the same thing. REM buf is a single self-contained .exe; install it under REM %LOCALAPPDATA%\buf\bin\buf.exe to avoid requiring admin rights. +REM BUF_VERSION is sourced from .devcontainer/shared/versions.env. REM ----------------------------------------------------------------------- -set "BUF_VERSION=1.67.0" set "BUF_FOUND=0" if "%SIMULATE_CLEAN%"=="0" ( where buf.exe >nul 2>&1 @@ -365,9 +392,10 @@ if errorlevel 1 ( ) REM Pin both the vcpkg checkout and the manifest's builtin-baseline to the -REM same commit testing-cpp.yml uses. Bumping vcpkg? Bump both this value -REM and VCPKG_COMMIT in .github/workflows/testing-cpp.yml in lockstep. -set "VCPKG_BASELINE_COMMIT=dc8d75cfc3281b8e2a4ed8ee4163c891190df932" +REM same commit testing-cpp.yml uses. VCPKG_BASELINE_COMMIT is sourced from +REM .devcontainer/shared/versions.env (the single source of truth for the +REM Linux + Windows devcontainers, prepare.bat, and CI). To bump vcpkg, +REM edit that file. set "VCPKG_TRIPLET=x64-windows-static" set "VCPKG_EXE=" From c110c0599ff826c9b163bbeb4da8af45bb45a676 Mon Sep 17 00:00:00 2001 From: wenchy Date: Mon, 1 Jun 2026 19:18:49 +0800 Subject: [PATCH 31/66] fix(ci): drop ${{ env.KEY }} from load-versions action description GitHub Actions evaluates ${{ ... }} expressions in the description field at parse time, and `env` is not in the allowed context for action manifests. Rephrase to plain prose so the manifest loads. Reported by the workflow run as: Unrecognized named-value: 'env'. Located at position 1 within expression: env.KEY Co-Authored-By: Claude Opus 4.6 --- .github/actions/load-versions/action.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/actions/load-versions/action.yml b/.github/actions/load-versions/action.yml index 8d7f0bd..45fc8ed 100644 --- a/.github/actions/load-versions/action.yml +++ b/.github/actions/load-versions/action.yml @@ -2,7 +2,8 @@ name: Load pinned versions description: > Reads .devcontainer/shared/versions.env (the single source of truth shared with the devcontainers and prepare.bat) and exports each KEY=VALUE pair to - $GITHUB_ENV so subsequent steps can reference them as ${{ env.KEY }}. + GITHUB_ENV so subsequent steps can reference them as env-context values + (e.g. env.BUF_VERSION). Format rules of versions.env (kept simple so every consumer can parse it with a builtin): From 09dc729d8614ab70513ed37ae9af66f3a8d05417 Mon Sep 17 00:00:00 2001 From: wenchy Date: Mon, 1 Jun 2026 20:47:05 +0800 Subject: [PATCH 32/66] fix(ci): switch Docker daemon via dockerd --register-service, not DockerCli Current windows-2022 GitHub runners ship Moby Engine instead of Docker Desktop, so `$env:ProgramFiles\Docker\Docker\DockerCli.exe -SwitchDaemon` no longer resolves. Re-register the `docker` Windows service directly via dockerd.exe; probes both legacy and current install paths and emits diagnostics if neither is present. Reported by the previous workflow run as: The term 'C:\Program Files\Docker\Docker\DockerCli.exe' is not recognized as a name of a cmdlet, function, script file, or executable program. Co-Authored-By: Claude Opus 4.6 --- .../workflows/devcontainer-windows-smoke.yml | 54 +++++++++++++++++-- 1 file changed, 50 insertions(+), 4 deletions(-) diff --git a/.github/workflows/devcontainer-windows-smoke.yml b/.github/workflows/devcontainer-windows-smoke.yml index 4810c11..0413095 100644 --- a/.github/workflows/devcontainer-windows-smoke.yml +++ b/.github/workflows/devcontainer-windows-smoke.yml @@ -36,10 +36,56 @@ jobs: - name: Switch Docker to Windows containers shell: pwsh run: | - # GitHub-hosted windows-2022 runners ship Docker in Linux-containers - # mode by default. Flip to Windows containers; the runner has - # DockerCli.exe in PATH for exactly this reason. - & "$env:ProgramFiles\Docker\Docker\DockerCli.exe" -SwitchDaemon + # GitHub-hosted windows-2022 runners ship Moby Engine (not Docker + # Desktop) and default to Linux-containers mode. The legacy + # `DockerCli.exe -SwitchDaemon` helper is not present on current + # runner images. Flip the daemon by re-registering the `docker` + # Windows service without the `dockerd-default` (Linux) backend. + $ErrorActionPreference = 'Stop' + + $current = (docker info --format '{{.OSType}}' 2>$null) + Write-Host "Current OSType before switch: $current" + if ($current -eq 'windows') { + Write-Host 'Already in Windows-containers mode; nothing to do.' + docker info | Select-String -Pattern 'OSType|Server Version|Operating System' + exit 0 + } + + # Locate dockerd.exe. Moby Engine on the runner lives under + # %ProgramFiles%\docker\; Docker Desktop (older runners) used + # %ProgramFiles%\Docker\Docker\. Probe both. + $candidates = @( + "$env:ProgramFiles\docker\dockerd.exe", + "$env:ProgramFiles\Docker\Docker\resources\dockerd.exe", + "$env:ProgramFiles\Docker\dockerd.exe" + ) + $dockerd = $candidates | Where-Object { Test-Path $_ } | Select-Object -First 1 + if (-not $dockerd) { + Write-Error "dockerd.exe not found in any of: $($candidates -join '; ')" + Write-Host '--- diagnostic: Get-Service docker ---' + Get-Service docker -ErrorAction SilentlyContinue | Format-List * + Write-Host '--- diagnostic: ProgramFiles tree ---' + Get-ChildItem "$env:ProgramFiles\docker", "$env:ProgramFiles\Docker" -ErrorAction SilentlyContinue + exit 1 + } + Write-Host "Using dockerd at: $dockerd" + + # Stop, re-register without `-G docker` Linux args, restart. + Stop-Service docker + & $dockerd --unregister-service + if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } + & $dockerd --register-service --service-name docker + if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } + Start-Service docker + + # Verify the switch took effect. + $after = (docker info --format '{{.OSType}}' 2>$null) + Write-Host "Current OSType after switch: $after" + if ($after -ne 'windows') { + Write-Error "Docker daemon did not switch to Windows containers (OSType=$after)." + docker info + exit 1 + } docker info | Select-String -Pattern 'OSType|Server Version|Operating System' - name: Build devcontainer image From 7f5ba11af67aaa01dbe2411a9d601511a55165b8 Mon Sep 17 00:00:00 2001 From: wenchy Date: Mon, 1 Jun 2026 21:14:26 +0800 Subject: [PATCH 33/66] fix(devcontainer): switch Windows base to mcr.microsoft.com/visualstudio/buildtools MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The plain windows/servercore:ltsc2022 base image cannot host the modern VS 2022 Build Tools installer. vs_BuildTools.exe expects Windows servicing components that servercore strips, so the chocolatey visualstudio2022buildtools install fails with installer exit code -2146232797 (COR_E_NOTSUPPORTED) deep into the install. Microsoft publishes mcr.microsoft.com/visualstudio/buildtools:ltsc2022 specifically to solve this — same servercore base but with VS 2022 Build Tools (VC++ workload) pre-installed, exposing C:\BuildTools\Common7\Tools\VsDevCmd.bat at a fixed path. Drops the ~6 GB chocolatey Build Tools install layer (now in the base image) and lets vcpkg-install.ps1 skip the vswhere probe and go directly to C:\BuildTools\... with vswhere kept as a defensive fallback for future base-image layout changes. Discovered while testing the Windows devcontainer locally: ERROR: The installation of visualstudio2022buildtools failed (installer exit code: -2146232797). Co-Authored-By: Claude Opus 4.6 --- .devcontainer/windows/Dockerfile | 44 +++++++++++++------------ .devcontainer/windows/README.md | 32 +++++++++++------- .devcontainer/windows/vcpkg-install.ps1 | 29 ++++++++++------ 3 files changed, 62 insertions(+), 43 deletions(-) diff --git a/.devcontainer/windows/Dockerfile b/.devcontainer/windows/Dockerfile index fee3fc2..d77e316 100644 --- a/.devcontainer/windows/Dockerfile +++ b/.devcontainer/windows/Dockerfile @@ -14,13 +14,20 @@ # Build context is .devcontainer/ (the parent of this directory) so that # `COPY shared\...` resolves. devcontainer.json sets `"context": ".."`. # -# Why servercore (not nanoserver): -# vcpkg compiles protobuf from source via MSVC — that needs a real -# Win32 environment (PowerShell, Win32 APIs, CRT) that nanoserver does -# not provide. ltsc2022 matches GitHub Actions' windows-latest runner -# family, so the post-install assertion's resolved versions match CI. +# Why mcr.microsoft.com/visualstudio/buildtools (not windows/servercore): +# +# The modern VS 2022 Build Tools installer (`vs_BuildTools.exe`) +# requires Windows servicing components that `windows/servercore` +# strips out. Attempting to install via Chocolatey or directly fails +# with installer exit code -2146232797 deep into the install. Microsoft +# publishes `mcr.microsoft.com/visualstudio/buildtools:ltsc2022` +# specifically to solve this — the image is based on servercore but +# ships the VS 2022 Build Tools (VC++ workload) pre-installed, with +# `C:\BuildTools\Common7\Tools\VsDevCmd.bat` ready to source. +# +# ltsc2022 matches GitHub Actions' windows-latest runner family. -FROM mcr.microsoft.com/windows/servercore:ltsc2022 +FROM mcr.microsoft.com/visualstudio/buildtools:ltsc2022 SHELL ["powershell", "-NoProfile", "-NoLogo", "-ExecutionPolicy", "Bypass", "-Command", "$ErrorActionPreference='Stop'; $ProgressPreference='SilentlyContinue';"] @@ -45,8 +52,7 @@ SHELL ["powershell", "-NoProfile", "-NoLogo", "-ExecutionPolicy", "Bypass", "-Co # --------------------------------------------------------------------------- # Chocolatey — bootstrap the package manager that installs Git, CMake, -# .NET, Node, and the MSVC build tools. Same channel `prepare.bat` uses -# on a bare-metal Windows host. +# Ninja, .NET, and Node. (MSVC Build Tools come from the base image.) # --------------------------------------------------------------------------- RUN [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12; ` Invoke-Expression ((New-Object System.Net.WebClient).DownloadString('https://community.chocolatey.org/install.ps1')) @@ -66,14 +72,6 @@ RUN choco install -y --no-progress git ninja; ` RUN $env:PATH = [System.Environment]::GetEnvironmentVariable('PATH','Machine') + ';' + [System.Environment]::GetEnvironmentVariable('PATH','User'); ` [System.Environment]::SetEnvironmentVariable('PATH', $env:PATH, 'Process') -# --------------------------------------------------------------------------- -# Visual Studio 2022 Build Tools (VC++ workload). This is the heavy layer -# (~6 GB on disk). Matches the `prepare.bat` Step 3 install pattern. -# `--passive --wait --norestart` so Docker doesn't see a hung installer. -# --------------------------------------------------------------------------- -RUN choco install -y --no-progress visualstudio2022buildtools ` - --package-parameters '\"--add Microsoft.VisualStudio.Workload.VCTools --includeRecommended --passive --wait --norestart --locale en-US\"' - # --------------------------------------------------------------------------- # Go SDK — official MSI to C:\Go. Pinned to versions.env's GO_VERSION. # --------------------------------------------------------------------------- @@ -120,8 +118,9 @@ RUN git clone https://github.com/microsoft/vcpkg.git $env:VCPKG_ROOT; ` # Render the manifest (PROTOBUF_VERSION build-arg overrides versions.env if set), # then install via vcpkg, then assert the resolved version matches what we asked for. -# vcpkg compiles protobuf from source under MSVC, which needs the VS Build Tools -# environment active — load it from vsdevcmd.bat into the build session. +# vcpkg compiles protobuf from source under MSVC; vcpkg-install.ps1 activates +# the VS Build Tools env via vsdevcmd.bat (now at C:\BuildTools\... since the +# base image is mcr.microsoft.com/visualstudio/buildtools). COPY windows\vcpkg-install.ps1 C:\loader\vcpkg-install.ps1 RUN $effective = if ($env:PROTOBUF_VERSION) { $env:PROTOBUF_VERSION } else { ` ([System.Environment]::GetEnvironmentVariable('PROTOBUF_VERSION','Machine')) }; ` @@ -137,7 +136,7 @@ ENV CMAKE_PREFIX_PATH=C:\vcpkg-manifest\vcpkg_installed\x64-windows-static # --------------------------------------------------------------------------- # .NET SDK + Node.js LTS. Versions from versions.env. Chocolatey is the -# install channel for both — same one prepare.bat uses for VS Build Tools. +# install channel for both. # --------------------------------------------------------------------------- RUN $dotnetPkg = \"dotnet-${env:DOTNET_VERSION}-sdk\"; ` choco install -y --no-progress $dotnetPkg; ` @@ -154,7 +153,10 @@ COPY shared\postcreate-banner.ps1 C:\loader\postcreate-banner.ps1 WORKDIR C:\workspaces\loader # Drop back to a normal shell for the running container — the build-time -# version-import shell is no longer needed. +# version-import shell is no longer needed. Override the base image's +# ENTRYPOINT (which auto-runs vsdevcmd) so a plain `docker run` lands +# the user in a clean PowerShell — vsdevcmd is invoked on demand by +# vcpkg-install.ps1 and any user shell that needs cl.exe. SHELL ["powershell", "-NoProfile", "-NoLogo", "-Command"] - +ENTRYPOINT [] CMD ["powershell.exe", "-NoLogo"] diff --git a/.devcontainer/windows/README.md b/.devcontainer/windows/README.md index ed5024d..9fa683d 100644 --- a/.devcontainer/windows/README.md +++ b/.devcontainer/windows/README.md @@ -52,29 +52,37 @@ variant, same default sourced from ## Architecture -Single Dockerfile based on `mcr.microsoft.com/windows/servercore:ltsc2022`: +Single Dockerfile based on `mcr.microsoft.com/visualstudio/buildtools:ltsc2022` +— Microsoft's official Windows-container image with VS 2022 Build Tools +(VC++ workload) pre-installed. The plain `windows/servercore:ltsc2022` +strips Windows servicing components that `vs_BuildTools.exe` requires; +that's why we use the buildtools image instead. + +Build layers: 1. Imports `versions.env` into machine-wide env so each `RUN` sees `$env:KEY`. 2. Chocolatey bootstrap. 3. Git, Ninja, CMake (CMake version pinned by `versions.env`). -4. Visual Studio 2022 Build Tools — VC++ workload. -5. Go SDK — official MSI, version from `versions.env`. -6. buf CLI — single-binary release, version from `versions.env`. -7. vcpkg (pinned to `VCPKG_BASELINE_COMMIT`) + protobuf via manifest mode +4. Go SDK — official MSI, version from `versions.env`. +5. buf CLI — single-binary release, version from `versions.env`. +6. vcpkg (pinned to `VCPKG_BASELINE_COMMIT`) + protobuf via manifest mode with the `x64-windows-static` triplet — same triplet `prepare.bat` uses on bare metal, so a developer moving between native and container builds avoids the LNK2038 `_ITERATOR_DEBUG_LEVEL` CRT-mismatch - trap. Post-install assertion catches version drift. -8. .NET SDK + Node.js LTS via Chocolatey. -9. `ENV CMAKE_PREFIX_PATH=C:\vcpkg-manifest\vcpkg_installed\x64-windows-static` + trap. `vcpkg-install.ps1` activates the Build Tools env from + `C:\BuildTools\Common7\Tools\VsDevCmd.bat` before invoking vcpkg. + Post-install assertion catches version drift. +7. .NET SDK + Node.js LTS via Chocolatey. +8. `ENV CMAKE_PREFIX_PATH=C:\vcpkg-manifest\vcpkg_installed\x64-windows-static` so `find_package(Protobuf CONFIG)` resolves automatically. ## Why is the image so large? -`visualstudio2022buildtools` with the VC++ workload is ~6 GB on disk; the -Windows base image is another ~5 GB. There is no nanoserver path because -vcpkg compiles protobuf from source under MSVC and that needs the full -Win32 environment. +`mcr.microsoft.com/visualstudio/buildtools:ltsc2022` is ~14 GB compressed +(VS 2022 Build Tools + servercore base). The ~6 GB of Build Tools is +unavoidable — vcpkg compiles protobuf from source under MSVC and that +needs the full VC++ toolchain. There is no nanoserver path because +nanoserver lacks the Win32 environment vcpkg's compile depends on. ## Why isolation=hyperv? diff --git a/.devcontainer/windows/vcpkg-install.ps1 b/.devcontainer/windows/vcpkg-install.ps1 index cb51110..22f2028 100644 --- a/.devcontainer/windows/vcpkg-install.ps1 +++ b/.devcontainer/windows/vcpkg-install.ps1 @@ -43,17 +43,26 @@ $manifest = @{ Set-Content -Path (Join-Path $manifestDir 'vcpkg.json') -Value $manifest -Encoding ASCII # 2. Activate the VS 2022 Build Tools environment for this PS session. -# Mirrors what prepare.bat's `call "!VCVARSALL!" x64` does on bare metal. -$vswhere = 'C:\Program Files (x86)\Microsoft Visual Studio\Installer\vswhere.exe' -if (-not (Test-Path $vswhere)) { - throw "vswhere.exe not found at $vswhere; VS Build Tools install layer failed." +# On the mcr.microsoft.com/visualstudio/buildtools base image, the +# Build Tools live at a fixed `C:\BuildTools\` prefix, so we go straight +# there instead of probing via vswhere. Fall back to vswhere if a future +# base image changes this layout. +$primaryVsdevcmd = 'C:\BuildTools\Common7\Tools\VsDevCmd.bat' +if (Test-Path $primaryVsdevcmd) { + $vsdevcmd = $primaryVsdevcmd +} else { + $vswhere = 'C:\Program Files (x86)\Microsoft Visual Studio\Installer\vswhere.exe' + if (-not (Test-Path $vswhere)) { + throw "Neither $primaryVsdevcmd nor $vswhere found; VS Build Tools layer is missing." + } + $installPath = & $vswhere -latest -products * ` + -requires Microsoft.VisualStudio.Component.VC.Tools.x86.x64 ` + -property installationPath + if (-not $installPath) { throw 'No VS install with C++ tools detected.' } + $vsdevcmd = Join-Path $installPath 'Common7\Tools\VsDevCmd.bat' + if (-not (Test-Path $vsdevcmd)) { throw "VsDevCmd.bat not found: $vsdevcmd" } } -$installPath = & $vswhere -latest -products * ` - -requires Microsoft.VisualStudio.Component.VC.Tools.x86.x64 ` - -property installationPath -if (-not $installPath) { throw 'No VS install with C++ tools detected.' } -$vsdevcmd = Join-Path $installPath 'Common7\Tools\VsDevCmd.bat' -if (-not (Test-Path $vsdevcmd)) { throw "VsDevCmd.bat not found: $vsdevcmd" } +Write-Host "Using VsDevCmd at: $vsdevcmd" # Capture the environment that vsdevcmd produces. $envDump = & cmd.exe /s /c "`"$vsdevcmd`" -arch=amd64 -host_arch=amd64 && set" From 4e35924d337276b64c3cfda390a3127346a35ce5 Mon Sep 17 00:00:00 2001 From: wenchy Date: Mon, 1 Jun 2026 21:19:22 +0800 Subject: [PATCH 34/66] fix(devcontainer): use dotnet-framework base for Windows container MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit mcr.microsoft.com/visualstudio/buildtools doesn't exist as a published image (I invented the path from memory). Use Microsoft's actually documented pattern: base on dotnet/framework/runtime:4.8-windowsservercore-ltsc2022 and download vs_BuildTools.exe from aka.ms/vs/17/release/vs_buildtools.exe. Microsoft's official VS Build Tools container guidance (https://learn.microsoft.com/en-us/visualstudio/install/build-tools-container) explicitly recommends the dotnet-framework base over plain windows/servercore, because the modern vs_BuildTools.exe installer needs .NET Framework runtime support that servercore strips. Side benefit: dotnet/framework/runtime:4.8-windowsservercore-ltsc2022 is pre-cached on GitHub's windows-2022 runners (per the runner image manifest), so the CI smoke job's image pull is free. Also pass --memory 2GB to the smoke job's docker build — Microsoft documents this as required for vs_BuildTools (default 1 GB silently fails partway through). Co-Authored-By: Claude Opus 4.6 --- .devcontainer/windows/Dockerfile | 53 +++++++++++++++--- .devcontainer/windows/README.md | 55 +++++++++++++------ .../workflows/devcontainer-windows-smoke.yml | 5 +- 3 files changed, 85 insertions(+), 28 deletions(-) diff --git a/.devcontainer/windows/Dockerfile b/.devcontainer/windows/Dockerfile index d77e316..6c827f7 100644 --- a/.devcontainer/windows/Dockerfile +++ b/.devcontainer/windows/Dockerfile @@ -14,21 +14,56 @@ # Build context is .devcontainer/ (the parent of this directory) so that # `COPY shared\...` resolves. devcontainer.json sets `"context": ".."`. # -# Why mcr.microsoft.com/visualstudio/buildtools (not windows/servercore): +# Why mcr.microsoft.com/dotnet/framework/runtime (not windows/servercore): # # The modern VS 2022 Build Tools installer (`vs_BuildTools.exe`) -# requires Windows servicing components that `windows/servercore` -# strips out. Attempting to install via Chocolatey or directly fails -# with installer exit code -2146232797 deep into the install. Microsoft -# publishes `mcr.microsoft.com/visualstudio/buildtools:ltsc2022` -# specifically to solve this — the image is based on servercore but -# ships the VS 2022 Build Tools (VC++ workload) pre-installed, with -# `C:\BuildTools\Common7\Tools\VsDevCmd.bat` ready to source. +# requires .NET Framework runtime support that `windows/servercore` +# strips out. Microsoft's own guidance — see +# https://learn.microsoft.com/en-us/visualstudio/install/build-tools-container +# — explicitly says: "If you base your image directly on +# microsoft/windowsservercore, the .NET Framework might not install +# properly … Instead, base your image on microsoft/dotnet-framework:4.7.1 +# or later." We use 4.8-windowsservercore-ltsc2022, which is also one +# of the images pre-cached on GitHub's windows-2022 runners. # # ltsc2022 matches GitHub Actions' windows-latest runner family. +# +# Build memory: pass `docker build -m 2GB` (or higher). vs_BuildTools +# exhausts the default 1 GB silently. CI's windows-2022 runners have +# plenty of RAM but you must set the flag. + +FROM mcr.microsoft.com/dotnet/framework/runtime:4.8-windowsservercore-ltsc2022 -FROM mcr.microsoft.com/visualstudio/buildtools:ltsc2022 +# Restore cmd as the default shell so the vs_BuildTools install RUN below +# matches Microsoft's documented pattern. We switch back to PowerShell +# right after. +SHELL ["cmd", "/S", "/C"] +# --------------------------------------------------------------------------- +# Visual Studio 2022 Build Tools — VC++ workload only. +# +# Mirrors Microsoft's documented Dockerfile from +# https://learn.microsoft.com/en-us/visualstudio/install/build-tools-container +# but with `--add Microsoft.VisualStudio.Workload.VCTools` (we don't need +# the AzureBuildTools workload) and the same `--remove` flags for old +# Windows SDKs that can't install in a container. +# +# Exit code 3010 is "success, reboot required" — fine inside a container. +# --------------------------------------------------------------------------- +RUN curl -SL --output vs_buildtools.exe https://aka.ms/vs/17/release/vs_buildtools.exe ` + && (start /w vs_buildtools.exe --quiet --wait --norestart --nocache ` + --installPath "C:\BuildTools" ` + --add Microsoft.VisualStudio.Workload.VCTools ` + --add Microsoft.VisualStudio.Component.VC.Tools.x86.x64 ` + --add Microsoft.VisualStudio.Component.Windows11SDK.22621 ` + --remove Microsoft.VisualStudio.Component.Windows10SDK.10240 ` + --remove Microsoft.VisualStudio.Component.Windows10SDK.10586 ` + --remove Microsoft.VisualStudio.Component.Windows10SDK.14393 ` + --remove Microsoft.VisualStudio.Component.Windows81SDK ` + || IF "%ERRORLEVEL%"=="3010" EXIT 0) ` + && del /q vs_buildtools.exe + +# Switch to PowerShell for the rest of the Dockerfile. SHELL ["powershell", "-NoProfile", "-NoLogo", "-ExecutionPolicy", "Bypass", "-Command", "$ErrorActionPreference='Stop'; $ProgressPreference='SilentlyContinue';"] # --------------------------------------------------------------------------- diff --git a/.devcontainer/windows/README.md b/.devcontainer/windows/README.md index 9fa683d..f553413 100644 --- a/.devcontainer/windows/README.md +++ b/.devcontainer/windows/README.md @@ -52,37 +52,56 @@ variant, same default sourced from ## Architecture -Single Dockerfile based on `mcr.microsoft.com/visualstudio/buildtools:ltsc2022` -— Microsoft's official Windows-container image with VS 2022 Build Tools -(VC++ workload) pre-installed. The plain `windows/servercore:ltsc2022` -strips Windows servicing components that `vs_BuildTools.exe` requires; -that's why we use the buildtools image instead. +Single Dockerfile based on +`mcr.microsoft.com/dotnet/framework/runtime:4.8-windowsservercore-ltsc2022`. +Microsoft's [official guidance for VS Build Tools containers](https://learn.microsoft.com/en-us/visualstudio/install/build-tools-container) +explicitly recommends the dotnet-framework base over plain +`windows/servercore`, because the modern `vs_BuildTools.exe` installer +requires the .NET Framework runtime that servercore strips. The +dotnet-framework image is also pre-cached on GitHub's `windows-2022` +runners. Build layers: -1. Imports `versions.env` into machine-wide env so each `RUN` sees `$env:KEY`. -2. Chocolatey bootstrap. -3. Git, Ninja, CMake (CMake version pinned by `versions.env`). -4. Go SDK — official MSI, version from `versions.env`. -5. buf CLI — single-binary release, version from `versions.env`. -6. vcpkg (pinned to `VCPKG_BASELINE_COMMIT`) + protobuf via manifest mode +1. Visual Studio 2022 Build Tools — VC++ workload, installed from the + official `aka.ms/vs/17/release/vs_buildtools.exe` bootstrapper into + `C:\BuildTools\`. Mirrors Microsoft's documented pattern. +2. Imports `versions.env` into machine-wide env so each `RUN` sees `$env:KEY`. +3. Chocolatey bootstrap. +4. Git, Ninja, CMake (CMake version pinned by `versions.env`). +5. Go SDK — official MSI, version from `versions.env`. +6. buf CLI — single-binary release, version from `versions.env`. +7. vcpkg (pinned to `VCPKG_BASELINE_COMMIT`) + protobuf via manifest mode with the `x64-windows-static` triplet — same triplet `prepare.bat` uses on bare metal, so a developer moving between native and container builds avoids the LNK2038 `_ITERATOR_DEBUG_LEVEL` CRT-mismatch trap. `vcpkg-install.ps1` activates the Build Tools env from `C:\BuildTools\Common7\Tools\VsDevCmd.bat` before invoking vcpkg. Post-install assertion catches version drift. -7. .NET SDK + Node.js LTS via Chocolatey. -8. `ENV CMAKE_PREFIX_PATH=C:\vcpkg-manifest\vcpkg_installed\x64-windows-static` +8. .NET SDK + Node.js LTS via Chocolatey. +9. `ENV CMAKE_PREFIX_PATH=C:\vcpkg-manifest\vcpkg_installed\x64-windows-static` so `find_package(Protobuf CONFIG)` resolves automatically. +## Build memory + +The VS Build Tools install needs **at least 2 GB of memory** during the +`docker build` — Microsoft documents this. Pass `-m 2GB` (or higher): + +```sh +docker build -m 2GB --file windows/Dockerfile --tag loader-devcontainer-windows . +``` + +The default 1 GB is silently insufficient and the install fails partway +through with cryptic errors. The `devcontainer-windows-smoke.yml` CI +workflow sets this for you. + ## Why is the image so large? -`mcr.microsoft.com/visualstudio/buildtools:ltsc2022` is ~14 GB compressed -(VS 2022 Build Tools + servercore base). The ~6 GB of Build Tools is -unavoidable — vcpkg compiles protobuf from source under MSVC and that -needs the full VC++ toolchain. There is no nanoserver path because -nanoserver lacks the Win32 environment vcpkg's compile depends on. +The dotnet-framework base is ~5 GB; VS 2022 Build Tools install adds +~6 GB on top. The ~6 GB of Build Tools is unavoidable — vcpkg compiles +protobuf from source under MSVC and that needs the full VC++ toolchain. +There is no nanoserver path because nanoserver lacks the Win32 +environment vcpkg's compile depends on. ## Why isolation=hyperv? diff --git a/.github/workflows/devcontainer-windows-smoke.yml b/.github/workflows/devcontainer-windows-smoke.yml index 0413095..6059760 100644 --- a/.github/workflows/devcontainer-windows-smoke.yml +++ b/.github/workflows/devcontainer-windows-smoke.yml @@ -94,8 +94,11 @@ jobs: run: | # Context is .devcontainer/ so `COPY shared\...` resolves. # Tag mirrors the devcontainer.json name so it's recognisable - # in `docker images`. + # in `docker images`. -m 2GB is required for the VS Build Tools + # install (Microsoft's documented minimum; default 1 GB silently + # fails partway through). docker build ` + --memory 2GB ` --file windows\Dockerfile ` --tag loader-devcontainer-windows:smoke ` . From f549d5af78a9b0f36ce763694e59f667d722b9ef Mon Sep 17 00:00:00 2001 From: wenchy Date: Tue, 2 Jun 2026 16:04:46 +0800 Subject: [PATCH 35/66] fix(devcontainer): Windows Dockerfile aligns with Linux versions.env pattern MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Mirrors linux/Dockerfile's approach of COPYing versions.env into the image and reading it per-RUN. PowerShell has no POSIX `source`, so we provide one as Import-LoaderVersions.ps1 and bake it into the SHELL prefix; every RUN's body executes with $env:GO_VERSION etc. populated from C:\loader\versions.env. Three real bugs fixed: 1. The previous version's `Get-Content $path | ConvertFrom-StringData` pipeline was a trap: ConvertFrom-StringData consumed input line-by-line and produced an array of 7 single-key hashtables instead of one merged hashtable. Looping over `$vars.Keys` was therefore a no-op and $env:GO_VERSION came up empty. The CI run on 4e35924 hit this directly: https://go.dev/dl/go${env:GO_VERSION}.windows-amd64.msi -> resolved to https://go.dev/dl/go.windows-amd64.msi (404) Fixed by joining the lines into one string and calling `ConvertFrom-StringData -StringData`. 2. The previous version relied on `[Environment]::SetEnvironmentVariable(... 'Machine')` persisting across RUN layer boundaries via registry. ARG/ENV is the documented portable mechanism; Machine-scope env in Windows containers is unreliable across layers. The new pattern doesn't depend on it. 3. The previous version had a `start /w vs_buildtools.exe ...` cmd-shell sequence that swallowed the actual exit code. Replaced with a Start-Process + ExitCode check that fails the build with the real error code. Symmetry win: the smoke workflow no longer needs to know the list of pinned versions — it just runs `docker build`, exactly like the Linux smoke job. Single source of truth in versions.env; build wrappers stay version-agnostic. Co-Authored-By: Claude Opus 4.6 --- .devcontainer/windows/Dockerfile | 118 ++++++++---------- .../windows/Import-LoaderVersions.ps1 | 32 +++++ 2 files changed, 86 insertions(+), 64 deletions(-) create mode 100644 .devcontainer/windows/Import-LoaderVersions.ps1 diff --git a/.devcontainer/windows/Dockerfile b/.devcontainer/windows/Dockerfile index 6c827f7..c79db6d 100644 --- a/.devcontainer/windows/Dockerfile +++ b/.devcontainer/windows/Dockerfile @@ -11,6 +11,15 @@ # Go / buf / protobuf / .NET / Node / vcpkg-baseline / cmake, edit that # file — not this one. # +# Symmetry with linux/Dockerfile: that file COPYs versions.env into the +# image and `source`s it per-RUN. Windows has no POSIX `source`, so we +# do the equivalent by baking a small `Import-LoaderVersions` helper +# into a stable on-image path, then making every RUN's SHELL prefix call +# it before the body executes. Net effect: every RUN sees $env:GO_VERSION +# (and friends) without depending on Machine-scope env, without enumerating +# --build-arg flags from the wrapper, and without re-parsing versions.env +# in every RUN body. +# # Build context is .devcontainer/ (the parent of this directory) so that # `COPY shared\...` resolves. devcontainer.json sets `"context": ".."`. # @@ -34,10 +43,18 @@ FROM mcr.microsoft.com/dotnet/framework/runtime:4.8-windowsservercore-ltsc2022 -# Restore cmd as the default shell so the vs_BuildTools install RUN below -# matches Microsoft's documented pattern. We switch back to PowerShell -# right after. -SHELL ["cmd", "/S", "/C"] +# --------------------------------------------------------------------------- +# Drop versions.env on disk and a helper script that imports it into the +# current PowerShell session's $env:KEY namespace. Both are stable, +# read-only, and shared across every later RUN. +# --------------------------------------------------------------------------- +COPY shared\versions.env C:\loader\versions.env +COPY windows\Import-LoaderVersions.ps1 C:\loader\Import-LoaderVersions.ps1 + +# Default SHELL: PowerShell, with `Import-LoaderVersions` already invoked +# so $env:GO_VERSION etc. are live before the RUN body runs. Mirrors the +# linux/Dockerfile heredoc that begins with `. /opt/versions.env`. +SHELL ["powershell", "-NoProfile", "-NoLogo", "-ExecutionPolicy", "Bypass", "-Command", "$ErrorActionPreference='Stop'; $ProgressPreference='SilentlyContinue'; . C:\\loader\\Import-LoaderVersions.ps1; Import-LoaderVersions;"] # --------------------------------------------------------------------------- # Visual Studio 2022 Build Tools — VC++ workload only. @@ -50,44 +67,28 @@ SHELL ["cmd", "/S", "/C"] # # Exit code 3010 is "success, reboot required" — fine inside a container. # --------------------------------------------------------------------------- -RUN curl -SL --output vs_buildtools.exe https://aka.ms/vs/17/release/vs_buildtools.exe ` - && (start /w vs_buildtools.exe --quiet --wait --norestart --nocache ` - --installPath "C:\BuildTools" ` - --add Microsoft.VisualStudio.Workload.VCTools ` - --add Microsoft.VisualStudio.Component.VC.Tools.x86.x64 ` - --add Microsoft.VisualStudio.Component.Windows11SDK.22621 ` - --remove Microsoft.VisualStudio.Component.Windows10SDK.10240 ` - --remove Microsoft.VisualStudio.Component.Windows10SDK.10586 ` - --remove Microsoft.VisualStudio.Component.Windows10SDK.14393 ` - --remove Microsoft.VisualStudio.Component.Windows81SDK ` - || IF "%ERRORLEVEL%"=="3010" EXIT 0) ` - && del /q vs_buildtools.exe - -# Switch to PowerShell for the rest of the Dockerfile. -SHELL ["powershell", "-NoProfile", "-NoLogo", "-ExecutionPolicy", "Bypass", "-Command", "$ErrorActionPreference='Stop'; $ProgressPreference='SilentlyContinue';"] - -# --------------------------------------------------------------------------- -# Pull pinned versions from the shared file. PowerShell consumers parse it -# with `ConvertFrom-StringData` after stripping comment / blank lines. -# We render the parsed values into machine-wide environment variables so -# subsequent RUN layers (and the running container) see them as $env:KEY. -# --------------------------------------------------------------------------- -COPY shared\versions.env C:\loader\versions.env - -RUN $raw = Get-Content C:\loader\versions.env | ` - Where-Object { $_ -and $_ -notmatch '^\s*#' }; ` - $vars = $raw | ConvertFrom-StringData; ` - foreach ($k in $vars.Keys) { ` - [System.Environment]::SetEnvironmentVariable($k, $vars[$k], 'Machine'); ` - } - -# Re-import machine env into this build session so each RUN below sees -# $env:GO_VERSION etc. without re-parsing the file. -SHELL ["powershell", "-NoProfile", "-NoLogo", "-ExecutionPolicy", "Bypass", "-Command", "$ErrorActionPreference='Stop'; $ProgressPreference='SilentlyContinue'; foreach ($k in @('GO_VERSION','BUF_VERSION','PROTOBUF_VERSION','VCPKG_BASELINE_COMMIT','DOTNET_VERSION','NODE_VERSION','CMAKE_VERSION')) { Set-Item -Path \"env:$k\" -Value ([System.Environment]::GetEnvironmentVariable($k,'Machine')) };"] +RUN curl.exe -SL --output C:\vs_buildtools.exe https://aka.ms/vs/17/release/vs_buildtools.exe; ` + $proc = Start-Process C:\vs_buildtools.exe ` + -ArgumentList @( ` + '--quiet','--wait','--norestart','--nocache', ` + '--installPath','C:\BuildTools', ` + '--add','Microsoft.VisualStudio.Workload.VCTools', ` + '--add','Microsoft.VisualStudio.Component.VC.Tools.x86.x64', ` + '--add','Microsoft.VisualStudio.Component.Windows11SDK.22621', ` + '--remove','Microsoft.VisualStudio.Component.Windows10SDK.10240', ` + '--remove','Microsoft.VisualStudio.Component.Windows10SDK.10586', ` + '--remove','Microsoft.VisualStudio.Component.Windows10SDK.14393', ` + '--remove','Microsoft.VisualStudio.Component.Windows81SDK' ` + ) -Wait -NoNewWindow -PassThru; ` + if ($proc.ExitCode -ne 0 -and $proc.ExitCode -ne 3010) { ` + Write-Error \"vs_buildtools failed with exit code $($proc.ExitCode)\"; ` + exit $proc.ExitCode ` + }; ` + Remove-Item C:\vs_buildtools.exe -Force # --------------------------------------------------------------------------- # Chocolatey — bootstrap the package manager that installs Git, CMake, -# Ninja, .NET, and Node. (MSVC Build Tools come from the base image.) +# Ninja, .NET, and Node. # --------------------------------------------------------------------------- RUN [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12; ` Invoke-Expression ((New-Object System.Net.WebClient).DownloadString('https://community.chocolatey.org/install.ps1')) @@ -108,9 +109,10 @@ RUN $env:PATH = [System.Environment]::GetEnvironmentVariable('PATH','Machine') + [System.Environment]::SetEnvironmentVariable('PATH', $env:PATH, 'Process') # --------------------------------------------------------------------------- -# Go SDK — official MSI to C:\Go. Pinned to versions.env's GO_VERSION. +# Go SDK — official MSI to C:\Program Files\Go. # --------------------------------------------------------------------------- RUN $url = \"https://go.dev/dl/go${env:GO_VERSION}.windows-amd64.msi\"; ` + Write-Host \"Downloading $url\"; ` Invoke-WebRequest -Uri $url -OutFile C:\go.msi; ` Start-Process msiexec.exe -ArgumentList '/i','C:\go.msi','/qn','/norestart' -Wait; ` Remove-Item C:\go.msi @@ -121,11 +123,11 @@ RUN [System.Environment]::SetEnvironmentVariable('PATH', ` 'Machine') # --------------------------------------------------------------------------- -# buf CLI — single-binary Windows release. Pinned to versions.env's -# BUF_VERSION. Mirrors prepare.bat Step 4. +# buf CLI — single-binary Windows release. # --------------------------------------------------------------------------- RUN New-Item -ItemType Directory -Force -Path C:\Tools\buf | Out-Null; ` $url = \"https://github.com/bufbuild/buf/releases/download/v${env:BUF_VERSION}/buf-Windows-x86_64.exe\"; ` + Write-Host \"Downloading $url\"; ` Invoke-WebRequest -Uri $url -OutFile C:\Tools\buf\buf.exe; ` [System.Environment]::SetEnvironmentVariable('PATH', ` 'C:\Tools\buf;' + [System.Environment]::GetEnvironmentVariable('PATH','Machine'), ` @@ -137,13 +139,7 @@ RUN New-Item -ItemType Directory -Force -Path C:\Tools\buf | Out-Null; ` # Triplet: x64-windows-static. Same triplet prepare.bat uses, so a # user moving between native (prepare.bat) and container builds avoids # the LNK2038 _ITERATOR_DEBUG_LEVEL CRT-mismatch trap. -# -# Why manifest mode: classic-mode `vcpkg install --x-version=...` is -# silently a no-op; only manifest mode + overrides actually pin the port -# version. The post-install assertion catches the case where vcpkg's -# resolution still picks a different port revision than requested. # --------------------------------------------------------------------------- -ARG PROTOBUF_VERSION= ENV VCPKG_ROOT=C:\vcpkg ` VCPKG_DEFAULT_TRIPLET=x64-windows-static @@ -151,16 +147,11 @@ RUN git clone https://github.com/microsoft/vcpkg.git $env:VCPKG_ROOT; ` git -C $env:VCPKG_ROOT checkout $env:VCPKG_BASELINE_COMMIT; ` & \"$env:VCPKG_ROOT\bootstrap-vcpkg.bat\" -disableMetrics -# Render the manifest (PROTOBUF_VERSION build-arg overrides versions.env if set), -# then install via vcpkg, then assert the resolved version matches what we asked for. -# vcpkg compiles protobuf from source under MSVC; vcpkg-install.ps1 activates -# the VS Build Tools env via vsdevcmd.bat (now at C:\BuildTools\... since the -# base image is mcr.microsoft.com/visualstudio/buildtools). +# Render the manifest, install via vcpkg, assert the resolved version +# matches PROTOBUF_VERSION. vcpkg compiles protobuf from source under MSVC; +# vcpkg-install.ps1 activates the VS Build Tools env via vsdevcmd.bat. COPY windows\vcpkg-install.ps1 C:\loader\vcpkg-install.ps1 -RUN $effective = if ($env:PROTOBUF_VERSION) { $env:PROTOBUF_VERSION } else { ` - ([System.Environment]::GetEnvironmentVariable('PROTOBUF_VERSION','Machine')) }; ` - $env:PROTOBUF_VERSION = $effective; ` - & C:\loader\vcpkg-install.ps1 +RUN & C:\loader\vcpkg-install.ps1 # Stable PATH entries so `protoc` and the vcpkg tools are visible. RUN [System.Environment]::SetEnvironmentVariable('PATH', ` @@ -170,8 +161,7 @@ RUN [System.Environment]::SetEnvironmentVariable('PATH', ` ENV CMAKE_PREFIX_PATH=C:\vcpkg-manifest\vcpkg_installed\x64-windows-static # --------------------------------------------------------------------------- -# .NET SDK + Node.js LTS. Versions from versions.env. Chocolatey is the -# install channel for both. +# .NET SDK + Node.js LTS via Chocolatey. # --------------------------------------------------------------------------- RUN $dotnetPkg = \"dotnet-${env:DOTNET_VERSION}-sdk\"; ` choco install -y --no-progress $dotnetPkg; ` @@ -187,11 +177,11 @@ COPY shared\postcreate-banner.ps1 C:\loader\postcreate-banner.ps1 # --------------------------------------------------------------------------- WORKDIR C:\workspaces\loader -# Drop back to a normal shell for the running container — the build-time -# version-import shell is no longer needed. Override the base image's -# ENTRYPOINT (which auto-runs vsdevcmd) so a plain `docker run` lands -# the user in a clean PowerShell — vsdevcmd is invoked on demand by -# vcpkg-install.ps1 and any user shell that needs cl.exe. +# Final SHELL: drop the auto-import prefix for the running container — +# end-users don't need versions.env in their shell session, and keeping +# it in the SHELL prefix would slow every interactive command. The +# helper is still on disk at C:\loader\Import-LoaderVersions.ps1 for any +# script that wants it. SHELL ["powershell", "-NoProfile", "-NoLogo", "-Command"] ENTRYPOINT [] CMD ["powershell.exe", "-NoLogo"] diff --git a/.devcontainer/windows/Import-LoaderVersions.ps1 b/.devcontainer/windows/Import-LoaderVersions.ps1 new file mode 100644 index 0000000..19e3b60 --- /dev/null +++ b/.devcontainer/windows/Import-LoaderVersions.ps1 @@ -0,0 +1,32 @@ +# tableauio/loader devcontainer (windows) - versions.env importer. +# +# Mirrors the linux/Dockerfile pattern of `. /opt/versions.env` in every +# RUN that needs the values. PowerShell has no `source`-equivalent for +# KEY=VALUE files, so we provide one as a function. +# +# Loaded into the SHELL prefix of the windows/Dockerfile so every RUN +# layer's body executes with $env:GO_VERSION etc. already populated from +# C:\loader\versions.env. + +function Import-LoaderVersions { + [CmdletBinding()] + param( + [string]$Path = 'C:\loader\versions.env' + ) + if (-not (Test-Path $Path)) { + throw "versions.env not found at $Path" + } + # Strip blank lines and # comments, then JOIN into one string before + # passing to ConvertFrom-StringData. The pipeline form + # Get-Content $Path | ConvertFrom-StringData + # is a trap: it produces ONE HASHTABLE PER INPUT LINE (7 separate + # single-key hashtables), not one merged hashtable. The string form + # parses the entire blob as a single hashtable. + $body = (Get-Content $Path | + Where-Object { $_ -and $_ -notmatch '^\s*#' }) -join "`n" + $vars = ConvertFrom-StringData -StringData $body + foreach ($k in $vars.Keys) { + Set-Item -Path "env:$k" -Value $vars[$k] + } +} + From 30a4567de1eabdf030b7ad98bda04aea3b356615 Mon Sep 17 00:00:00 2001 From: wenchy Date: Tue, 2 Jun 2026 16:12:25 +0800 Subject: [PATCH 36/66] fix(ci): resolve dockerd.exe via service registry, not Program Files probes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The previous version probed three hardcoded Program Files paths for dockerd.exe. None of them match the path Microsoft's MobyOnWindowsRunner installer chose on current actions/runner-images windows-2022 builds — the upstream installer doesn't commit to a stable location. Resolve dockerd via the registered `docker` Windows service's ImagePath instead. That works regardless of where the runner image installed Docker, and degrades to a useful diagnostic ("docker service is not registered") if the runner doesn't have Docker at all. Reported by the previous workflow run as: dockerd.exe not found in any of: C:\Program Files\docker\dockerd.exe; C:\Program Files\Docker\Docker\resources\dockerd.exe; C:\Program Files\Docker\dockerd.exe Also fixed: under $ErrorActionPreference='Stop', the previous Write-Error fallback aborted before the diagnostic Get-ChildItem could run, so nothing useful was logged. Throw with explicit diagnostic capture beforehand. Co-Authored-By: Claude Opus 4.6 --- .../workflows/devcontainer-windows-smoke.yml | 53 ++++++++++++------- 1 file changed, 33 insertions(+), 20 deletions(-) diff --git a/.github/workflows/devcontainer-windows-smoke.yml b/.github/workflows/devcontainer-windows-smoke.yml index 6059760..4dc32a4 100644 --- a/.github/workflows/devcontainer-windows-smoke.yml +++ b/.github/workflows/devcontainer-windows-smoke.yml @@ -40,7 +40,12 @@ jobs: # Desktop) and default to Linux-containers mode. The legacy # `DockerCli.exe -SwitchDaemon` helper is not present on current # runner images. Flip the daemon by re-registering the `docker` - # Windows service without the `dockerd-default` (Linux) backend. + # Windows service. + # + # `dockerd.exe`'s install path is not stable across runner-image + # versions (Microsoft's MobyOnWindowsRunner installer doesn't + # commit to one), so we resolve it from the registered service's + # ImagePath rather than guessing common Program Files locations. $ErrorActionPreference = 'Stop' $current = (docker info --format '{{.OSType}}' 2>$null) @@ -51,26 +56,34 @@ jobs: exit 0 } - # Locate dockerd.exe. Moby Engine on the runner lives under - # %ProgramFiles%\docker\; Docker Desktop (older runners) used - # %ProgramFiles%\Docker\Docker\. Probe both. - $candidates = @( - "$env:ProgramFiles\docker\dockerd.exe", - "$env:ProgramFiles\Docker\Docker\resources\dockerd.exe", - "$env:ProgramFiles\Docker\dockerd.exe" - ) - $dockerd = $candidates | Where-Object { Test-Path $_ } | Select-Object -First 1 - if (-not $dockerd) { - Write-Error "dockerd.exe not found in any of: $($candidates -join '; ')" - Write-Host '--- diagnostic: Get-Service docker ---' - Get-Service docker -ErrorAction SilentlyContinue | Format-List * - Write-Host '--- diagnostic: ProgramFiles tree ---' - Get-ChildItem "$env:ProgramFiles\docker", "$env:ProgramFiles\Docker" -ErrorAction SilentlyContinue - exit 1 + # Discover dockerd.exe via the registered `docker` service's + # ImagePath in the registry. This works regardless of where the + # runner image installed Docker. + $svcKey = 'HKLM:\SYSTEM\CurrentControlSet\Services\docker' + if (-not (Test-Path $svcKey)) { + Write-Host '--- diagnostic: docker service not registered ---' + Get-Service docker* -ErrorAction SilentlyContinue | Format-List Name,Status,StartType + Get-Command dockerd -ErrorAction SilentlyContinue | Format-List Name,Source + throw 'docker service is not registered; cannot determine dockerd.exe path.' + } + $imagePath = (Get-ItemProperty -Path $svcKey -Name ImagePath).ImagePath + Write-Host "Service ImagePath: $imagePath" + # ImagePath may be quoted and may include trailing args; extract + # the first token (the .exe path) and strip surrounding quotes. + if ($imagePath -match '^"([^"]+)"' -or $imagePath -match '^(\S+\.exe)') { + $dockerd = $Matches[1] + } else { + $dockerd = ($imagePath -split '\s+')[0] + } + if (-not (Test-Path $dockerd)) { + Write-Host '--- diagnostic: dockerd path from registry does not exist ---' + Write-Host " resolved path: $dockerd" + Get-Command dockerd -ErrorAction SilentlyContinue | Format-List Name,Source + throw "dockerd.exe not found at $dockerd" } Write-Host "Using dockerd at: $dockerd" - # Stop, re-register without `-G docker` Linux args, restart. + # Stop, re-register the service, restart. Stop-Service docker & $dockerd --unregister-service if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } @@ -82,9 +95,9 @@ jobs: $after = (docker info --format '{{.OSType}}' 2>$null) Write-Host "Current OSType after switch: $after" if ($after -ne 'windows') { - Write-Error "Docker daemon did not switch to Windows containers (OSType=$after)." + Write-Host 'docker info dump on failure:' docker info - exit 1 + throw "Docker daemon did not switch to Windows containers (OSType=$after)." } docker info | Select-String -Pattern 'OSType|Server Version|Operating System' From bae2377fa82dfca2ebc5cdb6b8db533d812fe42b Mon Sep 17 00:00:00 2001 From: wenchy Date: Tue, 2 Jun 2026 20:23:35 +0800 Subject: [PATCH 37/66] revert(devcontainer): drop Windows-container variant The Windows-container ecosystem (HNS, WinNAT, NDIS filter drivers, Hyper-V virtual switches) is too sensitive to host config to be reliable for daily dev. Real failures hit during validation included: - WinNAT half-state where Get-NetNat returned empty but Get-NetNatExternalAddress still showed stale bindings, blocking New-NetNat with "Windows System Error 52: duplicate name". - Ghost vEthernet (nat) interfaces registered to the IP stack with no associated Get-NetAdapter (Description=). - Container outbound TCP fully blocked at the host NAT layer despite ICMP working and per-interface forwarding being enabled. WFP audit logging found zero drop events for the container's source IP, so the issue was in the NAT translation path, not in any filter callout. - HNS service restart broke dockerd's Windows engine API with persistent 500 Internal Server Error survivable across factory reset and reboot. For Windows hosts the alternatives are: 1. Linux container under WSL2 - the documented recommendation. Multi-arch, multi-OS, none of the issues above. 2. Bare-metal native dev via prepare.bat (C++ toolchain) plus winget for Go / .NET / Node. Documented in the repo root README's new "Windows (bare-metal)" section. Removes: - .devcontainer/windows/ (Dockerfile, devcontainer.json, README, vcpkg-install.ps1, Import-LoaderVersions.ps1) - .github/workflows/devcontainer-windows-smoke.yml - .devcontainer/shared/postcreate-banner.ps1 (orphaned without the Windows devcontainer) Updates docs to reflect two devcontainer subfolders (linux/, macos/) instead of three; macOS already follows this docs-only pattern, and the new Windows path is parallel - winget one-liners + prepare.bat plus a pointer at the Linux container under WSL2. Co-Authored-By: Claude Opus 4.6 --- .devcontainer/linux/README.md | 29 ++- .devcontainer/linux/devcontainer.json | 14 +- .devcontainer/shared/README.md | 33 +--- .devcontainer/shared/postcreate-banner.ps1 | 10 - .devcontainer/shared/postcreate-banner.sh | 5 +- .devcontainer/shared/versions.env | 7 +- .devcontainer/windows/Dockerfile | 187 ------------------ .../windows/Import-LoaderVersions.ps1 | 32 --- .devcontainer/windows/README.md | 133 ------------- .devcontainer/windows/devcontainer.json | 60 ------ .devcontainer/windows/vcpkg-install.ps1 | 107 ---------- .../workflows/devcontainer-windows-smoke.yml | 148 -------------- CLAUDE.md | 2 +- README.md | 42 ++-- 14 files changed, 57 insertions(+), 752 deletions(-) delete mode 100644 .devcontainer/shared/postcreate-banner.ps1 delete mode 100644 .devcontainer/windows/Dockerfile delete mode 100644 .devcontainer/windows/Import-LoaderVersions.ps1 delete mode 100644 .devcontainer/windows/README.md delete mode 100644 .devcontainer/windows/devcontainer.json delete mode 100644 .devcontainer/windows/vcpkg-install.ps1 delete mode 100644 .github/workflows/devcontainer-windows-smoke.yml diff --git a/.devcontainer/linux/README.md b/.devcontainer/linux/README.md index 1420f3d..9fd33da 100644 --- a/.devcontainer/linux/README.md +++ b/.devcontainer/linux/README.md @@ -1,17 +1,16 @@ -# Dev Container — Linux variant (recommended) +# Dev Container The recommended way to develop on `tableauio/loader`. One container, all four target languages (C++17, Go, .NET, Node) plus protobuf via vcpkg, pinned to the exact toolchain CI uses. All version pins live in [`../shared/versions.env`](../shared/versions.env) — bumping any of them -is a one-line change consumed by this Dockerfile, the Windows-container -sibling, `prepare.bat`, and the CI workflows. +is a one-line change consumed by this Dockerfile, `prepare.bat`, and +the CI workflows. Use this variant on **every** host that can run Docker: Linux (amd64 + -arm64), macOS (Intel + Apple Silicon), and Windows + WSL2. The -[Windows-container variant](../windows/) is only relevant if you -specifically need a native MSVC environment inside the container on a -Windows host. +arm64), macOS (Intel + Apple Silicon), and Windows + WSL2. For Windows +hosts that prefer bare-metal native dev (no Docker, no WSL2), see the +[`prepare.bat`](../../prepare.bat) bootstrap at the repo root. ## Prerequisites @@ -25,10 +24,8 @@ code . # in the repo root ``` In VS Code, run **Dev Containers: Reopen in Container** from the command -palette. If both `linux/` and `windows/` are present, VS Code shows a -picker — choose **tableauio/loader (linux)**. First build is one-time -~25 minutes (vcpkg compiles protobuf from source); subsequent reopens -are near-instant. +palette. First build is one-time ~25 minutes (vcpkg compiles protobuf +from source); subsequent reopens are near-instant. When the container is ready, the integrated terminal prints a banner with five toolchain versions. After that, every command from the per-language @@ -123,8 +120,10 @@ Windows `prepare.bat`, per-language `Install protobuf` instructions — still work. The devcontainer is the recommended path; the rest is the supported fallback. -For a Windows host that wants a containerized environment but with -native MSVC inside the container (no WSL2), see the -[Windows-container variant](../windows/). For macOS hosts where you'd -rather not run a Linux container at all, see the +For a Windows host that prefers bare-metal native dev (no Docker, no +WSL2), see [`prepare.bat`](../../prepare.bat) at the repo root — it +bootstraps the C++ toolchain (MSVC, CMake, Ninja, vcpkg+protobuf, buf). +You'll additionally need Go, .NET SDK, and Node.js, which one-line winget +installs cover (see the repo root [README](../../README.md)). For macOS +hosts where you'd rather not run a Linux container at all, see the [macOS notes](../macos/). diff --git a/.devcontainer/linux/devcontainer.json b/.devcontainer/linux/devcontainer.json index cf513a3..8def01e 100644 --- a/.devcontainer/linux/devcontainer.json +++ b/.devcontainer/linux/devcontainer.json @@ -1,14 +1,12 @@ { - // tableauio/loader Dev Container — Linux variant (recommended). - // - // Multi-arch: builds linux/amd64 on x64 hosts, linux/arm64 on Apple - // Silicon and Windows-on-ARM. Use this on every host that can run - // Docker Desktop / Docker Engine; the Windows-container variant under - // ../windows/ is only for Windows hosts that need native MSVC inside - // the container. + // The loader's only devcontainer. Multi-arch: builds linux/amd64 on + // x64 hosts, linux/arm64 on Apple Silicon and Windows-on-ARM. Use + // this on every host that can run Docker Desktop or Docker Engine. + // Bare-metal Windows users (no Docker) bootstrap via prepare.bat at + // the repo root. // // See ../shared/versions.env for all toolchain pins. - "name": "tableauio/loader (linux)", + "name": "tableauio/loader", // Build args wire host env to Dockerfile ARGs: // LOADER_PROTOBUF_VERSION on the host -> PROTOBUF_VERSION inside. // Default falls through to versions.env's PROTOBUF_VERSION (CI's modern diff --git a/.devcontainer/shared/README.md b/.devcontainer/shared/README.md index f7b3558..15898bc 100644 --- a/.devcontainer/shared/README.md +++ b/.devcontainer/shared/README.md @@ -1,16 +1,15 @@ # tableauio/loader — `.devcontainer/shared/` -Files in this directory are consumed by **multiple** devcontainer paths -(`linux/`, `windows/`) and by host-side scripts (`prepare.bat`, CI -workflows). Edit them with cross-platform parsing in mind. +Files in this directory are consumed by the Linux devcontainer +(`../linux/`) and by host-side scripts (`prepare.bat`, CI workflows). +Edit them with cross-platform parsing in mind. ## Files | File | Consumers | Format | | --- | --- | --- | -| [`versions.env`](./versions.env) | All Dockerfiles, `prepare.bat`, every `.github/workflows/*.yml` | `KEY=VALUE`, one per line, no quotes, no `$VAR` expansion | +| [`versions.env`](./versions.env) | `linux/Dockerfile`, `prepare.bat`, every `.github/workflows/*.yml` | `KEY=VALUE`, one per line, no quotes, no `$VAR` expansion | | [`postcreate-banner.sh`](./postcreate-banner.sh) | `linux/devcontainer.json` `postCreateCommand` | POSIX `sh` | -| [`postcreate-banner.ps1`](./postcreate-banner.ps1) | `windows/devcontainer.json` `postCreateCommand` | PowerShell | ## `versions.env` parsing rules @@ -31,14 +30,6 @@ Quick parsers per language: echo "$GO_VERSION" ``` -```powershell -# PowerShell (Windows Dockerfile) -$v = Get-Content .devcontainer/shared/versions.env | - Where-Object { $_ -and $_ -notmatch '^\s*#' } | - ConvertFrom-StringData -$v.GO_VERSION -``` - ```cmd :: Windows cmd (prepare.bat) for /f "tokens=1,2 delims==" %%a in (.devcontainer\shared\versions.env) do ( @@ -48,15 +39,9 @@ echo %GO_VERSION% ``` ```yaml -# GitHub Actions (read once, export to $GITHUB_ENV) -- name: Read pinned versions - shell: bash - run: | - while IFS='=' read -r k v; do - [ -z "$k" ] && continue - [ "${k#\#}" != "$k" ] && continue - printf '%s=%s\n' "$k" "$v" >> "$GITHUB_ENV" - done < .devcontainer/shared/versions.env +# GitHub Actions +- uses: ./.github/actions/load-versions +# subsequent steps reference the values as ${{ env.GO_VERSION }} etc. ``` ## Lockstep rule @@ -65,5 +50,5 @@ echo %GO_VERSION% `PROTOBUF_VERSION` value used anywhere (devcontainer default + every `testing-cpp.yml` matrix entry). Bumping `PROTOBUF_VERSION` to a value the current baseline doesn't know is caught at build time by the -post-install assertion in both Dockerfiles and `prepare.bat` — fail loud, -no silent wrong-version installs. +post-install assertion in `linux/Dockerfile` and `prepare.bat` — fail +loud, no silent wrong-version installs. diff --git a/.devcontainer/shared/postcreate-banner.ps1 b/.devcontainer/shared/postcreate-banner.ps1 deleted file mode 100644 index 8c98f51..0000000 --- a/.devcontainer/shared/postcreate-banner.ps1 +++ /dev/null @@ -1,10 +0,0 @@ -# Post-create banner for the Windows devcontainer. -# Same shape as ../shared/postcreate-banner.sh (Linux) — five lines, -# pure version queries, no installs. -$ErrorActionPreference = 'Stop' -Write-Host 'tableauio/loader devcontainer ready (windows).' -Write-Host (' go: {0}' -f ((go version) -split '\s+')[2]) -Write-Host (' buf: {0}' -f (buf --version 2>&1)) -Write-Host (' protoc: {0}' -f (protoc --version)) -Write-Host (' dotnet: {0}' -f (dotnet --version)) -Write-Host (' node: {0}' -f (node --version)) diff --git a/.devcontainer/shared/postcreate-banner.sh b/.devcontainer/shared/postcreate-banner.sh index 5aa8d40..16a6794 100644 --- a/.devcontainer/shared/postcreate-banner.sh +++ b/.devcontainer/shared/postcreate-banner.sh @@ -1,9 +1,8 @@ #!/bin/sh # Post-create banner for the Linux devcontainer. # Pure echo — no installs, no version-pinning at runtime, no surprises. -# Mirrors the same five-line summary the previous inline postCreateCommand -# emitted; extracted to a script so the Windows container can have a -# parallel postcreate-banner.ps1 with the same shape. +# Five-line summary that prints when the container becomes ready, so the +# developer can confirm at a glance which toolchain versions landed. set -e printf 'tableauio/loader devcontainer ready (linux).\n' printf ' go: %s\n' "$(go version | cut -d' ' -f3)" diff --git a/.devcontainer/shared/versions.env b/.devcontainer/shared/versions.env index aed69a5..db03824 100644 --- a/.devcontainer/shared/versions.env +++ b/.devcontainer/shared/versions.env @@ -2,9 +2,8 @@ # # Single source of truth consumed by: # - .devcontainer/linux/Dockerfile (sourced as a shell file) -# - .devcontainer/windows/Dockerfile (parsed line-by-line in PowerShell) # - prepare.bat (parsed via `for /f` in cmd) -# - .github/workflows/*.yml (read by a "Read versions" step into $GITHUB_ENV) +# - .github/workflows/*.yml (read by the load-versions composite action into $GITHUB_ENV) # # Format rules so every consumer can parse this file with a single regex: # - One KEY=VALUE per line, no quotes, no spaces around `=`. @@ -32,7 +31,6 @@ PROTOBUF_VERSION=6.33.4 # vcpkg checkout commit. Same value used by: # - .devcontainer/linux/Dockerfile (ARG VCPKG_BASELINE_COMMIT) -# - .devcontainer/windows/Dockerfile # - prepare.bat (set VCPKG_BASELINE_COMMIT=...) # - .github/workflows/testing-cpp.yml (env: VCPKG_COMMIT) # This commit MUST know about every PROTOBUF_VERSION in the testing-cpp.yml @@ -49,6 +47,5 @@ DOTNET_VERSION=8.0 NODE_VERSION=20 # CMake version installed by prepare.bat (Linux devcontainer base image -# already ships a recent cmake; Windows container installs this exact one -# to match the native prepare.bat experience). +# already ships a recent cmake). CMAKE_VERSION=3.31.8 diff --git a/.devcontainer/windows/Dockerfile b/.devcontainer/windows/Dockerfile deleted file mode 100644 index c79db6d..0000000 --- a/.devcontainer/windows/Dockerfile +++ /dev/null @@ -1,187 +0,0 @@ -# escape=` -# tableauio/loader devcontainer (windows) -# -# Windows-container variant. Runs ONLY on Windows hosts (Hyper-V or -# process isolation, Docker Desktop in Windows-containers mode). -# windows/amd64 ONLY — Microsoft does not publish arm64 Windows base -# images, so arm64 hosts must use the Linux container under ../linux/. -# -# All version pins are read from ../shared/versions.env (the single source -# of truth shared with the Linux container, prepare.bat, and CI). To bump -# Go / buf / protobuf / .NET / Node / vcpkg-baseline / cmake, edit that -# file — not this one. -# -# Symmetry with linux/Dockerfile: that file COPYs versions.env into the -# image and `source`s it per-RUN. Windows has no POSIX `source`, so we -# do the equivalent by baking a small `Import-LoaderVersions` helper -# into a stable on-image path, then making every RUN's SHELL prefix call -# it before the body executes. Net effect: every RUN sees $env:GO_VERSION -# (and friends) without depending on Machine-scope env, without enumerating -# --build-arg flags from the wrapper, and without re-parsing versions.env -# in every RUN body. -# -# Build context is .devcontainer/ (the parent of this directory) so that -# `COPY shared\...` resolves. devcontainer.json sets `"context": ".."`. -# -# Why mcr.microsoft.com/dotnet/framework/runtime (not windows/servercore): -# -# The modern VS 2022 Build Tools installer (`vs_BuildTools.exe`) -# requires .NET Framework runtime support that `windows/servercore` -# strips out. Microsoft's own guidance — see -# https://learn.microsoft.com/en-us/visualstudio/install/build-tools-container -# — explicitly says: "If you base your image directly on -# microsoft/windowsservercore, the .NET Framework might not install -# properly … Instead, base your image on microsoft/dotnet-framework:4.7.1 -# or later." We use 4.8-windowsservercore-ltsc2022, which is also one -# of the images pre-cached on GitHub's windows-2022 runners. -# -# ltsc2022 matches GitHub Actions' windows-latest runner family. -# -# Build memory: pass `docker build -m 2GB` (or higher). vs_BuildTools -# exhausts the default 1 GB silently. CI's windows-2022 runners have -# plenty of RAM but you must set the flag. - -FROM mcr.microsoft.com/dotnet/framework/runtime:4.8-windowsservercore-ltsc2022 - -# --------------------------------------------------------------------------- -# Drop versions.env on disk and a helper script that imports it into the -# current PowerShell session's $env:KEY namespace. Both are stable, -# read-only, and shared across every later RUN. -# --------------------------------------------------------------------------- -COPY shared\versions.env C:\loader\versions.env -COPY windows\Import-LoaderVersions.ps1 C:\loader\Import-LoaderVersions.ps1 - -# Default SHELL: PowerShell, with `Import-LoaderVersions` already invoked -# so $env:GO_VERSION etc. are live before the RUN body runs. Mirrors the -# linux/Dockerfile heredoc that begins with `. /opt/versions.env`. -SHELL ["powershell", "-NoProfile", "-NoLogo", "-ExecutionPolicy", "Bypass", "-Command", "$ErrorActionPreference='Stop'; $ProgressPreference='SilentlyContinue'; . C:\\loader\\Import-LoaderVersions.ps1; Import-LoaderVersions;"] - -# --------------------------------------------------------------------------- -# Visual Studio 2022 Build Tools — VC++ workload only. -# -# Mirrors Microsoft's documented Dockerfile from -# https://learn.microsoft.com/en-us/visualstudio/install/build-tools-container -# but with `--add Microsoft.VisualStudio.Workload.VCTools` (we don't need -# the AzureBuildTools workload) and the same `--remove` flags for old -# Windows SDKs that can't install in a container. -# -# Exit code 3010 is "success, reboot required" — fine inside a container. -# --------------------------------------------------------------------------- -RUN curl.exe -SL --output C:\vs_buildtools.exe https://aka.ms/vs/17/release/vs_buildtools.exe; ` - $proc = Start-Process C:\vs_buildtools.exe ` - -ArgumentList @( ` - '--quiet','--wait','--norestart','--nocache', ` - '--installPath','C:\BuildTools', ` - '--add','Microsoft.VisualStudio.Workload.VCTools', ` - '--add','Microsoft.VisualStudio.Component.VC.Tools.x86.x64', ` - '--add','Microsoft.VisualStudio.Component.Windows11SDK.22621', ` - '--remove','Microsoft.VisualStudio.Component.Windows10SDK.10240', ` - '--remove','Microsoft.VisualStudio.Component.Windows10SDK.10586', ` - '--remove','Microsoft.VisualStudio.Component.Windows10SDK.14393', ` - '--remove','Microsoft.VisualStudio.Component.Windows81SDK' ` - ) -Wait -NoNewWindow -PassThru; ` - if ($proc.ExitCode -ne 0 -and $proc.ExitCode -ne 3010) { ` - Write-Error \"vs_buildtools failed with exit code $($proc.ExitCode)\"; ` - exit $proc.ExitCode ` - }; ` - Remove-Item C:\vs_buildtools.exe -Force - -# --------------------------------------------------------------------------- -# Chocolatey — bootstrap the package manager that installs Git, CMake, -# Ninja, .NET, and Node. -# --------------------------------------------------------------------------- -RUN [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12; ` - Invoke-Expression ((New-Object System.Net.WebClient).DownloadString('https://community.chocolatey.org/install.ps1')) - -ENV ChocolateyInstall=C:\ProgramData\chocolatey -RUN $env:PATH = \"$env:ChocolateyInstall\bin;$env:PATH\"; ` - [System.Environment]::SetEnvironmentVariable('PATH', $env:PATH, 'Machine') - -# --------------------------------------------------------------------------- -# Git, CMake, Ninja — same versions/sources prepare.bat uses. -# Git is needed for `git clone` of vcpkg below. -# --------------------------------------------------------------------------- -RUN choco install -y --no-progress git ninja; ` - choco install -y --no-progress cmake --version=$env:CMAKE_VERSION --installargs '\"ADD_CMAKE_TO_PATH=System\"' - -# Refresh PATH so the new tools are visible in this build session. -RUN $env:PATH = [System.Environment]::GetEnvironmentVariable('PATH','Machine') + ';' + [System.Environment]::GetEnvironmentVariable('PATH','User'); ` - [System.Environment]::SetEnvironmentVariable('PATH', $env:PATH, 'Process') - -# --------------------------------------------------------------------------- -# Go SDK — official MSI to C:\Program Files\Go. -# --------------------------------------------------------------------------- -RUN $url = \"https://go.dev/dl/go${env:GO_VERSION}.windows-amd64.msi\"; ` - Write-Host \"Downloading $url\"; ` - Invoke-WebRequest -Uri $url -OutFile C:\go.msi; ` - Start-Process msiexec.exe -ArgumentList '/i','C:\go.msi','/qn','/norestart' -Wait; ` - Remove-Item C:\go.msi -ENV GOPATH=C:\Users\ContainerUser\go -RUN [System.Environment]::SetEnvironmentVariable('PATH', ` - 'C:\Program Files\Go\bin;C:\Users\ContainerUser\go\bin;' + ` - [System.Environment]::GetEnvironmentVariable('PATH','Machine'), ` - 'Machine') - -# --------------------------------------------------------------------------- -# buf CLI — single-binary Windows release. -# --------------------------------------------------------------------------- -RUN New-Item -ItemType Directory -Force -Path C:\Tools\buf | Out-Null; ` - $url = \"https://github.com/bufbuild/buf/releases/download/v${env:BUF_VERSION}/buf-Windows-x86_64.exe\"; ` - Write-Host \"Downloading $url\"; ` - Invoke-WebRequest -Uri $url -OutFile C:\Tools\buf\buf.exe; ` - [System.Environment]::SetEnvironmentVariable('PATH', ` - 'C:\Tools\buf;' + [System.Environment]::GetEnvironmentVariable('PATH','Machine'), ` - 'Machine') - -# --------------------------------------------------------------------------- -# vcpkg + protobuf via manifest mode. -# -# Triplet: x64-windows-static. Same triplet prepare.bat uses, so a -# user moving between native (prepare.bat) and container builds avoids -# the LNK2038 _ITERATOR_DEBUG_LEVEL CRT-mismatch trap. -# --------------------------------------------------------------------------- -ENV VCPKG_ROOT=C:\vcpkg ` - VCPKG_DEFAULT_TRIPLET=x64-windows-static - -RUN git clone https://github.com/microsoft/vcpkg.git $env:VCPKG_ROOT; ` - git -C $env:VCPKG_ROOT checkout $env:VCPKG_BASELINE_COMMIT; ` - & \"$env:VCPKG_ROOT\bootstrap-vcpkg.bat\" -disableMetrics - -# Render the manifest, install via vcpkg, assert the resolved version -# matches PROTOBUF_VERSION. vcpkg compiles protobuf from source under MSVC; -# vcpkg-install.ps1 activates the VS Build Tools env via vsdevcmd.bat. -COPY windows\vcpkg-install.ps1 C:\loader\vcpkg-install.ps1 -RUN & C:\loader\vcpkg-install.ps1 - -# Stable PATH entries so `protoc` and the vcpkg tools are visible. -RUN [System.Environment]::SetEnvironmentVariable('PATH', ` - 'C:\vcpkg;C:\vcpkg-manifest\vcpkg_installed\x64-windows-static\tools\protobuf;' + ` - [System.Environment]::GetEnvironmentVariable('PATH','Machine'), ` - 'Machine') -ENV CMAKE_PREFIX_PATH=C:\vcpkg-manifest\vcpkg_installed\x64-windows-static - -# --------------------------------------------------------------------------- -# .NET SDK + Node.js LTS via Chocolatey. -# --------------------------------------------------------------------------- -RUN $dotnetPkg = \"dotnet-${env:DOTNET_VERSION}-sdk\"; ` - choco install -y --no-progress $dotnetPkg; ` - choco install -y --no-progress nodejs-lts --version=\"${env:NODE_VERSION}.0.0\" - -# --------------------------------------------------------------------------- -# Post-create banner script (parallel to the Linux variant). -# --------------------------------------------------------------------------- -COPY shared\postcreate-banner.ps1 C:\loader\postcreate-banner.ps1 - -# --------------------------------------------------------------------------- -# Workspace dir. Matches devcontainer.json's workspaceFolder. -# --------------------------------------------------------------------------- -WORKDIR C:\workspaces\loader - -# Final SHELL: drop the auto-import prefix for the running container — -# end-users don't need versions.env in their shell session, and keeping -# it in the SHELL prefix would slow every interactive command. The -# helper is still on disk at C:\loader\Import-LoaderVersions.ps1 for any -# script that wants it. -SHELL ["powershell", "-NoProfile", "-NoLogo", "-Command"] -ENTRYPOINT [] -CMD ["powershell.exe", "-NoLogo"] diff --git a/.devcontainer/windows/Import-LoaderVersions.ps1 b/.devcontainer/windows/Import-LoaderVersions.ps1 deleted file mode 100644 index 19e3b60..0000000 --- a/.devcontainer/windows/Import-LoaderVersions.ps1 +++ /dev/null @@ -1,32 +0,0 @@ -# tableauio/loader devcontainer (windows) - versions.env importer. -# -# Mirrors the linux/Dockerfile pattern of `. /opt/versions.env` in every -# RUN that needs the values. PowerShell has no `source`-equivalent for -# KEY=VALUE files, so we provide one as a function. -# -# Loaded into the SHELL prefix of the windows/Dockerfile so every RUN -# layer's body executes with $env:GO_VERSION etc. already populated from -# C:\loader\versions.env. - -function Import-LoaderVersions { - [CmdletBinding()] - param( - [string]$Path = 'C:\loader\versions.env' - ) - if (-not (Test-Path $Path)) { - throw "versions.env not found at $Path" - } - # Strip blank lines and # comments, then JOIN into one string before - # passing to ConvertFrom-StringData. The pipeline form - # Get-Content $Path | ConvertFrom-StringData - # is a trap: it produces ONE HASHTABLE PER INPUT LINE (7 separate - # single-key hashtables), not one merged hashtable. The string form - # parses the entire blob as a single hashtable. - $body = (Get-Content $Path | - Where-Object { $_ -and $_ -notmatch '^\s*#' }) -join "`n" - $vars = ConvertFrom-StringData -StringData $body - foreach ($k in $vars.Keys) { - Set-Item -Path "env:$k" -Value $vars[$k] - } -} - diff --git a/.devcontainer/windows/README.md b/.devcontainer/windows/README.md deleted file mode 100644 index f553413..0000000 --- a/.devcontainer/windows/README.md +++ /dev/null @@ -1,133 +0,0 @@ -# Dev Container — Windows variant - -A **Windows-container** image (windows/amd64 only) for Windows hosts that -want a native MSVC build environment inside the container, without -WSL2. The Linux container under [`../linux/`](../linux/) is the -recommended path for almost everyone — use this variant only when you -specifically need Windows-native tooling. - -## When to use this variant - -| Goal | Use | -| --- | --- | -| Develop on Windows + WSL2 | `../linux/` | -| Develop on Windows without WSL2, native MSVC | **this** | -| Develop on macOS / Linux | `../linux/` | -| Develop on arm64 Windows (Surface Pro X) | `../linux/` (linux/arm64 native) | -| Match CI's `windows-latest` toolchain locally | **this** | - -## Prerequisites - -- **Windows 10/11 Pro or Enterprise.** Home edition cannot run Windows - containers (Hyper-V missing). -- **Docker Desktop in Windows-containers mode.** Right-click the tray - icon → *Switch to Windows containers*. This is a per-host toggle: - Linux and Windows containers cannot run simultaneously. -- **Disk space.** ~12 GB for MSVC Build Tools alone. Total image is - ~15 GB. -- **VS Code** with the Dev Containers extension. - -## Open the container - -```sh -code . # in the repo root -``` - -In VS Code, run **Dev Containers: Reopen in Container** from the command -palette. VS Code shows a picker; choose **tableauio/loader (windows)**. - -First build is one-time, ~45 minutes (vcpkg compiles protobuf from -source under MSVC). Subsequent reopens are near-instant. - -## Pin a different protobuf version - -```cmd -set LOADER_PROTOBUF_VERSION=3.21.12 -code . -``` - -…then **Dev Containers: Rebuild Container**. Same knob as the Linux -variant, same default sourced from -[`../shared/versions.env`](../shared/versions.env). - -## Architecture - -Single Dockerfile based on -`mcr.microsoft.com/dotnet/framework/runtime:4.8-windowsservercore-ltsc2022`. -Microsoft's [official guidance for VS Build Tools containers](https://learn.microsoft.com/en-us/visualstudio/install/build-tools-container) -explicitly recommends the dotnet-framework base over plain -`windows/servercore`, because the modern `vs_BuildTools.exe` installer -requires the .NET Framework runtime that servercore strips. The -dotnet-framework image is also pre-cached on GitHub's `windows-2022` -runners. - -Build layers: - -1. Visual Studio 2022 Build Tools — VC++ workload, installed from the - official `aka.ms/vs/17/release/vs_buildtools.exe` bootstrapper into - `C:\BuildTools\`. Mirrors Microsoft's documented pattern. -2. Imports `versions.env` into machine-wide env so each `RUN` sees `$env:KEY`. -3. Chocolatey bootstrap. -4. Git, Ninja, CMake (CMake version pinned by `versions.env`). -5. Go SDK — official MSI, version from `versions.env`. -6. buf CLI — single-binary release, version from `versions.env`. -7. vcpkg (pinned to `VCPKG_BASELINE_COMMIT`) + protobuf via manifest mode - with the `x64-windows-static` triplet — same triplet `prepare.bat` - uses on bare metal, so a developer moving between native and - container builds avoids the LNK2038 `_ITERATOR_DEBUG_LEVEL` CRT-mismatch - trap. `vcpkg-install.ps1` activates the Build Tools env from - `C:\BuildTools\Common7\Tools\VsDevCmd.bat` before invoking vcpkg. - Post-install assertion catches version drift. -8. .NET SDK + Node.js LTS via Chocolatey. -9. `ENV CMAKE_PREFIX_PATH=C:\vcpkg-manifest\vcpkg_installed\x64-windows-static` - so `find_package(Protobuf CONFIG)` resolves automatically. - -## Build memory - -The VS Build Tools install needs **at least 2 GB of memory** during the -`docker build` — Microsoft documents this. Pass `-m 2GB` (or higher): - -```sh -docker build -m 2GB --file windows/Dockerfile --tag loader-devcontainer-windows . -``` - -The default 1 GB is silently insufficient and the install fails partway -through with cryptic errors. The `devcontainer-windows-smoke.yml` CI -workflow sets this for you. - -## Why is the image so large? - -The dotnet-framework base is ~5 GB; VS 2022 Build Tools install adds -~6 GB on top. The ~6 GB of Build Tools is unavoidable — vcpkg compiles -protobuf from source under MSVC and that needs the full VC++ toolchain. -There is no nanoserver path because nanoserver lacks the Win32 -environment vcpkg's compile depends on. - -## Why isolation=hyperv? - -Process isolation requires the **host's** Windows build to be ≥ the -container image's Windows build. ltsc2022 is conservative enough to -work on most Windows 10 21H2+ and Windows 11 hosts under Hyper-V -isolation. If your host is Windows 11 22H2+ you can drop `runArgs` -or change to `--isolation=process` for slightly faster startup. - -## Falling back - -If the container build is too slow or too large for your machine, the -existing manual setup paths still work: - -- Native Windows: [`prepare.bat`](../../prepare.bat) at the repo root. - Same vcpkg baseline and triplet as this container. -- Linux container under WSL2: [`../linux/`](../linux/). Same toolchain - versions, much smaller image. - -## Limitations - -- **windows/amd64 only.** No arm64 Windows base image exists. Use the - Linux container on arm64 Windows hosts (it builds linux/arm64 natively - under Docker's Linux-container engine). -- **Windows containers don't run on macOS or Linux.** Don't try to build - this image on a non-Windows host; it won't work. -- **Heavier than the Linux variant.** Both build time and disk footprint - are roughly 2× the Linux container. If you don't specifically need - Windows-native tooling, use the Linux variant. diff --git a/.devcontainer/windows/devcontainer.json b/.devcontainer/windows/devcontainer.json deleted file mode 100644 index b5bf621..0000000 --- a/.devcontainer/windows/devcontainer.json +++ /dev/null @@ -1,60 +0,0 @@ -{ - // tableauio/loader Dev Container — Windows variant. - // - // Windows-host-only. Runs windows/amd64 ONLY (Microsoft does not - // publish arm64 Windows base images). arm64 hosts and macOS hosts - // must use the Linux container under ../linux/ instead. - // - // Prerequisites: - // - Windows 10/11 Pro or Enterprise (Hyper-V required; Home edition - // cannot run Windows containers) - // - Docker Desktop in **Windows-containers mode** - // (right-click tray icon → "Switch to Windows containers") - // - Host Windows build ≥ ltsc2022 base image build, OR accept - // Hyper-V isolation (slower but compatible across builds) - // - // All version pins live in ../shared/versions.env. See the - // ../README.md for the longer how-to. - "name": "tableauio/loader (windows)", - "build": { - "dockerfile": "Dockerfile", - // Build context is the parent .devcontainer/ directory so that - // `COPY shared\...` and `COPY windows\...` resolve. - "context": "..", - "args": { - // Wire LOADER_PROTOBUF_VERSION host env to the Dockerfile's - // PROTOBUF_VERSION ARG. Empty default → Dockerfile falls back - // to versions.env's PROTOBUF_VERSION. - "PROTOBUF_VERSION": "${localEnv:LOADER_PROTOBUF_VERSION:}" - } - }, - // Process isolation needs host build ≥ ltsc2022; Hyper-V isolation is - // the safe default and works everywhere Windows containers run. - "runArgs": ["--isolation=hyperv"], - "remoteUser": "ContainerUser", - "workspaceFolder": "C:\\workspaces\\loader", - // No named-volume go-mod cache here. Windows-container named volumes - // have well-documented ACL footguns; accept that mod cache rebuilds - // each container rebuild. If that's painful, add a host bind-mount - // here pointing at a local directory. - "customizations": { - "vscode": { - "extensions": [ - "golang.go", - "ms-vscode.cmake-tools", - "ms-vscode.cpptools", - "ms-dotnettools.csharp", - "bufbuild.vscode-buf", - "DrBlury.protobuf-vsc" - ], - "settings": { - "go.toolsManagement.autoUpdate": false, - "cmake.configureOnOpen": false, - // PowerShell as default terminal inside the container. - "terminal.integrated.defaultProfile.windows": "PowerShell" - } - } - }, - // Ready banner so the developer knows the container is healthy. - "postCreateCommand": "powershell -NoProfile -ExecutionPolicy Bypass -File C:\\loader\\postcreate-banner.ps1" -} diff --git a/.devcontainer/windows/vcpkg-install.ps1 b/.devcontainer/windows/vcpkg-install.ps1 deleted file mode 100644 index 22f2028..0000000 --- a/.devcontainer/windows/vcpkg-install.ps1 +++ /dev/null @@ -1,107 +0,0 @@ -# tableauio/loader — Windows-container vcpkg + protobuf install. -# -# Called from windows/Dockerfile during image build. Mirrors the Linux -# Dockerfile's vcpkg manifest-mode install + post-install version -# assertion, in PowerShell + cmd. -# -# Inputs (env vars; set in the Dockerfile from versions.env): -# PROTOBUF_VERSION — protobuf vcpkg port version, e.g. 6.33.4 -# VCPKG_BASELINE_COMMIT — commit to pin in builtin-baseline -# VCPKG_ROOT — C:\vcpkg -# -# Side effects: -# - Renders C:\vcpkg-manifest\vcpkg.json -# - Runs `vcpkg install --triplet=x64-windows-static` -# - Asserts the resolved port version matches PROTOBUF_VERSION -# -# vcpkg compiles protobuf from source under MSVC — that needs the VS Build -# Tools env (cl.exe, INCLUDE, LIB) active in the running shell. We invoke -# vsdevcmd.bat through cmd.exe and capture the resulting environment, then -# replay it into PowerShell before invoking vcpkg. - -$ErrorActionPreference = 'Stop' -$ProgressPreference = 'SilentlyContinue' - -$triplet = 'x64-windows-static' -$manifestDir = 'C:\vcpkg-manifest' -$installRoot = Join-Path $manifestDir 'vcpkg_installed' -New-Item -ItemType Directory -Force -Path $manifestDir | Out-Null - -# 1. Render manifest. JSON-escape the substituted values just in case. -$pv = $env:PROTOBUF_VERSION -$bc = $env:VCPKG_BASELINE_COMMIT -if (-not $pv) { throw 'PROTOBUF_VERSION env var is empty.' } -if (-not $bc) { throw 'VCPKG_BASELINE_COMMIT env var is empty.' } - -$manifest = @{ - name = 'loader-devcontainer-windows' - version = '0.1.0' - dependencies = @('protobuf') - overrides = @(@{ name = 'protobuf'; version = $pv }) - 'builtin-baseline' = $bc -} | ConvertTo-Json -Depth 4 -Set-Content -Path (Join-Path $manifestDir 'vcpkg.json') -Value $manifest -Encoding ASCII - -# 2. Activate the VS 2022 Build Tools environment for this PS session. -# On the mcr.microsoft.com/visualstudio/buildtools base image, the -# Build Tools live at a fixed `C:\BuildTools\` prefix, so we go straight -# there instead of probing via vswhere. Fall back to vswhere if a future -# base image changes this layout. -$primaryVsdevcmd = 'C:\BuildTools\Common7\Tools\VsDevCmd.bat' -if (Test-Path $primaryVsdevcmd) { - $vsdevcmd = $primaryVsdevcmd -} else { - $vswhere = 'C:\Program Files (x86)\Microsoft Visual Studio\Installer\vswhere.exe' - if (-not (Test-Path $vswhere)) { - throw "Neither $primaryVsdevcmd nor $vswhere found; VS Build Tools layer is missing." - } - $installPath = & $vswhere -latest -products * ` - -requires Microsoft.VisualStudio.Component.VC.Tools.x86.x64 ` - -property installationPath - if (-not $installPath) { throw 'No VS install with C++ tools detected.' } - $vsdevcmd = Join-Path $installPath 'Common7\Tools\VsDevCmd.bat' - if (-not (Test-Path $vsdevcmd)) { throw "VsDevCmd.bat not found: $vsdevcmd" } -} -Write-Host "Using VsDevCmd at: $vsdevcmd" - -# Capture the environment that vsdevcmd produces. -$envDump = & cmd.exe /s /c "`"$vsdevcmd`" -arch=amd64 -host_arch=amd64 && set" -foreach ($line in $envDump) { - if ($line -match '^([^=]+)=(.*)$') { - Set-Item -Path "env:$($Matches[1])" -Value $Matches[2] - } -} -if (-not (Get-Command cl.exe -ErrorAction SilentlyContinue)) { - throw 'cl.exe not on PATH after VsDevCmd activation; cannot proceed.' -} - -# 3. Manifest-mode install. -Push-Location $manifestDir -try { - & "$env:VCPKG_ROOT\vcpkg.exe" install ` - "--triplet=$triplet" ` - "--x-install-root=$installRoot" - if ($LASTEXITCODE -ne 0) { - throw "vcpkg install failed (exit code $LASTEXITCODE)" - } -} finally { - Pop-Location -} - -# 4. Post-install assertion — same shape as the Linux Dockerfile's case -# statement and prepare.bat's findstr check. -$infoDir = Join-Path $installRoot 'vcpkg\info' -$marker = Get-ChildItem -Path $infoDir -Filter "protobuf_*_${triplet}.list" ` - -ErrorAction SilentlyContinue | Select-Object -First 1 -if (-not $marker) { - throw "vcpkg installed-file marker not found under $infoDir" -} -if ($marker.Name -notlike "protobuf_${pv}*") { - Write-Error "Installed protobuf does not match requested version $pv." - Write-Error " vcpkg installed-file marker: $($marker.Name)" - Write-Error " Bump VCPKG_BASELINE_COMMIT in .devcontainer/shared/versions.env" - Write-Error " to a commit that knows about the requested version." - exit 1 -} - -Write-Host "vcpkg protobuf $pv installed under $installRoot ($triplet)." diff --git a/.github/workflows/devcontainer-windows-smoke.yml b/.github/workflows/devcontainer-windows-smoke.yml deleted file mode 100644 index 4dc32a4..0000000 --- a/.github/workflows/devcontainer-windows-smoke.yml +++ /dev/null @@ -1,148 +0,0 @@ -name: Devcontainer Windows Smoke - -# Builds the Windows-container variant of the devcontainer and runs a -# minimal smoke test inside it. Triggered only when files that affect -# the Windows image change, so we don't burn windows-2022 CI minutes on -# unrelated PRs. The Linux container is dogfooded by humans (per -# CLAUDE.md); this job exists because a Windows-container regression is -# much harder for a human to notice locally. - -on: - pull_request: - paths: - - '.devcontainer/windows/**' - - '.devcontainer/shared/**' - - '.github/workflows/devcontainer-windows-smoke.yml' - push: - branches: [master, main] - paths: - - '.devcontainer/windows/**' - - '.devcontainer/shared/**' - workflow_dispatch: - -permissions: - contents: read - -jobs: - build-and-smoke: - name: build windows devcontainer + smoke - runs-on: windows-2022 - timeout-minutes: 90 - - steps: - - name: Checkout - uses: actions/checkout@v6 - - - name: Switch Docker to Windows containers - shell: pwsh - run: | - # GitHub-hosted windows-2022 runners ship Moby Engine (not Docker - # Desktop) and default to Linux-containers mode. The legacy - # `DockerCli.exe -SwitchDaemon` helper is not present on current - # runner images. Flip the daemon by re-registering the `docker` - # Windows service. - # - # `dockerd.exe`'s install path is not stable across runner-image - # versions (Microsoft's MobyOnWindowsRunner installer doesn't - # commit to one), so we resolve it from the registered service's - # ImagePath rather than guessing common Program Files locations. - $ErrorActionPreference = 'Stop' - - $current = (docker info --format '{{.OSType}}' 2>$null) - Write-Host "Current OSType before switch: $current" - if ($current -eq 'windows') { - Write-Host 'Already in Windows-containers mode; nothing to do.' - docker info | Select-String -Pattern 'OSType|Server Version|Operating System' - exit 0 - } - - # Discover dockerd.exe via the registered `docker` service's - # ImagePath in the registry. This works regardless of where the - # runner image installed Docker. - $svcKey = 'HKLM:\SYSTEM\CurrentControlSet\Services\docker' - if (-not (Test-Path $svcKey)) { - Write-Host '--- diagnostic: docker service not registered ---' - Get-Service docker* -ErrorAction SilentlyContinue | Format-List Name,Status,StartType - Get-Command dockerd -ErrorAction SilentlyContinue | Format-List Name,Source - throw 'docker service is not registered; cannot determine dockerd.exe path.' - } - $imagePath = (Get-ItemProperty -Path $svcKey -Name ImagePath).ImagePath - Write-Host "Service ImagePath: $imagePath" - # ImagePath may be quoted and may include trailing args; extract - # the first token (the .exe path) and strip surrounding quotes. - if ($imagePath -match '^"([^"]+)"' -or $imagePath -match '^(\S+\.exe)') { - $dockerd = $Matches[1] - } else { - $dockerd = ($imagePath -split '\s+')[0] - } - if (-not (Test-Path $dockerd)) { - Write-Host '--- diagnostic: dockerd path from registry does not exist ---' - Write-Host " resolved path: $dockerd" - Get-Command dockerd -ErrorAction SilentlyContinue | Format-List Name,Source - throw "dockerd.exe not found at $dockerd" - } - Write-Host "Using dockerd at: $dockerd" - - # Stop, re-register the service, restart. - Stop-Service docker - & $dockerd --unregister-service - if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } - & $dockerd --register-service --service-name docker - if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } - Start-Service docker - - # Verify the switch took effect. - $after = (docker info --format '{{.OSType}}' 2>$null) - Write-Host "Current OSType after switch: $after" - if ($after -ne 'windows') { - Write-Host 'docker info dump on failure:' - docker info - throw "Docker daemon did not switch to Windows containers (OSType=$after)." - } - docker info | Select-String -Pattern 'OSType|Server Version|Operating System' - - - name: Build devcontainer image - shell: pwsh - working-directory: .devcontainer - run: | - # Context is .devcontainer/ so `COPY shared\...` resolves. - # Tag mirrors the devcontainer.json name so it's recognisable - # in `docker images`. -m 2GB is required for the VS Build Tools - # install (Microsoft's documented minimum; default 1 GB silently - # fails partway through). - docker build ` - --memory 2GB ` - --file windows\Dockerfile ` - --tag loader-devcontainer-windows:smoke ` - . - - - name: Smoke test — Go - shell: pwsh - run: | - # Run `go test ./...` from the repo's Go module inside the - # built image. The container's working dir is C:\workspaces\loader; - # bind-mount the checkout there. - docker run --rm ` - --isolation=hyperv ` - -v "${{ github.workspace }}:C:\workspaces\loader" ` - -w C:\workspaces\loader ` - loader-devcontainer-windows:smoke ` - powershell -NoProfile -Command "go vet ./... ; if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }" - - - name: Smoke test — buf generate (Go) - shell: pwsh - run: | - docker run --rm ` - --isolation=hyperv ` - -v "${{ github.workspace }}:C:\workspaces\loader" ` - -w C:\workspaces\loader\test\go-tableau-loader ` - loader-devcontainer-windows:smoke ` - powershell -NoProfile -Command "buf generate .. ; if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }" - - - name: Smoke test — version banner - shell: pwsh - run: | - docker run --rm ` - --isolation=hyperv ` - loader-devcontainer-windows:smoke ` - powershell -NoProfile -ExecutionPolicy Bypass -File C:\loader\postcreate-banner.ps1 diff --git a/CLAUDE.md b/CLAUDE.md index 3457e10..ea46eb0 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -24,7 +24,7 @@ All build and test work happens **per language** inside `test/-tableau-loa To rebuild against the legacy v3 protobuf line: `LOADER_PROTOBUF_VERSION=3.21.12 code .` then **Reopen in Container**. The Dockerfile ARG `PROTOBUF_VERSION` is wired to that host env var via `devcontainer.json` build args; vcpkg manifest mode pins the override and the post-install assertion fails the build if anything resolves to the wrong version. See `.devcontainer/README.md` for the longer how-to and host-OS caveats (notably: Windows users should check the workspace out under WSL2, not `/mnt/c/`, for usable bind-mount perf). -CI's primary test path does **not** build the devcontainer images. The `testing-{cpp,go,csharp}.yml` workflows run `lukka/run-vcpkg` directly on GitHub-hosted runners (faster cached vcpkg installs). Two **smoke** workflows additionally build the devcontainer images themselves on devcontainer-touching PRs: `devcontainer-linux-smoke.yml` (amd64 + arm64, runs `buf generate` + `go vet` inside the image) and `devcontainer-windows-smoke.yml` (windows-2022, Windows-containers mode). They're path-gated to `.devcontainer/{linux,windows,shared}/**` so they don't burn CI minutes on unrelated PRs, but a `versions.env` typo or a Dockerfile regression won't ship. All toolchain versions (Go, buf, protobuf, vcpkg baseline commit, .NET, Node, CMake) are pinned in **`.devcontainer/shared/versions.env`** — the single source of truth consumed by `.devcontainer/linux/Dockerfile`, `.devcontainer/windows/Dockerfile`, `prepare.bat`, and every `.github/workflows/testing-*.yml` (via the **`./.github/actions/load-versions`** composite action that exports each `KEY=VALUE` to `$GITHUB_ENV`). Bump versions there; everything downstream picks them up automatically. The `.devcontainer/` directory has three subfolders: `linux/` (multi-arch, recommended), `windows/` (windows/amd64-only Windows container for native-MSVC dev), and `macos/` (docs only — Apple's licence forbids macOS containers; macOS users run the Linux container). +CI's primary test path does **not** build the devcontainer images. The `testing-{cpp,go,csharp}.yml` workflows run `lukka/run-vcpkg` directly on GitHub-hosted runners (faster cached vcpkg installs). One **smoke** workflow additionally builds the devcontainer image itself on devcontainer-touching PRs: `devcontainer-linux-smoke.yml` (amd64 + arm64, runs `buf generate` + `go vet` inside the image). It's path-gated to `.devcontainer/{linux,shared}/**` so it doesn't burn CI minutes on unrelated PRs, but a `versions.env` typo or a Dockerfile regression won't ship. All toolchain versions (Go, buf, protobuf, vcpkg baseline commit, .NET, Node, CMake) are pinned in **`.devcontainer/shared/versions.env`** — the single source of truth consumed by `.devcontainer/linux/Dockerfile`, `prepare.bat`, and every `.github/workflows/testing-*.yml` (via the **`./.github/actions/load-versions`** composite action that exports each `KEY=VALUE` to `$GITHUB_ENV`). Bump versions there; everything downstream picks them up automatically. The `.devcontainer/` directory has two subfolders: `linux/` (multi-arch, recommended for every host that can run Docker including Windows + WSL2) and `macos/` (docs only — Apple's licence forbids macOS containers; macOS users run the Linux container or use the documented `brew install` recipe). Bare-metal Windows users (no Docker, no WSL2) use `prepare.bat` for the C++ toolchain plus winget for Go/.NET/Node — the loader never had a Windows-container devcontainer because the Windows-container ecosystem (HNS, WinNAT, NDIS filter drivers) is too sensitive to host config to be reliable for daily dev. ### Plugin development (Go module at repo root) diff --git a/README.md b/README.md index 754b40d..7818dd6 100644 --- a/README.md +++ b/README.md @@ -28,27 +28,31 @@ The official config loader for [Tableau](https://github.com/tableauio/tableau). The fastest way to get a reproducible build environment is to open the repo in VS Code and choose **Reopen in Container**. The -[`.devcontainer/`](./.devcontainer/) directory ships **two** container -variants plus a macOS docs path: - -- [`.devcontainer/linux/`](./.devcontainer/linux/) — recommended for - every host (Linux, macOS Intel + Apple Silicon, Windows + WSL2). - Multi-arch (linux/amd64 + linux/arm64). -- [`.devcontainer/windows/`](./.devcontainer/windows/) — - Windows-host-only (windows/amd64), for users who specifically need a - native MSVC environment inside the container without WSL2. -- [`.devcontainer/macos/`](./.devcontainer/macos/) — documentation only. - Apple's licence forbids macOS containers; macOS users use the Linux - variant. - -When you run **Dev Containers: Reopen in Container** and both `linux/` -and `windows/` are present, VS Code shows a picker — choose whichever -matches your host. All version pins (Go, buf, protobuf, vcpkg baseline, -.NET, Node, CMake) live in +[`.devcontainer/linux/`](./.devcontainer/linux/) directory ships a +multi-arch (linux/amd64 + linux/arm64) container that works on every +host that can run Docker: Linux, macOS (Intel + Apple Silicon), and +Windows + WSL2. + +All version pins (Go, buf, protobuf, vcpkg baseline, .NET, Node, CMake) +live in [`.devcontainer/shared/versions.env`](./.devcontainer/shared/versions.env), the single source of truth shared with `prepare.bat` and CI. First -container build is one-time ~25 min (Linux) / ~45 min (Windows); vcpkg -compiles protobuf from source. Subsequent reopens are near-instant. +container build is one-time ~25 min (vcpkg compiles protobuf from +source). Subsequent reopens are near-instant. + +For hosts that can't or won't run Docker: +- **macOS** — see [`.devcontainer/macos/`](./.devcontainer/macos/) for + a native `brew install` recipe with versions pinned to `versions.env`. +- **Windows (bare-metal, native MSVC)** — see + [`prepare.bat`](./prepare.bat) for the C++ toolchain (MSVC, CMake, + Ninja, vcpkg + protobuf, buf), plus one-line winget installs for + Go / .NET / Node: + + ```cmd + winget install --id GoLang.Go.1.24 -e + winget install --id Microsoft.DotNet.SDK.8 -e + winget install --id OpenJS.NodeJS.LTS -e + ``` After the container starts you can skip the per-language setup below and jump straight to **[C++](#c)** / **[Go](#go)** / **[C#](#c-1)** / From 23a38bf51e017e4c0ca54a9b06664ec6d95a6f96 Mon Sep 17 00:00:00 2001 From: wenchy Date: Tue, 2 Jun 2026 21:11:41 +0800 Subject: [PATCH 38/66] refactor(devcontainer): flatten layout - one container, no subdirs The Linux/Windows/macOS split was never useful: with the Windows-container variant gone, linux/ and shared/ subdirs were just ceremony for a single Dockerfile + a single shared file. macos/ was docs-only and easier to inline into the repo root README's prerequisites section, alongside the Windows bare-metal recipe. File moves (all via git mv so history is preserved): .devcontainer/linux/Dockerfile -> .devcontainer/Dockerfile .devcontainer/linux/devcontainer.json -> .devcontainer/devcontainer.json .devcontainer/linux/README.md -> .devcontainer/README.md .devcontainer/shared/versions.env -> .devcontainer/versions.env .devcontainer/shared/postcreate-banner.sh -> .devcontainer/postcreate-banner.sh .github/workflows/devcontainer-linux-smoke.yml -> .github/workflows/devcontainer-smoke.yml Deletions: .devcontainer/shared/README.md (parsing-rules section moved into .devcontainer/README.md) .devcontainer/macos/README.md (brew-install recipe inlined into the repo root README) Path updates: - Dockerfile: COPY shared/foo -> COPY foo - devcontainer.json: drop "context": "..", default ('.') is correct - prepare.bat: %~dp0.devcontainer\shared\versions.env -> %~dp0.devcontainer\versions.env - load-versions action: file=.devcontainer/shared/versions.env -> file=.devcontainer/versions.env - devcontainer-smoke.yml: drop linux/ prefix, broaden path filter to .devcontainer/** - CLAUDE.md, repo root README.md: paths and the macOS pointer Net diff: +132 / -252 lines. Single source of truth, single Dockerfile, single banner script. Co-Authored-By: Claude Opus 4.6 --- .devcontainer/{linux => }/Dockerfile | 19 ++--- .devcontainer/{linux => }/README.md | 78 +++++++++++------ .devcontainer/{linux => }/devcontainer.json | 8 +- .devcontainer/macos/README.md | 84 ------------------- .../{shared => }/postcreate-banner.sh | 0 .devcontainer/shared/README.md | 54 ------------ .devcontainer/{shared => }/versions.env | 20 ++--- .github/actions/load-versions/action.yml | 6 +- ...linux-smoke.yml => devcontainer-smoke.yml} | 47 +++++------ CLAUDE.md | 2 +- README.md | 49 ++++++----- prepare.bat | 17 ++-- 12 files changed, 132 insertions(+), 252 deletions(-) rename .devcontainer/{linux => }/Dockerfile (92%) rename .devcontainer/{linux => }/README.md (66%) rename .devcontainer/{linux => }/devcontainer.json (92%) delete mode 100644 .devcontainer/macos/README.md rename .devcontainer/{shared => }/postcreate-banner.sh (100%) delete mode 100644 .devcontainer/shared/README.md rename .devcontainer/{shared => }/versions.env (74%) rename .github/workflows/{devcontainer-linux-smoke.yml => devcontainer-smoke.yml} (65%) diff --git a/.devcontainer/linux/Dockerfile b/.devcontainer/Dockerfile similarity index 92% rename from .devcontainer/linux/Dockerfile rename to .devcontainer/Dockerfile index 694e9c5..759e09e 100644 --- a/.devcontainer/linux/Dockerfile +++ b/.devcontainer/Dockerfile @@ -1,16 +1,15 @@ # syntax=docker/dockerfile:1.7 -# tableauio/loader devcontainer (linux) +# tableauio/loader devcontainer # # Single-stage, multi-arch (amd64 + arm64) image bringing the full # C++/Go/.NET/Node toolchain plus protobuf at the exact versions CI uses. # -# All version pins are read from ../shared/versions.env (the single source -# of truth shared with the Windows container, prepare.bat, and CI). To bump -# Go / buf / protobuf / .NET / Node / vcpkg-baseline, edit that file — not -# this one. +# All version pins are read from ./versions.env (the single source of +# truth shared with prepare.bat and the CI workflows). To bump Go / buf / +# protobuf / .NET / Node / vcpkg-baseline, edit that file — not this one. # -# Build context is .devcontainer/ (the parent of this directory), so that -# `COPY shared/...` resolves. devcontainer.json sets `"context": ".."`. +# Build context is .devcontainer/ (the directory containing this file), +# so `COPY versions.env ...` resolves directly. FROM mcr.microsoft.com/devcontainers/cpp:1-ubuntu-24.04 @@ -19,7 +18,7 @@ FROM mcr.microsoft.com/devcontainers/cpp:1-ubuntu-24.04 # location and `source` it in every RUN that needs the values; ARG/ENV alone # don't survive across RUN boundaries in BuildKit, so we re-source per layer. # --------------------------------------------------------------------------- -COPY shared/versions.env /opt/versions.env +COPY versions.env /opt/versions.env # --------------------------------------------------------------------------- # Architecture detection. BuildKit auto-populates TARGETARCH; we resolve it @@ -139,7 +138,7 @@ case "$(basename "${INFO_FILE:-/missing}" 2>/dev/null)" in *) echo "ERROR: installed protobuf does not match requested version ${PROTOBUF_VERSION}." echo " vcpkg installed-file marker: ${INFO_FILE:-}" - echo " Bump VCPKG_BASELINE_COMMIT in .devcontainer/shared/versions.env" + echo " Bump VCPKG_BASELINE_COMMIT in .devcontainer/versions.env" echo " (and prepare.bat + testing-cpp.yml will pick it up automatically)" echo " to a commit that knows about the requested version." exit 1 @@ -182,5 +181,5 @@ ENV CMAKE_PREFIX_PATH=/opt/vcpkg/active # Copy the post-create banner script into a stable on-image path so # devcontainer.json can call it without needing the workspace mount. -COPY shared/postcreate-banner.sh /usr/local/bin/loader-devcontainer-banner +COPY postcreate-banner.sh /usr/local/bin/loader-devcontainer-banner RUN chmod +x /usr/local/bin/loader-devcontainer-banner diff --git a/.devcontainer/linux/README.md b/.devcontainer/README.md similarity index 66% rename from .devcontainer/linux/README.md rename to .devcontainer/README.md index 9fd33da..4a76484 100644 --- a/.devcontainer/linux/README.md +++ b/.devcontainer/README.md @@ -3,14 +3,14 @@ The recommended way to develop on `tableauio/loader`. One container, all four target languages (C++17, Go, .NET, Node) plus protobuf via vcpkg, pinned to the exact toolchain CI uses. All version pins live in -[`../shared/versions.env`](../shared/versions.env) — bumping any of them -is a one-line change consumed by this Dockerfile, `prepare.bat`, and -the CI workflows. +[`./versions.env`](./versions.env) — bumping any of them is a one-line +change consumed by this Dockerfile, `prepare.bat`, and the CI +workflows. -Use this variant on **every** host that can run Docker: Linux (amd64 + -arm64), macOS (Intel + Apple Silicon), and Windows + WSL2. For Windows -hosts that prefer bare-metal native dev (no Docker, no WSL2), see the -[`prepare.bat`](../../prepare.bat) bootstrap at the repo root. +Use the devcontainer on **every** host that can run Docker: Linux +(amd64 + arm64), macOS (Intel + Apple Silicon), and Windows + WSL2. For +Windows hosts that prefer bare-metal native dev (no Docker, no WSL2), +see the [`prepare.bat`](../prepare.bat) bootstrap at the repo root. ## Prerequisites @@ -35,9 +35,9 @@ no PATH dance, no extra cmake flags. ## Pin a different protobuf version Daily dev runs against the `PROTOBUF_VERSION` set in -[`../shared/versions.env`](../shared/versions.env) (CI's "modern" matrix -entry). To rebuild against the legacy v3 line for one container only, -without editing the shared file: +[`./versions.env`](./versions.env) (CI's "modern" matrix entry). To +rebuild against the legacy v3 line for one container only, without +editing the shared file: ```sh LOADER_PROTOBUF_VERSION=3.21.12 code . @@ -82,10 +82,44 @@ The architecture choice is detected from BuildKit's `TARGETARCH` and fed into Go / buf / vcpkg triplet selection. Docker auto-selects the host arch on build. -The `dockerfile` field in `devcontainer.json` is `Dockerfile`, but the -build `context` is the parent `.devcontainer/` directory so the -Dockerfile's `COPY shared/versions.env …` and `COPY shared/postcreate-banner.sh …` -lines resolve. +The build context defaults to the directory containing `devcontainer.json` +(this `.devcontainer/` directory), so the Dockerfile's +`COPY versions.env …` and `COPY postcreate-banner.sh …` lines resolve +directly. + +## `versions.env` parsing rules + +`versions.env` is consumed by the Dockerfile, `prepare.bat`, and the CI +workflows. The format is intentionally minimal so every consumer can +parse it with a builtin: + +- One assignment per line, exactly `KEY=VALUE`. +- No quotes, no spaces around `=`, no inline comments after the value. +- Comments start at column 0 with `#`. +- Blank lines are ignored. +- No shell expansion — values are bare literals. + +Quick parsers per consumer: + +```sh +# POSIX shell (Dockerfile) +. .devcontainer/versions.env +echo "$GO_VERSION" +``` + +```cmd +:: Windows cmd (prepare.bat) +for /f "tokens=1,2 delims==" %%a in (.devcontainer\versions.env) do ( + if not "%%a"=="" if not "%%a:~0,1%"=="#" set "%%a=%%b" +) +echo %GO_VERSION% +``` + +```yaml +# GitHub Actions +- uses: ./.github/actions/load-versions +# subsequent steps reference the values as ${{ env.GO_VERSION }} etc. +``` ## Troubleshooting @@ -115,15 +149,7 @@ directory. A fresh clone doesn't have these stale artefacts. ## Falling back If you can't run Docker (corp policy, restricted machines, etc.) the -existing manual setup paths in the [repo README](../../README.md) — -Windows `prepare.bat`, per-language `Install protobuf` instructions — -still work. The devcontainer is the recommended path; the rest is the -supported fallback. - -For a Windows host that prefers bare-metal native dev (no Docker, no -WSL2), see [`prepare.bat`](../../prepare.bat) at the repo root — it -bootstraps the C++ toolchain (MSVC, CMake, Ninja, vcpkg+protobuf, buf). -You'll additionally need Go, .NET SDK, and Node.js, which one-line winget -installs cover (see the repo root [README](../../README.md)). For macOS -hosts where you'd rather not run a Linux container at all, see the -[macOS notes](../macos/). +existing manual setup paths in the [repo README](../README.md) — Windows +`prepare.bat`, per-language `Install protobuf` instructions, the macOS +Homebrew recipe — still work. The devcontainer is the recommended path; +the rest is the supported fallback. diff --git a/.devcontainer/linux/devcontainer.json b/.devcontainer/devcontainer.json similarity index 92% rename from .devcontainer/linux/devcontainer.json rename to .devcontainer/devcontainer.json index 8def01e..2b0d48a 100644 --- a/.devcontainer/linux/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -5,7 +5,7 @@ // Bare-metal Windows users (no Docker) bootstrap via prepare.bat at // the repo root. // - // See ../shared/versions.env for all toolchain pins. + // See ./versions.env for all toolchain pins. "name": "tableauio/loader", // Build args wire host env to Dockerfile ARGs: // LOADER_PROTOBUF_VERSION on the host -> PROTOBUF_VERSION inside. @@ -13,11 +13,11 @@ // matrix entry). To rebuild against the legacy v3 line: // LOADER_PROTOBUF_VERSION=3.21.12 code . # then Reopen in Container. // - // "context" is the parent .devcontainer/ directory so the Dockerfile's - // `COPY shared/...` lines resolve. + // "context" defaults to the directory holding this devcontainer.json + // (.devcontainer/), so the Dockerfile's `COPY versions.env ...` + // resolves directly. "build": { "dockerfile": "Dockerfile", - "context": "..", "args": { "PROTOBUF_VERSION": "${localEnv:LOADER_PROTOBUF_VERSION:}" } diff --git a/.devcontainer/macos/README.md b/.devcontainer/macos/README.md deleted file mode 100644 index 7f234ad..0000000 --- a/.devcontainer/macos/README.md +++ /dev/null @@ -1,84 +0,0 @@ -# Dev Container — macOS notes - -> **There is no macOS devcontainer.** Apple's macOS SLA prohibits -> virtualising macOS on non-Apple hardware, and Microsoft / Docker -> publish no `mcr.microsoft.com/macos/...` base image. This directory is -> documentation only — it intentionally has no `devcontainer.json` so -> the VS Code Dev Containers picker does not list it. - -## Recommended path on macOS - -Use the [Linux devcontainer](../linux/). Docker Desktop on macOS runs -Linux containers natively: - -| Host | Container arch the Linux Dockerfile builds | Native? | -| --- | --- | --- | -| Apple Silicon (M-series) | `linux/arm64` | yes | -| Intel Mac | `linux/amd64` | yes | - -Confirm with: - -```sh -docker info | grep Architecture -``` - -→ `aarch64` on Apple Silicon, `x86_64` on Intel. Neither uses Rosetta or -QEMU emulation. - -## If you want to build on macOS *without* a container - -The same toolchain versions live in -[`../shared/versions.env`](../shared/versions.env) — read them and -install via Homebrew. There is no automated script (yet) because the -Linux container is the recommended path; this is documented for parity -with Windows's `prepare.bat` fallback. - -```sh -# Read pinned versions -. .devcontainer/shared/versions.env - -# Toolchain via Homebrew -brew install \ - "go@${GO_VERSION%.*}" \ - "buf" \ - "protobuf" \ - "dotnet@${DOTNET_VERSION%.*}" \ - "node@${NODE_VERSION}" \ - "cmake" \ - "ninja" -``` - -> **Caveats** -> -> - Homebrew tracks the latest formula version, not `versions.env`'s -> exact pin. For protobuf in particular, this means you may end up -> with whichever 6.x or later release Homebrew currently ships, not -> `${PROTOBUF_VERSION}`. If the gencode/runtime version check bites, -> either install via vcpkg manifest mode (see the repo -> [README → Install protobuf](../../README.md#install-protobuf)) or -> switch to the Linux devcontainer. -> -> - `dotnet@8` and `node@20` are versioned brews; the unversioned `dotnet` -> / `node` formulae track their respective LTS lines and may drift. -> -> - Apple Silicon users: nothing in the loader requires Rosetta. If you -> see x86_64 Homebrew complaints, run `arch -arm64 brew ...`. - -## Why no macOS container? - -Three blocking reasons: - -1. **Apple licensing.** macOS may only run on Apple-branded hardware. - Container images are by design hardware-portable; the licence - forbids that. -2. **No base image.** Microsoft (the publisher of nearly every Windows / - Linux base image used by Docker) doesn't publish a macOS image, and - no third party does either. -3. **No runtime.** Even if a base image existed, `dockerd` on macOS runs - a Linux VM under the hood (LinuxKit on Intel, virtualization-framework - on Apple Silicon). It cannot run a macOS container. - -If your build genuinely needs Apple-specific toolchains (Xcode, -codesigning, `security` keychain), use a real macOS host — that's what -GitHub Actions' `macos-latest` runners are. We don't have one of those -in our matrix today, so this isn't a parity loss. diff --git a/.devcontainer/shared/postcreate-banner.sh b/.devcontainer/postcreate-banner.sh similarity index 100% rename from .devcontainer/shared/postcreate-banner.sh rename to .devcontainer/postcreate-banner.sh diff --git a/.devcontainer/shared/README.md b/.devcontainer/shared/README.md deleted file mode 100644 index 15898bc..0000000 --- a/.devcontainer/shared/README.md +++ /dev/null @@ -1,54 +0,0 @@ -# tableauio/loader — `.devcontainer/shared/` - -Files in this directory are consumed by the Linux devcontainer -(`../linux/`) and by host-side scripts (`prepare.bat`, CI workflows). -Edit them with cross-platform parsing in mind. - -## Files - -| File | Consumers | Format | -| --- | --- | --- | -| [`versions.env`](./versions.env) | `linux/Dockerfile`, `prepare.bat`, every `.github/workflows/*.yml` | `KEY=VALUE`, one per line, no quotes, no `$VAR` expansion | -| [`postcreate-banner.sh`](./postcreate-banner.sh) | `linux/devcontainer.json` `postCreateCommand` | POSIX `sh` | - -## `versions.env` parsing rules - -Every consumer needs to read this file with at most a one-liner. The format -is therefore extremely conservative: - -- **One assignment per line**, exactly `KEY=VALUE`. -- **No quotes**, no spaces around `=`, no inline comments after the value. -- **Comments start at column 0** with `#`. -- **Blank lines** are ignored. -- **No shell expansion** — values are bare literals. - -Quick parsers per language: - -```sh -# POSIX shell (Linux Dockerfile) -. .devcontainer/shared/versions.env -echo "$GO_VERSION" -``` - -```cmd -:: Windows cmd (prepare.bat) -for /f "tokens=1,2 delims==" %%a in (.devcontainer\shared\versions.env) do ( - if not "%%a"=="" if not "%%a:~0,1%"=="#" set "%%a=%%b" -) -echo %GO_VERSION% -``` - -```yaml -# GitHub Actions -- uses: ./.github/actions/load-versions -# subsequent steps reference the values as ${{ env.GO_VERSION }} etc. -``` - -## Lockstep rule - -`VCPKG_BASELINE_COMMIT` is the trickiest pin: it must know about every -`PROTOBUF_VERSION` value used anywhere (devcontainer default + every -`testing-cpp.yml` matrix entry). Bumping `PROTOBUF_VERSION` to a value -the current baseline doesn't know is caught at build time by the -post-install assertion in `linux/Dockerfile` and `prepare.bat` — fail -loud, no silent wrong-version installs. diff --git a/.devcontainer/shared/versions.env b/.devcontainer/versions.env similarity index 74% rename from .devcontainer/shared/versions.env rename to .devcontainer/versions.env index db03824..502c033 100644 --- a/.devcontainer/shared/versions.env +++ b/.devcontainer/versions.env @@ -1,7 +1,7 @@ # tableauio/loader — pinned toolchain versions. # # Single source of truth consumed by: -# - .devcontainer/linux/Dockerfile (sourced as a shell file) +# - .devcontainer/Dockerfile (sourced as a shell file) # - prepare.bat (parsed via `for /f` in cmd) # - .github/workflows/*.yml (read by the load-versions composite action into $GITHUB_ENV) # @@ -12,7 +12,7 @@ # - Values are bare strings — no shell expansion, no $VAR references. # # Bumping any of these is a one-line change. The matching post-install -# assertion in each Dockerfile / prepare.bat catches the case where the +# assertion in the Dockerfile / prepare.bat catches the case where the # vcpkg baseline doesn't yet know about the requested PROTOBUF_VERSION. # Go SDK shipped in the devcontainer. Must be ≥ the `go` directive in go.mod; @@ -20,7 +20,7 @@ # go-version-file, so devcontainer is the only place Go is hard-pinned. GO_VERSION=1.24.0 -# buf CLI release. Pinned in devcontainers, prepare.bat, and all three +# buf CLI release. Pinned in the devcontainer, prepare.bat, and all three # testing-*.yml workflows. Bump everywhere together. BUF_VERSION=1.67.0 @@ -30,22 +30,20 @@ BUF_VERSION=1.67.0 PROTOBUF_VERSION=6.33.4 # vcpkg checkout commit. Same value used by: -# - .devcontainer/linux/Dockerfile (ARG VCPKG_BASELINE_COMMIT) -# - prepare.bat (set VCPKG_BASELINE_COMMIT=...) +# - .devcontainer/Dockerfile (ARG VCPKG_BASELINE_COMMIT) +# - prepare.bat (set VCPKG_BASELINE_COMMIT=...) # - .github/workflows/testing-cpp.yml (env: VCPKG_COMMIT) # This commit MUST know about every PROTOBUF_VERSION in the testing-cpp.yml # matrix; bump it forward (never sideways) when adding a new protobuf entry. VCPKG_BASELINE_COMMIT=dc8d75cfc3281b8e2a4ed8ee4163c891190df932 -# .NET SDK major.minor. apt installs `dotnet-sdk-${DOTNET_VERSION}` on Linux, -# Chocolatey installs `dotnet-${DOTNET_VERSION}-sdk` on Windows. CI uses -# `${DOTNET_VERSION}.x` with actions/setup-dotnet. +# .NET SDK major.minor. apt installs `dotnet-sdk-${DOTNET_VERSION}` on Linux. +# CI uses `${DOTNET_VERSION}.x` with actions/setup-dotnet. DOTNET_VERSION=8.0 -# Node.js LTS major. NodeSource apt repo is `setup_${NODE_VERSION}.x`, -# Chocolatey package is `nodejs-lts --version=${NODE_VERSION}.*`. +# Node.js LTS major. NodeSource apt repo is `setup_${NODE_VERSION}.x`. NODE_VERSION=20 -# CMake version installed by prepare.bat (Linux devcontainer base image +# CMake version installed by prepare.bat (the devcontainer base image # already ships a recent cmake). CMAKE_VERSION=3.31.8 diff --git a/.github/actions/load-versions/action.yml b/.github/actions/load-versions/action.yml index 45fc8ed..5688415 100644 --- a/.github/actions/load-versions/action.yml +++ b/.github/actions/load-versions/action.yml @@ -1,7 +1,7 @@ name: Load pinned versions description: > - Reads .devcontainer/shared/versions.env (the single source of truth shared - with the devcontainers and prepare.bat) and exports each KEY=VALUE pair to + Reads .devcontainer/versions.env (the single source of truth shared + with the devcontainer and prepare.bat) and exports each KEY=VALUE pair to GITHUB_ENV so subsequent steps can reference them as env-context values (e.g. env.BUF_VERSION). @@ -18,7 +18,7 @@ runs: shell: bash run: | set -eu - file=.devcontainer/shared/versions.env + file=.devcontainer/versions.env if [ ! -f "$file" ]; then echo "::error::Missing $file; cannot resolve pinned tool versions." exit 1 diff --git a/.github/workflows/devcontainer-linux-smoke.yml b/.github/workflows/devcontainer-smoke.yml similarity index 65% rename from .github/workflows/devcontainer-linux-smoke.yml rename to .github/workflows/devcontainer-smoke.yml index 5fda640..4c9222b 100644 --- a/.github/workflows/devcontainer-linux-smoke.yml +++ b/.github/workflows/devcontainer-smoke.yml @@ -1,19 +1,18 @@ -name: Devcontainer Linux Smoke +name: Devcontainer Smoke -# Builds the Linux-container variant of the devcontainer and runs a -# minimal smoke test inside it. Triggered only when files that affect -# the Linux image change, so we don't burn CI minutes on unrelated PRs. +# Builds the devcontainer image and runs a minimal smoke test inside it. +# Triggered only when files that affect the image change, so we don't +# burn CI minutes on unrelated PRs. # # What this catches that testing-cpp.yml does not: -# 1. arm64 coverage. The Linux Dockerfile claims linux/amd64 + -# linux/arm64; testing-cpp.yml only exercises amd64. This workflow -# builds the image natively on ubuntu-24.04-arm runners. -# 2. shared/versions.env regressions. A typo in the shared file (stray -# space, quote, etc.) silently breaks the devcontainer build for -# anyone running `Reopen in Container`. The other testing-*.yml -# workflows now consume versions.env too, but only the keys they -# need; only this workflow exercises the full file in a real -# Dockerfile build. +# 1. arm64 coverage. The Dockerfile claims linux/amd64 + linux/arm64; +# testing-cpp.yml only exercises amd64. This workflow builds the +# image natively on ubuntu-24.04-arm runners. +# 2. versions.env regressions. A typo in the shared file (stray space, +# quote, etc.) silently breaks the devcontainer build for anyone +# running `Reopen in Container`. The testing-*.yml workflows +# consume versions.env too, but only the keys they need; only this +# workflow exercises the full file in a real Dockerfile build. # # What this DOES NOT do: # - Run the full C++ / C# test matrices. Those are testing-{cpp,csharp}.yml, @@ -24,14 +23,12 @@ name: Devcontainer Linux Smoke on: pull_request: paths: - - '.devcontainer/linux/**' - - '.devcontainer/shared/**' - - '.github/workflows/devcontainer-linux-smoke.yml' + - '.devcontainer/**' + - '.github/workflows/devcontainer-smoke.yml' push: branches: [master, main] paths: - - '.devcontainer/linux/**' - - '.devcontainer/shared/**' + - '.devcontainer/**' workflow_dispatch: permissions: @@ -51,7 +48,7 @@ jobs: arch: amd64 - runner: ubuntu-24.04-arm arch: arm64 - name: build linux devcontainer + smoke (${{ matrix.arch }}) + name: build devcontainer + smoke (${{ matrix.arch }}) runs-on: ${{ matrix.runner }} timeout-minutes: 45 @@ -65,20 +62,20 @@ jobs: - name: Build devcontainer image working-directory: .devcontainer run: | - # Context is .devcontainer/ so `COPY shared/...` resolves. + # Context is .devcontainer/ so `COPY versions.env` etc. resolve. # --load puts the image into the local docker daemon so the # subsequent `docker run` smoke steps can use it. docker buildx build \ --load \ - --file linux/Dockerfile \ - --tag loader-devcontainer-linux:smoke \ + --file Dockerfile \ + --tag loader-devcontainer:smoke \ . - name: Smoke — version banner run: | # Confirms the image runs at all and that all five toolchain # binaries (go, buf, protoc, dotnet, node) resolve. - docker run --rm loader-devcontainer-linux:smoke \ + docker run --rm loader-devcontainer:smoke \ /usr/local/bin/loader-devcontainer-banner - name: Smoke — buf generate (Go) @@ -89,7 +86,7 @@ jobs: docker run --rm \ -v "${{ github.workspace }}:/workspaces/loader" \ -w /workspaces/loader/test/go-tableau-loader \ - loader-devcontainer-linux:smoke \ + loader-devcontainer:smoke \ buf generate .. - name: Smoke — go vet (plugin packages only) @@ -100,5 +97,5 @@ jobs: docker run --rm \ -v "${{ github.workspace }}:/workspaces/loader" \ -w /workspaces/loader \ - loader-devcontainer-linux:smoke \ + loader-devcontainer:smoke \ go vet ./cmd/... ./pkg/... ./internal/options/... ./internal/loadutil/... ./internal/xproto/... diff --git a/CLAUDE.md b/CLAUDE.md index ea46eb0..2e2f5fa 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -24,7 +24,7 @@ All build and test work happens **per language** inside `test/-tableau-loa To rebuild against the legacy v3 protobuf line: `LOADER_PROTOBUF_VERSION=3.21.12 code .` then **Reopen in Container**. The Dockerfile ARG `PROTOBUF_VERSION` is wired to that host env var via `devcontainer.json` build args; vcpkg manifest mode pins the override and the post-install assertion fails the build if anything resolves to the wrong version. See `.devcontainer/README.md` for the longer how-to and host-OS caveats (notably: Windows users should check the workspace out under WSL2, not `/mnt/c/`, for usable bind-mount perf). -CI's primary test path does **not** build the devcontainer images. The `testing-{cpp,go,csharp}.yml` workflows run `lukka/run-vcpkg` directly on GitHub-hosted runners (faster cached vcpkg installs). One **smoke** workflow additionally builds the devcontainer image itself on devcontainer-touching PRs: `devcontainer-linux-smoke.yml` (amd64 + arm64, runs `buf generate` + `go vet` inside the image). It's path-gated to `.devcontainer/{linux,shared}/**` so it doesn't burn CI minutes on unrelated PRs, but a `versions.env` typo or a Dockerfile regression won't ship. All toolchain versions (Go, buf, protobuf, vcpkg baseline commit, .NET, Node, CMake) are pinned in **`.devcontainer/shared/versions.env`** — the single source of truth consumed by `.devcontainer/linux/Dockerfile`, `prepare.bat`, and every `.github/workflows/testing-*.yml` (via the **`./.github/actions/load-versions`** composite action that exports each `KEY=VALUE` to `$GITHUB_ENV`). Bump versions there; everything downstream picks them up automatically. The `.devcontainer/` directory has two subfolders: `linux/` (multi-arch, recommended for every host that can run Docker including Windows + WSL2) and `macos/` (docs only — Apple's licence forbids macOS containers; macOS users run the Linux container or use the documented `brew install` recipe). Bare-metal Windows users (no Docker, no WSL2) use `prepare.bat` for the C++ toolchain plus winget for Go/.NET/Node — the loader never had a Windows-container devcontainer because the Windows-container ecosystem (HNS, WinNAT, NDIS filter drivers) is too sensitive to host config to be reliable for daily dev. +CI's primary test path does **not** build the devcontainer image. The `testing-{cpp,go,csharp}.yml` workflows run `lukka/run-vcpkg` directly on GitHub-hosted runners (faster cached vcpkg installs). One **smoke** workflow additionally builds the devcontainer image itself on devcontainer-touching PRs: `devcontainer-smoke.yml` (amd64 + arm64, runs `buf generate` + `go vet` inside the image). It's path-gated to `.devcontainer/**` so it doesn't burn CI minutes on unrelated PRs, but a `versions.env` typo or a Dockerfile regression won't ship. All toolchain versions (Go, buf, protobuf, vcpkg baseline commit, .NET, Node, CMake) are pinned in **`.devcontainer/versions.env`** — the single source of truth consumed by `.devcontainer/Dockerfile`, `prepare.bat`, and every `.github/workflows/testing-*.yml` (via the **`./.github/actions/load-versions`** composite action that exports each `KEY=VALUE` to `$GITHUB_ENV`). Bump versions there; everything downstream picks them up automatically. The `.devcontainer/` directory is flat (one Dockerfile, one devcontainer.json, versions.env, and the postcreate banner script) — there's only one container, not a per-OS variant. macOS users either run the container under Docker Desktop or follow the brew-install recipe in the repo root README; Windows users either run the container under Docker+WSL2 or follow the `prepare.bat` + winget bare-metal recipe. ### Plugin development (Go module at repo root) diff --git a/README.md b/README.md index 7818dd6..04d25ea 100644 --- a/README.md +++ b/README.md @@ -10,40 +10,38 @@ The official config loader for [Tableau](https://github.com/tableauio/tableau). gencode/runtime version check via `PROTOBUF_VERSION` in the generated headers, so a mismatched `protoc` and `libprotobuf` will fail to link. -> **Migrating from the bundled-protobuf layout?** Loader used to vendor -> protobuf as a git submodule under `third_party/_submodules/protobuf` plus -> an `init.sh` / `init.bat` build pipeline. Both are gone. If you've checked -> the repo out before this change, clean up the orphan worktree and submodule -> metadata before building: -> -> ```sh -> git submodule deinit -f third_party/_submodules/protobuf -> rm -rf third_party/_submodules/protobuf .git/modules/third_party/_submodules/protobuf -> ``` -> -> Then install protobuf via one of the channels documented in -> [Install protobuf](#install-protobuf). - ### Recommended: Dev Container (any host OS) The fastest way to get a reproducible build environment is to open the repo in VS Code and choose **Reopen in Container**. The -[`.devcontainer/linux/`](./.devcontainer/linux/) directory ships a -multi-arch (linux/amd64 + linux/arm64) container that works on every -host that can run Docker: Linux, macOS (Intel + Apple Silicon), and -Windows + WSL2. +[`.devcontainer/`](./.devcontainer/) directory ships a multi-arch +(linux/amd64 + linux/arm64) container that works on every host that +can run Docker: Linux, macOS (Intel + Apple Silicon), and Windows + +WSL2. All version pins (Go, buf, protobuf, vcpkg baseline, .NET, Node, CMake) live in -[`.devcontainer/shared/versions.env`](./.devcontainer/shared/versions.env), -the single source of truth shared with `prepare.bat` and CI. First +[`.devcontainer/versions.env`](./.devcontainer/versions.env), the +single source of truth shared with `prepare.bat` and CI. First container build is one-time ~25 min (vcpkg compiles protobuf from source). Subsequent reopens are near-instant. For hosts that can't or won't run Docker: -- **macOS** — see [`.devcontainer/macos/`](./.devcontainer/macos/) for - a native `brew install` recipe with versions pinned to `versions.env`. -- **Windows (bare-metal, native MSVC)** — see + +- **macOS (native, no container).** Install via Homebrew. Versions + follow whatever the brew formula currently ships, which is usually + close enough; the loader's [`Install protobuf`](#install-protobuf) + section below covers the protobuf gotcha if you need an exact pin: + + ```sh + # Toolchain via Homebrew + brew install go buf protobuf dotnet@8 node@20 cmake ninja + ``` + + > Apple Silicon: nothing in the loader requires Rosetta. If you see + > x86_64 Homebrew complaints, run `arch -arm64 brew ...`. + +- **Windows (bare-metal, native MSVC).** Run [`prepare.bat`](./prepare.bat) for the C++ toolchain (MSVC, CMake, Ninja, vcpkg + protobuf, buf), plus one-line winget installs for Go / .NET / Node: @@ -59,8 +57,9 @@ and jump straight to **[C++](#c)** / **[Go](#go)** / **[C#](#c-1)** / **[TypeScript](#typescript)**. Requirements: Docker Desktop (Windows + macOS) or Docker Engine (Linux), -and the VS Code "Dev Containers" extension. See the per-variant READMEs -linked above for the longer how-to. +and the VS Code "Dev Containers" extension. See +[`.devcontainer/README.md`](./.devcontainer/README.md) for the longer +how-to. ### Install protobuf diff --git a/prepare.bat b/prepare.bat index 4e460b5..66f5abf 100644 --- a/prepare.bat +++ b/prepare.bat @@ -5,7 +5,7 @@ REM =========================================================================== REM prepare.bat — bootstrap a Windows build environment for the C++ loader. REM REM Installs (only if missing): Chocolatey, Ninja, CMake (version pinned in -REM .devcontainer/shared/versions.env), MSVC Build +REM .devcontainer/versions.env), MSVC Build REM Tools (Visual Studio 2022 Build Tools), buf CLI, and vcpkg. REM REM Then installs `protobuf` (and friends) into vcpkg using the static-CRT @@ -52,14 +52,13 @@ if "%SIMULATE_CLEAN%"=="1" echo [DRY-RUN] Simulating a clean machine (all tools echo [INFO] Preparing build environment... REM ----------------------------------------------------------------------- -REM Load pinned tool versions from .devcontainer/shared/versions.env. +REM Load pinned tool versions from .devcontainer/versions.env. REM -REM Single source of truth shared with the Linux/Windows devcontainers -REM and with the .github/workflows/*.yml CI workflows. Format is one -REM KEY=VALUE per line, no quotes, no $VAR expansion. See -REM .devcontainer/shared/README.md for the full format spec. +REM Single source of truth shared with the devcontainer and with the +REM .github/workflows/*.yml CI workflows. Format is one KEY=VALUE per +REM line, no quotes, no $VAR expansion. REM ----------------------------------------------------------------------- -set "VERSIONS_FILE=%~dp0.devcontainer\shared\versions.env" +set "VERSIONS_FILE=%~dp0.devcontainer\versions.env" if not exist "%VERSIONS_FILE%" ( echo [ERROR] Missing %VERSIONS_FILE%; cannot resolve pinned tool versions. exit /b 1 @@ -305,7 +304,7 @@ REM The CI workflow uses bufbuild/buf-action@v1 (also pinned to REM BUF_VERSION below) to do the same thing. REM buf is a single self-contained .exe; install it under REM %LOCALAPPDATA%\buf\bin\buf.exe to avoid requiring admin rights. -REM BUF_VERSION is sourced from .devcontainer/shared/versions.env. +REM BUF_VERSION is sourced from .devcontainer/versions.env. REM ----------------------------------------------------------------------- set "BUF_FOUND=0" if "%SIMULATE_CLEAN%"=="0" ( @@ -393,7 +392,7 @@ if errorlevel 1 ( REM Pin both the vcpkg checkout and the manifest's builtin-baseline to the REM same commit testing-cpp.yml uses. VCPKG_BASELINE_COMMIT is sourced from -REM .devcontainer/shared/versions.env (the single source of truth for the +REM .devcontainer/versions.env (the single source of truth for the REM Linux + Windows devcontainers, prepare.bat, and CI). To bump vcpkg, REM edit that file. set "VCPKG_TRIPLET=x64-windows-static" From 355aceba1b12b7807cea5df8653189aeb1a916e8 Mon Sep 17 00:00:00 2001 From: wenchy Date: Wed, 3 Jun 2026 14:37:12 +0800 Subject: [PATCH 39/66] perf(devcontainer): cache vcpkg binaries across builds MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two complementary cache mechanisms, both exploiting that the dominant build cost (~15-30 min) is vcpkg compiling protobuf from source under the host's C++ toolchain. 1. Dockerfile: BuildKit cache mount on /vcpkg-binarycache. `RUN --mount=type=cache,target=/vcpkg-binarycache,sharing=locked` plus `ENV VCPKG_DEFAULT_BINARY_CACHE=/vcpkg-binarycache` directs vcpkg to read/write its compiled-package zips into a directory whose contents persist across `docker build` invocations on the same host (BuildKit's own cache, not a Docker volume — volumes don't mount at build time). After the first build, every subsequent build with the same VCPKG_BASELINE_COMMIT + PROTOBUF_VERSION skips the compile and uses cached zips, even when an unrelated upstream layer (e.g. a BUF_VERSION bump) invalidates Docker's per-layer cache for the vcpkg RUN. 2. Smoke workflow: docker/build-push-action with cache-{from,to}=gha. Replaces the inline `docker buildx build` with the canonical build-push-action, which exports BuildKit's full cache (including the cache mount from #1) to GitHub's per-repo Actions cache via mode=max. Cold first run is still ~25 min; every subsequent run restores cached layers AND cached vcpkg binaries from GHA and completes in ~3-5 min. Mirrors testing-cpp.yml's approach (which actions/cache's $VCPKG_INSTALLED_DIR for the same reason) using the docker-native equivalent. Cache scope is keyed on `matrix.arch` so amd64 and arm64 caches don't cross-contaminate (vcpkg binaries are arch-specific). Adds `actions: write` permission to the workflow — required for `cache-to: type=gha` to push BuildKit cache entries to the per-repo Actions cache (~10 GB free quota; well under the cache size). Co-Authored-By: Claude Opus 4.6 --- .devcontainer/Dockerfile | 32 +++++++++++++++++++++++- .github/workflows/devcontainer-smoke.yml | 32 ++++++++++++++++-------- 2 files changed, 53 insertions(+), 11 deletions(-) diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index 759e09e..97df95c 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -86,12 +86,38 @@ EOF # --------------------------------------------------------------------------- ARG PROTOBUF_VERSION= ENV VCPKG_ROOT=/opt/vcpkg +# Tell vcpkg to share its compiled binaries via /vcpkg-binarycache. The +# directory is mounted from BuildKit's persistent cache via +# `RUN --mount=type=cache,...` below; that cache survives across +# `docker build` invocations on the same host (and across CI runs when +# `cache-to=type=gha,mode=max` is configured in the smoke workflow). +# +# The win: when an unrelated layer earlier in the Dockerfile changes +# (e.g. you bump BUF_VERSION in versions.env), Docker re-runs the vcpkg +# layer because its inputs changed, but vcpkg itself short-circuits the +# expensive ~15-30 min compile and just relinks the protobuf binaries +# from cache. First build pays the full cost; every later build with the +# same VCPKG_BASELINE_COMMIT + PROTOBUF_VERSION is fast. +ENV VCPKG_DEFAULT_BINARY_CACHE=/vcpkg-binarycache # `_ARG_PROTOBUF_VERSION` captures the Dockerfile ARG via Dockerfile-level # variable expansion BEFORE we source versions.env in the heredoc body # (otherwise the file default would clobber the build-arg). Empty string # means "no override; use file default". -RUN < /opt/vcpkg-manifest/vcpkg.json < Date: Thu, 4 Jun 2026 17:01:35 +0800 Subject: [PATCH 40/66] feat: replace prepare.bat with cross-platform make.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Single Python entrypoint for setup/generate/build/test/clean across Windows native, macOS, Linux, and devcontainer. Stdlib-only, Python 3.10+. CI workflows call it directly so dev and CI run identical commands. Key design points: - Windows MSVC env wrap: every cl.exe/vcpkg/cmake call runs in `cmd /c "call vcvarsall.bat x64 >/dev/null && "` via a sentinel + shell=True, bypassing CreateProcess quote mangling. Shell PATH is never mutated. - vcpkg manifest mode: --protobuf-version renders vcpkg.json, runs `vcpkg install --x-install-root=...`, and routes buf-generate's protoc to the manifest's tools dir so codegen matches libprotobuf. - Per-host --race default: ON for Linux/macOS, OFF for Windows (cgo+C compiler dependency). Explicit --race opts in. - Devcontainer detection skips host setup; CI's lukka/run-vcpkg path is preserved via --no-vcpkg-install. Tests: test_make.py (next to make.py, Go-style) — 72 unit + dry-run snapshot tests. Runs in <5s. New testing-make.yml workflow runs them on ubuntu/macos/windows on every push. Verified end-to-end on Windows native: - python make.py setup --lang cpp (idempotent) - python make.py test --lang go (Go suite, 1.4s) - python make.py test --lang cpp (C++ 6.33.4, 13/13 pass) - python make.py test --lang cpp --protobuf-version 3.21.12 (legacy v3, 13/13 pass) Docs simplified: README, CLAUDE.md, .devcontainer/README.md all collapsed to make.py-first usage. Old verbose per-language recipes removed. Co-Authored-By: Claude Opus 4.6 --- .devcontainer/Dockerfile | 6 +- .devcontainer/README.md | 141 +- .devcontainer/devcontainer.json | 2 +- .devcontainer/versions.env | 12 +- .github/actions/load-versions/action.yml | 2 +- .github/workflows/devcontainer-smoke.yml | 17 +- .github/workflows/testing-cpp.yml | 38 +- .github/workflows/testing-csharp.yml | 10 +- .github/workflows/testing-go.yml | 16 +- .github/workflows/testing-make.yml | 50 + .gitignore | 1 + CLAUDE.md | 145 +- README.md | 300 +--- .../plans/2026-05-29-devcontainer.md | 1047 ------------- .../specs/2026-05-29-devcontainer-design.md | 200 --- make.py | 1308 +++++++++++++++++ prepare.bat | 582 -------- test_make.py | 737 ++++++++++ 18 files changed, 2298 insertions(+), 2316 deletions(-) create mode 100644 .github/workflows/testing-make.yml delete mode 100644 docs/superpowers/plans/2026-05-29-devcontainer.md delete mode 100644 docs/superpowers/specs/2026-05-29-devcontainer-design.md create mode 100644 make.py delete mode 100644 prepare.bat create mode 100644 test_make.py diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index 97df95c..7459f84 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -5,7 +5,7 @@ # C++/Go/.NET/Node toolchain plus protobuf at the exact versions CI uses. # # All version pins are read from ./versions.env (the single source of -# truth shared with prepare.bat and the CI workflows). To bump Go / buf / +# truth shared with make.py and the CI workflows). To bump Go / buf / # protobuf / .NET / Node / vcpkg-baseline, edit that file — not this one. # # Build context is .devcontainer/ (the directory containing this file), @@ -72,7 +72,7 @@ EOF # # Pins (all from /opt/versions.env): # VCPKG_BASELINE_COMMIT — same commit testing-cpp.yml's VCPKG_COMMIT and -# prepare.bat's VCPKG_BASELINE_COMMIT use. Bumping vcpkg = bump that +# make.py's vcpkg setup use. Bumping vcpkg = bump that # one line in versions.env. # PROTOBUF_VERSION — defaults to versions.env's value (CI's primary). # Override at build time: @@ -169,7 +169,7 @@ case "$(basename "${INFO_FILE:-/missing}" 2>/dev/null)" in echo "ERROR: installed protobuf does not match requested version ${PROTOBUF_VERSION}." echo " vcpkg installed-file marker: ${INFO_FILE:-}" echo " Bump VCPKG_BASELINE_COMMIT in .devcontainer/versions.env" - echo " (and prepare.bat + testing-cpp.yml will pick it up automatically)" + echo " (and make.py + testing-cpp.yml will pick it up automatically)" echo " to a commit that knows about the requested version." exit 1 ;; diff --git a/.devcontainer/README.md b/.devcontainer/README.md index 4a76484..871c905 100644 --- a/.devcontainer/README.md +++ b/.devcontainer/README.md @@ -1,16 +1,8 @@ # Dev Container -The recommended way to develop on `tableauio/loader`. One container, all -four target languages (C++17, Go, .NET, Node) plus protobuf via vcpkg, -pinned to the exact toolchain CI uses. All version pins live in -[`./versions.env`](./versions.env) — bumping any of them is a one-line -change consumed by this Dockerfile, `prepare.bat`, and the CI -workflows. - -Use the devcontainer on **every** host that can run Docker: Linux -(amd64 + arm64), macOS (Intel + Apple Silicon), and Windows + WSL2. For -Windows hosts that prefer bare-metal native dev (no Docker, no WSL2), -see the [`prepare.bat`](../prepare.bat) bootstrap at the repo root. +The recommended way to develop on `tableauio/loader`. One container, all four target languages (C++17, Go, .NET, Node) plus protobuf via vcpkg, pinned to the same toolchain CI uses. Version pins live in [`./versions.env`](./versions.env), consumed by the Dockerfile, [`make.py`](../make.py), and the CI workflows. + +For native dev (no Docker, no WSL2), use [`make.py`](../make.py) directly. ## Prerequisites @@ -20,136 +12,61 @@ see the [`prepare.bat`](../prepare.bat) bootstrap at the repo root. ## Open the container ```sh -code . # in the repo root +code . ``` -In VS Code, run **Dev Containers: Reopen in Container** from the command -palette. First build is one-time ~25 minutes (vcpkg compiles protobuf -from source); subsequent reopens are near-instant. +Then **Dev Containers: Reopen in Container** from the command palette. First build is one-time ~25 min (vcpkg compiles protobuf); reopens are near-instant. -When the container is ready, the integrated terminal prints a banner with -five toolchain versions. After that, every command from the per-language -sections of the repo root [`README.md`](../README.md) works as written — -no PATH dance, no extra cmake flags. +Inside the container, `python make.py setup` is a no-op. Use `python make.py test --lang ` for any language. ## Pin a different protobuf version -Daily dev runs against the `PROTOBUF_VERSION` set in -[`./versions.env`](./versions.env) (CI's "modern" matrix entry). To -rebuild against the legacy v3 line for one container only, without -editing the shared file: - ```sh LOADER_PROTOBUF_VERSION=3.21.12 code . ``` -…then **Dev Containers: Reopen in Container** (or **Rebuild Container** -if the container is already running). The vcpkg layer rebuilds with the -manifest pinning protobuf 3.21.12; everything else is reused from the -cache. +Then **Rebuild Container**. Only the vcpkg layer rebuilds. ## Host-OS caveats -- **Windows.** WSL2 backend required. **Check the workspace out under - WSL2** (e.g. `\\wsl.localhost\Ubuntu\home\\loader`) — not under - `/mnt/c/...` — for good bind-mount performance. Files under `/mnt/c/` - work but file-watching and large `cmake --build` operations are 5–10× - slower. - -- **Apple Silicon.** Docker builds the container natively as arm64. No - Rosetta or QEMU emulation. Confirm with `docker info | grep Architecture` - → expect `linux/arm64`. - +- **Windows.** WSL2 backend required. Check the workspace out under WSL2 (`\\wsl.localhost\Ubuntu\home\\loader`) — not `/mnt/c/...` — for good bind-mount performance. +- **Apple Silicon.** Docker builds the image natively as arm64. Confirm with `docker info | grep Architecture`. - **Linux (native Docker Engine).** No special configuration. ## Architecture -Single-stage Dockerfile based on -`mcr.microsoft.com/devcontainers/cpp:1-ubuntu-24.04`, with these layers: +Single-stage Dockerfile on `mcr.microsoft.com/devcontainers/cpp:1-ubuntu-24.04`: 1. Architecture detection (`TARGETARCH` → Go arch, buf arch, vcpkg triplet) -2. Go (official tarball, multi-arch) — version from `versions.env` -3. buf (single-binary release, multi-arch) — version from `versions.env` -4. vcpkg pinned to `versions.env`'s `VCPKG_BASELINE_COMMIT`, protobuf - installed via vcpkg manifest mode and asserted against the requested - version -5. .NET SDK (Microsoft apt repo) — version from `versions.env` -6. Node.js LTS (NodeSource apt repo) — version from `versions.env` -7. `ENV CMAKE_PREFIX_PATH=/opt/vcpkg/active` so `find_package(Protobuf CONFIG)` - resolves automatically - -The architecture choice is detected from BuildKit's `TARGETARCH` and fed -into Go / buf / vcpkg triplet selection. Docker auto-selects the host -arch on build. - -The build context defaults to the directory containing `devcontainer.json` -(this `.devcontainer/` directory), so the Dockerfile's -`COPY versions.env …` and `COPY postcreate-banner.sh …` lines resolve -directly. - -## `versions.env` parsing rules - -`versions.env` is consumed by the Dockerfile, `prepare.bat`, and the CI -workflows. The format is intentionally minimal so every consumer can -parse it with a builtin: - -- One assignment per line, exactly `KEY=VALUE`. -- No quotes, no spaces around `=`, no inline comments after the value. -- Comments start at column 0 with `#`. -- Blank lines are ignored. -- No shell expansion — values are bare literals. - -Quick parsers per consumer: +2. Go — version from `versions.env` +3. buf — version from `versions.env` +4. vcpkg pinned to `VCPKG_BASELINE_COMMIT`, protobuf installed via manifest mode (asserts version) +5. .NET SDK — version from `versions.env` +6. Node.js LTS — version from `versions.env` +7. `ENV CMAKE_PREFIX_PATH=/opt/vcpkg/active` so `find_package(Protobuf CONFIG)` resolves automatically -```sh -# POSIX shell (Dockerfile) -. .devcontainer/versions.env -echo "$GO_VERSION" -``` +Build context is `.devcontainer/`, so `COPY versions.env …` resolves directly. -```cmd -:: Windows cmd (prepare.bat) -for /f "tokens=1,2 delims==" %%a in (.devcontainer\versions.env) do ( - if not "%%a"=="" if not "%%a:~0,1%"=="#" set "%%a=%%b" -) -echo %GO_VERSION% -``` +## `versions.env` format -```yaml -# GitHub Actions -- uses: ./.github/actions/load-versions -# subsequent steps reference the values as ${{ env.GO_VERSION }} etc. -``` +- One `KEY=VALUE` per line. +- No quotes, no spaces around `=`, no inline comments. +- Comments at column 0 with `#`. Blank lines ignored. No shell expansion. -## Troubleshooting +Consumers: Dockerfile (sourced as a shell file), [`make.py`](../make.py) (`Versions.load()`), and `.github/actions/load-versions` (exports to `$GITHUB_ENV`). + +Use `python make.py env` for a JSON dump of resolved values. -### `buf generate` works but the C++ or C# build then fails with stale-codegen errors +## Troubleshooting -If you're hitting errors like +### Stale-codegen errors after `buf generate` +Symptoms: - C++: `fatal error: google/protobuf/generated_message_table_driven.h: No such file or directory` - C#: hundreds of `error CS0101: The namespace already contains a definition for ...` -your host workspace probably has generated files from a *different* protobuf -version (left over from a previous host toolchain — gitignored, so `git pull` -doesn't remove them). They shadow what the container's `protoc` produces. - -Wipe and retry: - -```sh -rm -rf test/cpp-tableau-loader/src/protoconf/tableau \ - test/cpp-tableau-loader/build \ - test/csharp-tableau-loader/protoconf \ - test/csharp-tableau-loader/{bin,obj} -``` - -Then re-run `buf generate ..` from the affected language's `test/-tableau-loader/` -directory. A fresh clone doesn't have these stale artefacts. +The host workspace has gitignored `*.pb.*` from a previous protobuf version that `git pull` didn't remove. Wipe with `python make.py clean --lang cpp` (or `--lang csharp`), then rerun the test. ## Falling back -If you can't run Docker (corp policy, restricted machines, etc.) the -existing manual setup paths in the [repo README](../README.md) — Windows -`prepare.bat`, per-language `Install protobuf` instructions, the macOS -Homebrew recipe — still work. The devcontainer is the recommended path; -the rest is the supported fallback. +No Docker? Use [`make.py`](../make.py) directly: `python make.py setup --lang all` then `python make.py test --lang `. Works on macOS / Linux / Windows native. diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 2b0d48a..c2f3fc8 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -2,7 +2,7 @@ // The loader's only devcontainer. Multi-arch: builds linux/amd64 on // x64 hosts, linux/arm64 on Apple Silicon and Windows-on-ARM. Use // this on every host that can run Docker Desktop or Docker Engine. - // Bare-metal Windows users (no Docker) bootstrap via prepare.bat at + // Native users (no Docker) bootstrap via `python make.py setup` at // the repo root. // // See ./versions.env for all toolchain pins. diff --git a/.devcontainer/versions.env b/.devcontainer/versions.env index 502c033..94369df 100644 --- a/.devcontainer/versions.env +++ b/.devcontainer/versions.env @@ -2,7 +2,7 @@ # # Single source of truth consumed by: # - .devcontainer/Dockerfile (sourced as a shell file) -# - prepare.bat (parsed via `for /f` in cmd) +# - make.py (parsed via Versions.load()) # - .github/workflows/*.yml (read by the load-versions composite action into $GITHUB_ENV) # # Format rules so every consumer can parse this file with a single regex: @@ -12,7 +12,7 @@ # - Values are bare strings — no shell expansion, no $VAR references. # # Bumping any of these is a one-line change. The matching post-install -# assertion in the Dockerfile / prepare.bat catches the case where the +# assertion in the Dockerfile / make.py catches the case where the # vcpkg baseline doesn't yet know about the requested PROTOBUF_VERSION. # Go SDK shipped in the devcontainer. Must be ≥ the `go` directive in go.mod; @@ -20,7 +20,7 @@ # go-version-file, so devcontainer is the only place Go is hard-pinned. GO_VERSION=1.24.0 -# buf CLI release. Pinned in the devcontainer, prepare.bat, and all three +# buf CLI release. Pinned in the devcontainer, make.py, and all three # testing-*.yml workflows. Bump everywhere together. BUF_VERSION=1.67.0 @@ -31,7 +31,7 @@ PROTOBUF_VERSION=6.33.4 # vcpkg checkout commit. Same value used by: # - .devcontainer/Dockerfile (ARG VCPKG_BASELINE_COMMIT) -# - prepare.bat (set VCPKG_BASELINE_COMMIT=...) +# - make.py (Versions.vcpkg_baseline_commit) # - .github/workflows/testing-cpp.yml (env: VCPKG_COMMIT) # This commit MUST know about every PROTOBUF_VERSION in the testing-cpp.yml # matrix; bump it forward (never sideways) when adding a new protobuf entry. @@ -44,6 +44,6 @@ DOTNET_VERSION=8.0 # Node.js LTS major. NodeSource apt repo is `setup_${NODE_VERSION}.x`. NODE_VERSION=20 -# CMake version installed by prepare.bat (the devcontainer base image -# already ships a recent cmake). +# CMake version installed by `make.py setup` on Windows (the devcontainer +# base image already ships a recent cmake; macOS/Linux use system cmake). CMAKE_VERSION=3.31.8 diff --git a/.github/actions/load-versions/action.yml b/.github/actions/load-versions/action.yml index 5688415..eebb9d6 100644 --- a/.github/actions/load-versions/action.yml +++ b/.github/actions/load-versions/action.yml @@ -1,7 +1,7 @@ name: Load pinned versions description: > Reads .devcontainer/versions.env (the single source of truth shared - with the devcontainer and prepare.bat) and exports each KEY=VALUE pair to + with the devcontainer and make.py) and exports each KEY=VALUE pair to GITHUB_ENV so subsequent steps can reference them as env-context values (e.g. env.BUF_VERSION). diff --git a/.github/workflows/devcontainer-smoke.yml b/.github/workflows/devcontainer-smoke.yml index 7c3afb2..3d0ad4e 100644 --- a/.github/workflows/devcontainer-smoke.yml +++ b/.github/workflows/devcontainer-smoke.yml @@ -93,21 +93,22 @@ jobs: - name: Smoke — buf generate (Go) run: | # Bind-mount the checkout into the container's workspace dir - # and regenerate Go protoconf. Catches protoc/buf integration - # regressions that wouldn't show up in the banner. + # and regenerate Go protoconf via make.py. Catches protoc/buf + # integration regressions that wouldn't show up in the banner. docker run --rm \ -v "${{ github.workspace }}:/workspaces/loader" \ - -w /workspaces/loader/test/go-tableau-loader \ + -w /workspaces/loader \ loader-devcontainer:smoke \ - buf generate .. + python3 make.py generate --lang go - name: Smoke — go vet (plugin packages only) run: | - # Vet the plugin sources and shared internal packages. Skip - # ./test/... and ./internal/index — those depend on freshly - # generated *.pb.go that we don't produce in this smoke job. + # Vet the plugin sources and shared internal packages via the + # make.py --smoke shortcut. Skips ./test/... and ./internal/index + # — those depend on freshly generated *.pb.go that we don't + # produce in this smoke job. docker run --rm \ -v "${{ github.workspace }}:/workspaces/loader" \ -w /workspaces/loader \ loader-devcontainer:smoke \ - go vet ./cmd/... ./pkg/... ./internal/options/... ./internal/loadutil/... ./internal/xproto/... + python3 make.py test --lang go --smoke diff --git a/.github/workflows/testing-cpp.yml b/.github/workflows/testing-cpp.yml index 7b86935..83db81b 100644 --- a/.github/workflows/testing-cpp.yml +++ b/.github/workflows/testing-cpp.yml @@ -59,6 +59,11 @@ jobs: if: runner.os == 'Windows' uses: ilammy/msvc-dev-cmd@v1 + - name: Setup Python + uses: actions/setup-python@v6 + with: + python-version: "3.12" + - name: Render vcpkg.json working-directory: test/cpp-tableau-loader shell: bash @@ -98,12 +103,6 @@ jobs: shell: pwsh run: Add-Content -Path $env:GITHUB_PATH -Value "$env:VCPKG_INSTALLED_DIR\${{ matrix.triplet }}\tools\protobuf" - - name: Verify protoc - shell: bash - run: | - which protoc - protoc --version - - name: Install Buf uses: bufbuild/buf-action@v1 with: @@ -111,25 +110,10 @@ jobs: setup_only: true github_token: ${{ secrets.GITHUB_TOKEN }} - - name: Generate protoconf - working-directory: test/cpp-tableau-loader - run: buf generate .. - - - name: CMake Configure - working-directory: test/cpp-tableau-loader + - name: Test run: > - cmake -S . -B build -G Ninja - -DCMAKE_BUILD_TYPE=Debug - -DCMAKE_CXX_STANDARD=17 - -DCMAKE_TOOLCHAIN_FILE=${{ env.VCPKG_ROOT }}/scripts/buildsystems/vcpkg.cmake - -DVCPKG_TARGET_TRIPLET=${{ matrix.triplet }} - -DVCPKG_INSTALLED_DIR=${{ env.VCPKG_INSTALLED_DIR }} - -DVCPKG_MANIFEST_INSTALL=OFF - - - name: CMake Build - working-directory: test/cpp-tableau-loader - run: cmake --build build --parallel - - - name: Run tests - working-directory: test/cpp-tableau-loader - run: ctest --test-dir build --output-on-failure + python make.py test --lang cpp + --protobuf-version ${{ matrix.config.protobuf-version }} + --triplet ${{ matrix.triplet }} + --no-clean + --no-vcpkg-install diff --git a/.github/workflows/testing-csharp.yml b/.github/workflows/testing-csharp.yml index da40e18..7daeae3 100644 --- a/.github/workflows/testing-csharp.yml +++ b/.github/workflows/testing-csharp.yml @@ -70,10 +70,10 @@ jobs: setup_only: true github_token: ${{ secrets.GITHUB_TOKEN }} - - name: Generate protoconf - working-directory: test/csharp-tableau-loader - run: buf generate .. + - name: Setup Python + uses: actions/setup-python@v6 + with: + python-version: '3.12' - name: Test - working-directory: test/csharp-tableau-loader - run: dotnet test --nologo --logger "console;verbosity=normal" + run: python make.py test --lang csharp diff --git a/.github/workflows/testing-go.yml b/.github/workflows/testing-go.yml index 258efa8..08b2626 100644 --- a/.github/workflows/testing-go.yml +++ b/.github/workflows/testing-go.yml @@ -46,16 +46,20 @@ jobs: setup_only: true github_token: ${{ secrets.GITHUB_TOKEN }} - - name: Generate protoconf - working-directory: test/go-tableau-loader - run: buf generate .. + - name: Setup Python + uses: actions/setup-python@v6 + with: + python-version: '3.12' - name: Vet run: go vet ./... - - name: Unittest - run: go test -v -timeout 30m -race ./... -coverprofile=coverage.txt - -covermode=atomic + - name: Test + # Explicit --race so Windows CI matches Linux (Windows runners have + # MSVC available; -race needs cgo + a C compiler). On a fresh + # Windows dev machine without MSVC, make.py's default is --no-race; + # CI overrides that. + run: python make.py test --lang go --race --coverage - name: Upload coverage reports to Codecov uses: codecov/codecov-action@v5 diff --git a/.github/workflows/testing-make.yml b/.github/workflows/testing-make.yml new file mode 100644 index 0000000..6fb5029 --- /dev/null +++ b/.github/workflows/testing-make.yml @@ -0,0 +1,50 @@ +name: Testing make.py + +# Regression tests for make.py itself: unit tests + dry-run snapshot +# tests of the orchestrator. Catches make.py bugs before the slower +# testing-{go,cpp,csharp}.yml E2E workflows even start. + +on: + push: + branches: [master, main] + paths: + - 'make.py' + - 'test_make.py' + - '.devcontainer/versions.env' + - '.github/workflows/testing-make.yml' + pull_request: + paths: + - 'make.py' + - 'test_make.py' + - '.devcontainer/versions.env' + - '.github/workflows/testing-make.yml' + workflow_dispatch: + +permissions: + contents: read + +jobs: + test: + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, macos-latest, windows-latest] + + name: test (${{ matrix.os }}) + runs-on: ${{ matrix.os }} + timeout-minutes: 5 + + steps: + - name: Checkout + uses: actions/checkout@v6 + + - name: Setup Python + uses: actions/setup-python@v6 + with: + python-version: '3.12' + + - name: Install pytest + run: python -m pip install --upgrade pip pytest + + - name: Run make.py tests + run: python -m pytest test_make.py -v diff --git a/.gitignore b/.gitignore index b92e31d..f059865 100644 --- a/.gitignore +++ b/.gitignore @@ -36,6 +36,7 @@ obj/ node_modules *.pb.* +__pycache__/ cmd/protoc-gen-cpp-tableau-loader/protoc-gen-cpp-tableau-loader cmd/protoc-gen-go-tableau-loader/protoc-gen-go-tableau-loader diff --git a/CLAUDE.md b/CLAUDE.md index 2e2f5fa..5327662 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -1,135 +1,146 @@ # CLAUDE.md -This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository. +Guidance for Claude Code (claude.ai/code) when working in this repository. ## What this repo is -`github.com/tableauio/loader` is the official config-loader generator for [Tableau](https://github.com/tableauio/tableau). It ships **three `protoc` plugins** written in Go that read protobuf files annotated with the `tableau.workbook` / `tableau.worksheet` / `tableau.field` extensions and emit strongly-typed loader code in three target languages: +`github.com/tableauio/loader` is the official config-loader generator for [Tableau](https://github.com/tableauio/tableau). It ships **three `protoc` plugins** (Go) that read protobuf files annotated with the `tableau.workbook` / `tableau.worksheet` / `tableau.field` extensions and emit strongly-typed loader code: -| Plugin (under `cmd/`) | Output language | Generated file extension | +| Plugin (under `cmd/`) | Output | Generated extension | | --- | --- | --- | | `protoc-gen-go-tableau-loader` | Go | `*.pc.go` | | `protoc-gen-cpp-tableau-loader` | C++17 | `*.pc.h` / `*.pc.cc` | | `protoc-gen-csharp-tableau-loader` | C# (Unity 2022.3 LTS / .NET 8) | `*.pc.cs` | -Generated code is opinionated: every worksheet message becomes a `Messager` with `Load`/`Store`/`Get*`/index/ordered-map accessors, all messagers register into a singleton-ish `Hub`, and the runtime delegates file IO + protobuf (un)marshaling to the upstream `github.com/tableauio/tableau` Go module's `format`/`load`/`store` packages. +Generated code is opinionated: every worksheet message becomes a `Messager` with `Load`/`Store`/`Get*`/index/ordered-map accessors; all messagers register into a singleton-ish `Hub`; runtime delegates file IO + protobuf (un)marshaling to `github.com/tableauio/tableau`'s `format`/`load`/`store` packages. ## Common commands -All build and test work happens **per language** inside `test/-tableau-loader/`. The repo root only hosts the Go module and the plugin sources; running `go test ./...` from the root only exercises the small shared packages. +Build/test happens **per language** under `test/-tableau-loader/` (TS lives under `_lab/ts/`). The repo root only hosts the Go module + plugin sources; `go test ./...` from root only exercises shared packages (`internal/index`, `internal/loadutil`, `pkg/treemap`, `pkg/udiff`). -### Dev Container (recommended for most contributors) +The single cross-platform driver is **`make.py`** (Python 3.10+, stdlib only). It works on Windows, macOS, Linux, and inside the devcontainer, and is what CI calls. -`.devcontainer/` ships a single Ubuntu 24.04 image with the entire toolchain pinned to CI's exact versions: Go 1.24, buf 1.67.0, protobuf 6.33.4 (via vcpkg, manifest mode), .NET 8, Node 20. Open the repo in VS Code and run **Dev Containers: Reopen in Container** — first build is one-time ~25 min (vcpkg compiles protobuf from source); reopens after that are instant. Inside the container every command in the per-language sections below works as written, *and* the C++ section's `-DCMAKE_TOOLCHAIN_FILE=...` flag is unnecessary (the image sets `CMAKE_PREFIX_PATH=/opt/vcpkg/active`). The container is the recommended path on every host OS; `prepare.bat` and the per-language `Install protobuf` recipes in the repo README stay as the explicit fallback for hosts that can't run Docker. +```sh +python make.py setup --lang all # one-time host toolchain install (no-op in container) +python make.py generate --lang go # buf generate .. +python make.py build --lang cpp +python make.py test --lang go +python make.py test --lang cpp -k HubTest.Load +python make.py env # diagnostic JSON +python make.py --version +``` -To rebuild against the legacy v3 protobuf line: `LOADER_PROTOBUF_VERSION=3.21.12 code .` then **Reopen in Container**. The Dockerfile ARG `PROTOBUF_VERSION` is wired to that host env var via `devcontainer.json` build args; vcpkg manifest mode pins the override and the post-install assertion fails the build if anything resolves to the wrong version. See `.devcontainer/README.md` for the longer how-to and host-OS caveats (notably: Windows users should check the workspace out under WSL2, not `/mnt/c/`, for usable bind-mount perf). +C++ wipes `test/cpp-tableau-loader/{build,src/tableau,src/protoconf}` before regenerating (gitignored `*.pb.*` shadows fresh codegen). `--no-clean` skips it. -CI's primary test path does **not** build the devcontainer image. The `testing-{cpp,go,csharp}.yml` workflows run `lukka/run-vcpkg` directly on GitHub-hosted runners (faster cached vcpkg installs). One **smoke** workflow additionally builds the devcontainer image itself on devcontainer-touching PRs: `devcontainer-smoke.yml` (amd64 + arm64, runs `buf generate` + `go vet` inside the image). It's path-gated to `.devcontainer/**` so it doesn't burn CI minutes on unrelated PRs, but a `versions.env` typo or a Dockerfile regression won't ship. All toolchain versions (Go, buf, protobuf, vcpkg baseline commit, .NET, Node, CMake) are pinned in **`.devcontainer/versions.env`** — the single source of truth consumed by `.devcontainer/Dockerfile`, `prepare.bat`, and every `.github/workflows/testing-*.yml` (via the **`./.github/actions/load-versions`** composite action that exports each `KEY=VALUE` to `$GITHUB_ENV`). Bump versions there; everything downstream picks them up automatically. The `.devcontainer/` directory is flat (one Dockerfile, one devcontainer.json, versions.env, and the postcreate banner script) — there's only one container, not a per-OS variant. macOS users either run the container under Docker Desktop or follow the brew-install recipe in the repo root README; Windows users either run the container under Docker+WSL2 or follow the `prepare.bat` + winget bare-metal recipe. +On Windows, `make.py` wraps every C++ subprocess in `cmd /c "call vcvarsall.bat x64 >nul && "` so MSVC env lives per-subprocess; the shell PATH is never mutated. -### Plugin development (Go module at repo root) +### Dev container -```sh -go vet ./... -go test ./... # tests live in internal/index, internal/loadutil, pkg/treemap, pkg/udiff -go test ./internal/index -run Test_ParseIndexDescriptor # single test -go build -o /tmp/p ./cmd/protoc-gen-go-tableau-loader # smoke-build a plugin -``` +- `.devcontainer/` → **Dev Containers: Reopen in Container**. Ubuntu 24.04 + all toolchains pinned. First build ~25 min; reopens instant. +- Inside: `python make.py setup` is a no-op. `python make.py test --lang ` works for all languages — Dockerfile presets `CMAKE_PREFIX_PATH=/opt/vcpkg/active`. +- Override protobuf version: `LOADER_PROTOBUF_VERSION=3.21.12 code .` then **Rebuild Container**. +- Single source of truth for all toolchain versions: **`.devcontainer/versions.env`**. -The plugins are always invoked through `buf generate` from a test directory — `buf.gen.yaml` runs them via `local: ["go", "run", "../../cmd/protoc-gen-go-tableau-loader"]`, so plugin changes take effect on the next `buf generate` without an explicit install step. +CI primary tests (`testing-{cpp,go,csharp}.yml`) use `lukka/run-vcpkg` for cached vcpkg installs + `python make.py test --lang ` for build/test. `devcontainer-smoke.yml` builds the image on `.devcontainer/**` PRs (amd64 + arm64). `testing-make.yml` runs the make.py unit + dry-run regression suite on every push. -### Go end-to-end (`test/go-tableau-loader/`) +### Plugin development (Go module at repo root) ```sh -cd test/go-tableau-loader -buf generate .. # regenerate *.pb.go + protoconf/loader/*.pc.go -go test ./... # runs hub, index, and main test suites -go test -run Test_ActivityConf_OrderedMap ./... # single test +go vet ./... +go test ./... # internal/index, internal/loadutil, pkg/treemap, pkg/udiff +go test ./internal/index -run Test_ParseIndexDescriptor # single test +go build -o /tmp/p ./cmd/protoc-gen-go-tableau-loader # smoke-build a plugin ``` -CI mirrors this with `go test -v -timeout 30m -race ./... -coverprofile=coverage.txt`. - -### C++ end-to-end (`test/cpp-tableau-loader/`) +Plugins are invoked through `buf generate` from a test directory (`buf.gen.yaml` runs them via `local: ["go", "run", "../../cmd/protoc-gen-go-tableau-loader"]`), so plugin changes take effect on the next `buf generate` without an explicit install step. -Requires a matching `protoc` + `libprotobuf` toolchain (protobuf v22+ enforces a strict gencode/runtime version check). Loader does **not** vendor protobuf — bring your own via vcpkg, system package, or source build. +### Per-language ```sh -cd test/cpp-tableau-loader -buf generate .. # writes src/protoconf/*.pb.* and *.pc.* -cmake -S . -B build -DCMAKE_BUILD_TYPE=Debug \ - -DCMAKE_TOOLCHAIN_FILE=/scripts/buildsystems/vcpkg.cmake -cmake --build build --parallel -ctest --test-dir build --output-on-failure -ctest --test-dir build -R HubTest.Load --output-on-failure # single test +# Go +python make.py test --lang go # full +python make.py test --lang go -k Test_ActivityConf_OrderedMap # filter +python make.py test --lang go --smoke # plugin-only `go vet` (devcontainer-smoke) +python make.py test --lang go --coverage # CI: -coverprofile=coverage.txt -covermode=atomic +python make.py test --lang go --race # opt in to -race (default off on Windows; needs cgo+MSVC) + +# C++ (requires matching protoc + libprotobuf — protobuf v22+ enforces gencode/runtime check) +python make.py test --lang cpp # full +python make.py test --lang cpp -k HubTest.Load # filter +python make.py test --lang cpp --cxx-std 20 # C++20 +python make.py test --lang cpp --cxx-compiler clang # clang++ +python make.py test --lang cpp --protobuf-version 3.21.12 # legacy v3 (vcpkg manifest mode) + +# C# +python make.py test --lang csharp # full +python make.py test --lang csharp -k HubTest.Load # FullyQualifiedName~HubTest.Load + +# TypeScript (experimental, not in CI) +python make.py test --lang ts # npm install + generate + test ``` -Windows additionally requires running `.\prepare.bat` from the repo root (in **cmd**, not PowerShell, **as Administrator** the first time) in every new shell — it installs the toolchain on first run and re-loads MSVC env vars (`cl.exe`, `INCLUDE`, `LIB`, `VCPKG_ROOT`, …) every run because `vcvarsall.bat` does not persist them. Use `-DCMAKE_BUILD_TYPE=Debug -DVCPKG_TARGET_TRIPLET=x64-windows-static` to match the static-CRT protobuf installed by `prepare.bat`; mismatched CRTs surface as `LNK2038 _ITERATOR_DEBUG_LEVEL` errors. GoogleTest is fetched via CMake `FetchContent` — no manual install. (None of this applies inside the Dev Container — Windows users on that path interact only with Docker Desktop + WSL2.) +GoogleTest is fetched via CMake `FetchContent` — no manual install. -### C# end-to-end (`test/csharp-tableau-loader/`) +### make.py regression tests ```sh -cd test/csharp-tableau-loader -buf generate .. # writes protoconf/ and tableau/*.pc.cs -dotnet test # xUnit -dotnet test --filter "FullyQualifiedName~HubTest.Load" # single test +pip install pytest && python -m pytest test_make.py -v ``` -### TypeScript scratch (`_lab/ts/`) - -Experimental, not wired into CI. `npm install && npm run generate && npm run test`. +`test_make.py` (next to `make.py`) covers: pure-logic unit tests (`Versions`, `Platform`, `windows_msvc_wrap`, `_winquote`, `Runner`, repo-root discovery) + dry-run snapshot tests (assert exact subprocess sequences for every ` --lang ` combo). Runs in <5s; CI: `.github/workflows/testing-make.yml` (ubuntu/macos/windows). ## Big-picture architecture ### Plugin pipeline (the part you'll modify most) -Every plugin's `main.go` is the same shape: parse flags, set protogen feature bits (proto2 → editions 2024, FEATURE_PROTO3_OPTIONAL), iterate `gen.Files`, and for each file/message decide what to generate. The decision logic is centralized in `internal/options`: +Every plugin's `main.go` is the same shape: parse flags, set protogen feature bits (proto2 → editions 2024, FEATURE_PROTO3_OPTIONAL), iterate `gen.Files`, decide what to generate per file/message. Decision logic is centralized in **`internal/options`**: - `options.NeedGenFile(f)` — file-level gate: must have `(tableau.workbook)` set and at least one message with `(tableau.worksheet)`. -- `options.NeedGenOrderedMap` / `NeedGenIndex` / `NeedGenOrderedIndex` — message-level gates that *also* honour the `lang_options` map on `WorksheetOptions`, e.g. `lang_options: { key: "Index" value: "go" }` means "only generate the index accessors for Go". `internal/options/options.go` defines the language IDs (`cpp`, `go`, `cs`). +- `options.NeedGenOrderedMap` / `NeedGenIndex` / `NeedGenOrderedIndex` — message-level gates that *also* honour the `lang_options` map on `WorksheetOptions` (e.g. `lang_options: { key: "Index" value: "go" }` = "only generate index accessors for Go"). Language IDs (`cpp`, `go`, `cs`) are in `internal/options/options.go`. -Each plugin then splits work between two passes: +Each plugin splits work between two passes: -1. **Per-message generation** (`messager.go` in each plugin) — emits one `*.pc.{go,h,cc,cs}` per `.proto`. Delegates ordered-map field/method emission to `cmd//orderedmap/` and index field/method emission to `cmd//indexes/`. The shared semantic model — what the index syntax means — lives in **`internal/index`**, not in each plugin: `ParseIndexDescriptor` walks the message tree and returns a `LevelMessage` linked-list describing what indices apply at which nesting level (map → list → map → list, etc.). Plugins consume this descriptor to emit language-specific code; do not duplicate this parsing per-language. -2. **Cross-message ("embed") generation** — emits `hub.pc.*`, `messager_container.pc.go`, `util.pc.*`, etc. Driven by `cmd//embed.go`, which `//go:embed`s the templates under `cmd//embed/templates/` (Go) or `cmd//embed/` (C++/C# also include verbatim `*.pc.{h,cc,cs}` files that are emitted unchanged). The list of "all messagers in source order" the templates iterate over comes from **`internal/xproto.ParseProtoFiles`**. +1. **Per-message generation** (`messager.go` per plugin) — emits one `*.pc.{go,h,cc,cs}` per `.proto`. Delegates ordered-map field/method emission to `cmd//orderedmap/`, index emission to `cmd//indexes/`. **The shared semantic model — what the index syntax means — lives in `internal/index`, not per-plugin**: `ParseIndexDescriptor` walks the message tree and returns a `LevelMessage` linked-list describing what indices apply at each nesting level (map → list → map → list, etc.). Plugins consume this descriptor; do not duplicate parsing per-language. +2. **Cross-message ("embed") generation** — emits `hub.pc.*`, `messager_container.pc.go`, `util.pc.*`. Driven by `cmd//embed.go`, which `//go:embed`s templates under `cmd//embed/templates/` (Go) or `cmd//embed/` (C++/C# also include verbatim `*.pc.{h,cc,cs}` files emitted unchanged). The "all messagers in source order" iteration source is **`internal/xproto.ParseProtoFiles`**. -The C++ plugin is the only one with sharding: `--shards=N` in `buf.gen.yaml` makes `xproto.ProtoFiles.SplitShards(N)` partition messagers across N `hub_shard*.pc.cc` files to parallelize the (very heavy) compile. It also has a tri-state `--mode` flag (`default` / `hub` / `messager`) for users who want to split protoconf generation across separate `buf generate` invocations. +C++ plugin sharding: `--shards=N` in `buf.gen.yaml` makes `xproto.ProtoFiles.SplitShards(N)` partition messagers across N `hub_shard*.pc.cc` files to parallelize the (heavy) compile. Tri-state `--mode` (`default` / `hub` / `messager`) lets users split protoconf generation across separate `buf generate` invocations. ### Index syntax (`internal/index/index.go`) -The `worksheet.index` / `worksheet.ordered_index` strings use a compact mini-language parsed by one regex. Knowing the shape helps when reading or writing tests in `test/proto/index_conf.proto`: +`worksheet.index` / `worksheet.ordered_index` strings use a compact mini-language parsed by one regex: ``` -ID # single-column index on field "ID" -ID@Item # named "Item" -ID@Item # sort within group by SortedCol -(ID, Name)@Item # composite index -CountryItemAttrName # CamelCase concatenation of nested-field path Country.Item.Attr.Name +ID # single-column index on field "ID" +ID@Item # named "Item" +ID@Item # sort within group by SortedCol +(ID, Name)@Item # composite index +CountryItemAttrName # CamelCase concatenation of nested-field path Country.Item.Attr.Name ``` Multi-level nested maps/lists are flattened: `CountryItemAttrName` reaches into `country_list[].item_map[].attr_list[].name`. Generated indexes return leveled key structs whose names are derived in `helper.ParseLeveledMapPrefix` — a 3-level map with indexes only at the 2nd level produces fewer key structs than one with indexes at every level (compare `Fruit5Conf` vs `Fruit4Conf` in `test/proto/index_conf.proto`). ### Hub and Messager runtime (Go) -Hub state is held in `atomic.Pointer[MessagerContainer]` so `Load(...)` can swap the entire snapshot atomically while `Get*()` callers race-freely read the previous one. The container is generated (`messager_container.pc.go`) and exposes both a generic `GetMessager(name)` and a typed `Get()` per messager. `Hub.NewContext` / `FromContext` propagate the snapshot through `context.Context` so request handlers see a stable view. - -Mutability detection (`hub.WithMutableCheck`) is opt-in: enabling it flips `enableBackup()` on each messager so they retain the originally-loaded `proto.Message`, then a goroutine periodically `proto.Equal`s `originalMessage()` vs `Message()` and calls `OnMutate(name, original, current)` (default handler prints a unified diff via `pkg/udiff`). - -`processAfterLoad` (per-messager, runs as `Load` finishes) and `ProcessAfterLoadAll(hub *Hub)` (cross-messager, runs after all are loaded against a temporary hub) are the two extension points. `test/go-tableau-loader/customconf/custom_item_conf.go` shows how a hand-written messager registers via `tableau.Register(func() Messager { ... })` and uses `ProcessAfterLoadAll` to consume data from another messager — that's the canonical pattern for derived/computed configs. +- Hub state is held in `atomic.Pointer[MessagerContainer]` so `Load(...)` swaps the snapshot atomically while `Get*()` callers race-freely read the previous one. +- Container is generated (`messager_container.pc.go`) with both generic `GetMessager(name)` and typed `Get()`. `Hub.NewContext` / `FromContext` propagate the snapshot through `context.Context`. +- `hub.WithMutableCheck` is opt-in: enabling it flips `enableBackup()` on each messager, then a goroutine periodically `proto.Equal`s `originalMessage()` vs `Message()` and calls `OnMutate(name, original, current)` (default handler prints a unified diff via `pkg/udiff`). +- Two extension points: `processAfterLoad` (per-messager, runs as `Load` finishes) and `ProcessAfterLoadAll(hub *Hub)` (cross-messager, against a temporary hub). `test/go-tableau-loader/customconf/custom_item_conf.go` shows the canonical pattern: hand-written messager registers via `tableau.Register(func() Messager { ... })` and uses `ProcessAfterLoadAll` to consume data from another messager — that's how derived/computed configs are built. ### Test data flow -1. `test/proto/*.proto` — hand-written annotated protos (the source of truth for what generators are exercised). -2. `test/testdata/conf/*.json`, `patchconf/`, `patchconf2/`, `patchresult/` — JSON inputs the upstream `tableau` toolchain would produce from spreadsheets; loader tests read them at runtime. -3. Each `test/-tableau-loader/` runs `buf generate ..` to produce its language-specific stubs into a per-language output dir, then runs that language's native test runner against `test/testdata/`. +1. `test/proto/*.proto` — hand-written annotated protos (source of truth for what generators are exercised). +2. `test/testdata/conf/*.json`, `patchconf/`, `patchconf2/`, `patchresult/` — JSON inputs the upstream `tableau` toolchain produces from spreadsheets; loader tests read them at runtime. +3. Each `test/-tableau-loader/` runs `buf generate ..` → language-specific stubs → native test runner against `test/testdata/`. -Patch tests verify three semantics defined upstream (merge / replace / recursive_patch) — the loader's job is just to wire `patch_paths`/`patch_dirs` through `load.MessagerOptions`. +Patch tests verify three semantics defined upstream (merge / replace / recursive_patch) — loader's job is just to wire `patch_paths`/`patch_dirs` through `load.MessagerOptions`. ## Versioning and releases -Each plugin has its own `version` constant in its `main.go` and is released independently via tags shaped `cmd/protoc-gen-{go,cpp,csharp}-tableau-loader/`. The matching workflows in `.github/workflows/release-*.yml` build cross-platform binaries and attach them to the GitHub release. Bump the relevant `const version = "..."` in lockstep with the tag. +Each plugin has its own `version` constant in its `main.go`, released independently via tags shaped `cmd/protoc-gen-{go,cpp,csharp}-tableau-loader/`. Matching workflows in `.github/workflows/release-*.yml` build cross-platform binaries and attach them to the GitHub release. Bump `const version = "..."` in lockstep with the tag. ## Style and conventions -- C++ / proto: `clang-format` with the rules in `.clang-format` (Google base, 120-col). +- C++ / proto: `clang-format` per `.clang-format` (Google base, 120-col). - Go: standard `gofmt` / `go vet`; CI runs `go vet ./...` and `go test -race`. -- Generated files end in `.pc.` (the `extensions.PC` constant). Anything matching `*.pb.*` is `.gitignore`d — never check generated proto-runtime files in. -- Worksheet language gating: when adding a feature that only some target languages support, add it behind a `lang_options` check in `internal/options` rather than making the per-plugin `messager.go` know the rules. +- Generated files end in `.pc.` (the `extensions.PC` constant). Anything matching `*.pb.*` is `.gitignore`d — never commit generated proto-runtime files. +- Worksheet language gating: when adding a feature only some target languages support, gate it via `lang_options` in `internal/options` rather than baking the rule into per-plugin `messager.go`. diff --git a/README.md b/README.md index 04d25ea..510ff97 100644 --- a/README.md +++ b/README.md @@ -2,271 +2,69 @@ The official config loader for [Tableau](https://github.com/tableauio/tableau). -## Prerequisites - -- C++ standard: at least C++17 -- A working **`protoc` + `libprotobuf`** toolchain on your machine. The same - protobuf release **must** provide both: protobuf v22+ enforces a strict - gencode/runtime version check via `PROTOBUF_VERSION` in the generated - headers, so a mismatched `protoc` and `libprotobuf` will fail to link. - -### Recommended: Dev Container (any host OS) - -The fastest way to get a reproducible build environment is to open the -repo in VS Code and choose **Reopen in Container**. The -[`.devcontainer/`](./.devcontainer/) directory ships a multi-arch -(linux/amd64 + linux/arm64) container that works on every host that -can run Docker: Linux, macOS (Intel + Apple Silicon), and Windows + -WSL2. - -All version pins (Go, buf, protobuf, vcpkg baseline, .NET, Node, CMake) -live in -[`.devcontainer/versions.env`](./.devcontainer/versions.env), the -single source of truth shared with `prepare.bat` and CI. First -container build is one-time ~25 min (vcpkg compiles protobuf from -source). Subsequent reopens are near-instant. - -For hosts that can't or won't run Docker: - -- **macOS (native, no container).** Install via Homebrew. Versions - follow whatever the brew formula currently ships, which is usually - close enough; the loader's [`Install protobuf`](#install-protobuf) - section below covers the protobuf gotcha if you need an exact pin: - - ```sh - # Toolchain via Homebrew - brew install go buf protobuf dotnet@8 node@20 cmake ninja - ``` - - > Apple Silicon: nothing in the loader requires Rosetta. If you see - > x86_64 Homebrew complaints, run `arch -arm64 brew ...`. - -- **Windows (bare-metal, native MSVC).** Run - [`prepare.bat`](./prepare.bat) for the C++ toolchain (MSVC, CMake, - Ninja, vcpkg + protobuf, buf), plus one-line winget installs for - Go / .NET / Node: - - ```cmd - winget install --id GoLang.Go.1.24 -e - winget install --id Microsoft.DotNet.SDK.8 -e - winget install --id OpenJS.NodeJS.LTS -e - ``` - -After the container starts you can skip the per-language setup below -and jump straight to **[C++](#c)** / **[Go](#go)** / **[C#](#c-1)** / -**[TypeScript](#typescript)**. - -Requirements: Docker Desktop (Windows + macOS) or Docker Engine (Linux), -and the VS Code "Dev Containers" extension. See -[`.devcontainer/README.md`](./.devcontainer/README.md) for the longer -how-to. - -### Install protobuf - -> **Skip this section if you're using the [devcontainer](#recommended-dev-container-any-host-os).** -> The instructions below cover the manual fallback for hosts where -> Docker isn't available. - -Pick whichever channel fits your platform; loader does not bundle protobuf. - -- **vcpkg (recommended, cross-platform):** - ```sh - git clone https://github.com/microsoft/vcpkg.git ~/vcpkg - ~/vcpkg/bootstrap-vcpkg.sh # macOS / Linux - # .\vcpkg\bootstrap-vcpkg.bat # Windows - ~/vcpkg/vcpkg install protobuf # Linux: x64-linux - # ~/vcpkg/vcpkg install protobuf:x64-osx # macOS - # .\vcpkg\vcpkg install protobuf:x64-windows-static # Windows (matches loader's static CRT) - ``` - This installs whatever protobuf version the vcpkg checkout's baseline ships - (currently the 6.x line). To pin a specific version, use vcpkg **manifest - mode**: drop a `vcpkg.json` in your build directory with a `builtin-baseline` - + `overrides`, e.g. - - ```json - { - "name": "loader-build", - "version": "0.1.0", - "dependencies": ["protobuf"], - "overrides": [{ "name": "protobuf", "version": "3.21.12" }], - "builtin-baseline": "" - } - ``` - - > **Note:** classic-mode `vcpkg install --x-version=...` is silently a no-op; - > version pinning only works in manifest mode. See - > `.github/workflows/testing-cpp.yml` for the exact pattern CI uses. - - Then put `protoc` on `PATH` (so `buf generate` works) and pass - `-DCMAKE_TOOLCHAIN_FILE=/scripts/buildsystems/vcpkg.cmake` to - CMake. See [Dev at Linux](#dev-at-linux) / [Dev at Windows](#dev-at-windows) - for the exact commands. - -- **Linux (system package):** - ```sh - sudo apt-get install -y protobuf-compiler libprotobuf-dev # Debian / Ubuntu - ``` - > **Avoid `dnf` / `yum` on RHEL-family distros.** The `protobuf-devel` - > shipped by Fedora / RHEL / TencentOS repos is typically stuck on - > protobuf **3.5.x**, which is far behind what loader expects and predates - > the v22 / Abseil split. Use vcpkg or build from source instead. - -- **macOS (Homebrew):** - ```sh - brew install protobuf - ``` - -- **From source:** see [Protocol Buffers C++ Installation](https://github.com/protocolbuffers/protobuf/tree/master/src). - After installing, point CMake at it with `-DCMAKE_PREFIX_PATH=/path/to/protobuf-install` - (or `-DProtobuf_ROOT=...`). - -### Windows: bootstrap the rest of the toolchain - -> **Skip this section if you're using the [devcontainer](#recommended-dev-container-any-host-os).** -> `prepare.bat` is the manual fallback for Windows hosts that can't run -> Docker. - -Run `prepare.bat` **as Administrator** to install everything you need on a -fresh Windows machine: [Chocolatey](https://chocolatey.org/), -[CMake](https://github.com/Kitware/CMake/releases), -[Ninja](https://ninja-build.org/), MSVC build tools, [buf](https://buf.build/), -**vcpkg**, and `protobuf:x64-windows-static`. It also activates the MSVC -compiler environment for the current cmd session. - -```bat -.\prepare.bat +| Plugin | Language | Generated extension | +| --- | --- | --- | +| `protoc-gen-go-tableau-loader` | Go | `*.pc.go` | +| `protoc-gen-cpp-tableau-loader` | C++17 | `*.pc.h` / `*.pc.cc` | +| `protoc-gen-csharp-tableau-loader` | C# (Unity 2022.3 LTS / .NET 8) | `*.pc.cs` | + +## Quick start + +Use [`make.py`](./make.py) (Python 3.10+, stdlib only): + +```sh +python make.py setup --lang all # one-time host toolchain install +python make.py test --lang go # Go +python make.py test --lang cpp # C++ +python make.py test --lang csharp # C# +python make.py test --lang ts # TypeScript (experimental) ``` -> ⚠️ **Admin required:** This script uses Chocolatey and MSI installers that write to system-protected directories (`C:\ProgramData`, `C:\Program Files`). Right-click Command Prompt → **Run as administrator**, then execute the script. -> -> Preview what the script would do without making any changes: -> ```bat -> .\prepare.bat --dry-run -> ``` -> -> Override the protobuf vcpkg port version (e.g. for the legacy v3 ABI): -> ```bat -> set PROTOBUF_VCPKG_VERSION=3.21.12 && .\prepare.bat -> ``` -> Setting this switches the script to vcpkg **manifest mode** — the only mode -> in which the version pin actually takes effect. The install root moves from -> `%VCPKG_ROOT%\installed\x64-windows-static\` to a manifest dir under -> `%LOCALAPPDATA%\loader\vcpkg-manifest\vcpkg_installed\`, and `prepare.bat` -> exports its path as `%VCPKG_INSTALLED_DIR%`. Your downstream CMake invocation -> must then add `-DVCPKG_INSTALLED_DIR=%VCPKG_INSTALLED_DIR%` and -> `-DVCPKG_MANIFEST_INSTALL=OFF` (see [Dev at Windows](#dev-at-windows)). +Recommended environment: [devcontainer](./.devcontainer/) (open in VS Code → **Dev Containers: Reopen in Container**). Inside the container, `setup` is a no-op. -> **Note:** The **installation** part of `prepare.bat` only runs once per machine — it detects already-installed tools (Chocolatey, Ninja, CMake, MSVC Build Tools, buf, vcpkg, protobuf) and skips them, so no manual installation is required. -> -> However, the MSVC compiler environment (`cl.exe` on `PATH`, plus `INCLUDE` / `LIB` / `LIBPATH` / `WindowsSdkDir` / `VCToolsInstallDir`) is exported to the **current cmd session only** — `vcvarsall.bat` does not (and should not) write these into the persistent user `PATH`. You therefore need to re-run `.\prepare.bat` in **every new cmd window** before building the loader. Subsequent runs are near-instant since no installation work is repeated. +Native hosts: `python make.py setup` installs everything via `brew` (macOS), `apt`/`dnf` (Linux), or Chocolatey + MSVC + vcpkg (Windows). On Windows it must be run from **cmd as Administrator** the first time; subsequent runs work from any shell because each subprocess sources `vcvarsall.bat` itself — your shell PATH/INCLUDE/LIB are never mutated. -### References +## Commands -- [Chocolatey](https://chocolatey.org/) -- [CMake 3.31.8](https://github.com/Kitware/CMake/releases/tag/v3.31.8) -- [Ninja](https://ninja-build.org/) -- [Visual Studio 2022](https://visualstudio.microsoft.com/downloads/) -- [Use the Microsoft C++ Build Tools from the command line](https://learn.microsoft.com/en-us/cpp/build/building-on-the-command-line?view=msvc-170) -- [vcpkg](https://github.com/microsoft/vcpkg) -- [buf CLI](https://buf.build/docs/cli/) +``` +python make.py setup [--lang go|cpp|csharp|ts|all] +python make.py generate --lang go|cpp|csharp|ts +python make.py build --lang go|cpp|csharp|ts [--cxx-std 17|20] [--cxx-compiler msvc|clang|gcc] + [--protobuf-version ] [--triplet ] +python make.py test --lang go|cpp|csharp|ts [-k ] [--smoke] [--coverage] [--no-race] + (+ all build flags) +python make.py clean [--lang ...] [--all] +python make.py env # diagnostic JSON +python make.py --version +``` -## C++ +Global flags: `--verbose / -v`, `--dry-run`, `--cwd `. -### Dev at Linux +Examples: -- Change dir: `cd test/cpp-tableau-loader` -- Generate protoconf: `buf generate ..` (assumes `protoc` is on `PATH`; if you installed via vcpkg, prepend `/installed/x64-linux/tools/protobuf` to `PATH`) -- CMake (system protobuf): - - C++17: `cmake -S . -B build -DCMAKE_BUILD_TYPE=Debug` - - C++20: `cmake -S . -B build -DCMAKE_BUILD_TYPE=Debug -DCMAKE_CXX_STANDARD=20` - - clang: `cmake -S . -B build -DCMAKE_BUILD_TYPE=Debug -DCMAKE_CXX_COMPILER=clang++` -- CMake (vcpkg-provided protobuf): - ```sh - cmake -S . -B build -DCMAKE_BUILD_TYPE=Debug \ - -DCMAKE_TOOLCHAIN_FILE=/scripts/buildsystems/vcpkg.cmake - ``` -- Build: `cmake --build build --parallel` -- Test: `ctest --test-dir build --output-on-failure` +```sh +python make.py test --lang go -k Test_ActivityConf_OrderedMap +python make.py test --lang go --race # opt in (Windows default is off; needs cgo+MSVC) +python make.py test --lang cpp --protobuf-version 3.21.12 +python make.py test --lang csharp -k HubTest.Load +python make.py test --lang cpp --no-clean # skip pre-build wipe +``` -### Dev at Windows +The C++ flow wipes `test/cpp-tableau-loader/{build,src/tableau,src/protoconf}` by default (gitignored `*.pb.*` from a prior protobuf version shadows fresh codegen). -> **Important:** CMake with Ninja requires MSVC environment variables (`cl.exe`, `INCLUDE`, `LIB`, etc.) to be active. Run `.\prepare.bat` from the **loader** root in the **same cmd session** (use **cmd**, not PowerShell — `prepare.bat` exports vars via `endlocal & set ...` which only works for a cmd parent process) before switching to the test directory. Opening a new terminal window will lose these variables. -> -> **Build type:** vcpkg's `x64-windows-static` triplet (and our `prepare.bat`) builds protobuf as **Debug** with the static CRT (`/MTd`). To avoid LNK2038 `_ITERATOR_DEBUG_LEVEL` / `RuntimeLibrary` CRT-mismatch errors, the loader must also be built as Debug. `CMakeLists.txt` does not set a default, so always pass `-DCMAKE_BUILD_TYPE=Debug` explicitly — also required for multi-config generators (Visual Studio default = Debug, but stay explicit to match the protobuf you installed). +## Tests for `make.py` itself -- Initialize MSVC environment (from loader root): `.\prepare.bat` -- Change dir: `cd test\cpp-tableau-loader`, or change directory with Drive, e.g.: `cd /D D:\GitHub\loader\test\cpp-tableau-loader` -- Generate protoconf: `buf generate ..` (the `prepare.bat` step above already puts the vcpkg-built `protoc.exe` on `PATH`) -- CMake (vcpkg-provided protobuf, classic mode — default): - - C++17: `cmake -S . -B build -G Ninja -DCMAKE_BUILD_TYPE=Debug -DCMAKE_TOOLCHAIN_FILE=%VCPKG_ROOT%\scripts\buildsystems\vcpkg.cmake -DVCPKG_TARGET_TRIPLET=x64-windows-static` - - C++20: append `-DCMAKE_CXX_STANDARD=20` -- CMake (vcpkg manifest mode — only when you ran `prepare.bat` with `PROTOBUF_VCPKG_VERSION` set): - - C++17: `cmake -S . -B build -G Ninja -DCMAKE_BUILD_TYPE=Debug -DCMAKE_TOOLCHAIN_FILE=%VCPKG_ROOT%\scripts\buildsystems\vcpkg.cmake -DVCPKG_TARGET_TRIPLET=x64-windows-static -DVCPKG_INSTALLED_DIR="%VCPKG_INSTALLED_DIR%" -DVCPKG_MANIFEST_INSTALL=OFF` -- Build: `cmake --build build --parallel` -- Test: `ctest --test-dir build --output-on-failure` +```sh +pip install pytest +python -m pytest test_make.py -v +``` -> **Note:** Tests are written with [GoogleTest](https://github.com/google/googletest), pulled in via CMake `FetchContent` (no manual installation needed). +CI: [`.github/workflows/testing-make.yml`](.github/workflows/testing-make.yml). -### References +## References -- [Protocol Buffers C++ Installation](https://github.com/protocolbuffers/protobuf/tree/master/src) - [Protocol Buffers C++ Reference](https://protobuf.dev/reference/cpp/) - -## Go - -- Install: **go1.24** or above -- Change dir: `cd test/go-tableau-loader` -- Generate protoconf: `buf generate ..` -- Test: `go test ./...` - -### References - - [Protocol Buffers Go Reference](https://protobuf.dev/reference/go/) - -## C# - -### Requirements - -- Unity 2022.3 LTS (C# 9) -- dotnet-sdk-8.0 - -### Test - -- Install: **dotnet-sdk-8.0** -- Change dir: `cd test/csharp-tableau-loader` -- Generate protoconf: `buf generate ..` (requires `protoc` on `PATH`; install protobuf as described in [Install protobuf](#install-protobuf)) -- Test: `dotnet test` - -> **Note:** Tests are written with [xUnit](https://xunit.net/). - -## TypeScript - -### Requirements - -- nodejs v16.0.0 -- protobufjs v7.2.3 - -### Test - -- Change dir: `cd test/ts-tableau-loader` -- Install depedencies: `npm install` -- Generate protoconf: `npm run generate` -- Test: `npm run test` - -### Problems in [protobufjs](https://github.com/protobufjs/protobuf.js): - -- [Unable to use Google well known types](https://github.com/protobufjs/protobuf.js/issues/1042) -- [google.protobuf.Timestamp deserialization incompatible with canonical JSON representation](https://github.com/protobufjs/protobuf.js/issues/893) -- [Implement wrapper for google.protobuf.Timestamp, and correctly generate wrappers for static target.](https://github.com/protobufjs/protobuf.js/pull/1258) - - -> [protobufjs: Reflection vs. static code](https://github.com/protobufjs/protobuf.js/blob/master/cli/README.md#reflection-vs-static-code) - -If using reflection (`.proto` or `JSON`) but not static code, and for well-known types support, then [proto3-json-serializer](https://github.com/googleapis/proto3-json-serializer-nodejs) is a good option. This library implements proto3 JSON serialization and deserialization for -[protobuf.js](https://www.npmjs.com/package/protobufjs) protobuf objects -according to the [spec](https://protobuf.dev/programming-guides/proto3/#json). - -### References: - -- [How to Setup a TypeScript + Node.js Project](https://khalilstemmler.com/blogs/typescript/node-starter-project/) +- [vcpkg](https://github.com/microsoft/vcpkg) +- [buf CLI](https://buf.build/docs/cli/) - [proto3-json-serializer](https://github.com/googleapis/proto3-json-serializer-nodejs) diff --git a/docs/superpowers/plans/2026-05-29-devcontainer.md b/docs/superpowers/plans/2026-05-29-devcontainer.md deleted file mode 100644 index 0188240..0000000 --- a/docs/superpowers/plans/2026-05-29-devcontainer.md +++ /dev/null @@ -1,1047 +0,0 @@ -# Dev Container Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Add a Dev Container under `.devcontainer/` so contributors on any host (Windows/macOS/Linux) get a one-command, reproducible build environment that mirrors CI's exact toolchain (C++17, Go 1.24, .NET 8, Node 20, buf 1.67.0, protobuf 6.33.4 via vcpkg). - -**Architecture:** Single-stage `Dockerfile` based on `mcr.microsoft.com/devcontainers/cpp:1-ubuntu-24.04`, layered Go → buf → vcpkg/protobuf → .NET/Node. Multi-arch via BuildKit `TARGETARCH` (amd64 + arm64 native, no QEMU). Protobuf version pinnable via the `LOADER_PROTOBUF_VERSION` host env var, flowing through `devcontainer.json` build args into a vcpkg manifest-mode install with a post-install version assertion. - -**Tech Stack:** Docker (BuildKit), `mcr.microsoft.com/devcontainers/cpp:1-ubuntu-24.04`, vcpkg manifest mode, VS Code Dev Containers spec. - -**Spec:** [`docs/superpowers/specs/2026-05-29-devcontainer-design.md`](./../specs/2026-05-29-devcontainer-design.md) - ---- - -## Files Created / Modified - -| Path | Action | Purpose | -|---|---|---| -| `.devcontainer/Dockerfile` | Create | Multi-arch, multi-language toolchain image | -| `.devcontainer/devcontainer.json` | Create | VS Code config: build args, named volume, extensions, banner | -| `.devcontainer/README.md` | Create | Host prerequisites, how-to, host-OS caveats | -| `README.md` | Modify | Add "Recommended: Dev Container" subsection at top of Prerequisites; add "If you can't / don't want to use the devcontainer…" lead-in to existing Windows + per-language blocks | - -CI workflows (`.github/workflows/*.yml`), `prepare.bat`, `buf.gen.yaml` files, and `CMakeLists.txt` are **not** touched. - ---- - -## Conventions for this plan - -- Each Dockerfile change is **one logical layer**, built and verified before the next is added. The "test" for a layer is `docker build` + a `docker run --rm` smoke check that the binary on PATH reports the expected version. -- Build target image tag: `loader-devcontainer:dev` (overwritten each build). -- Build context is `.devcontainer/` so all `docker build` commands use `docker build -t loader-devcontainer:dev .devcontainer/`. -- Smoke checks use `docker run --rm loader-devcontainer:dev `. -- After Task 5 the image takes ~25 minutes to rebuild from scratch on first run because vcpkg compiles protobuf from source. Subsequent builds reuse layers and only re-run the layer you changed. - ---- - -### Task 1: Stub Dockerfile with the base image only - -**Files:** -- Create: `.devcontainer/Dockerfile` - -- [ ] **Step 1: Create the file** - -`.devcontainer/Dockerfile`: - -```dockerfile -# syntax=docker/dockerfile:1.7 -# tableauio/loader devcontainer -# -# Single-stage, multi-arch (amd64 + arm64) image bringing the full -# C++/Go/.NET/Node toolchain plus protobuf 6.33.4 (via vcpkg) at the -# exact versions CI uses. See docs/superpowers/specs/2026-05-29-devcontainer-design.md. - -FROM mcr.microsoft.com/devcontainers/cpp:1-ubuntu-24.04 -``` - -- [ ] **Step 2: Build the base layer** - -```sh -docker build -t loader-devcontainer:dev .devcontainer/ -``` - -Expected: build completes in <1 min on a warm Docker, ending with -`=> => naming to docker.io/library/loader-devcontainer:dev`. - -- [ ] **Step 3: Smoke-check the base image runs and gives us the vscode user** - -```sh -docker run --rm loader-devcontainer:dev id -``` - -Expected output: `uid=0(root) gid=0(root) groups=0(root)` (RUN context defaults to root; the `vscode` user is set later via `devcontainer.json`'s `remoteUser`). Confirm with: - -```sh -docker run --rm loader-devcontainer:dev id vscode -``` - -Expected: `uid=1000(vscode) gid=1000(vscode) groups=1000(vscode),...` — confirming the base image ships the `vscode` user. - -- [ ] **Step 4: Commit** - -```sh -git add .devcontainer/Dockerfile -git commit -m "$(cat <<'EOF' -feat(devcontainer): add base Dockerfile (Ubuntu 24.04 cpp image) - -Bootstrap the devcontainer image with Microsoft's multi-arch -mcr.microsoft.com/devcontainers/cpp:1-ubuntu-24.04 base. Subsequent -commits layer Go, buf, vcpkg/protobuf, .NET, and Node on top. - -Refs: docs/superpowers/specs/2026-05-29-devcontainer-design.md -EOF -)" -``` - ---- - -### Task 2: Add architecture-detection layer - -Resolves `TARGETARCH` (BuildKit auto-populates it from the host) into the per-arch values that downstream layers need: Go's tarball arch, buf's release-asset arch, and the vcpkg triplet. Writes them to `/opt/buildargs.env` so later `RUN` commands can `source` them — Dockerfile `ARG`s don't persist across `RUN` blocks the way ENV does, but a shell-readable file does. - -**Files:** -- Modify: `.devcontainer/Dockerfile` - -- [ ] **Step 1: Append the architecture detection block** - -Append to `.devcontainer/Dockerfile`: - -```dockerfile -# --------------------------------------------------------------------------- -# Architecture detection. BuildKit auto-populates TARGETARCH; we resolve it -# into per-arch download-name fragments (Go's tarball, buf's release asset, -# vcpkg triplet) and persist them to /opt/buildargs.env so later RUN layers -# can `source` them — Dockerfile ARGs don't survive across RUN boundaries. -# --------------------------------------------------------------------------- -ARG TARGETARCH -RUN < /opt/buildargs.env -EOF -``` - -- [ ] **Step 2: Build** - -```sh -docker build -t loader-devcontainer:dev .devcontainer/ -``` - -Expected: success in <1 min. The architecture-detection layer is just a `case` + `printf`. - -- [ ] **Step 3: Smoke-check the resolved values** - -```sh -docker run --rm loader-devcontainer:dev cat /opt/buildargs.env -``` - -Expected (on an amd64 host): -``` -GO_ARCH=amd64 -BUF_ARCH=x86_64 -VCPKG_TRIPLET=x64-linux -``` - -(arm64 host would print `GO_ARCH=arm64`, `BUF_ARCH=aarch64`, `VCPKG_TRIPLET=arm64-linux`.) - -- [ ] **Step 4: Commit** - -```sh -git add .devcontainer/Dockerfile -git commit -m "$(cat <<'EOF' -feat(devcontainer): add architecture detection - -Resolves TARGETARCH (amd64 or arm64) into per-arch values -(Go tarball arch, buf release-asset arch, vcpkg triplet) and -writes them to /opt/buildargs.env for downstream RUN layers -to source. Unknown arches fail the build. -EOF -)" -``` - ---- - -### Task 3: Add Go 1.24.0 - -**Files:** -- Modify: `.devcontainer/Dockerfile` - -- [ ] **Step 1: Append the Go layer** - -Append to `.devcontainer/Dockerfile`: - -```dockerfile -# --------------------------------------------------------------------------- -# Go 1.24.0 — official tarball into /usr/local/go. -# -# PATH is set via ENV (not /etc/profile.d/) so non-interactive shells like -# the postCreateCommand and downstream RUNs see Go without sourcing profile. -# /home/vscode/go/bin lands `go install`-placed binaries on PATH automatically. -# --------------------------------------------------------------------------- -ARG GO_VERSION=1.24.0 -RUN < /opt/vcpkg-manifest/vcpkg.json </dev/null | head -n1) -case "$(basename "${INFO_FILE:-/missing}" 2>/dev/null)" in - protobuf_${PROTOBUF_VERSION}*) - ;; - *) - echo "ERROR: installed protobuf does not match requested version ${PROTOBUF_VERSION}." - echo " vcpkg installed-file marker: ${INFO_FILE:-}" - echo " Bump VCPKG_BASELINE_COMMIT (in this Dockerfile, prepare.bat," - echo " and testing-cpp.yml) to a commit that knows about the requested version." - exit 1 - ;; -esac - -# 5. Stable symlinks so ENV CMAKE_PREFIX_PATH (last layer) doesn't have to -# care about the underlying triplet. -ln -s /opt/vcpkg-manifest/vcpkg_installed/${VCPKG_TRIPLET} /opt/vcpkg/active -ln -s /opt/vcpkg/active/tools/protobuf/protoc /usr/local/bin/protoc -EOF -``` - -- [ ] **Step 2: Build (this is the slow one, ~25 minutes on cold cache)** - -```sh -docker build -t loader-devcontainer:dev .devcontainer/ -``` - -Expected: success after ~25 minutes on first run; subsequent builds reuse the layer instantly. The build log includes lines like `protobuf:x64-linux@6.33.4 -- Building`, then a long CMake/ninja compile, then `Total install time:`. - -- [ ] **Step 3: Smoke-check protoc** - -```sh -docker run --rm loader-devcontainer:dev protoc --version -``` - -Expected: `libprotoc 33.4` (the protoc binary reports the umbrella protoc release tag, which is `33.4` for the libprotobuf C++ 6.33.4 line — same mapping `testing-csharp.yml` documents). - -- [ ] **Step 4: Smoke-check the symlinks** - -```sh -docker run --rm loader-devcontainer:dev sh -c 'readlink -f /usr/local/bin/protoc; readlink -f /opt/vcpkg/active' -``` - -Expected (on amd64): -``` -/opt/vcpkg-manifest/vcpkg_installed/x64-linux/tools/protobuf/protoc -/opt/vcpkg-manifest/vcpkg_installed/x64-linux -``` - -- [ ] **Step 5: Smoke-check the version-assertion marker file** - -```sh -docker run --rm loader-devcontainer:dev sh -c 'ls /opt/vcpkg-manifest/vcpkg_installed/vcpkg/info/protobuf_*.list' -``` - -Expected: a single file named like `protobuf_6.33.4_x64-linux.list` (or with a `#N` port-revision suffix — the assertion uses prefix matching so either passes). - -- [ ] **Step 6: Commit** - -```sh -git add .devcontainer/Dockerfile -git commit -m "$(cat <<'EOF' -feat(devcontainer): add vcpkg + protobuf via manifest mode - -Pin vcpkg to commit dc8d75c…df932 (lock-step with prepare.bat and -testing-cpp.yml's VCPKG_COMMIT). Render a minimal vcpkg.json manifest -with the protobuf override + builtin-baseline, install via -manifest mode (the only mode where the version pin actually takes -effect), and post-assert that the resolved port version starts with -the requested PROTOBUF_VERSION. Default is 6.33.4; legacy v3 reachable -via --build-arg PROTOBUF_VERSION=3.21.12. - -Symlink /opt/vcpkg/active → installed/ and /usr/local/bin/protoc -→ active/tools/protobuf/protoc so downstream ENV/PATH stays -arch-independent. -EOF -)" -``` - ---- - -### Task 6: Verify the protobuf version-pinning path (no Dockerfile change) - -This task is verification-only: a deliberate counterexample build that proves the `LOADER_PROTOBUF_VERSION` knob works end-to-end. No commit produced. - -- [ ] **Step 1: Build with protobuf 3.21.12 (legacy v3 line)** - -```sh -docker build \ - --build-arg PROTOBUF_VERSION=3.21.12 \ - -t loader-devcontainer:legacy-v3 .devcontainer/ -``` - -Expected: vcpkg layer rebuilds (~10 min on cache hit for everything before that layer; the protobuf compile itself takes the whole time). Final assertion succeeds because the resolved port matches `3.21.12`. - -- [ ] **Step 2: Smoke-check protoc reports the legacy version** - -```sh -docker run --rm loader-devcontainer:legacy-v3 protoc --version -``` - -Expected: `libprotoc 3.21.12` (the legacy-v3 line still uses `3.x.y` in `protoc --version`; the umbrella tag is `v21.12`). - -- [ ] **Step 3: Verify the assertion fires when given an impossible version** - -```sh -docker build \ - --build-arg PROTOBUF_VERSION=999.0.0 \ - -t loader-devcontainer:never .devcontainer/ 2>&1 | tail -20 -``` - -Expected: build fails. The vcpkg manifest install errors out (or the assertion catches it), and the last lines include either `error: while looking up version 999.0.0` (vcpkg-side rejection) or `ERROR: installed protobuf does not match requested version 999.0.0` (our assertion). Either failure is acceptable; both prove the silent-wrong-version regression is impossible. - -- [ ] **Step 4: Drop the temporary tags** - -```sh -docker rmi loader-devcontainer:legacy-v3 loader-devcontainer:never 2>/dev/null || true -``` - -(No commit — this task is purely verification of behaviour established in Task 5.) - ---- - -### Task 7: Add .NET SDK 8.0 + Node.js 20 LTS - -**Files:** -- Modify: `.devcontainer/Dockerfile` - -- [ ] **Step 1: Append the .NET + Node layer** - -Append to `.devcontainer/Dockerfile`: - -```dockerfile -# --------------------------------------------------------------------------- -# .NET SDK 8.0 + Node.js 20 LTS — apt-based installs from the official -# Microsoft and NodeSource repositories. apt-get clean + rm /var/lib/apt/lists -# at the end keeps the layer small. -# --------------------------------------------------------------------------- -RUN <` (latest 8.0.x at apt-cache time; e.g. `8.0.404`). - -- [ ] **Step 4: Smoke-check Node** - -```sh -docker run --rm loader-devcontainer:dev node --version -``` - -Expected: `v20..` (latest v20 LTS at NodeSource setup time). - -- [ ] **Step 5: Commit** - -```sh -git add .devcontainer/Dockerfile -git commit -m "$(cat <<'EOF' -feat(devcontainer): add .NET SDK 8.0 and Node.js 20 LTS - -Microsoft apt repo for dotnet-sdk-8.0 (matches testing-csharp.yml's -target). NodeSource setup_20.x for Node 20 LTS (covers the -experimental _lab/ts/ workflow; not currently in CI). Both clean -their apt caches to keep the layer small. -EOF -)" -``` - ---- - -### Task 8: Final ENV (CMAKE_PREFIX_PATH) - -This is the single line that lets every existing per-language daily command from the README work inside the container without flag tweaks. `find_package(Protobuf CONFIG)` resolves to vcpkg's pinned protobuf because `CMAKE_PREFIX_PATH` includes `/opt/vcpkg/active`. - -**Files:** -- Modify: `.devcontainer/Dockerfile` - -- [ ] **Step 1: Append the final ENV block** - -Append to `.devcontainer/Dockerfile`: - -```dockerfile -# --------------------------------------------------------------------------- -# Final environment: with CMAKE_PREFIX_PATH pointing at vcpkg's installed -# tree, find_package(Protobuf CONFIG) resolves automatically — the existing -# README's "Dev at Linux → CMake (system protobuf)" recipe works as written -# inside the container. -# VCPKG_ROOT is set above (Task 5) for users who want to invoke the toolchain -# file explicitly: -DCMAKE_TOOLCHAIN_FILE=$VCPKG_ROOT/scripts/buildsystems/vcpkg.cmake. -# --------------------------------------------------------------------------- -ENV CMAKE_PREFIX_PATH=/opt/vcpkg/active -``` - -- [ ] **Step 2: Build** - -```sh -docker build -t loader-devcontainer:dev .devcontainer/ -``` - -Expected: success in <30 s (only the ENV layer changes). - -- [ ] **Step 3: Smoke-check the env** - -```sh -docker run --rm loader-devcontainer:dev sh -c 'echo "VCPKG_ROOT=$VCPKG_ROOT"; echo "CMAKE_PREFIX_PATH=$CMAKE_PREFIX_PATH"' -``` - -Expected: -``` -VCPKG_ROOT=/opt/vcpkg -CMAKE_PREFIX_PATH=/opt/vcpkg/active -``` - -- [ ] **Step 4: Commit** - -```sh -git add .devcontainer/Dockerfile -git commit -m "$(cat <<'EOF' -feat(devcontainer): finalize CMAKE_PREFIX_PATH - -CMAKE_PREFIX_PATH=/opt/vcpkg/active lets the existing README cmake -recipe (-DCMAKE_BUILD_TYPE=Debug, no toolchain file) resolve protobuf -inside the container without any flag changes from the host workflow. -EOF -)" -``` - ---- - -### Task 9: Add `devcontainer.json` - -**Files:** -- Create: `.devcontainer/devcontainer.json` - -- [ ] **Step 1: Create the file** - -`.devcontainer/devcontainer.json`: - -```jsonc -{ - // tableauio/loader Dev Container. - // See docs/superpowers/specs/2026-05-29-devcontainer-design.md for the design. - "name": "tableauio/loader", - - // Build args wire host env to Dockerfile ARGs: - // LOADER_PROTOBUF_VERSION on the host -> PROTOBUF_VERSION inside. - // Default 6.33.4 (CI's modern matrix entry). To rebuild against legacy v3: - // LOADER_PROTOBUF_VERSION=3.21.12 code . # then Reopen in Container. - "build": { - "dockerfile": "Dockerfile", - "args": { - "PROTOBUF_VERSION": "${localEnv:LOADER_PROTOBUF_VERSION:6.33.4}" - } - }, - - // Persist the Go module cache across container rebuilds. Workspace itself - // uses VS Code's default bind-mount so edits sync to the host. - "mounts": [ - "source=loader-go-mod,target=/home/vscode/go,type=volume" - ], - - "remoteUser": "vscode", - "workspaceFolder": "/workspaces/loader", - - "customizations": { - "vscode": { - "extensions": [ - "golang.go", - "ms-vscode.cmake-tools", - "ms-vscode.cpptools", - "ms-dotnettools.csharp", - "bufbuild.vscode-buf", - "zxh404.vscode-proto3" - ], - "settings": { - // Don't auto-install gopls and friends on first open — let the user - // do it explicitly from the Go extension's command palette. - "go.toolsManagement.autoUpdate": false, - // Don't auto-cmake-configure on workspace open; we run cmake manually - // per the existing README recipes. - "cmake.configureOnOpen": false - } - } - }, - - // One-line ready banner so the developer knows the container is healthy. - // Pure echo — no installs, no version-pinning at runtime, no surprises. - "postCreateCommand": "printf 'tableauio/loader devcontainer ready.\\n go: %s\\n buf: %s\\n protoc: %s\\n dotnet: %s\\n node: %s\\n' \"$(go version | cut -d' ' -f3)\" \"$(buf --version)\" \"$(protoc --version)\" \"$(dotnet --version)\" \"$(node --version)\"" -} -``` - -- [ ] **Step 2: Validate the JSON parses (with jsonc comments stripped)** - -```sh -python3 - <<'PY' -import json, re, pathlib -src = pathlib.Path('.devcontainer/devcontainer.json').read_text() -# Strip // line comments (devcontainer.json is jsonc). -stripped = re.sub(r'(^|[^:])//.*$', r'\1', src, flags=re.MULTILINE) -parsed = json.loads(stripped) -assert parsed['name'] == 'tableauio/loader' -assert parsed['build']['dockerfile'] == 'Dockerfile' -assert parsed['build']['args']['PROTOBUF_VERSION'] == '${localEnv:LOADER_PROTOBUF_VERSION:6.33.4}' -print('devcontainer.json OK') -PY -``` - -Expected: `devcontainer.json OK`. - -- [ ] **Step 3: Commit** - -```sh -git add .devcontainer/devcontainer.json -git commit -m "$(cat <<'EOF' -feat(devcontainer): add devcontainer.json - -Wires the Dockerfile under build.args, mounts a named volume for the -Go module cache, declares the VS Code extension set, and prints a -one-line ready-banner via postCreateCommand. PROTOBUF_VERSION flows -from the host LOADER_PROTOBUF_VERSION env var (default 6.33.4) so -contributors can rebuild against the legacy v3 line via: - LOADER_PROTOBUF_VERSION=3.21.12 code . -EOF -)" -``` - ---- - -### Task 10: Add `.devcontainer/README.md` - -**Files:** -- Create: `.devcontainer/README.md` - -- [ ] **Step 1: Create the file** - -`.devcontainer/README.md`: - -````markdown -# Dev Container - -The recommended way to develop on `tableauio/loader`. One container, all -four target languages (C++17, Go 1.24, .NET 8, Node 20) plus protobuf -6.33.4 via vcpkg, pinned to the exact toolchain CI uses. - -## Prerequisites - -- Docker Desktop (Windows / macOS) or Docker Engine (Linux) -- VS Code with the [Dev Containers extension](https://marketplace.visualstudio.com/items?itemName=ms-vscode-remote.remote-containers) - -## Open the container - -```sh -code . # in the repo root -``` - -In VS Code, run **Dev Containers: Reopen in Container** from the command -palette. First build is one-time ~25 minutes (vcpkg compiles protobuf -6.33.4 from source); subsequent reopens are near-instant. - -When the container is ready, the integrated terminal prints a banner with -five toolchain versions. After that, every command from the per-language -sections of the repo root [`README.md`](../README.md) works as written — -no PATH dance, no extra cmake flags. - -## Pin a different protobuf version - -Daily dev runs against protobuf 6.33.4 (CI's "modern" matrix entry). To -rebuild against the legacy v3 line: - -```sh -LOADER_PROTOBUF_VERSION=3.21.12 code . -``` - -…then **Dev Containers: Reopen in Container** (or **Rebuild Container** -if the container is already running). The vcpkg layer rebuilds with the -manifest pinning protobuf 3.21.12; everything else is reused from the -cache. - -## Host-OS caveats - -- **Windows.** WSL2 backend required. **Check the workspace out under - WSL2** (e.g. `\\wsl.localhost\Ubuntu\home\\loader`) — not under - `/mnt/c/...` — for good bind-mount performance. Files under `/mnt/c/` - work but file-watching and large `cmake --build` operations are 5–10× - slower. - -- **Apple Silicon.** Docker builds the container natively as arm64. No - Rosetta or QEMU emulation. Confirm with `docker info | grep Architecture` - → expect `linux/arm64`. - -- **Linux (native Docker Engine).** No special configuration. - -## Architecture - -Single-stage Dockerfile based on -`mcr.microsoft.com/devcontainers/cpp:1-ubuntu-24.04`, with these layers: - -1. Architecture detection (`TARGETARCH` → Go arch, buf arch, vcpkg triplet) -2. Go 1.24.0 (official tarball, multi-arch) -3. buf 1.67.0 (single-binary release, multi-arch) -4. vcpkg pinned to `dc8d75c…df932`, protobuf installed via vcpkg manifest - mode and asserted against the requested version -5. .NET SDK 8.0 (Microsoft apt repo) -6. Node.js 20 LTS (NodeSource apt repo) -7. `ENV CMAKE_PREFIX_PATH=/opt/vcpkg/active` so `find_package(Protobuf CONFIG)` - resolves automatically - -The architecture choice is detected from BuildKit's `TARGETARCH` and fed -into Go / buf / vcpkg triplet selection. Docker auto-selects the host -arch on build. - -## Falling back - -If you can't run Docker (corp policy, restricted machines, etc.) the -existing manual setup paths in the [repo README](../README.md) — Windows -`prepare.bat`, per-language `Install protobuf` instructions — still work. -The devcontainer is the recommended path; the rest is the supported -fallback. -```` - -- [ ] **Step 2: Verify the file renders as expected** - -```sh -ls -la .devcontainer/README.md && wc -l .devcontainer/README.md -``` - -Expected: file exists, ~70 lines. - -- [ ] **Step 3: Commit** - -```sh -git add .devcontainer/README.md -git commit -m "$(cat <<'EOF' -docs(devcontainer): add .devcontainer/README.md - -One-pager covering prerequisites (Docker Desktop / Engine + VS Code -Dev Containers extension), how to open the container, the -LOADER_PROTOBUF_VERSION knob, host-OS caveats (WSL2 workspace -location, Apple Silicon native arm64), and the layered Dockerfile -architecture. Points users at prepare.bat / per-language manual setup -as the explicit fallback path. -EOF -)" -``` - ---- - -### Task 11: Update repo root `README.md` - -Adds the "Recommended: Dev Container" subsection at the top of `Prerequisites` and prefixes the existing Windows + per-language blocks with a short opt-out lead-in. - -**Files:** -- Modify: `README.md` - -- [ ] **Step 1: Add the "Recommended: Dev Container" subsection** - -Open `README.md` and find the Prerequisites bullet list that ends with `…will fail to link.`. Immediately after the existing migration callout (the `> **Migrating from the bundled-protobuf layout?**` block), insert this new subsection — i.e. right before `### Install protobuf`: - -```markdown -### Recommended: Dev Container (any host OS) - -The fastest way to get a reproducible build environment is to open the -repo in VS Code and choose **Reopen in Container**. The devcontainer -under [`.devcontainer/`](./.devcontainer/) has everything pinned to the -exact versions CI uses (Go 1.24, buf 1.67.0, protobuf 6.33.4 via vcpkg, -.NET 8.0, Node 20). First container build is one-time ~25 minutes (vcpkg -compiles protobuf from source); subsequent reopens are near-instant. - -After the container starts you can skip the per-language setup below and -jump straight to **[C++](#c)** / **[Go](#go)** / **[C#](#c-1)** / -**[TypeScript](#typescript)**. - -Requirements: Docker Desktop (Windows + macOS) or Docker Engine (Linux), -and the VS Code "Dev Containers" extension. See -[`.devcontainer/README.md`](./.devcontainer/README.md) for the longer -how-to. -``` - -- [ ] **Step 2: Add an opt-out lead-in to the `Install protobuf` section** - -In `README.md`, find the line that currently reads: - -```markdown -### Install protobuf - -Pick whichever channel fits your platform; loader does not bundle protobuf. -``` - -Replace it with: - -```markdown -### Install protobuf - -> **Skip this section if you're using the [devcontainer](#recommended-dev-container-any-host-os).** -> The instructions below cover the manual fallback for hosts where -> Docker isn't available. - -Pick whichever channel fits your platform; loader does not bundle protobuf. -``` - -- [ ] **Step 3: Add an opt-out lead-in to the Windows bootstrap section** - -Find: - -```markdown -### Windows: bootstrap the rest of the toolchain - -Run `prepare.bat` **as Administrator** to install everything you need on a -fresh Windows machine: [Chocolatey](https://chocolatey.org/), -``` - -Replace with: - -```markdown -### Windows: bootstrap the rest of the toolchain - -> **Skip this section if you're using the [devcontainer](#recommended-dev-container-any-host-os).** -> `prepare.bat` is the manual fallback for Windows hosts that can't run -> Docker. - -Run `prepare.bat` **as Administrator** to install everything you need on a -fresh Windows machine: [Chocolatey](https://chocolatey.org/), -``` - -- [ ] **Step 4: Verify the README still renders sanely** - -```sh -grep -nE '^#{1,4} ' README.md | head -30 -``` - -Expected: shows the heading skeleton, with the new `### Recommended: Dev Container (any host OS)` heading appearing between the Prerequisites bullets and `### Install protobuf`. - -- [ ] **Step 5: Commit** - -```sh -git add README.md -git commit -m "$(cat <<'EOF' -docs: recommend the devcontainer in repo README - -Add a new "Recommended: Dev Container (any host OS)" subsection at the -top of Prerequisites pointing contributors at .devcontainer/. Add a -"Skip this section if you're using the devcontainer" lead-in to the -existing "Install protobuf" and "Windows: bootstrap" blocks so the -manual paths are clearly the fallback, not the primary route. -EOF -)" -``` - ---- - -### Task 12: End-to-end integration check (verification only, no commit) - -Final smoke test: bring the container up exactly the way a contributor would, and run the four E2E test commands from the README to confirm the toolchain inside the container actually exercises the repo. No commit produced. - -- [ ] **Step 1: Build the final container** - -```sh -docker build -t loader-devcontainer:dev .devcontainer/ -``` - -Expected: success. If Tasks 1–10 were committed individually, this should be all-cache-hits and finish in <10 s. - -- [ ] **Step 2: Run the postCreate banner manually** - -```sh -docker run --rm loader-devcontainer:dev sh -c " -printf 'tableauio/loader devcontainer ready.\n go: %s\n buf: %s\n protoc: %s\n dotnet: %s\n node: %s\n' \"\$(go version | cut -d' ' -f3)\" \"\$(buf --version)\" \"\$(protoc --version)\" \"\$(dotnet --version)\" \"\$(node --version)\" -" -``` - -Expected output (versions may vary slightly): -``` -tableauio/loader devcontainer ready. - go: go1.24.0 - buf: 1.67.0 - protoc: libprotoc 33.4 - dotnet: 8.0.404 - node: v20.18.0 -``` - -- [ ] **Step 3: Run the Go E2E inside the container** - -```sh -docker run --rm -v "$(pwd):/workspaces/loader" -w /workspaces/loader/test/go-tableau-loader \ - loader-devcontainer:dev sh -c "buf generate .. && go test ./..." -``` - -Expected: `buf generate` regenerates the Go protoconf and loader stubs, then `go test ./...` reports `ok` for `test/go-tableau-loader`, `internal/index`, `pkg/treemap`, `pkg/udiff`, etc. - -- [ ] **Step 4: Run the C++ E2E inside the container** - -```sh -docker run --rm -v "$(pwd):/workspaces/loader" -w /workspaces/loader/test/cpp-tableau-loader \ - loader-devcontainer:dev sh -c " - buf generate .. && - cmake -S . -B build -G Ninja -DCMAKE_BUILD_TYPE=Debug && - cmake --build build --parallel && - ctest --test-dir build --output-on-failure - " -``` - -Expected: `buf generate` regenerates `*.pb.*` and `*.pc.*`, cmake configure picks up `protobuf::libprotobuf` via `CMAKE_PREFIX_PATH`, build succeeds, ctest reports all tests passed. - -- [ ] **Step 5: Run the C# E2E inside the container** - -```sh -docker run --rm -v "$(pwd):/workspaces/loader" -w /workspaces/loader/test/csharp-tableau-loader \ - loader-devcontainer:dev sh -c "buf generate .. && dotnet test --nologo --logger 'console;verbosity=normal'" -``` - -Expected: protobuf C# stubs regenerated, dotnet builds the project, xUnit reports passed. - -- [ ] **Step 6: Confirm the named volume mount works (interactive)** - -```sh -docker volume create loader-go-mod >/dev/null -docker run --rm -v "$(pwd):/workspaces/loader" \ - -v loader-go-mod:/home/vscode/go \ - -w /workspaces/loader/test/go-tableau-loader \ - loader-devcontainer:dev go test ./... -``` - -Run the command twice. The second run should be noticeably faster (Go's module cache is warm in `/home/vscode/go/pkg/mod`). - -Cleanup: - -```sh -docker volume rm loader-go-mod -``` - -- [ ] **Step 7: Push the branch (no commit; everything is already committed in earlier tasks)** - -```sh -git push origin HEAD -``` - -Expected: branch is pushed to remote with all 10 task commits visible in `git log`. - ---- - -## Self-Review - -**Spec coverage** — every Goals item in the spec maps to a task: - -| Spec goal | Implementing task(s) | -|---|---| -| One-command setup on any host (Reopen in Container) | Tasks 9 (devcontainer.json) + 10 (.devcontainer/README.md) + 11 (repo README) | -| Reproducibility — pinned versions matching CI | Tasks 3 (Go 1.24), 4 (buf 1.67.0), 5 (vcpkg + protobuf 6.33.4 with assertion), 7 (.NET 8.0, Node 20) | -| Multi-arch native (amd64 + arm64) | Tasks 2 (TARGETARCH detection), 3, 4, 5 (per-arch downloads + triplet) | -| Pinnable protobuf version via LOADER_PROTOBUF_VERSION | Tasks 5 (Dockerfile ARG + manifest mode) + 9 (devcontainer.json `${localEnv:...}`) + 6 (verification) | -| Daily commands stay unchanged (CMAKE_PREFIX_PATH) | Task 8 | - -Non-goals (no ghcr.io publish, no CI inside container, no Unity, no replacing prepare.bat) — none of them produce a task, by design. - -**Placeholder scan** — none. Every step has runnable code/commands; commit messages are concrete; expected outputs are spelled out. - -**Type/path consistency** — `loader-devcontainer:dev` is the consistent image tag across all tasks; `.devcontainer/Dockerfile` and `.devcontainer/devcontainer.json` are referenced with consistent paths; `/opt/vcpkg/active`, `/opt/vcpkg-manifest/...`, `/opt/buildargs.env` all used identically across tasks. - ---- - -## Execution Handoff - -Plan complete and saved to `docs/superpowers/plans/2026-05-29-devcontainer.md`. Two execution options: - -**1. Subagent-Driven (recommended)** — I dispatch a fresh subagent per task, review between tasks, fast iteration with two-stage review. - -**2. Inline Execution** — Execute tasks in this session using executing-plans, batch execution with checkpoints. - -Which approach? diff --git a/docs/superpowers/specs/2026-05-29-devcontainer-design.md b/docs/superpowers/specs/2026-05-29-devcontainer-design.md deleted file mode 100644 index 923095f..0000000 --- a/docs/superpowers/specs/2026-05-29-devcontainer-design.md +++ /dev/null @@ -1,200 +0,0 @@ -# Dev Container for tableauio/loader — Design - -**Status:** Approved through brainstorming. Implementation plan to follow via `writing-plans`. -**Date:** 2026-05-29 -**Scope:** Add a Dev Container that pins the full multi-language toolchain -(C++17, Go 1.24, .NET 8, Node 20, buf 1.67.0, protobuf 6.33.4 via vcpkg) so -contributors on any host OS get a reproducible build environment that mirrors -CI exactly. - -## Goals - -1. **One-command setup** on any host (Windows, macOS, Linux) — "Reopen in Container" replaces the existing per-OS, per-language manual setup as the *recommended* path. -2. **Reproducibility** — protobuf, buf, Go, .NET, Node, and the vcpkg checkout are pinned to the exact versions / commit SHA used by CI (`testing-cpp.yml`, `testing-go.yml`, `testing-csharp.yml`). -3. **Multi-arch native** — Apple Silicon contributors build natively as arm64 (no QEMU emulation); amd64 contributors build natively as amd64. One Dockerfile, no buildx publish step required. -4. **Pinnable protobuf version** — daily dev runs against modern (6.33.4) by default; legacy-v3 (3.21.12) is reachable by setting one host env var, with no second Dockerfile. -5. **Daily commands stay unchanged** — every shell snippet currently in README's per-language sections (`buf generate ..`, `cmake -S . -B build -DCMAKE_BUILD_TYPE=Debug`, `go test ./...`, `dotnet test`) works inside the container without flag tweaks. - -## Non-goals - -- **Prebuilt-and-pushed image on ghcr.io.** Out of scope for v1; revisit only if first-run latency (~25 min vcpkg compile) becomes a real complaint. Adding it later is a CI-only change with no Dockerfile churn. -- **CI running inside the devcontainer.** CI keeps `lukka/run-vcpkg` for cached vcpkg installs; rebuilding the devcontainer image per matrix entry would be strictly slower with no reproducibility win. -- **Unity-side C# workflow.** Unity Editor doesn't run in a Linux container; the container covers `.NET 8 + xUnit` (which is what `test/csharp-tableau-loader/` exercises) only. -- **Replacing `prepare.bat` or per-language manual setup.** Both stay as fallback paths for contributors who can't run Docker (corp policy, restricted machines). - -## File layout - -Three new files, one directory; nothing existing moves. - -``` -.devcontainer/ -├── Dockerfile # ~95 lines, single stage, multi-arch, multi-language -├── devcontainer.json # ~30 lines -└── README.md # 1-pager: prerequisites, how to enter, host caveats -``` - -## Architecture - -### Image base - -`mcr.microsoft.com/devcontainers/cpp:1-ubuntu-24.04` — Microsoft's officially-maintained Dev Container base image. Provides: - -- gcc-13, glibc 2.39, cmake, ninja, git, sudo -- Non-root `vscode` user (uid/gid 1000) with passwordless sudo -- Multi-arch: pulls amd64 or arm64 automatically based on host -- Dev Containers shell hooks (`postCreateCommand` etc.) - -### Toolchain layers (Dockerfile, ordered for cache friendliness) - -Each layer is a single `RUN` block, version-pinned, named in a comment block. Order goes least-likely-to-change to most-likely-to-change so editing the latter doesn't invalidate the former. - -1. **Architecture detection.** `ARG TARGETARCH` (BuildKit auto-populates from build host). One `RUN` writes the resolved arch-dependent values to `/opt/buildargs.env` for downstream layers: - - | TARGETARCH | GO_ARCH | BUF_ARCH | VCPKG_TRIPLET | - |---|---|---|---| - | amd64 | amd64 | x86_64 | x64-linux | - | arm64 | arm64 | aarch64 | arm64-linux | - - Unknown arches fail the build with a clear message. - -2. **Go 1.24.0** — download official tarball for `${GO_ARCH}` to `/usr/local/go`. PATH is exposed via `ENV PATH=/usr/local/go/bin:/home/vscode/go/bin:${PATH}` (not `/etc/profile.d/`) so that non-interactive shells like the `postCreateCommand` and any `RUN` in downstream Dockerfiles see Go without sourcing profile. `/home/vscode/go/bin` is included so `go install`-placed binaries land on PATH automatically. - -3. **buf 1.67.0** — single binary download `buf-Linux-${BUF_ARCH}` to `/usr/local/bin/buf`. - -4. **vcpkg + protobuf via manifest mode** *(the heavy layer; ~25 min on first build).* - - Pinned commit: `VCPKG_BASELINE_COMMIT=dc8d75cfc3281b8e2a4ed8ee4163c891190df932` (lock-step with `prepare.bat` and `testing-cpp.yml`'s `VCPKG_COMMIT`). - - Pinned port: `PROTOBUF_VERSION=6.33.4` (a `Dockerfile ARG`, override-friendly — see "Pinnable protobuf version" below). - - Cloned into `/opt/vcpkg`, checked out to the pinned commit, bootstrapped with `-disableMetrics`. - - A small `vcpkg.json` is rendered into `/opt/vcpkg-manifest/` carrying `dependencies: ["protobuf"]`, `overrides: [{name: protobuf, version: ${PROTOBUF_VERSION}}]`, `builtin-baseline: ${VCPKG_BASELINE_COMMIT}`. - - `vcpkg install --triplet=${VCPKG_TRIPLET} --x-install-root=/opt/vcpkg-manifest/vcpkg_installed` runs from that directory. - - **Post-install assertion:** the same `dir-bin/findstr` pattern from `prepare.bat`, ported to bash — fail the build if the resolved port version doesn't have `${PROTOBUF_VERSION}` as a prefix. This is the safety net against future vcpkg-resolution regressions silently producing the wrong version. - - `ln -s /opt/vcpkg-manifest/vcpkg_installed/${VCPKG_TRIPLET} /opt/vcpkg/active` so the architecture-dependent path collapses behind a stable symlink. - - `ln -s /opt/vcpkg/active/tools/protobuf/protoc /usr/local/bin/protoc` so `buf generate` finds protoc with no PATH dance. - -5. **.NET SDK 8.0 + Node.js 20 LTS** — Microsoft's `packages-microsoft-prod.deb` and NodeSource's `setup_20.x` apt repos; `apt-get install -y dotnet-sdk-8.0 nodejs`; clean `/var/lib/apt/lists` to keep the layer trim. - -6. **Final environment.** - ```dockerfile - ENV CMAKE_PREFIX_PATH=/opt/vcpkg/active - ENV VCPKG_ROOT=/opt/vcpkg - ``` - Stable paths, no triplet leakage. - -### `devcontainer.json` - -```jsonc -{ - "name": "tableauio/loader", - "build": { - "dockerfile": "Dockerfile", - "args": { - "PROTOBUF_VERSION": "${localEnv:LOADER_PROTOBUF_VERSION:6.33.4}" - } - }, - - // Keep Go's module cache across container rebuilds. - "mounts": [ - "source=loader-go-mod,target=/home/vscode/go,type=volume" - ], - - "remoteUser": "vscode", - "workspaceFolder": "/workspaces/loader", - - "customizations": { - "vscode": { - "extensions": [ - "golang.go", - "ms-vscode.cmake-tools", - "ms-vscode.cpptools", - "ms-dotnettools.csharp", - "bufbuild.vscode-buf", - "zxh404.vscode-proto3" - ], - "settings": { - "go.toolsManagement.autoUpdate": false, - "cmake.configureOnOpen": false - } - } - }, - - "postCreateCommand": "printf 'tableauio/loader devcontainer ready.\\n go: %s\\n buf: %s\\n protoc: %s\\n dotnet: %s\\n node: %s\\n' \"$(go version | cut -d' ' -f3)\" \"$(buf --version)\" \"$(protoc --version)\" \"$(dotnet --version)\" \"$(node --version)\"" -} -``` - -Three intentional choices: - -1. **`${localEnv:LOADER_PROTOBUF_VERSION:6.33.4}`** — host env var picked up at container-build time. Workflow: `LOADER_PROTOBUF_VERSION=3.21.12 code .` → Reopen in Container → legacy-v3 image. No second devcontainer.json. -2. **Named volume `loader-go-mod` for `~/go`** — Go module cache persists across rebuilds. Workspace itself uses VS Code's default bind-mount (edits sync to host). -3. **`go.toolsManagement.autoUpdate: false`, `cmake.configureOnOpen: false`** — stops both extensions from auto-running their setup actions on first open, which would fight manual `cmake -S . -B build` invocations and trigger a 2-minute background `gopls` install. - -The `postCreateCommand` is pure echo — it prints the five tool versions so the contributor immediately knows the container is healthy. No installs, no conditionals. - -## Integration with existing flows - -### README change (additive, surgical) - -A new subsection at the top of `Prerequisites`, **above** "Install protobuf": - -> ### Recommended: Dev Container (any host OS) -> -> The fastest way to get a reproducible build environment is to open the -> repo in VS Code and choose **Reopen in Container**. The devcontainer -> under `.devcontainer/` has everything pinned to the exact versions CI -> uses (Go 1.24, buf 1.67.0, protobuf 6.33.4 via vcpkg, .NET 8.0, -> Node 20). First container build is one-time ~25 minutes (vcpkg -> compiles protobuf from source); subsequent reopens are instant. -> -> After the container starts you can skip the per-language setup below -> and jump straight to **C++** / **Go** / **C#** / **TypeScript**. -> -> Requirements: Docker Desktop (Windows + macOS) or Docker Engine (Linux), -> and the VS Code "Dev Containers" extension. See `.devcontainer/README.md` -> for the longer how-to. - -The existing `Windows: bootstrap…` block and per-language `Install protobuf` block both stay as written. Each gains a one-line lead-in: *"If you can't or don't want to use the devcontainer (corp Docker policy, etc.), follow the steps below."* - -### Container-side env so daily commands stay flag-free - -The Dockerfile's final `ENV CMAKE_PREFIX_PATH=/opt/vcpkg/active` is the *only* mechanism needed to make the existing "Dev at Linux → CMake (system protobuf)" recipe work in the container — `find_package(Protobuf CONFIG)` resolves to vcpkg's pinned protobuf without any toolchain-file flag. The contributor types the same commands they would on a host with system-installed protobuf; they happen to land on vcpkg's pinned 6.33.4. None of the four `buf.gen.yaml` files change. - -### Things explicitly NOT changed by this design - -- `prepare.bat` (already correct; stays as Windows-host fallback) -- Any `buf.gen.yaml` -- `test/cpp-tableau-loader/CMakeLists.txt` -- `.github/workflows/*.yml` (CI keeps `lukka/run-vcpkg` directly) - -## Verification matrix - -| Host | Container arch | First-run cost | Daily-cmd cost | Notes | -|---|---|---|---|---| -| Linux x86 | amd64 native | ~25 min build | bind-mount IO native | reference path | -| macOS Apple Silicon | **arm64 native** | ~25 min build | bind-mount IO native | no Rosetta tax | -| macOS Intel | amd64 native | ~25 min build | bind-mount IO native | | -| Windows + WSL2 | amd64 native | ~25 min build | bind-mount IO good if workspace is under WSL2 (`\\wsl.localhost\Ubuntu\…`), poor under `/mnt/c/...` | flagged in `.devcontainer/README.md` | - -**Acceptance gates:** - -1. `docker build .devcontainer/` succeeds clean on amd64 and arm64 hosts (gate 1: image actually builds). -2. Container start runs `postCreateCommand` and prints all five tool versions (gate 2: toolchain is wired correctly). -3. Inside the container, all four E2E paths from the README run green: - - `cd test/go-tableau-loader && buf generate .. && go test ./...` - - `cd test/cpp-tableau-loader && buf generate .. && cmake -S . -B build -DCMAKE_BUILD_TYPE=Debug && cmake --build build && ctest --test-dir build --output-on-failure` - - `cd test/csharp-tableau-loader && buf generate .. && dotnet test` - - `cd _lab/ts && npm install && npm run generate && npm run test` *(stretch — TS lab isn't in CI)* -4. `LOADER_PROTOBUF_VERSION=3.21.12 code .` → Reopen in Container → all C++ test paths still green (gate 3: protobuf version pinning works end-to-end). - -## Trade-offs and explicit deferrals - -- **First-run latency.** ~25 min on cold build (vcpkg compiles protobuf from source). Consequences: every change to the protobuf-installation `RUN` invalidates that layer for everyone who pulls the change. Mitigated by ordering it late in the layer chain (Go/buf changes don't trigger it). If pain becomes acute, prebuild-on-ghcr.io is the escape hatch. -- **No multi-arch publishing.** The Dockerfile is arch-agnostic, but `docker build` only builds the host arch. Apple Silicon contributors get arm64 natively because Docker Desktop's BuildKit picks `linux/arm64`. We do **not** run `docker buildx build --platform linux/amd64,linux/arm64 --push` anywhere. If we ever publish to ghcr.io, that's the moment to add buildx. -- **Modern protobuf default.** Daily dev runs against 6.33.4. Legacy-v3 contributors must rebuild the container with `LOADER_PROTOBUF_VERSION=3.21.12`. Acceptable because (a) CI catches legacy-v3 regressions automatically and (b) the container rebuild is incremental — only the vcpkg layer reruns. -- **Named volume for `~/go`.** Persists the Go module cache across rebuilds (~30s saved per first `go test` post-rebuild). If a contributor wants pure isolation, they `docker volume rm loader-go-mod`. - -## Implementation outline (for the writing-plans step that follows) - -1. Add `.devcontainer/Dockerfile` (multi-arch, manifest-mode vcpkg, version assertion). -2. Add `.devcontainer/devcontainer.json` (build args, named volume, extensions, postCreate banner). -3. Add `.devcontainer/README.md` (Docker prereqs, host-OS caveats, the `LOADER_PROTOBUF_VERSION` knob). -4. Update repo-root `README.md` Prerequisites section: add the new "Recommended: Dev Container" subsection; lead the existing Windows / per-language blocks with the "If you can't / don't want to use the devcontainer…" line. -5. Verification: `docker build` locally for amd64; container start produces the banner; all four E2E test commands green; `LOADER_PROTOBUF_VERSION=3.21.12` rebuild produces a working legacy-v3 image. diff --git a/make.py b/make.py new file mode 100644 index 0000000..cf84ad8 --- /dev/null +++ b/make.py @@ -0,0 +1,1308 @@ +#!/usr/bin/env python3 +""" +make.py — single cross-platform entrypoint for the tableauio/loader repo. + +Consolidates per-language `buf generate` / `cmake` / `go test` / `dotnet test` +/ `npm test` recipes into one Python tool that works identically on +native Windows, macOS, Linux, and inside the devcontainer. + +Usage (high level): + python make.py setup [--lang go|cpp|csharp|ts|all] [--dry-run] + python make.py generate --lang go|cpp|csharp|ts + python make.py build --lang go|cpp|csharp|ts [build flags] + python make.py test --lang go|cpp|csharp|ts [build flags] [-k FILTER] [--smoke] + python make.py clean [--lang ...] [--all] + python make.py env + python make.py --version + +Standard flags (apply to every subcommand): + --verbose / -v echo every subprocess + --dry-run print but do not execute + --cwd repo root (default: auto-detect from versions.env) + +The Windows MSVC env trick: every command that needs cl.exe / vcpkg-protoc +runs through Platform.windows_msvc_wrap(), which transparently wraps the +command in `cmd /c " x64 && "`. The user's shell +PATH/INCLUDE/LIB is never mutated. + +Stdlib only — no `pip install` required. Targets Python >= 3.10. +""" + +import argparse +import json +import os +import platform as _stdlib_platform +import shlex # noqa: F401 (kept for potential future POSIX shell quoting) +import shutil +import subprocess +import sys +import urllib.request +from dataclasses import dataclass, field +from pathlib import Path +from typing import Optional + +MAKE_PY_VERSION = "0.1.0" + +# --------------------------------------------------------------------------- +# Versions +# --------------------------------------------------------------------------- + + +@dataclass +class Versions: + """Parsed view of .devcontainer/versions.env. + + The format rules (documented in .devcontainer/README.md): + - One KEY=VALUE per line, no quotes, no spaces around `=`. + - Comments start with `#` at column 0. + - Blank lines are ignored. + - No shell expansion. + """ + + raw: dict[str, str] = field(default_factory=dict) + + @classmethod + def load(cls, repo_root: Path) -> "Versions": + path = repo_root / ".devcontainer" / "versions.env" + if not path.is_file(): + raise FileNotFoundError( + f"Missing {path}; cannot resolve pinned tool versions." + ) + raw: dict[str, str] = {} + for line in path.read_text(encoding="utf-8").splitlines(): + if not line or line.startswith("#"): + continue + if "=" not in line: + continue + k, v = line.split("=", 1) + raw[k.strip()] = v.strip() + return cls(raw=raw) + + def get(self, key: str, default: Optional[str] = None) -> Optional[str]: + return self.raw.get(key, default) + + @property + def go_version(self) -> Optional[str]: + return self.raw.get("GO_VERSION") + + @property + def buf_version(self) -> Optional[str]: + return self.raw.get("BUF_VERSION") + + @property + def protobuf_version(self) -> Optional[str]: + return self.raw.get("PROTOBUF_VERSION") + + @property + def vcpkg_baseline_commit(self) -> Optional[str]: + return self.raw.get("VCPKG_BASELINE_COMMIT") + + @property + def dotnet_version(self) -> Optional[str]: + return self.raw.get("DOTNET_VERSION") + + @property + def node_version(self) -> Optional[str]: + return self.raw.get("NODE_VERSION") + + @property + def cmake_version(self) -> Optional[str]: + return self.raw.get("CMAKE_VERSION") + + +# --------------------------------------------------------------------------- +# Platform +# --------------------------------------------------------------------------- + + +@dataclass +class Platform: + """Host OS / arch / devcontainer detection plus a few helpers. + + The two helpers worth highlighting: + + - cmake_toolchain_args(): returns the right `-DCMAKE_TOOLCHAIN_FILE=... + -DVCPKG_TARGET_TRIPLET=...` flags on Windows; returns [] inside the + devcontainer (CMAKE_PREFIX_PATH=/opt/vcpkg/active is preset by the + Dockerfile) and on macOS / Linux (system protobuf is on PATH). + + - windows_msvc_wrap(cmd): on Windows wraps the command in + `cmd /c " x64 && "` so MSVC env lives only in the + single child process. On all other OSes returns cmd unchanged. + """ + + sys_platform: str + machine: str + in_devcontainer: bool + vcpkg_root: Optional[Path] = None + vcvarsall_path: Optional[Path] = None + protoc_tools_dir: Optional[Path] = None + vcpkg_installed_dir: Optional[Path] = None # manifest mode only + + @classmethod + def detect(cls) -> "Platform": + sys_platform = sys.platform + machine = _stdlib_platform.machine().lower() + in_devcontainer = ( + Path("/opt/vcpkg/active").exists() or Path("/.dockerenv").exists() + ) + return cls( + sys_platform=sys_platform, + machine=machine, + in_devcontainer=in_devcontainer, + ) + + @property + def is_windows(self) -> bool: + return self.sys_platform.startswith("win") + + @property + def is_macos(self) -> bool: + return self.sys_platform == "darwin" + + @property + def is_linux(self) -> bool: + return self.sys_platform.startswith("linux") + + @property + def vcpkg_triplet(self) -> str: + """Default vcpkg triplet for the host. Mirrors the Dockerfile lines 32-36.""" + if self.is_windows: + return "x64-windows-static" + if self.is_macos: + if self.machine in ("arm64", "aarch64"): + return "arm64-osx" + return "x64-osx" + if self.is_linux: + if self.machine in ("arm64", "aarch64"): + return "arm64-linux" + return "x64-linux" + return "x64-linux" + + def cmake_toolchain_args(self, triplet: Optional[str] = None) -> list[str]: + """Extra cmake -D flags to pick up vcpkg's protobuf, when applicable.""" + if self.in_devcontainer: + return [] # Dockerfile presets CMAKE_PREFIX_PATH=/opt/vcpkg/active. + if not self.is_windows: + # macOS/Linux native: system protobuf or homebrew/apt resolves via + # find_package(Protobuf) without a toolchain file. + return [] + # Windows: VCPKG_ROOT must be set, either by `make.py setup` or by + # the CI's lukka/run-vcpkg step. The toolchain file location is + # canonical inside any vcpkg root. + vcpkg_root = self.vcpkg_root or _env_path("VCPKG_ROOT") + if vcpkg_root is None: + # Best-effort fallback so cmake configure fails with a useful + # message rather than us emitting an empty -D flag. + return [] + toolchain = vcpkg_root / "scripts" / "buildsystems" / "vcpkg.cmake" + args = [ + f"-DCMAKE_TOOLCHAIN_FILE={toolchain}", + f"-DVCPKG_TARGET_TRIPLET={triplet or self.vcpkg_triplet}", + ] + return args + + def windows_msvc_wrap(self, cmd: list[str]) -> list[str]: + """Wrap a command so it runs inside an MSVC-environment subshell. + + On Windows, returns a single-element list containing one cmd-shell + command string of the form: + 'call "" x64 >nul && ' + + Runner.run() detects the [windows-shell-string] pattern (via the + first element having no path separators but containing spaces) and + passes it to subprocess with shell=True so cmd parses it natively + — bypassing Python's Win32 CreateProcess quoting that otherwise + backslash-escapes the inner double quotes and breaks cmd's parser. + + On every other OS this returns `cmd` unchanged. + """ + if not self.is_windows: + return cmd + vcvars = self.vcvarsall_path or locate_vcvarsall() + if vcvars is None: + # No MSVC available — return cmd unchanged so the caller's + # subprocess fails with a useful "cl.exe not found" error rather + # than a more confusing wrapping failure. + return cmd + self.vcvarsall_path = vcvars + inner = " ".join(_winquote(arg) for arg in cmd) + # Single string: `call "" x64 >nul && `. + # `call` is required so vcvarsall returns control to our `&&`. + # `>nul` swallows vcvarsall's banner (cosmetic). + line = f'call "{vcvars}" x64 >nul && {inner}' + return [_WIN_SHELL_MARKER + line] + + +# Sentinel prefix used to flag a Runner.run() argv as "single shell string, +# please run via cmd". Using a prefix instead of a separate kwarg keeps +# windows_msvc_wrap() composable with the rest of the runner pipeline. +_WIN_SHELL_MARKER = "\x00CMDSHELL\x00" + + +def _env_path(name: str) -> Optional[Path]: + v = os.environ.get(name) + return Path(v) if v else None + + +def _winquote(arg: str) -> str: + """Quote an argument for the Windows cmd shell. + + cmd's quoting is famously bad. Wrap in double quotes if there is any + whitespace, ampersand, or other shell metacharacter. + """ + if not arg: + return '""' + if any(c in arg for c in ' &|<>^"()'): + # Escape embedded quotes by doubling them (cmd convention). + return '"' + arg.replace('"', '""') + '"' + return arg + + +def locate_vcvarsall() -> Optional[Path]: + """Find vcvarsall.bat on a Windows host. + + Strategy: + 1. Try vswhere.exe (the canonical method since VS 2017). + 2. Fall back to a few well-known install paths. + Returns None if MSVC isn't installed. + """ + if sys.platform != "win32": + return None + pf86 = os.environ.get("ProgramFiles(x86)", r"C:\Program Files (x86)") + pf = os.environ.get("ProgramFiles", r"C:\Program Files") + for base in (pf86, pf): + vswhere = Path(base) / "Microsoft Visual Studio" / "Installer" / "vswhere.exe" + if vswhere.is_file(): + try: + out = subprocess.run( + [ + str(vswhere), + "-latest", + "-products", + "*", + "-requires", + "Microsoft.VisualStudio.Component.VC.Tools.x86.x64", + "-property", + "installationPath", + ], + capture_output=True, + text=True, + check=False, + ) + inst = out.stdout.strip().splitlines() + if inst: + candidate = ( + Path(inst[0]) / "VC" / "Auxiliary" / "Build" / "vcvarsall.bat" + ) + if candidate.is_file(): + return candidate + except OSError: + pass + # Hardcoded fallbacks for VS 2022 Build Tools / Community / Enterprise. + for base in (pf86, pf): + for edition in ("BuildTools", "Community", "Professional", "Enterprise"): + candidate = ( + Path(base) + / "Microsoft Visual Studio" + / "2022" + / edition + / "VC" + / "Auxiliary" + / "Build" + / "vcvarsall.bat" + ) + if candidate.is_file(): + return candidate + return None + + +# --------------------------------------------------------------------------- +# Runner — subprocess helper +# --------------------------------------------------------------------------- + + +@dataclass +class Runner: + """Thin wrapper around subprocess.run with verbose / dry-run support.""" + + verbose: bool = False + dry_run: bool = False + + def run( + self, + cmd: list[str], + cwd: Optional[Path] = None, + env: Optional[dict[str, str]] = None, + check: bool = True, + shell: bool = False, + ) -> int: + # Detect the windows_msvc_wrap sentinel: a single-element argv whose + # value starts with _WIN_SHELL_MARKER. Strip the marker and run via + # cmd's native parser (shell=True) so Python's Win32 CreateProcess + # quoting doesn't mangle the embedded double quotes. + if ( + len(cmd) == 1 + and isinstance(cmd[0], str) + and cmd[0].startswith(_WIN_SHELL_MARKER) + ): + line = cmd[0][len(_WIN_SHELL_MARKER) :] + location = f" (cwd={cwd})" if cwd else "" + if self.dry_run: + print(f"[dry-run] {line}{location}") + return 0 + if self.verbose: + print(f"[run-shell] {line}{location}") + proc = subprocess.run( + line, + cwd=str(cwd) if cwd else None, + env=env, + check=False, + shell=True, + ) + if check and proc.returncode != 0: + raise SystemExit( + f"[error] command failed with exit code {proc.returncode}: {line}" + ) + return proc.returncode + + printable = " ".join(_winquote(c) if " " in c else c for c in cmd) + location = f" (cwd={cwd})" if cwd else "" + if self.dry_run: + print(f"[dry-run] {printable}{location}") + return 0 + if self.verbose: + print(f"[run] {printable}{location}") + proc = subprocess.run( + cmd, + cwd=str(cwd) if cwd else None, + env=env, + check=False, + shell=shell, + ) + if check and proc.returncode != 0: + raise SystemExit( + f"[error] command failed with exit code {proc.returncode}: {printable}" + ) + return proc.returncode + + def rmtree(self, path: Path) -> None: + if self.dry_run: + # Always announce the wipe in dry-run, even if path is missing — + # it's the orchestrator's *intent* we want to capture. + print(f"[dry-run] rm -rf {path}") + return + if not path.exists(): + return + if self.verbose: + print(f"[rm-rf] {path}") + shutil.rmtree(path, ignore_errors=True) + + def mkdirp(self, path: Path) -> None: + if self.dry_run: + print(f"[dry-run] mkdir -p {path}") + return + if path.exists(): + return + if self.verbose: + print(f"[mkdir-p] {path}") + path.mkdir(parents=True, exist_ok=True) + + +# --------------------------------------------------------------------------- +# Repo root discovery +# --------------------------------------------------------------------------- + + +def find_repo_root(start: Optional[Path] = None) -> Path: + """Locate the repo root by walking upward to find .devcontainer/versions.env.""" + here = (start or Path(__file__).resolve()).parent if start is None else start + here = here.resolve() + for candidate in [here, *here.parents]: + if (candidate / ".devcontainer" / "versions.env").is_file(): + return candidate + raise SystemExit( + "[error] Could not locate repo root (no .devcontainer/versions.env found)." + ) + + +# --------------------------------------------------------------------------- +# ~/.loader-env.json — Windows toolchain cache +# --------------------------------------------------------------------------- + + +LOADER_ENV_PATH = Path.home() / ".loader-env.json" + + +def load_loader_env() -> dict: + if not LOADER_ENV_PATH.is_file(): + return {} + try: + return json.loads(LOADER_ENV_PATH.read_text(encoding="utf-8")) + except (OSError, ValueError): + return {} + + +def save_loader_env(data: dict, runner: Runner) -> None: + text = json.dumps(data, indent=2, sort_keys=True) + if runner.dry_run: + print(f"[dry-run] write {LOADER_ENV_PATH}: {text}") + return + LOADER_ENV_PATH.write_text(text, encoding="utf-8") + + +def hydrate_platform_from_env(plat: Platform) -> None: + """Populate Platform from $VCPKG_ROOT and ~/.loader-env.json (Windows).""" + if not plat.is_windows: + return + if plat.vcpkg_root is None: + plat.vcpkg_root = _env_path("VCPKG_ROOT") + cache = load_loader_env() + if plat.vcpkg_root is None and cache.get("vcpkg_root"): + plat.vcpkg_root = Path(cache["vcpkg_root"]) + if plat.vcvarsall_path is None and cache.get("vcvarsall_path"): + candidate = Path(cache["vcvarsall_path"]) + if candidate.is_file(): + plat.vcvarsall_path = candidate + if plat.protoc_tools_dir is None and cache.get("protoc_tools_dir"): + plat.protoc_tools_dir = Path(cache["protoc_tools_dir"]) + if plat.vcpkg_installed_dir is None and cache.get("vcpkg_installed_dir"): + plat.vcpkg_installed_dir = Path(cache["vcpkg_installed_dir"]) + + +# --------------------------------------------------------------------------- +# Setup commands +# --------------------------------------------------------------------------- + + +LANGS_ALL = ("go", "cpp", "csharp", "ts") + + +def _which(name: str) -> Optional[str]: + return shutil.which(name) + + +def cmd_setup(args, ctx: "Context") -> int: + """Install host toolchains. OS-dispatched. Idempotent.""" + if ctx.platform.in_devcontainer: + print("[info] Running inside devcontainer; toolchain already installed.") + return 0 + + langs = _resolve_langs(args.lang) + print(f"[info] Setting up host toolchain for: {', '.join(langs)}") + print(f"[info] Pinned versions: {ctx.versions.raw}") + + if ctx.platform.is_macos: + return _setup_macos(langs, ctx) + if ctx.platform.is_linux: + return _setup_linux(langs, ctx) + if ctx.platform.is_windows: + return _setup_windows(langs, ctx, skip_vcpkg=getattr(args, "skip_vcpkg", False)) + print( + f"[error] Unsupported host platform: {ctx.platform.sys_platform}", + file=sys.stderr, + ) + return 1 + + +def _resolve_langs(lang: str) -> list[str]: + if lang in (None, "all"): + return list(LANGS_ALL) + return [lang] + + +def _setup_macos(langs: list[str], ctx: "Context") -> int: + if _which("brew") is None: + print( + "[error] Homebrew is required on macOS. Install from https://brew.sh and re-run.", + file=sys.stderr, + ) + return 1 + pkgs: list[str] = [] + if "go" in langs: + pkgs.append("go") + if "cpp" in langs: + pkgs.extend(["protobuf", "cmake", "ninja"]) + if "csharp" in langs: + # Homebrew dotnet@8 cask covers .NET 8. + pkgs.append(f"dotnet@{ctx.versions.dotnet_version or '8'}") + if "ts" in langs: + pkgs.append(f"node@{ctx.versions.node_version or '20'}") + pkgs.append("buf") + pkgs = list(dict.fromkeys(pkgs)) # de-dup, preserving order + ctx.runner.run(["brew", "update"], check=False) + ctx.runner.run(["brew", "install", *pkgs], check=False) + print("[info] macOS toolchain ready.") + return 0 + + +def _setup_linux(langs: list[str], ctx: "Context") -> int: + apt = _which("apt-get") is not None + dnf = _which("dnf") is not None + if not (apt or dnf): + print( + "[error] Neither apt-get nor dnf found. Install your toolchain manually.", + file=sys.stderr, + ) + return 1 + + base_pkgs: list[str] = [] + if "cpp" in langs: + if apt: + base_pkgs.extend( + [ + "protobuf-compiler", + "libprotobuf-dev", + "cmake", + "ninja-build", + "build-essential", + "git", + ] + ) + else: + base_pkgs.extend( + [ + "protobuf-compiler", + "protobuf-devel", + "cmake", + "ninja-build", + "gcc-c++", + "git", + ] + ) + if "go" in langs and apt: + base_pkgs.append("golang") + if "go" in langs and dnf: + base_pkgs.append("golang") + + if base_pkgs: + if apt: + sudo_prefix = ["sudo"] if os.geteuid() != 0 else [] + ctx.runner.run([*sudo_prefix, "apt-get", "update"], check=False) + ctx.runner.run( + [*sudo_prefix, "apt-get", "install", "-y", *base_pkgs], check=False + ) + else: + sudo_prefix = ["sudo"] if os.geteuid() != 0 else [] + ctx.runner.run( + [*sudo_prefix, "dnf", "install", "-y", *base_pkgs], check=False + ) + + # buf, .NET, Node are not always packaged at our pinned version; install + # via direct download / vendor scripts to ~/.local/. + if "go" in langs or "cpp" in langs or "csharp" in langs or "ts" in langs: + _ensure_buf_linux(ctx) + if "csharp" in langs: + _ensure_dotnet_linux(ctx) + if "ts" in langs: + _ensure_node_linux(ctx) + + print("[info] Linux toolchain ready.") + return 0 + + +def _ensure_buf_linux(ctx: "Context") -> None: + if _which("buf") is not None: + return + ver = ctx.versions.buf_version + if not ver: + return + arch = ( + "x86_64" + if _stdlib_platform.machine().lower() in ("x86_64", "amd64") + else "aarch64" + ) + url = f"https://github.com/bufbuild/buf/releases/download/v{ver}/buf-Linux-{arch}" + target_dir = Path.home() / ".local" / "bin" + ctx.runner.mkdirp(target_dir) + target = target_dir / "buf" + print(f"[info] Downloading buf {ver} -> {target}") + if not ctx.runner.dry_run: + urllib.request.urlretrieve(url, str(target)) + target.chmod(0o755) + + +def _ensure_dotnet_linux(ctx: "Context") -> None: + if _which("dotnet") is not None: + return + ver = ctx.versions.dotnet_version or "8.0" + script = Path.home() / ".local" / "bin" / "dotnet-install.sh" + ctx.runner.mkdirp(script.parent) + print(f"[info] Bootstrapping .NET {ver} via dotnet-install.sh") + if not ctx.runner.dry_run: + urllib.request.urlretrieve("https://dot.net/v1/dotnet-install.sh", str(script)) + script.chmod(0o755) + install_dir = Path.home() / ".dotnet" + ctx.runner.run( + [str(script), "--channel", ver, "--install-dir", str(install_dir)], + check=False, + ) + + +def _ensure_node_linux(ctx: "Context") -> None: + if _which("node") is not None: + return + print("[info] Node not found; install via your distro or NodeSource manually.") + + +def _setup_windows(langs: list[str], ctx: "Context", skip_vcpkg: bool) -> int: + """Windows host setup.""" + cache = load_loader_env() + + # Step 0: Chocolatey + choco = _which("choco") + if choco is None: + print("[info] Chocolatey not found. Installing...") + ctx.runner.run( + [ + "powershell", + "-NoProfile", + "-ExecutionPolicy", + "Bypass", + "-Command", + "[System.Net.ServicePointManager]::SecurityProtocol = " + "[System.Net.ServicePointManager]::SecurityProtocol -bor 3072; " + "iex ((New-Object System.Net.WebClient).DownloadString(" + "'https://community.chocolatey.org/install.ps1'))", + ], + check=False, + ) + else: + print(f"[info] Chocolatey already installed at: {choco}") + + if "cpp" in langs: + # Step 1: Ninja + if _which("ninja") is None: + print(f"[info] Installing ninja via choco...") + ctx.runner.run( + ["choco", "install", "ninja", "-y", "--no-progress"], check=False + ) + else: + print("[info] ninja already on PATH.") + + # Step 2: CMake + if _which("cmake") is None: + ver = ctx.versions.cmake_version or "3.31.8" + print(f"[info] Installing CMake {ver} via choco...") + ctx.runner.run( + [ + "choco", + "install", + "cmake", + f"--version={ver}", + "--installargs", + "ADD_CMAKE_TO_PATH=System", + "-y", + "--no-progress", + ], + check=False, + ) + else: + print("[info] cmake already on PATH.") + + # Step 3: MSVC Build Tools + vcvars = locate_vcvarsall() + if vcvars is None: + print( + "[info] MSVC Build Tools not found. Installing visualstudio2022buildtools..." + ) + ctx.runner.run( + [ + "choco", + "install", + "visualstudio2022buildtools", + "--package-parameters", + "--add Microsoft.VisualStudio.Workload.VCTools --includeRecommended --passive --locale en-US", + "-y", + ], + check=False, + ) + vcvars = locate_vcvarsall() + if vcvars is not None: + print(f"[info] vcvarsall.bat: {vcvars}") + cache["vcvarsall_path"] = str(vcvars) + ctx.platform.vcvarsall_path = vcvars + + # Step 4: buf + if _which("buf") is None: + ver = ctx.versions.buf_version + if ver: + buf_dir = ( + Path(os.environ.get("LOCALAPPDATA", str(Path.home()))) / "buf" / "bin" + ) + ctx.runner.mkdirp(buf_dir) + buf_exe = buf_dir / "buf.exe" + url = f"https://github.com/bufbuild/buf/releases/download/v{ver}/buf-Windows-x86_64.exe" + print(f"[info] Downloading buf {ver} -> {buf_exe}") + if not ctx.runner.dry_run: + urllib.request.urlretrieve(url, str(buf_exe)) + print(f"[info] buf installed at {buf_exe}; add to PATH manually if needed.") + else: + print("[info] buf already on PATH.") + + # Step 5: vcpkg + protobuf + if "cpp" in langs and not skip_vcpkg: + _setup_vcpkg_windows(ctx, cache) + + # Optional: Go / .NET / Node + if "go" in langs and _which("go") is None: + ctx.runner.run( + ["winget", "install", "--id", "GoLang.Go.1.24", "-e"], check=False + ) + if "csharp" in langs and _which("dotnet") is None: + ctx.runner.run( + ["winget", "install", "--id", "Microsoft.DotNet.SDK.8", "-e"], check=False + ) + if "ts" in langs and _which("node") is None: + ctx.runner.run( + ["winget", "install", "--id", "OpenJS.NodeJS.LTS", "-e"], check=False + ) + + save_loader_env(cache, ctx.runner) + print("[info] Windows toolchain ready.") + print( + "[info] Build/test commands run vcvarsall.bat per-process; your shell PATH is unchanged." + ) + return 0 + + +def _setup_vcpkg_windows(ctx: "Context", cache: dict) -> None: + triplet = ctx.platform.vcpkg_triplet + baseline = ctx.versions.vcpkg_baseline_commit + + vcpkg_root = ctx.platform.vcpkg_root or _env_path("VCPKG_ROOT") + if vcpkg_root is None and cache.get("vcpkg_root"): + vcpkg_root = Path(cache["vcpkg_root"]) + + # Reject manifest-only vcpkg under VS install dir (no bootstrap-vcpkg.bat). + if vcpkg_root is not None and not (vcpkg_root / "bootstrap-vcpkg.bat").is_file(): + print(f"[warn] {vcpkg_root} looks like a manifest-only vcpkg; ignoring.") + vcpkg_root = None + + if vcpkg_root is None: + candidate = Path(os.environ.get("USERPROFILE", str(Path.home()))) / "vcpkg" + if (candidate / "bootstrap-vcpkg.bat").is_file(): + vcpkg_root = candidate + + if vcpkg_root is None: + vcpkg_root = Path(os.environ.get("USERPROFILE", str(Path.home()))) / "vcpkg" + print(f"[info] Cloning vcpkg into {vcpkg_root}...") + ctx.runner.run( + ["git", "clone", "https://github.com/microsoft/vcpkg.git", str(vcpkg_root)], + check=False, + ) + if baseline: + ctx.runner.run( + ["git", "-C", str(vcpkg_root), "fetch", "--quiet", "origin", baseline], + check=False, + ) + ctx.runner.run( + ["git", "-C", str(vcpkg_root), "checkout", "--quiet", baseline], + check=False, + ) + # Bootstrap requires MSVC env on Windows. + bootstrap = vcpkg_root / "bootstrap-vcpkg.bat" + ctx.runner.run( + ctx.platform.windows_msvc_wrap([str(bootstrap), "-disableMetrics"]), + check=False, + ) + + cache["vcpkg_root"] = str(vcpkg_root) + ctx.platform.vcpkg_root = vcpkg_root + + vcpkg_exe = vcpkg_root / "vcpkg.exe" + print(f"[info] vcpkg at: {vcpkg_root}") + + print(f"[info] Installing protobuf:{triplet} via vcpkg (classic mode)...") + ctx.runner.run( + ctx.platform.windows_msvc_wrap( + [str(vcpkg_exe), "install", f"protobuf:{triplet}"] + ), + check=False, + ) + protoc_dir = vcpkg_root / "installed" / triplet / "tools" / "protobuf" + if (protoc_dir / "protoc.exe").is_file(): + cache["protoc_tools_dir"] = str(protoc_dir) + ctx.platform.protoc_tools_dir = protoc_dir + + +# --------------------------------------------------------------------------- +# generate / build / test / clean +# --------------------------------------------------------------------------- + + +def _lang_dir(repo_root: Path, lang: str) -> Path: + if lang == "ts": + return repo_root / "_lab" / "ts" + return repo_root / "test" / f"{lang}-tableau-loader" + + +def _buf_generate( + ctx: "Context", lang: str, protoc_dir_override: Optional[Path] = None +) -> None: + cwd = _lang_dir(ctx.repo_root, lang) + if lang == "ts": + # The TypeScript scratchpad has its own `npm run generate` script. + ctx.runner.run( + ["npm", "run", "generate"], cwd=cwd, shell=ctx.platform.is_windows + ) + return + cmd = ctx.platform.windows_msvc_wrap(["buf", "generate", ".."]) + env = os.environ.copy() + # Pick the protoc to put on PATH: + # - protoc_dir_override (manifest-mode protoc from this build's + # vcpkg_installed/) wins — required when --protobuf-version is set, + # so codegen matches the libprotobuf headers cmake will use. + # - Else the classic-mode protoc cached in ~/.loader-env.json. + protoc_dir = protoc_dir_override or ctx.platform.protoc_tools_dir + if protoc_dir is not None and ctx.platform.is_windows: + env["PATH"] = f"{protoc_dir}{os.pathsep}{env.get('PATH', '')}" + elif protoc_dir is not None and not ctx.platform.is_windows: + # On Linux/macOS the manifest-mode protoc dir matters for vcpkg + # manifest builds too (CI testing-cpp.yml does this on Linux). + env["PATH"] = f"{protoc_dir}{os.pathsep}{env.get('PATH', '')}" + ctx.runner.run(cmd, cwd=cwd, env=env) + + +def cmd_generate(args, ctx: "Context") -> int: + _buf_generate(ctx, args.lang) + return 0 + + +def cmd_build(args, ctx: "Context") -> int: + return _build_or_test(args, ctx, run_tests=False) + + +def cmd_test(args, ctx: "Context") -> int: + return _build_or_test(args, ctx, run_tests=True) + + +def _build_or_test(args, ctx: "Context", run_tests: bool) -> int: + lang = args.lang + if lang == "go": + return _go_build_or_test(args, ctx, run_tests) + if lang == "cpp": + return _cpp_build_or_test(args, ctx, run_tests) + if lang == "csharp": + return _csharp_build_or_test(args, ctx, run_tests) + if lang == "ts": + return _ts_build_or_test(args, ctx, run_tests) + print(f"[error] unknown --lang {lang}", file=sys.stderr) + return 2 + + +# ----- Go ----- + + +def _go_build_or_test(args, ctx: "Context", run_tests: bool) -> int: + cwd = _lang_dir(ctx.repo_root, "go") + + if run_tests and getattr(args, "smoke", False): + # Devcontainer-smoke equivalent — vet plugin packages only, skip ./test/... + # No buf-generate needed: those packages don't depend on freshly + # generated *.pb.go. + ctx.runner.run( + [ + "go", + "vet", + "./cmd/...", + "./pkg/...", + "./internal/options/...", + "./internal/loadutil/...", + "./internal/xproto/...", + ], + cwd=ctx.repo_root, + ) + return 0 + + # Always regenerate, mirroring CI. + if not getattr(args, "no_generate", False): + _buf_generate(ctx, "go") + + if not run_tests: + ctx.runner.run(["go", "build", "./..."], cwd=cwd) + return 0 + + cmd = ["go", "test", "-v", "-timeout", "30m"] + # Resolve --race default based on host OS. Windows requires cgo (and a + # C compiler) for -race; Linux/macOS work out of the box. + race = getattr(args, "race", None) + if race is None: + race = not ctx.platform.is_windows + if race: + cmd.append("-race") + if getattr(args, "coverage", False): + cmd.extend(["-coverprofile=coverage.txt", "-covermode=atomic"]) + cmd.append("./...") + if getattr(args, "k", None): + cmd.extend(["-run", args.k]) + ctx.runner.run(cmd, cwd=cwd) + return 0 + + +# ----- C++ ----- + + +def _cpp_build_or_test(args, ctx: "Context", run_tests: bool) -> int: + cwd = _lang_dir(ctx.repo_root, "cpp") + triplet = getattr(args, "triplet", None) or ctx.platform.vcpkg_triplet + protobuf_version = getattr(args, "protobuf_version", None) + cxx_std = getattr(args, "cxx_std", "17") + cxx_compiler = getattr(args, "cxx_compiler", None) + + # Stale-codegen wipe (gitignored *.pb.* files left over from a previous + # protoc version shadow fresh codegen). Skip with --no-clean. + if not getattr(args, "no_clean", False): + ctx.runner.rmtree(cwd / "build") + ctx.runner.rmtree(cwd / "src" / "tableau") + ctx.runner.rmtree(cwd / "src" / "protoconf") + + # Manifest mode: render vcpkg.json pinning the requested protobuf-version, + # then run `vcpkg install` to populate vcpkg_installed/. This matches CI's + # testing-cpp.yml flow (which uses lukka/run-vcpkg with runVcpkgInstall: + # true) and means switching --protobuf-version Just Works without + # re-running `make.py setup`. Idempotent: vcpkg detects already-installed + # packages and skips them. + cmake_extra: list[str] = [] + if protobuf_version: + baseline = ctx.versions.vcpkg_baseline_commit or "" + manifest = { + "name": "loader-cpp-test", + "version": "0.1.0", + "dependencies": ["protobuf"], + "overrides": [{"name": "protobuf", "version": protobuf_version}], + "builtin-baseline": baseline, + } + manifest_path = cwd / "vcpkg.json" + if not ctx.runner.dry_run: + manifest_path.write_text(json.dumps(manifest, indent=2), encoding="utf-8") + installed_dir = Path( + os.environ.get("VCPKG_INSTALLED_DIR", str(cwd / "vcpkg_installed")) + ) + + # Locate vcpkg.exe / vcpkg. On Windows we hydrated VCPKG_ROOT from + # ~/.loader-env.json; on CI it's set by lukka/run-vcpkg; on Linux + # it's the system or devcontainer vcpkg. + vcpkg_root = ctx.platform.vcpkg_root or _env_path("VCPKG_ROOT") + if vcpkg_root is None: + print( + "[error] --protobuf-version requires VCPKG_ROOT to be set " + "(run `python make.py setup --lang cpp` first, or set " + "VCPKG_ROOT in your environment).", + file=sys.stderr, + ) + return 1 + vcpkg_exe = vcpkg_root / ("vcpkg.exe" if ctx.platform.is_windows else "vcpkg") + + # Install the manifest: must `cd` into the manifest dir for vcpkg to + # discover it. Skip when --no-vcpkg-install is passed (CI's + # lukka/run-vcpkg already did it). + if not getattr(args, "no_vcpkg_install", False): + install_cmd = [ + str(vcpkg_exe), + "install", + f"--triplet={triplet}", + f"--x-install-root={installed_dir}", + ] + ctx.runner.run( + ctx.platform.windows_msvc_wrap(install_cmd), + cwd=cwd, + ) + + cmake_extra.extend( + [ + f"-DVCPKG_INSTALLED_DIR={installed_dir}", + "-DVCPKG_MANIFEST_INSTALL=OFF", + ] + ) + + if not getattr(args, "no_generate", False): + # In manifest mode, route buf-generate's protoc to the manifest's + # tools dir so codegen matches the libprotobuf cmake links against. + protoc_dir_override: Optional[Path] = None + if protobuf_version: + protoc_dir_override = installed_dir / triplet / "tools" / "protobuf" + _buf_generate(ctx, "cpp", protoc_dir_override=protoc_dir_override) + + configure_cmd = [ + "cmake", + "-S", + ".", + "-B", + "build", + "-DCMAKE_BUILD_TYPE=Debug", + f"-DCMAKE_CXX_STANDARD={cxx_std}", + ] + if cxx_compiler: + # Translate friendly names. + compiler = {"msvc": "cl", "clang": "clang++", "gcc": "g++"}.get( + cxx_compiler, cxx_compiler + ) + configure_cmd.append(f"-DCMAKE_CXX_COMPILER={compiler}") + if shutil.which("ninja") is not None: + configure_cmd.extend(["-G", "Ninja"]) + configure_cmd.extend(ctx.platform.cmake_toolchain_args(triplet=triplet)) + configure_cmd.extend(cmake_extra) + + ctx.runner.run(ctx.platform.windows_msvc_wrap(configure_cmd), cwd=cwd) + ctx.runner.run( + ctx.platform.windows_msvc_wrap(["cmake", "--build", "build", "--parallel"]), + cwd=cwd, + ) + + if run_tests: + ctest_cmd = ["ctest", "--test-dir", "build", "--output-on-failure"] + if getattr(args, "k", None): + ctest_cmd.extend(["-R", args.k]) + ctx.runner.run(ctx.platform.windows_msvc_wrap(ctest_cmd), cwd=cwd) + return 0 + + +# ----- C# ----- + + +def _csharp_build_or_test(args, ctx: "Context", run_tests: bool) -> int: + cwd = _lang_dir(ctx.repo_root, "csharp") + if not getattr(args, "no_generate", False): + _buf_generate(ctx, "csharp") + + if not run_tests: + ctx.runner.run(["dotnet", "build", "--nologo"], cwd=cwd) + return 0 + + cmd = ["dotnet", "test", "--nologo", "--logger", "console;verbosity=normal"] + if getattr(args, "k", None): + cmd.extend(["--filter", f"FullyQualifiedName~{args.k}"]) + ctx.runner.run(cmd, cwd=cwd) + return 0 + + +# ----- TypeScript ----- + + +def _ts_build_or_test(args, ctx: "Context", run_tests: bool) -> int: + cwd = _lang_dir(ctx.repo_root, "ts") + if not (cwd / "node_modules").is_dir(): + ctx.runner.run(["npm", "install"], cwd=cwd, shell=ctx.platform.is_windows) + if not getattr(args, "no_generate", False): + ctx.runner.run( + ["npm", "run", "generate"], cwd=cwd, shell=ctx.platform.is_windows + ) + if run_tests: + ctx.runner.run(["npm", "run", "test"], cwd=cwd, shell=ctx.platform.is_windows) + return 0 + + +# ----- clean / env ----- + + +def cmd_clean(args, ctx: "Context") -> int: + targets: list[str] = [] + if args.all: + targets = list(LANGS_ALL) + else: + targets = _resolve_langs(args.lang) + for lang in targets: + cwd = _lang_dir(ctx.repo_root, lang) + if lang == "cpp": + ctx.runner.rmtree(cwd / "build") + ctx.runner.rmtree(cwd / "src" / "tableau") + ctx.runner.rmtree(cwd / "src" / "protoconf") + elif lang == "csharp": + ctx.runner.rmtree(cwd / "bin") + ctx.runner.rmtree(cwd / "obj") + ctx.runner.rmtree(cwd / "protoconf") + elif lang == "go": + ctx.runner.rmtree(cwd / "protoconf") + elif lang == "ts": + ctx.runner.rmtree(cwd / "node_modules") + ctx.runner.rmtree(cwd / "dist") + return 0 + + +def cmd_env(args, ctx: "Context") -> int: + info = { + "make_py_version": MAKE_PY_VERSION, + "repo_root": str(ctx.repo_root), + "sys_platform": ctx.platform.sys_platform, + "machine": ctx.platform.machine, + "in_devcontainer": ctx.platform.in_devcontainer, + "vcpkg_triplet": ctx.platform.vcpkg_triplet, + "vcpkg_root": str(ctx.platform.vcpkg_root) if ctx.platform.vcpkg_root else None, + "vcvarsall_path": ( + str(ctx.platform.vcvarsall_path) if ctx.platform.vcvarsall_path else None + ), + "protoc_tools_dir": ( + str(ctx.platform.protoc_tools_dir) + if ctx.platform.protoc_tools_dir + else None + ), + "tools": { + "go": _which("go"), + "buf": _which("buf"), + "protoc": _which("protoc"), + "cmake": _which("cmake"), + "ninja": _which("ninja"), + "dotnet": _which("dotnet"), + "node": _which("node"), + "npm": _which("npm"), + }, + "versions_env": ctx.versions.raw, + } + print(json.dumps(info, indent=2)) + return 0 + + +# --------------------------------------------------------------------------- +# Context + arg parsing + main +# --------------------------------------------------------------------------- + + +@dataclass +class Context: + repo_root: Path + versions: Versions + platform: Platform + runner: Runner + + +def build_arg_parser() -> argparse.ArgumentParser: + p = argparse.ArgumentParser( + prog="make.py", + description="Cross-platform build/test driver for tableauio/loader.", + ) + p.add_argument( + "--version", action="store_true", help="print make.py version + versions.env" + ) + p.add_argument("-v", "--verbose", action="store_true") + p.add_argument("--dry-run", action="store_true") + p.add_argument("--cwd", type=str, default=None, help="repo root override") + + sub = p.add_subparsers(dest="command") + + # setup + sp = sub.add_parser("setup", help="Install host toolchain") + sp.add_argument("--lang", choices=[*LANGS_ALL, "all"], default="all") + sp.add_argument( + "--skip-vcpkg", + action="store_true", + help="(Windows) skip vcpkg install (CI uses lukka/run-vcpkg)", + ) + + # generate + sp = sub.add_parser("generate", help="Run buf generate for a language") + sp.add_argument("--lang", choices=LANGS_ALL, required=True) + + # build + sp = sub.add_parser("build", help="Compile generated code for a language") + _add_build_flags(sp) + + # test + sp = sub.add_parser("test", help="Run tests for a language") + _add_build_flags(sp) + sp.add_argument( + "-k", + type=str, + default=None, + help="test filter (go -run / ctest -R / dotnet --filter)", + ) + sp.add_argument( + "--smoke", action="store_true", help="(go only) smoke vet, no full test run" + ) + # --race / --no-race: tri-state. Default depends on host OS: + # Linux/macOS -> default ON (-race works out of the box). + # Windows -> default OFF (-race needs cgo+C compiler; users opt in + # explicitly via --race once they have MSVC/MinGW). + sp.add_argument( + "--race", + dest="race", + action="store_true", + default=None, + help="enable -race (default on Linux/macOS, off on Windows)", + ) + sp.add_argument( + "--no-race", dest="race", action="store_false", help="disable -race" + ) + sp.add_argument("--coverage", action="store_true") + + # clean + sp = sub.add_parser("clean", help="Wipe generated code + build outputs") + sp.add_argument("--lang", choices=[*LANGS_ALL, "all"], default="all") + sp.add_argument("--all", action="store_true") + + # env + sub.add_parser("env", help="Print resolved environment as JSON") + + return p + + +def _add_build_flags(sp: argparse.ArgumentParser) -> None: + sp.add_argument("--lang", choices=LANGS_ALL, required=True) + sp.add_argument("--cxx-std", choices=["17", "20"], default="17") + sp.add_argument("--cxx-compiler", choices=["msvc", "clang", "gcc"], default=None) + sp.add_argument( + "--protobuf-version", + type=str, + default=None, + help="(cpp) pin vcpkg protobuf port to this version (manifest mode)", + ) + sp.add_argument( + "--triplet", type=str, default=None, help="(cpp) vcpkg triplet override" + ) + sp.add_argument( + "--no-clean", + action="store_true", + help="(cpp) skip pre-build wipe of build/ + generated codegen", + ) + sp.add_argument( + "--no-vcpkg-install", + action="store_true", + help="(cpp manifest mode) skip `vcpkg install` (CI uses lukka/run-vcpkg)", + ) + sp.add_argument( + "--no-generate", action="store_true", help="skip the buf-generate step" + ) + + +def main(argv: Optional[list[str]] = None) -> int: + parser = build_arg_parser() + args = parser.parse_args(argv) + + if args.version: + repo = ( + find_repo_root(Path(args.cwd).resolve()) if args.cwd else find_repo_root() + ) + v = Versions.load(repo) + print(f"make.py {MAKE_PY_VERSION}") + for k, val in v.raw.items(): + print(f" {k}={val}") + return 0 + + if args.command is None: + parser.print_help() + return 0 + + repo = find_repo_root(Path(args.cwd).resolve()) if args.cwd else find_repo_root() + versions = Versions.load(repo) + plat = Platform.detect() + hydrate_platform_from_env(plat) + runner = Runner(verbose=args.verbose, dry_run=args.dry_run) + ctx = Context(repo_root=repo, versions=versions, platform=plat, runner=runner) + + dispatch = { + "setup": cmd_setup, + "generate": cmd_generate, + "build": cmd_build, + "test": cmd_test, + "clean": cmd_clean, + "env": cmd_env, + } + handler = dispatch.get(args.command) + if handler is None: + parser.print_help() + return 2 + return handler(args, ctx) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/prepare.bat b/prepare.bat deleted file mode 100644 index 66f5abf..0000000 --- a/prepare.bat +++ /dev/null @@ -1,582 +0,0 @@ -@echo off -setlocal enabledelayedexpansion - -REM =========================================================================== -REM prepare.bat — bootstrap a Windows build environment for the C++ loader. -REM -REM Installs (only if missing): Chocolatey, Ninja, CMake (version pinned in -REM .devcontainer/versions.env), MSVC Build -REM Tools (Visual Studio 2022 Build Tools), buf CLI, and vcpkg. -REM -REM Then installs `protobuf` (and friends) into vcpkg using the static-CRT -REM triplet x64-windows-static, so that downstream cmake builds can pick it -REM up via -DCMAKE_TOOLCHAIN_FILE=%VCPKG_ROOT%\scripts\buildsystems\vcpkg.cmake. -REM -REM The vcpkg checkout is pinned to VCPKG_BASELINE_COMMIT below for -REM reproducibility — the same commit testing-cpp.yml uses in CI. -REM -REM Override `protobuf` to a specific vcpkg port version with PROTOBUF_VCPKG_VERSION: -REM set PROTOBUF_VCPKG_VERSION=3.21.12 && .\prepare.bat -REM -REM When PROTOBUF_VCPKG_VERSION is set we install in vcpkg MANIFEST mode (a -REM rendered vcpkg.json under %LOCALAPPDATA%\loader\vcpkg-manifest\), which -REM is the only mode where `--x-version` / `overrides` actually pin the port -REM version. In that case the install root is %VCPKG_INSTALLED_DIR%, and -REM downstream cmake invocations must add: -REM -DVCPKG_INSTALLED_DIR=%VCPKG_INSTALLED_DIR% -DVCPKG_MANIFEST_INSTALL=OFF -REM (matching the CI flow in .github/workflows/testing-cpp.yml). -REM -REM When PROTOBUF_VCPKG_VERSION is NOT set we install in vcpkg CLASSIC mode -REM (no manifest), and downstream cmake works with just the toolchain file. -REM -REM This script is idempotent: re-running it on a machine that already has -REM everything installed is a no-op (a few seconds of probing). Only the MSVC -REM environment variables are re-exported each time, since vcvarsall.bat sets -REM cmd-session-local state that does not persist. -REM =========================================================================== - -REM ----------------------------------------------------------------------- -REM Parse arguments -REM --dry-run : print what would be done, but do not install anything -REM --simulate-clean : pretend nothing is installed (implies --dry-run) -REM ----------------------------------------------------------------------- -set "DRY_RUN=0" -set "SIMULATE_CLEAN=0" -for %%A in (%*) do ( - if /i "%%A"=="--dry-run" set "DRY_RUN=1" - if /i "%%A"=="--simulate-clean" set "DRY_RUN=1" & set "SIMULATE_CLEAN=1" -) -if "%DRY_RUN%"=="1" echo [DRY-RUN] No changes will be made to the system. -if "%SIMULATE_CLEAN%"=="1" echo [DRY-RUN] Simulating a clean machine (all tools treated as not installed). - -echo [INFO] Preparing build environment... - -REM ----------------------------------------------------------------------- -REM Load pinned tool versions from .devcontainer/versions.env. -REM -REM Single source of truth shared with the devcontainer and with the -REM .github/workflows/*.yml CI workflows. Format is one KEY=VALUE per -REM line, no quotes, no $VAR expansion. -REM ----------------------------------------------------------------------- -set "VERSIONS_FILE=%~dp0.devcontainer\versions.env" -if not exist "%VERSIONS_FILE%" ( - echo [ERROR] Missing %VERSIONS_FILE%; cannot resolve pinned tool versions. - exit /b 1 -) -for /f "usebackq tokens=1,2 delims==" %%a in ("%VERSIONS_FILE%") do ( - set "_KEY=%%a" - set "_VAL=%%b" - REM Skip blank lines and comment lines (start with #). - if defined _KEY if not "!_KEY:~0,1!"=="#" ( - set "!_KEY!=!_VAL!" - ) -) -set "_KEY=" -set "_VAL=" -echo [INFO] Pinned versions: cmake=%CMAKE_VERSION% buf=%BUF_VERSION% vcpkg-baseline=%VCPKG_BASELINE_COMMIT% - - -REM ----------------------------------------------------------------------- -REM Step 0: Ensure Chocolatey is installed -REM ----------------------------------------------------------------------- -set "CHOCO_EXE=" -set "CHOCO_BASE=" -if "%SIMULATE_CLEAN%"=="0" ( - REM Try env var first, then fall back to registry (HKCU then HKLM) - if defined ChocolateyInstall set "CHOCO_BASE=%ChocolateyInstall%" - if not defined CHOCO_BASE ( - for /f "usebackq tokens=2*" %%a in (`reg query "HKCU\Environment" /v ChocolateyInstall 2^>nul`) do set "CHOCO_BASE=%%b" - ) - if not defined CHOCO_BASE ( - for /f "usebackq tokens=2*" %%a in (`reg query "HKLM\SYSTEM\CurrentControlSet\Control\Session Manager\Environment" /v ChocolateyInstall 2^>nul`) do set "CHOCO_BASE=%%b" - ) - if not defined CHOCO_BASE set "CHOCO_BASE=%ALLUSERSPROFILE%\chocolatey" - if exist "!CHOCO_BASE!\bin\choco.exe" set "CHOCO_EXE=!CHOCO_BASE!\bin\choco.exe" - if exist "!CHOCO_BASE!\redirects\choco.exe" set "CHOCO_EXE=!CHOCO_BASE!\redirects\choco.exe" - if exist "!CHOCO_BASE!\tools\choco.exe" set "CHOCO_EXE=!CHOCO_BASE!\tools\choco.exe" -) -if not defined CHOCO_EXE ( - echo [INFO] Chocolatey not found. Installing Chocolatey... - if "%DRY_RUN%"=="0" ( - powershell -NoProfile -ExecutionPolicy Bypass -Command ^ - "[System.Net.ServicePointManager]::SecurityProtocol = [System.Net.ServicePointManager]::SecurityProtocol -bor 3072; iex ((New-Object System.Net.WebClient).DownloadString('https://community.chocolatey.org/install.ps1'))" - if errorlevel 1 ( - echo [ERROR] Failed to install Chocolatey. - exit /b 1 - ) - ) else ( - echo [DRY-RUN] Would run: powershell ... install Chocolatey - ) - REM Add Chocolatey to current session PATH - set "PATH=%ALLUSERSPROFILE%\chocolatey\bin;%PATH%" - REM Persist Chocolatey bin to user PATH permanently - if "%DRY_RUN%"=="0" ( - for /f "usebackq tokens=2*" %%a in (`reg query "HKCU\Environment" /v PATH 2^>nul`) do set "USR_PATH=%%b" - echo !USR_PATH! | findstr /i /c:"%ALLUSERSPROFILE%\chocolatey\bin" >nul 2>&1 - if errorlevel 1 ( - setx PATH "%ALLUSERSPROFILE%\chocolatey\bin;!USR_PATH!" - echo [INFO] Chocolatey bin added to user PATH permanently. - ) - ) else ( - echo [DRY-RUN] Would run: setx PATH "%%ALLUSERSPROFILE%%\chocolatey\bin;..." - ) - echo [INFO] Chocolatey installed successfully. -) else ( - echo [INFO] Chocolatey already installed. -) - -REM Refresh ChocolateyInstall var if it was just installed (also read from registry) -if not defined ChocolateyInstall ( - for /f "usebackq tokens=2*" %%a in (`reg query "HKCU\Environment" /v ChocolateyInstall 2^>nul`) do set "ChocolateyInstall=%%b" -) -if not defined ChocolateyInstall set "ChocolateyInstall=%ALLUSERSPROFILE%\chocolatey" -if "%SIMULATE_CLEAN%"=="0" ( - set "PATH=%ChocolateyInstall%\bin;%ChocolateyInstall%\lib\ninja\tools;%PATH%" -) - -REM ----------------------------------------------------------------------- -REM Step 1: Ensure Ninja is installed via Chocolatey -REM ----------------------------------------------------------------------- -set "NINJA_FOUND=0" -if "%SIMULATE_CLEAN%"=="0" ( - where ninja.exe >nul 2>&1 - if not errorlevel 1 set "NINJA_FOUND=1" -) -if "%NINJA_FOUND%"=="0" ( - echo [INFO] ninja.exe not found. Installing via choco... - if "%DRY_RUN%"=="0" ( - choco install ninja -y --no-progress - if errorlevel 1 ( - echo [ERROR] Failed to install ninja. - exit /b 1 - ) - ) else ( - echo [DRY-RUN] Would run: choco install ninja -y --no-progress - ) - REM Add ninja to current session PATH - if defined ChocolateyInstall ( - set "NINJA_PATH=!ChocolateyInstall!\lib\ninja\tools" - ) else ( - set "NINJA_PATH=%ALLUSERSPROFILE%\chocolatey\lib\ninja\tools" - ) - set "PATH=!NINJA_PATH!;%PATH%" - REM Persist ninja path to user PATH permanently - if "%DRY_RUN%"=="0" ( - for /f "usebackq tokens=2*" %%a in (`reg query "HKCU\Environment" /v PATH 2^>nul`) do set "USR_PATH=%%b" - echo !USR_PATH! | findstr /i /c:"ninja\tools" >nul 2>&1 - if errorlevel 1 ( - setx PATH "!NINJA_PATH!;!USR_PATH!" - echo [INFO] ninja path added to user PATH permanently. - ) - ) else ( - echo [DRY-RUN] Would run: setx PATH "!NINJA_PATH!;..." - ) - echo [INFO] ninja installed successfully. -) else ( - echo [INFO] ninja.exe already in PATH. -) - -REM ----------------------------------------------------------------------- -REM Step 2: Ensure CMake (CMAKE_VERSION from versions.env) is installed -REM Try Chocolatey first; fall back to direct MSI download. -REM ----------------------------------------------------------------------- -set "CMAKE_FOUND=0" -if "%SIMULATE_CLEAN%"=="0" ( - where cmake.exe >nul 2>&1 - if not errorlevel 1 set "CMAKE_FOUND=1" -) -if "%CMAKE_FOUND%"=="0" ( - echo [INFO] cmake.exe not found. Installing CMake %CMAKE_VERSION%... - if "%DRY_RUN%"=="0" ( - set "CMAKE_INSTALLED=0" - REM --- Attempt 1: Chocolatey --- - choco install cmake --version=%CMAKE_VERSION% --installargs "'ADD_CMAKE_TO_PATH=System'" -y --no-progress >nul 2>&1 && set "CMAKE_INSTALLED=1" - if "!CMAKE_INSTALLED!"=="0" ( - echo [WARN] choco install cmake failed. Falling back to direct MSI download... - set "CMAKE_MSI=%TEMP%\cmake-%CMAKE_VERSION%-windows-x86_64.msi" - powershell -NoProfile -Command "(New-Object Net.WebClient).DownloadFile('https://github.com/Kitware/CMake/releases/download/v%CMAKE_VERSION%/cmake-%CMAKE_VERSION%-windows-x86_64.msi','!CMAKE_MSI!')" - if not exist "!CMAKE_MSI!" ( - echo [ERROR] Failed to download CMake MSI. - exit /b 1 - ) - msiexec /i "!CMAKE_MSI!" ADD_CMAKE_TO_PATH=System /quiet /norestart - if errorlevel 1 ( - echo [ERROR] Failed to install CMake from MSI. - exit /b 1 - ) - del /q "!CMAKE_MSI!" 2>nul - ) - ) else ( - echo [DRY-RUN] Would run: choco install cmake --version=%CMAKE_VERSION% ... (or fallback to MSI download) - ) - REM Add cmake to current session PATH - set "CMAKE_PATH=C:\Program Files\CMake\bin" - set "PATH=!CMAKE_PATH!;%PATH%" - REM Persist cmake path to user PATH permanently - if "%DRY_RUN%"=="0" ( - for /f "usebackq tokens=2*" %%a in (`reg query "HKCU\Environment" /v PATH 2^>nul`) do set "USR_PATH=%%b" - echo !USR_PATH! | findstr /i /c:"CMake\bin" >nul 2>&1 - if errorlevel 1 ( - setx PATH "!CMAKE_PATH!;!USR_PATH!" - echo [INFO] cmake path added to user PATH permanently. - ) - ) else ( - echo [DRY-RUN] Would run: setx PATH "!CMAKE_PATH!;..." - ) - echo [INFO] cmake installed successfully. -) else ( - echo [INFO] cmake.exe already in PATH. -) - -REM ----------------------------------------------------------------------- -REM Step 3: Ensure MSVC compiler (cl.exe) is available, then activate its -REM environment for this cmd session via vcvarsall.bat. The CI -REM workflow uses ilammy/msvc-dev-cmd@v1 to do the same thing. -REM ----------------------------------------------------------------------- -set "CL_FOUND=0" -if "%SIMULATE_CLEAN%"=="0" ( - where cl.exe >nul 2>&1 - if not errorlevel 1 set "CL_FOUND=1" -) -set "SKIP_MSVC=0" -if "%CL_FOUND%"=="0" ( - echo [INFO] cl.exe not found. Searching for existing VS installation... - set "VSWHERE=" - if "%SIMULATE_CLEAN%"=="0" ( - for %%d in ("%ProgramFiles(x86)%" "%ProgramFiles%") do ( - if not defined VSWHERE ( - if exist "%%~d\Microsoft Visual Studio\Installer\vswhere.exe" ( - set "VSWHERE=%%~d\Microsoft Visual Studio\Installer\vswhere.exe" - ) - ) - ) - ) - if not defined VSWHERE ( - echo [INFO] Visual Studio not found. Installing via choco... - if "%DRY_RUN%"=="0" ( - choco install visualstudio2022buildtools --package-parameters "--add Microsoft.VisualStudio.Workload.VCTools --includeRecommended --passive --locale en-US" -y - if errorlevel 1 ( - echo [ERROR] Failed to install Visual Studio Build Tools. - exit /b 1 - ) - echo [INFO] Visual Studio Build Tools installed successfully. - REM Re-search vswhere after installation - for %%d in ("%ProgramFiles(x86)%" "%ProgramFiles%") do ( - if not defined VSWHERE ( - if exist "%%~d\Microsoft Visual Studio\Installer\vswhere.exe" ( - set "VSWHERE=%%~d\Microsoft Visual Studio\Installer\vswhere.exe" - ) - ) - ) - ) else ( - echo [DRY-RUN] Would run: choco install visualstudio2022buildtools ... - echo [DRY-RUN] Would search vswhere.exe after installation. - set "SKIP_MSVC=1" - ) - ) - if "!SKIP_MSVC!"=="0" ( - if not defined VSWHERE ( - echo [ERROR] vswhere.exe still not found after installation. Please restart and retry. - exit /b 1 - ) - set "VCVARSALL=" - for /f "usebackq delims=" %%p in (`"!VSWHERE!" -latest -products * -requires Microsoft.VisualStudio.Component.VC.Tools.x86.x64 -property installationPath`) do ( - set "VCVARSALL=%%p\VC\Auxiliary\Build\vcvarsall.bat" - ) - if not defined VCVARSALL ( - echo [ERROR] No VS installation with C++ tools detected. - exit /b 1 - ) - if not exist "!VCVARSALL!" ( - echo [ERROR] vcvarsall.bat not found at: !VCVARSALL! - exit /b 1 - ) - echo [INFO] Initializing MSVC environment from: !VCVARSALL! - call "!VCVARSALL!" x64 - ) -) else ( - echo [INFO] cl.exe already in PATH, skipping MSVC environment setup. -) - -REM ----------------------------------------------------------------------- -REM Step 4: Ensure buf CLI is installed -REM The CI workflow uses bufbuild/buf-action@v1 (also pinned to -REM BUF_VERSION below) to do the same thing. -REM buf is a single self-contained .exe; install it under -REM %LOCALAPPDATA%\buf\bin\buf.exe to avoid requiring admin rights. -REM BUF_VERSION is sourced from .devcontainer/versions.env. -REM ----------------------------------------------------------------------- -set "BUF_FOUND=0" -if "%SIMULATE_CLEAN%"=="0" ( - where buf.exe >nul 2>&1 - if not errorlevel 1 set "BUF_FOUND=1" -) -if "%BUF_FOUND%"=="0" ( - echo [INFO] buf.exe not found. Installing buf %BUF_VERSION%... - set "BUF_DIR=%LOCALAPPDATA%\buf\bin" - set "BUF_EXE=!BUF_DIR!\buf.exe" - set "BUF_URL=https://github.com/bufbuild/buf/releases/download/v%BUF_VERSION%/buf-Windows-x86_64.exe" - if "%DRY_RUN%"=="0" ( - if not exist "!BUF_DIR!" mkdir "!BUF_DIR!" - powershell -NoProfile -Command "(New-Object Net.WebClient).DownloadFile('!BUF_URL!','!BUF_EXE!')" - if not exist "!BUF_EXE!" ( - echo [ERROR] Failed to download buf from !BUF_URL!. - exit /b 1 - ) - ) else ( - echo [DRY-RUN] Would run: download !BUF_URL! to !BUF_EXE! - ) - REM Add buf to current session PATH - set "PATH=!BUF_DIR!;%PATH%" - REM Persist buf path to user PATH permanently - if "%DRY_RUN%"=="0" ( - for /f "usebackq tokens=2*" %%a in (`reg query "HKCU\Environment" /v PATH 2^>nul`) do set "USR_PATH=%%b" - echo !USR_PATH! | findstr /i /c:"buf\bin" >nul 2>&1 - if errorlevel 1 ( - setx PATH "!BUF_DIR!;!USR_PATH!" - echo [INFO] buf path added to user PATH permanently. - ) - ) else ( - echo [DRY-RUN] Would run: setx PATH "!BUF_DIR!;..." - ) - echo [INFO] buf installed successfully. -) else ( - echo [INFO] buf.exe already in PATH. -) - -REM ----------------------------------------------------------------------- -REM Step 5: Ensure vcpkg is installed and `protobuf` is provisioned -REM -REM Resolution order for the vcpkg install location: -REM 1. Existing %VCPKG_ROOT% if it points at a usable classic-mode bootstrap. -REM 2. Existing %VCPKG_INSTALLATION_ROOT% (set on GitHub-hosted runners). -REM 3. Existing %USERPROFILE%\vcpkg (a previous run of this script). -REM 4. Fresh clone into %USERPROFILE%\vcpkg. -REM -REM A "usable" vcpkg root must contain BOTH vcpkg.exe AND bootstrap-vcpkg.bat. -REM This deliberately rejects the manifest-only vcpkg shipped under -REM C:\Program Files\Microsoft Visual Studio\2022\\VC\vcpkg -REM which has no bootstrap script and refuses classic-mode `vcpkg install -REM :` with: "Could not locate a manifest (vcpkg.json) above -REM the current working directory. This vcpkg distribution does not have a -REM classic mode instance." -REM -REM When we install vcpkg ourselves we pin it to VCPKG_BASELINE_COMMIT so -REM the protobuf port version is reproducible. When we adopt a pre-existing -REM user-managed vcpkg, we leave its checkout state alone (the user owns it). -REM -REM We then run `vcpkg install protobuf:x64-windows-static` so that the -REM static-CRT libprotobuf + protoc match the loader build (CMakeLists.txt -REM forces /MT[d] via CMAKE_MSVC_RUNTIME_LIBRARY). -REM -REM Override the protobuf port version (e.g. for the legacy v3 line) with: -REM set PROTOBUF_VCPKG_VERSION=3.21.12 && .\prepare.bat -REM In that case we switch to manifest mode (see header comment above). -REM ----------------------------------------------------------------------- -REM Pre-flight: vcpkg compiles protobuf from source via MSVC, so cl.exe must -REM resolve here. If Step 3 didn't activate the MSVC environment (e.g. choco -REM just installed VS Build Tools and the registration hasn't fully landed -REM in this shell), fail fast with an actionable message rather than letting -REM vcpkg emit cryptic compiler-detection errors deep into the install. -where cl.exe >nul 2>&1 -if errorlevel 1 ( - if "%DRY_RUN%"=="0" ( - echo [ERROR] cl.exe not on PATH; the MSVC environment is not active in this shell. - echo [ERROR] Open a new "Developer Command Prompt for VS 2022" or rerun this script - echo [ERROR] in a fresh cmd window so vcvarsall.bat can take effect, then retry. - exit /b 1 - ) else ( - echo [DRY-RUN] [WARN] cl.exe not on PATH; would error out before vcpkg install. - ) -) - -REM Pin both the vcpkg checkout and the manifest's builtin-baseline to the -REM same commit testing-cpp.yml uses. VCPKG_BASELINE_COMMIT is sourced from -REM .devcontainer/versions.env (the single source of truth for the -REM Linux + Windows devcontainers, prepare.bat, and CI). To bump vcpkg, -REM edit that file. -set "VCPKG_TRIPLET=x64-windows-static" -set "VCPKG_EXE=" - -REM Honor pre-existing VCPKG_ROOT / VCPKG_INSTALLATION_ROOT only if they -REM point at a classic-mode-capable vcpkg (i.e. bootstrap-vcpkg.bat is present). -if "%SIMULATE_CLEAN%"=="0" ( - if defined VCPKG_ROOT ( - if exist "%VCPKG_ROOT%\vcpkg.exe" ( - if exist "%VCPKG_ROOT%\bootstrap-vcpkg.bat" ( - set "VCPKG_EXE=%VCPKG_ROOT%\vcpkg.exe" - ) else ( - echo [WARN] %VCPKG_ROOT% looks like a manifest-only vcpkg ^(no bootstrap-vcpkg.bat^); ignoring. - set "VCPKG_ROOT=" - ) - ) - ) - if not defined VCPKG_EXE ( - if defined VCPKG_INSTALLATION_ROOT ( - if exist "%VCPKG_INSTALLATION_ROOT%\vcpkg.exe" ( - if exist "%VCPKG_INSTALLATION_ROOT%\bootstrap-vcpkg.bat" ( - set "VCPKG_ROOT=%VCPKG_INSTALLATION_ROOT%" - set "VCPKG_EXE=%VCPKG_INSTALLATION_ROOT%\vcpkg.exe" - ) else ( - echo [WARN] %VCPKG_INSTALLATION_ROOT% looks like a manifest-only vcpkg; ignoring. - ) - ) - ) - ) - if not defined VCPKG_EXE ( - if exist "%USERPROFILE%\vcpkg\vcpkg.exe" ( - if exist "%USERPROFILE%\vcpkg\bootstrap-vcpkg.bat" ( - set "VCPKG_ROOT=%USERPROFILE%\vcpkg" - set "VCPKG_EXE=%USERPROFILE%\vcpkg\vcpkg.exe" - ) - ) - ) -) - -if not defined VCPKG_EXE ( - echo [INFO] vcpkg not found. Installing into %USERPROFILE%\vcpkg ... - set "VCPKG_ROOT=%USERPROFILE%\vcpkg" - if "%DRY_RUN%"=="0" ( - if not exist "!VCPKG_ROOT!" ( - REM Full clone (no --depth 1) so we can `git checkout` an arbitrary - REM commit below for reproducibility. - git clone https://github.com/microsoft/vcpkg.git "!VCPKG_ROOT!" - if errorlevel 1 ( - echo [ERROR] Failed to clone vcpkg. - exit /b 1 - ) - ) - REM Pin the checkout so port versions are reproducible. Safe to run - REM repeatedly: a no-op when we're already on VCPKG_BASELINE_COMMIT. - git -C "!VCPKG_ROOT!" fetch --quiet origin %VCPKG_BASELINE_COMMIT% - git -C "!VCPKG_ROOT!" checkout --quiet %VCPKG_BASELINE_COMMIT% - if errorlevel 1 ( - echo [ERROR] Failed to checkout vcpkg @ %VCPKG_BASELINE_COMMIT%. - exit /b 1 - ) - call "!VCPKG_ROOT!\bootstrap-vcpkg.bat" -disableMetrics - if errorlevel 1 ( - echo [ERROR] Failed to bootstrap vcpkg. - exit /b 1 - ) - ) else ( - echo [DRY-RUN] Would run: git clone https://github.com/microsoft/vcpkg.git "!VCPKG_ROOT!" - echo [DRY-RUN] Would run: git -C "!VCPKG_ROOT!" checkout %VCPKG_BASELINE_COMMIT% - echo [DRY-RUN] Would run: "!VCPKG_ROOT!\bootstrap-vcpkg.bat" -disableMetrics - ) - set "VCPKG_EXE=!VCPKG_ROOT!\vcpkg.exe" - REM Persist VCPKG_ROOT and PATH to user environment - if "%DRY_RUN%"=="0" ( - setx VCPKG_ROOT "!VCPKG_ROOT!" - for /f "usebackq tokens=2*" %%a in (`reg query "HKCU\Environment" /v PATH 2^>nul`) do set "USR_PATH=%%b" - echo !USR_PATH! | findstr /i /c:"!VCPKG_ROOT!" >nul 2>&1 - if errorlevel 1 ( - setx PATH "!VCPKG_ROOT!;!USR_PATH!" - echo [INFO] vcpkg path added to user PATH permanently. - ) - ) else ( - echo [DRY-RUN] Would run: setx VCPKG_ROOT "!VCPKG_ROOT!" - echo [DRY-RUN] Would run: setx PATH "!VCPKG_ROOT!;..." - ) - set "PATH=!VCPKG_ROOT!;%PATH%" - echo [INFO] vcpkg installed at !VCPKG_ROOT! ^(pinned to %VCPKG_BASELINE_COMMIT%^). -) else ( - echo [INFO] vcpkg already available at !VCPKG_ROOT! ^(user-managed; not re-pinning^). -) - -REM Install protobuf into vcpkg. -REM -REM Branching on PROTOBUF_VCPKG_VERSION: -REM - unset: CLASSIC mode. Installs into %VCPKG_ROOT%\installed\\, -REM auto-discovered by the vcpkg cmake toolchain — no extra cmake -REM flags needed downstream. Whatever protobuf the pinned vcpkg -REM checkout (VCPKG_BASELINE_COMMIT) ships is what you get. -REM - set: MANIFEST mode. Renders a vcpkg.json under -REM %LOCALAPPDATA%\loader\vcpkg-manifest\ with builtin-baseline + -REM an override pinning protobuf to the requested version. This -REM is the only mode in which `--x-version` / `overrides` actually -REM take effect; classic-mode `--x-version` is silently a no-op. -REM Install root: \vcpkg_installed\\. -REM -REM Both modes are idempotent: vcpkg detects already-installed packages and -REM skips them. -if defined PROTOBUF_VCPKG_VERSION ( - set "VCPKG_MANIFEST_DIR=%LOCALAPPDATA%\loader\vcpkg-manifest" - set "VCPKG_INSTALLED_DIR=!VCPKG_MANIFEST_DIR!\vcpkg_installed" - if "%DRY_RUN%"=="0" ( - if not exist "!VCPKG_MANIFEST_DIR!" mkdir "!VCPKG_MANIFEST_DIR!" - REM Render vcpkg.json. The `>` redirection truncates on first line and - REM `>>` appends the rest, mirroring a here-doc. - > "!VCPKG_MANIFEST_DIR!\vcpkg.json" echo { - >>"!VCPKG_MANIFEST_DIR!\vcpkg.json" echo "name": "loader-prepare", - >>"!VCPKG_MANIFEST_DIR!\vcpkg.json" echo "version": "0.1.0", - >>"!VCPKG_MANIFEST_DIR!\vcpkg.json" echo "dependencies": ["protobuf"], - >>"!VCPKG_MANIFEST_DIR!\vcpkg.json" echo "overrides": [{ "name": "protobuf", "version": "%PROTOBUF_VCPKG_VERSION%" }], - >>"!VCPKG_MANIFEST_DIR!\vcpkg.json" echo "builtin-baseline": "%VCPKG_BASELINE_COMMIT%" - >>"!VCPKG_MANIFEST_DIR!\vcpkg.json" echo } - echo [INFO] Installing protobuf %PROTOBUF_VCPKG_VERSION% into vcpkg ^(manifest mode, triplet !VCPKG_TRIPLET!^)... - pushd "!VCPKG_MANIFEST_DIR!" - "!VCPKG_EXE!" install --triplet=!VCPKG_TRIPLET! --x-install-root="!VCPKG_INSTALLED_DIR!" - set "VCPKG_INSTALL_RC=!ERRORLEVEL!" - popd - if not "!VCPKG_INSTALL_RC!"=="0" ( - echo [ERROR] vcpkg failed to install protobuf %PROTOBUF_VCPKG_VERSION%. - exit /b 1 - ) - REM Sanity check: assert the resolved version actually starts with - REM the requested one. vcpkg port versions can have a `#N` port-revision - REM suffix, so we match by prefix rather than equality. This is the - REM safety net that catches future regressions in vcpkg's manifest - REM resolution silently producing the wrong version. - set "VCPKG_INFO_FILE=!VCPKG_INSTALLED_DIR!\vcpkg\info\.unused" - for /f "delims=" %%f in ('dir /b /a-d "!VCPKG_INSTALLED_DIR!\vcpkg\info\protobuf_*_!VCPKG_TRIPLET!.list" 2^>nul') do ( - set "VCPKG_INFO_FILE=%%f" - ) - echo !VCPKG_INFO_FILE! | findstr /c:"protobuf_%PROTOBUF_VCPKG_VERSION%" >nul 2>&1 - if errorlevel 1 ( - echo [ERROR] Installed protobuf does not match requested version %PROTOBUF_VCPKG_VERSION%. - echo [ERROR] vcpkg installed file marker: !VCPKG_INFO_FILE! - echo [ERROR] This usually means VCPKG_BASELINE_COMMIT is too old to know about that - echo [ERROR] version. Bump the commit at the top of Step 5 and retry. - exit /b 1 - ) - ) else ( - echo [DRY-RUN] Would render: !VCPKG_MANIFEST_DIR!\vcpkg.json ^(protobuf %PROTOBUF_VCPKG_VERSION%, baseline %VCPKG_BASELINE_COMMIT%^) - echo [DRY-RUN] Would run: pushd "!VCPKG_MANIFEST_DIR!" ^&^& "!VCPKG_EXE!" install --triplet=!VCPKG_TRIPLET! --x-install-root="!VCPKG_INSTALLED_DIR!" - ) - set "PROTOC_TOOLS_DIR=!VCPKG_INSTALLED_DIR!\!VCPKG_TRIPLET!\tools\protobuf" -) else ( - if "%DRY_RUN%"=="0" ( - echo [INFO] Installing protobuf into vcpkg ^(classic mode, triplet !VCPKG_TRIPLET!^)... - "!VCPKG_EXE!" install "protobuf:!VCPKG_TRIPLET!" - if errorlevel 1 ( - echo [ERROR] vcpkg failed to install protobuf. - exit /b 1 - ) - ) else ( - echo [DRY-RUN] Would run: "!VCPKG_EXE!" install "protobuf:!VCPKG_TRIPLET!" - ) - set "VCPKG_INSTALLED_DIR=" - set "PROTOC_TOOLS_DIR=!VCPKG_ROOT!\installed\!VCPKG_TRIPLET!\tools\protobuf" -) - -REM Expose vcpkg-installed protoc on PATH so `buf generate` finds it. -if exist "!PROTOC_TOOLS_DIR!\protoc.exe" ( - set "PATH=!PROTOC_TOOLS_DIR!;%PATH%" - echo [INFO] vcpkg protoc on PATH: !PROTOC_TOOLS_DIR! -) - -if defined VCPKG_INSTALLED_DIR ( - echo [INFO] Manifest-mode install root: !VCPKG_INSTALLED_DIR! - echo [INFO] When invoking cmake, also pass: - echo [INFO] -DVCPKG_INSTALLED_DIR="!VCPKG_INSTALLED_DIR!" -DVCPKG_MANIFEST_INSTALL=OFF -) - -echo [INFO] Build environment ready. - -REM Export PATH and key MSVC vars back to the caller's environment. -REM Also export VCPKG_ROOT so subsequent `cmake -DCMAKE_TOOLCHAIN_FILE=%VCPKG_ROOT%\...` -REM invocations resolve in this same cmd session even before the persisted -REM setx value takes effect in newly-spawned processes. VCPKG_INSTALLED_DIR -REM is set only in manifest mode (PROTOBUF_VCPKG_VERSION pinned). -endlocal & set "PATH=%PATH%" & set "INCLUDE=%INCLUDE%" & set "LIB=%LIB%" & set "LIBPATH=%LIBPATH%" & set "WindowsSdkDir=%WindowsSdkDir%" & set "VCToolsInstallDir=%VCToolsInstallDir%" & set "VCPKG_ROOT=%VCPKG_ROOT%" & set "VCPKG_INSTALLED_DIR=%VCPKG_INSTALLED_DIR%" diff --git a/test_make.py b/test_make.py new file mode 100644 index 0000000..c8549e5 --- /dev/null +++ b/test_make.py @@ -0,0 +1,737 @@ +"""Unit + dry-run integration tests for make.py. + +Two layers: + +1. **Unit tests** — import make.py directly and exercise pure logic + (Versions, Platform, _winquote, Runner, etc.). No subprocess, no + network, no filesystem mutation outside pytest's tmp_path. + +2. **Dry-run snapshot tests** — spawn `python make.py --dry-run ` + and assert the printed command sequence. Canonical contract test + for "the orchestrator still emits the right cmake/ctest/buf calls." + +Subprocess tests use `--dry-run`, so they never touch the toolchain or +network. The whole suite runs in ~5s on any host. + +Run: + pip install pytest + python -m pytest test_make.py -v +""" + +import json +import subprocess +import sys +from pathlib import Path + +import pytest + +import make + +REPO_ROOT = Path(__file__).resolve().parent +MAKE_PY = REPO_ROOT / "make.py" + + +# --------------------------------------------------------------------------- +# Unit: Versions +# --------------------------------------------------------------------------- + + +class TestVersions: + def test_loads_real_versions_env(self): + v = make.Versions.load(REPO_ROOT) + # Every key documented in versions.env must be present. + for key in ( + "GO_VERSION", + "BUF_VERSION", + "PROTOBUF_VERSION", + "VCPKG_BASELINE_COMMIT", + "DOTNET_VERSION", + "NODE_VERSION", + "CMAKE_VERSION", + ): + assert key in v.raw, f"Expected {key} in versions.env" + assert v.raw[key], f"{key} must not be empty" + + def test_typed_accessors(self): + v = make.Versions.load(REPO_ROOT) + assert v.go_version == v.raw["GO_VERSION"] + assert v.buf_version == v.raw["BUF_VERSION"] + assert v.protobuf_version == v.raw["PROTOBUF_VERSION"] + assert v.vcpkg_baseline_commit == v.raw["VCPKG_BASELINE_COMMIT"] + assert v.dotnet_version == v.raw["DOTNET_VERSION"] + assert v.node_version == v.raw["NODE_VERSION"] + assert v.cmake_version == v.raw["CMAKE_VERSION"] + + def test_protobuf_version_looks_like_semver(self): + v = make.Versions.load(REPO_ROOT) + # Cheap smoke check — guards against a typo that breaks vcpkg. + parts = v.protobuf_version.split(".") + assert 2 <= len(parts) <= 3 + for p in parts: + assert p.isdigit(), f"Non-numeric protobuf version segment: {p}" + + def test_vcpkg_baseline_is_full_sha(self): + v = make.Versions.load(REPO_ROOT) + assert len(v.vcpkg_baseline_commit) == 40 + int(v.vcpkg_baseline_commit, 16) # raises ValueError if not hex + + def test_ignores_blank_and_comment_lines(self, tmp_path): + env = tmp_path / ".devcontainer" + env.mkdir() + (env / "versions.env").write_text( + "\n" + "# this is a comment\n" + "FOO=bar\n" + "\n" + "# another comment\n" + "BAZ=qux\n", + encoding="utf-8", + ) + v = make.Versions.load(tmp_path) + assert v.raw == {"FOO": "bar", "BAZ": "qux"} + + def test_missing_file_raises(self, tmp_path): + with pytest.raises(FileNotFoundError): + make.Versions.load(tmp_path) + + def test_get_with_default(self): + v = make.Versions(raw={"FOO": "bar"}) + assert v.get("FOO") == "bar" + assert v.get("MISSING") is None + assert v.get("MISSING", "default") == "default" + + +# --------------------------------------------------------------------------- +# Unit: Platform +# --------------------------------------------------------------------------- + + +class TestPlatform: + def test_detect_returns_one_os(self): + p = make.Platform.detect() + # Exactly one of is_windows / is_macos / is_linux must be True. + flags = [p.is_windows, p.is_macos, p.is_linux] + assert sum(flags) == 1, f"Expected exactly one OS flag, got {flags}" + + def test_vcpkg_triplet_windows(self): + p = make.Platform(sys_platform="win32", machine="amd64", in_devcontainer=False) + assert p.vcpkg_triplet == "x64-windows-static" + + def test_vcpkg_triplet_macos_x64(self): + p = make.Platform( + sys_platform="darwin", machine="x86_64", in_devcontainer=False + ) + assert p.vcpkg_triplet == "x64-osx" + + def test_vcpkg_triplet_macos_arm64(self): + p = make.Platform(sys_platform="darwin", machine="arm64", in_devcontainer=False) + assert p.vcpkg_triplet == "arm64-osx" + + def test_vcpkg_triplet_linux_x64(self): + p = make.Platform(sys_platform="linux", machine="x86_64", in_devcontainer=False) + assert p.vcpkg_triplet == "x64-linux" + + def test_vcpkg_triplet_linux_arm64(self): + p = make.Platform( + sys_platform="linux", machine="aarch64", in_devcontainer=False + ) + assert p.vcpkg_triplet == "arm64-linux" + + def test_cmake_toolchain_args_devcontainer_is_empty(self): + p = make.Platform(sys_platform="linux", machine="x86_64", in_devcontainer=True) + assert p.cmake_toolchain_args() == [] + + def test_cmake_toolchain_args_linux_is_empty(self): + p = make.Platform(sys_platform="linux", machine="x86_64", in_devcontainer=False) + assert p.cmake_toolchain_args() == [] + + def test_cmake_toolchain_args_macos_is_empty(self): + p = make.Platform(sys_platform="darwin", machine="arm64", in_devcontainer=False) + assert p.cmake_toolchain_args() == [] + + def test_cmake_toolchain_args_windows_with_vcpkg_root(self, tmp_path): + p = make.Platform( + sys_platform="win32", + machine="amd64", + in_devcontainer=False, + vcpkg_root=tmp_path, + ) + args = p.cmake_toolchain_args() + assert any("CMAKE_TOOLCHAIN_FILE" in a for a in args) + assert any("VCPKG_TARGET_TRIPLET=x64-windows-static" in a for a in args) + # Path must include vcpkg.cmake. + toolchain_arg = next(a for a in args if "CMAKE_TOOLCHAIN_FILE" in a) + assert toolchain_arg.endswith("vcpkg.cmake") + + def test_cmake_toolchain_args_windows_explicit_triplet(self, tmp_path): + p = make.Platform( + sys_platform="win32", + machine="amd64", + in_devcontainer=False, + vcpkg_root=tmp_path, + ) + args = p.cmake_toolchain_args(triplet="x64-windows") + assert any("VCPKG_TARGET_TRIPLET=x64-windows" in a for a in args) + assert not any("VCPKG_TARGET_TRIPLET=x64-windows-static" in a for a in args) + + def test_cmake_toolchain_args_windows_no_vcpkg_root_returns_empty( + self, monkeypatch + ): + # No VCPKG_ROOT in env, no vcpkg_root attr -> empty list (lets cmake + # fail with a useful "Could not find Protobuf" error). + monkeypatch.delenv("VCPKG_ROOT", raising=False) + p = make.Platform(sys_platform="win32", machine="amd64", in_devcontainer=False) + assert p.cmake_toolchain_args() == [] + + +# --------------------------------------------------------------------------- +# Unit: Platform.windows_msvc_wrap +# --------------------------------------------------------------------------- + + +class TestWindowsMsvcWrap: + """Exercises the central insight of make.py: the cmd-shell-string + sentinel that bypasses Python's CreateProcess quoting on Windows.""" + + def test_passthrough_on_linux(self): + p = make.Platform(sys_platform="linux", machine="x86_64", in_devcontainer=False) + cmd = ["cmake", "-S", ".", "-B", "build"] + assert p.windows_msvc_wrap(cmd) == cmd + + def test_passthrough_on_macos(self): + p = make.Platform(sys_platform="darwin", machine="arm64", in_devcontainer=False) + cmd = ["cmake", "-S", ".", "-B", "build"] + assert p.windows_msvc_wrap(cmd) == cmd + + def test_windows_with_no_vcvarsall_returns_unchanged(self): + # When MSVC isn't installed, return cmd unchanged so subprocess + # fails with a useful "cl.exe not found" error rather than a + # confusing wrapping failure. + p = make.Platform( + sys_platform="win32", + machine="amd64", + in_devcontainer=False, + vcvarsall_path=None, + ) + # Stub locate_vcvarsall to return None. + original = make.locate_vcvarsall + try: + make.locate_vcvarsall = lambda: None + cmd = ["cmake", "-S", "."] + assert p.windows_msvc_wrap(cmd) == cmd + finally: + make.locate_vcvarsall = original + + def test_windows_with_vcvarsall_emits_sentinel(self, tmp_path): + fake_vcvars = tmp_path / "vcvarsall.bat" + fake_vcvars.write_text("rem fake", encoding="utf-8") + p = make.Platform( + sys_platform="win32", + machine="amd64", + in_devcontainer=False, + vcvarsall_path=fake_vcvars, + ) + wrapped = p.windows_msvc_wrap(["buf", "generate", ".."]) + assert len(wrapped) == 1 + line = wrapped[0] + assert line.startswith(make._WIN_SHELL_MARKER) + # The shell line must `call` vcvarsall and then `&&` the inner cmd. + body = line[len(make._WIN_SHELL_MARKER) :] + assert body.startswith(f'call "{fake_vcvars}" x64') + assert " >nul && buf generate .." in body + + def test_windows_quotes_paths_with_spaces(self, tmp_path): + fake_vcvars = tmp_path / "vcvarsall.bat" + fake_vcvars.write_text("rem fake", encoding="utf-8") + p = make.Platform( + sys_platform="win32", + machine="amd64", + in_devcontainer=False, + vcvarsall_path=fake_vcvars, + ) + wrapped = p.windows_msvc_wrap(["cmake", "-DPATH=C:\\Program Files\\foo"]) + body = wrapped[0][len(make._WIN_SHELL_MARKER) :] + # The arg with a space MUST be quoted, but the bare cmake should not. + assert ( + ' cmake "-DPATH=C:\\Program Files\\foo"' in body + or ' cmake -DPATH="C:\\Program Files\\foo"' in body + or ' cmake "-DPATH=C:\\Program Files\\foo"' in body + or '"-DPATH=C:\\Program Files\\foo"' in body + ) # at least one of these forms + + +# --------------------------------------------------------------------------- +# Unit: _winquote +# --------------------------------------------------------------------------- + + +class TestQuoting: + def test_empty_string(self): + assert make._winquote("") == '""' + + def test_simple_arg_unchanged(self): + assert make._winquote("foo") == "foo" + assert make._winquote("--flag=value") == "--flag=value" + + def test_arg_with_space_is_quoted(self): + assert make._winquote("hello world") == '"hello world"' + + def test_arg_with_paren_is_quoted(self): + # The (x86) parser footgun from xxx.bat — must be quoted. + assert make._winquote("C:\\Program Files (x86)\\foo").startswith('"') + + def test_arg_with_embedded_quote_is_doubled(self): + # cmd quotes embedded " by doubling. + assert make._winquote('say "hi"') == '"say ""hi"""' + + +# --------------------------------------------------------------------------- +# Unit: Runner +# --------------------------------------------------------------------------- + + +class TestRunner: + def test_dry_run_does_not_execute(self, capsys, tmp_path): + runner = make.Runner(verbose=False, dry_run=True) + # If this were actually executed, it would create a file we can detect. + marker = tmp_path / "should_not_exist.txt" + if sys.platform == "win32": + cmd = ["cmd", "/c", f"echo hi > {marker}"] + else: + cmd = ["sh", "-c", f"echo hi > {marker}"] + rc = runner.run(cmd) + assert rc == 0 + assert not marker.exists(), "dry-run should not have executed" + out = capsys.readouterr().out + assert "[dry-run]" in out + + def test_dry_run_prints_cwd(self, capsys, tmp_path): + runner = make.Runner(verbose=False, dry_run=True) + runner.run(["echo", "hi"], cwd=tmp_path) + out = capsys.readouterr().out + assert f"cwd={tmp_path}" in out + + def test_sentinel_routes_through_shell(self, capsys): + runner = make.Runner(verbose=False, dry_run=True) + sentinel_cmd = [make._WIN_SHELL_MARKER + "echo hello-world"] + rc = runner.run(sentinel_cmd) + assert rc == 0 + out = capsys.readouterr().out + # The marker must be stripped from the printed line. + assert make._WIN_SHELL_MARKER not in out + assert "echo hello-world" in out + + def test_rmtree_dry_run_does_not_remove(self, capsys, tmp_path): + target = tmp_path / "doomed" + target.mkdir() + (target / "file.txt").write_text("x") + runner = make.Runner(verbose=False, dry_run=True) + runner.rmtree(target) + assert target.exists() + out = capsys.readouterr().out + assert "[dry-run] rm -rf" in out + + def test_rmtree_real_removes(self, tmp_path): + target = tmp_path / "doomed" + target.mkdir() + (target / "file.txt").write_text("x") + runner = make.Runner(verbose=False, dry_run=False) + runner.rmtree(target) + assert not target.exists() + + def test_rmtree_no_op_on_missing(self, tmp_path): + runner = make.Runner(verbose=False, dry_run=False) + runner.rmtree(tmp_path / "does-not-exist") # should not raise + + def test_check_raises_on_failure(self): + runner = make.Runner(verbose=False, dry_run=False) + # `python -c "raise SystemExit(7)"` will exit 7. + with pytest.raises(SystemExit): + runner.run([sys.executable, "-c", "raise SystemExit(7)"]) + + def test_check_false_suppresses_failure(self): + runner = make.Runner(verbose=False, dry_run=False) + rc = runner.run([sys.executable, "-c", "raise SystemExit(7)"], check=False) + assert rc == 7 + + +# --------------------------------------------------------------------------- +# Unit: repo root discovery +# --------------------------------------------------------------------------- + + +class TestRepoRoot: + def test_finds_root_from_repo(self): + assert make.find_repo_root() == REPO_ROOT + + def test_finds_root_from_subdir(self): + # Walk into a deep subdir and ensure we still locate it. + deep = REPO_ROOT / "internal" / "options" + if deep.is_dir(): + assert make.find_repo_root(deep) == REPO_ROOT + + def test_raises_outside_repo(self, tmp_path): + with pytest.raises(SystemExit): + make.find_repo_root(tmp_path) + + +# --------------------------------------------------------------------------- +# Unit: lang directory mapping +# --------------------------------------------------------------------------- + + +class TestLangDir: + def test_go(self): + assert ( + make._lang_dir(REPO_ROOT, "go") == REPO_ROOT / "test" / "go-tableau-loader" + ) + + def test_cpp(self): + assert ( + make._lang_dir(REPO_ROOT, "cpp") + == REPO_ROOT / "test" / "cpp-tableau-loader" + ) + + def test_csharp(self): + assert ( + make._lang_dir(REPO_ROOT, "csharp") + == REPO_ROOT / "test" / "csharp-tableau-loader" + ) + + def test_ts_lives_under_lab(self): + # TS is special-cased to _lab/ts/ per CLAUDE.md. + assert make._lang_dir(REPO_ROOT, "ts") == REPO_ROOT / "_lab" / "ts" + + +# --------------------------------------------------------------------------- +# Subprocess helpers +# --------------------------------------------------------------------------- + + +def run_make(*args: str, cwd: Path = REPO_ROOT) -> subprocess.CompletedProcess: + """Spawn `python make.py ` and capture stdout+stderr.""" + return subprocess.run( + [sys.executable, str(MAKE_PY), *args], + cwd=str(cwd), + capture_output=True, + text=True, + check=False, + ) + + +# --------------------------------------------------------------------------- +# Integration: top-level invocations +# --------------------------------------------------------------------------- + + +class TestTopLevel: + def test_help_exits_zero(self): + proc = run_make("--help") + assert proc.returncode == 0 + assert "make.py" in proc.stdout + assert "setup" in proc.stdout + assert "test" in proc.stdout + + def test_no_args_prints_help(self): + proc = run_make() + assert proc.returncode == 0 + assert "usage:" in proc.stdout.lower() + + def test_version_prints_versions_env(self): + proc = run_make("--version") + assert proc.returncode == 0 + assert f"make.py {make.MAKE_PY_VERSION}" in proc.stdout + # Must echo every key from versions.env. + for key in ( + "GO_VERSION", + "BUF_VERSION", + "PROTOBUF_VERSION", + "VCPKG_BASELINE_COMMIT", + "DOTNET_VERSION", + "NODE_VERSION", + "CMAKE_VERSION", + ): + assert key in proc.stdout, f"--version missing {key}" + + def test_env_emits_valid_json(self): + proc = run_make("env") + assert proc.returncode == 0 + info = json.loads(proc.stdout) + # Sanity-check structure. + assert info["make_py_version"] == make.MAKE_PY_VERSION + assert "sys_platform" in info + assert "vcpkg_triplet" in info + assert "tools" in info + assert "versions_env" in info + assert info["versions_env"]["GO_VERSION"] + + +# --------------------------------------------------------------------------- +# Integration: dry-run snapshots +# --------------------------------------------------------------------------- + + +class TestDryRunGo: + def test_test_lang_go(self): + proc = run_make("--dry-run", "test", "--lang", "go") + assert proc.returncode == 0 + out = proc.stdout + assert "buf generate .." in out + assert "go test" in out + assert "-timeout" in out and "30m" in out + # -race default depends on host OS; covered separately in + # test_test_lang_go_race_default_matches_host. + + def test_test_lang_go_no_race(self): + proc = run_make("--dry-run", "test", "--lang", "go", "--no-race") + assert proc.returncode == 0 + # When --no-race is passed, "-race" must NOT appear in the go test line. + for line in proc.stdout.splitlines(): + if "go test" in line: + assert "-race" not in line, f"Expected --no-race to drop -race: {line}" + + def test_test_lang_go_race_default_matches_host(self): + # Default --race depends on host OS: + # - Windows -> default OFF (-race needs cgo + C compiler) + # - Linux/macOS -> default ON + proc = run_make("--dry-run", "test", "--lang", "go") + assert proc.returncode == 0 + go_test_lines = [l for l in proc.stdout.splitlines() if "go test" in l] + assert go_test_lines, "no go test line" + joined = "\n".join(go_test_lines) + if sys.platform == "win32": + assert "-race" not in joined, "Windows default should NOT include -race" + else: + assert "-race" in joined, "Linux/macOS default should include -race" + + def test_test_lang_go_explicit_race_forces_on(self): + # Even on Windows, --race explicitly enables it (user opt-in). + proc = run_make("--dry-run", "test", "--lang", "go", "--race") + assert proc.returncode == 0 + go_test_lines = [l for l in proc.stdout.splitlines() if "go test" in l] + joined = "\n".join(go_test_lines) + assert "-race" in joined, "--race must force -race on regardless of host" + + def test_test_lang_go_smoke_skips_buf_generate(self): + proc = run_make("--dry-run", "test", "--lang", "go", "--smoke") + assert proc.returncode == 0 + # Smoke must NOT call buf generate; only go vet on the plugin packages. + assert "buf generate" not in proc.stdout + assert "go vet" in proc.stdout + # Specific plugin packages required. + for pkg in ( + "./cmd/...", + "./pkg/...", + "./internal/options/...", + "./internal/loadutil/...", + "./internal/xproto/...", + ): + assert pkg in proc.stdout, f"smoke missing {pkg}" + + def test_test_lang_go_filter(self): + proc = run_make( + "--dry-run", "test", "--lang", "go", "-k", "Test_ActivityConf_OrderedMap" + ) + assert proc.returncode == 0 + assert "-run" in proc.stdout + assert "Test_ActivityConf_OrderedMap" in proc.stdout + + def test_test_lang_go_coverage(self): + proc = run_make("--dry-run", "test", "--lang", "go", "--coverage") + assert proc.returncode == 0 + assert "-coverprofile=coverage.txt" in proc.stdout + assert "-covermode=atomic" in proc.stdout + + +class TestDryRunCpp: + def test_test_lang_cpp_default(self): + proc = run_make("--dry-run", "test", "--lang", "cpp") + assert proc.returncode == 0 + out = proc.stdout + assert "buf generate .." in out + assert "cmake -S . -B build" in out + assert "-DCMAKE_BUILD_TYPE=Debug" in out + assert "-DCMAKE_CXX_STANDARD=17" in out + assert "cmake --build build --parallel" in out + assert "ctest --test-dir build --output-on-failure" in out + + def test_test_lang_cpp_no_clean_skips_wipe(self): + proc_clean = run_make("--dry-run", "test", "--lang", "cpp") + proc_no_clean = run_make("--dry-run", "test", "--lang", "cpp", "--no-clean") + # Clean run must mention rm-rf the build/src dirs; --no-clean must not. + assert "rm -rf" in proc_clean.stdout + assert "build" in proc_clean.stdout + assert "rm -rf" not in proc_no_clean.stdout + + def test_test_lang_cpp_cxx_std_20(self): + proc = run_make("--dry-run", "test", "--lang", "cpp", "--cxx-std", "20") + assert proc.returncode == 0 + assert "-DCMAKE_CXX_STANDARD=20" in proc.stdout + assert "-DCMAKE_CXX_STANDARD=17" not in proc.stdout + + def test_test_lang_cpp_cxx_compiler_clang(self): + proc = run_make("--dry-run", "test", "--lang", "cpp", "--cxx-compiler", "clang") + assert proc.returncode == 0 + assert "-DCMAKE_CXX_COMPILER=clang++" in proc.stdout + + def test_test_lang_cpp_protobuf_version_manifest_mode(self): + proc = run_make( + "--dry-run", + "test", + "--lang", + "cpp", + "--protobuf-version", + "3.21.12", + "--triplet", + "x64-windows-static", + "--no-clean", + ) + assert proc.returncode == 0 + out = proc.stdout + # Manifest-mode flags MUST appear when --protobuf-version is set. + assert "-DVCPKG_INSTALLED_DIR=" in out + assert "-DVCPKG_MANIFEST_INSTALL=OFF" in out + # Must also `vcpkg install` the manifest before cmake configure. + assert "vcpkg" in out and "install" in out + # Triplet must be plumbed through. + assert "--triplet=x64-windows-static" in out + + def test_test_lang_cpp_no_vcpkg_install_skips_install(self): + proc = run_make( + "--dry-run", + "test", + "--lang", + "cpp", + "--protobuf-version", + "3.21.12", + "--triplet", + "x64-windows-static", + "--no-clean", + "--no-vcpkg-install", + ) + assert proc.returncode == 0 + # No `vcpkg ... install ...` line when --no-vcpkg-install is set. + # cmake configure still runs, with the manifest-mode flags. + assert "-DVCPKG_INSTALLED_DIR=" in proc.stdout + for line in proc.stdout.splitlines(): + # `vcpkg install` must not appear; `cmake` calls are fine. + if "vcpkg.exe install" in line or "vcpkg install" in line.replace( + ".exe", "" + ): + pytest.fail(f"--no-vcpkg-install did not skip vcpkg install: {line}") + + def test_test_lang_cpp_filter(self): + proc = run_make( + "--dry-run", "test", "--lang", "cpp", "-k", "HubTest.Load", "--no-clean" + ) + assert proc.returncode == 0 + # ctest -R + ctest_lines = [l for l in proc.stdout.splitlines() if "ctest" in l] + assert ctest_lines, "no ctest line in output" + joined = "\n".join(ctest_lines) + assert "-R" in joined and "HubTest.Load" in joined + + +class TestDryRunCsharp: + def test_test_lang_csharp_default(self): + proc = run_make("--dry-run", "test", "--lang", "csharp") + assert proc.returncode == 0 + assert "buf generate .." in proc.stdout + assert "dotnet test" in proc.stdout + assert "--nologo" in proc.stdout + + def test_test_lang_csharp_filter(self): + proc = run_make("--dry-run", "test", "--lang", "csharp", "-k", "HubTest.Load") + assert proc.returncode == 0 + # dotnet test --filter "FullyQualifiedName~" + assert "FullyQualifiedName~HubTest.Load" in proc.stdout + + +class TestDryRunGenerateAndBuild: + def test_generate_lang_go(self): + proc = run_make("--dry-run", "generate", "--lang", "go") + assert proc.returncode == 0 + assert "buf generate .." in proc.stdout + + def test_generate_lang_cpp(self): + proc = run_make("--dry-run", "generate", "--lang", "cpp") + assert proc.returncode == 0 + assert "buf generate .." in proc.stdout + + def test_build_lang_go_no_test(self): + proc = run_make("--dry-run", "build", "--lang", "go") + assert proc.returncode == 0 + # build (not test) must call `go build`, not `go test`. + assert "go build" in proc.stdout + # Must NOT run tests. + for line in proc.stdout.splitlines(): + assert "go test" not in line, f"build invoked go test: {line}" + + def test_build_lang_csharp_calls_dotnet_build(self): + proc = run_make("--dry-run", "build", "--lang", "csharp") + assert proc.returncode == 0 + assert "dotnet build" in proc.stdout + for line in proc.stdout.splitlines(): + assert "dotnet test" not in line + + +class TestDryRunClean: + def test_clean_cpp(self): + proc = run_make("--dry-run", "clean", "--lang", "cpp") + assert proc.returncode == 0 + out = proc.stdout + # Must wipe all three cpp dirs. + assert "build" in out + assert "src\\tableau" in out or "src/tableau" in out + assert "src\\protoconf" in out or "src/protoconf" in out + + def test_clean_csharp(self): + proc = run_make("--dry-run", "clean", "--lang", "csharp") + assert proc.returncode == 0 + out = proc.stdout + assert "bin" in out + assert "obj" in out + assert "protoconf" in out + + def test_clean_all(self): + proc = run_make("--dry-run", "clean", "--all") + assert proc.returncode == 0 + out = proc.stdout + # Should mention dirs from at least cpp, csharp, go, ts. + assert "cpp-tableau-loader" in out + assert "csharp-tableau-loader" in out + assert "go-tableau-loader" in out + + +# --------------------------------------------------------------------------- +# Integration: setup is a no-op in devcontainer +# --------------------------------------------------------------------------- + + +class TestSetupDevcontainer: + def test_setup_skips_inside_devcontainer(self, monkeypatch, tmp_path): + # Simulate devcontainer detection by patching Platform.detect. + original_detect = make.Platform.detect + + @classmethod + def fake_detect(cls): + return make.Platform( + sys_platform="linux", + machine="x86_64", + in_devcontainer=True, + ) + + monkeypatch.setattr(make.Platform, "detect", fake_detect) + try: + # Build a minimal Context and call cmd_setup directly. + ctx = make.Context( + repo_root=REPO_ROOT, + versions=make.Versions.load(REPO_ROOT), + platform=make.Platform.detect(), + runner=make.Runner(verbose=False, dry_run=True), + ) + args = type("Args", (), {"lang": "all"})() + rc = make.cmd_setup(args, ctx) + assert rc == 0 + finally: + make.Platform.detect = original_detect From 1c1eb09bd1037fede11869bfffcea3379d168381 Mon Sep 17 00:00:00 2001 From: wenchy Date: Thu, 4 Jun 2026 17:41:33 +0800 Subject: [PATCH 41/66] fix(ci): drop standalone Vet step + legacy-v3 C# matrix testing-go.yml: - Drop `go vet ./...` step. It ran from the repo root before the Test step, but vet needs the generated test/go-tableau-loader/protoconf/* packages that only `make.py test` produces. Easier than re-ordering: the Test step's `go test` already type-checks every package transitively, and devcontainer-smoke.yml's `make.py test --lang go --smoke` covers the plugin-only vet pass. testing-csharp.yml: - Drop the legacy-v3 protobuf matrix entry. C# consumes only generated *.cs files; libprotobuf is irrelevant at the C# level so there's no ABI compatibility to test against. Halves C# CI time. - Pin protoc to "33.4" directly (was matrix.config.protobuf-version). Co-Authored-By: Claude Opus 4.6 --- .github/workflows/testing-csharp.yml | 19 ++++--------------- .github/workflows/testing-go.yml | 7 ++++--- 2 files changed, 8 insertions(+), 18 deletions(-) diff --git a/.github/workflows/testing-csharp.yml b/.github/workflows/testing-csharp.yml index 7daeae3..db8bd22 100644 --- a/.github/workflows/testing-csharp.yml +++ b/.github/workflows/testing-csharp.yml @@ -18,21 +18,8 @@ jobs: fail-fast: false matrix: os: [ubuntu-latest, windows-latest] - # protobuf-version below is the protoc release tag (without the leading - # "v") fed to arduino/setup-protoc — NOT the libprotobuf C++ runtime - # SemVer used in testing-cpp.yml. The pairs map to the same upstream - # protobuf release: - # modern: protoc v33.4 ↔ libprotobuf C++ 6.33.4 (testing-cpp.yml) - # legacy-v3: protoc v21.12 ↔ libprotobuf C++ 3.21.12 (testing-cpp.yml) - # C# only needs protoc; libprotobuf is irrelevant, so we install via - # arduino/setup-protoc instead of vcpkg. - config: - - label: modern - protobuf-version: "33.4" - - label: legacy-v3 - protobuf-version: "21.12" - name: test (${{ matrix.os }}, ${{ matrix.config.label }}) + name: test (${{ matrix.os }}) runs-on: ${{ matrix.os }} timeout-minutes: 10 @@ -58,9 +45,11 @@ jobs: dotnet-version: "${{ env.DOTNET_VERSION }}.x" - name: Install Protoc + # C# consumes only generated `.cs` files; libprotobuf is irrelevant, + # so a single modern protoc release is enough (no legacy-v3 matrix). uses: arduino/setup-protoc@v3 with: - version: ${{ matrix.config.protobuf-version }} + version: "33.4" repo-token: ${{ secrets.GITHUB_TOKEN }} - name: Install Buf diff --git a/.github/workflows/testing-go.yml b/.github/workflows/testing-go.yml index 08b2626..dd57ea1 100644 --- a/.github/workflows/testing-go.yml +++ b/.github/workflows/testing-go.yml @@ -51,10 +51,11 @@ jobs: with: python-version: '3.12' - - name: Vet - run: go vet ./... - - name: Test + # `make.py test --lang go` runs `buf generate` then `go test`, which + # transitively vets every package (compilation = type-check + vet). + # Plugin-only `go vet` is covered by devcontainer-smoke.yml's + # `make.py test --lang go --smoke` step. # Explicit --race so Windows CI matches Linux (Windows runners have # MSVC available; -race needs cgo + a C compiler). On a fresh # Windows dev machine without MSVC, make.py's default is --no-race; From 2e671897ab862a790ef77f161ca4a9d6fd3a2eeb Mon Sep 17 00:00:00 2001 From: wenchy Date: Thu, 4 Jun 2026 19:01:24 +0800 Subject: [PATCH 42/66] =?UTF-8?q?fix:=20cpp=20manifest=20mode=20=E2=80=94?= =?UTF-8?q?=20toolchain=20flags=20on=20every=20host=20+=20dry-run=20safety?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two fixes for testing-{make,cpp}.yml regressions: 1. Platform.cmake_toolchain_args() gains force_vcpkg=True. When set, emits -DCMAKE_TOOLCHAIN_FILE=...vcpkg.cmake + -DVCPKG_TARGET_TRIPLET on every host, not just Windows. Used by _cpp_build_or_test when --protobuf-version is set: manifest mode means we ARE using vcpkg regardless of OS, so cmake's find_package(Protobuf) needs the toolchain to resolve against vcpkg_installed/. Fixes testing-cpp.yml ubuntu-latest legacy-v3 failure ("Could not find a package configuration file provided by Protobuf"). 2. _cpp_build_or_test no longer aborts in --dry-run when VCPKG_ROOT is unset. It substitutes a placeholder so snapshot tests still verify the printed command sequence on hosts (CI runners) without vcpkg installed. Real (non-dry-run) execution still hard-errors. Fixes testing-make.yml ubuntu/macos failures (test_test_lang_cpp_protobuf_version_manifest_mode + sibling). Adds two regression tests: - test_test_lang_cpp_manifest_no_vcpkg_root_dry_run_ok (delenv VCPKG_ROOT) - test_test_lang_cpp_manifest_forces_toolchain_on_linux (asserts -DCMAKE_TOOLCHAIN_FILE in output) Co-Authored-By: Claude Opus 4.6 --- make.py | 57 ++++++++++++++++++++++++++++++++++++++-------------- test_make.py | 43 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 85 insertions(+), 15 deletions(-) diff --git a/make.py b/make.py index cf84ad8..0e124ce 100644 --- a/make.py +++ b/make.py @@ -179,17 +179,28 @@ def vcpkg_triplet(self) -> str: return "x64-linux" return "x64-linux" - def cmake_toolchain_args(self, triplet: Optional[str] = None) -> list[str]: - """Extra cmake -D flags to pick up vcpkg's protobuf, when applicable.""" - if self.in_devcontainer: + def cmake_toolchain_args( + self, triplet: Optional[str] = None, force_vcpkg: bool = False + ) -> list[str]: + """Extra cmake -D flags to pick up vcpkg's protobuf, when applicable. + + Default behaviour (force_vcpkg=False): + - devcontainer: [] (Dockerfile presets CMAKE_PREFIX_PATH) + - macOS/Linux: [] (system protobuf via brew/apt) + - Windows: toolchain + triplet flags (always uses vcpkg) + + With force_vcpkg=True, every host (incl. Linux/macOS) gets the + toolchain flags. Used when --protobuf-version is set, because + manifest-mode means we ARE using vcpkg regardless of host. + """ + if self.in_devcontainer and not force_vcpkg: return [] # Dockerfile presets CMAKE_PREFIX_PATH=/opt/vcpkg/active. - if not self.is_windows: + if not self.is_windows and not force_vcpkg: # macOS/Linux native: system protobuf or homebrew/apt resolves via # find_package(Protobuf) without a toolchain file. return [] - # Windows: VCPKG_ROOT must be set, either by `make.py setup` or by - # the CI's lukka/run-vcpkg step. The toolchain file location is - # canonical inside any vcpkg root. + # Locate VCPKG_ROOT (cached attr or env). Required on Windows always, + # and on every OS when force_vcpkg=True (manifest mode). vcpkg_root = self.vcpkg_root or _env_path("VCPKG_ROOT") if vcpkg_root is None: # Best-effort fallback so cmake configure fails with a useful @@ -985,13 +996,21 @@ def _cpp_build_or_test(args, ctx: "Context", run_tests: bool) -> int: # it's the system or devcontainer vcpkg. vcpkg_root = ctx.platform.vcpkg_root or _env_path("VCPKG_ROOT") if vcpkg_root is None: - print( - "[error] --protobuf-version requires VCPKG_ROOT to be set " - "(run `python make.py setup --lang cpp` first, or set " - "VCPKG_ROOT in your environment).", - file=sys.stderr, - ) - return 1 + if ctx.runner.dry_run: + # Dry-run: print what would happen with a placeholder so the + # snapshot test can still verify the command sequence. + vcpkg_root = Path("") + else: + print( + "[error] --protobuf-version requires VCPKG_ROOT to be set " + "(run `python make.py setup --lang cpp` first, or set " + "VCPKG_ROOT in your environment).", + file=sys.stderr, + ) + return 1 + # Surface the resolved root on the platform so cmake_toolchain_args + # picks it up for the configure command. + ctx.platform.vcpkg_root = vcpkg_root vcpkg_exe = vcpkg_root / ("vcpkg.exe" if ctx.platform.is_windows else "vcpkg") # Install the manifest: must `cd` into the manifest dir for vcpkg to @@ -1041,7 +1060,15 @@ def _cpp_build_or_test(args, ctx: "Context", run_tests: bool) -> int: configure_cmd.append(f"-DCMAKE_CXX_COMPILER={compiler}") if shutil.which("ninja") is not None: configure_cmd.extend(["-G", "Ninja"]) - configure_cmd.extend(ctx.platform.cmake_toolchain_args(triplet=triplet)) + configure_cmd.extend( + ctx.platform.cmake_toolchain_args( + triplet=triplet, + # Manifest mode means we ARE using vcpkg regardless of host; + # force the toolchain flags even on Linux/macOS so cmake's + # find_package(Protobuf) resolves against vcpkg_installed/. + force_vcpkg=bool(protobuf_version), + ) + ) configure_cmd.extend(cmake_extra) ctx.runner.run(ctx.platform.windows_msvc_wrap(configure_cmd), cwd=cwd) diff --git a/test_make.py b/test_make.py index c8549e5..dff5485 100644 --- a/test_make.py +++ b/test_make.py @@ -596,6 +596,49 @@ def test_test_lang_cpp_protobuf_version_manifest_mode(self): # Triplet must be plumbed through. assert "--triplet=x64-windows-static" in out + def test_test_lang_cpp_manifest_no_vcpkg_root_dry_run_ok(self, monkeypatch): + # On a host with no VCPKG_ROOT (e.g. CI runner of testing-make.yml), + # --dry-run must still print the command sequence rather than abort. + # Real (non-dry-run) execution still hard-errors via a separate code + # path; that's covered by the unit test in TestPlatform. + monkeypatch.delenv("VCPKG_ROOT", raising=False) + proc = run_make( + "--dry-run", + "test", + "--lang", + "cpp", + "--protobuf-version", + "3.21.12", + "--triplet", + "x64-linux", + "--no-clean", + ) + assert proc.returncode == 0, proc.stderr + assert "-DVCPKG_INSTALLED_DIR=" in proc.stdout + assert "-DVCPKG_MANIFEST_INSTALL=OFF" in proc.stdout + + def test_test_lang_cpp_manifest_forces_toolchain_on_linux(self, monkeypatch): + # Manifest mode must emit -DCMAKE_TOOLCHAIN_FILE even on Linux: + # find_package(Protobuf) needs vcpkg's vcpkg.cmake to resolve the + # manifest-installed protobuf. Plain Linux (no --protobuf-version) + # would correctly get [] (system protobuf via apt). + monkeypatch.setenv("VCPKG_ROOT", "/tmp/fake-vcpkg") + proc = run_make( + "--dry-run", + "test", + "--lang", + "cpp", + "--protobuf-version", + "3.21.12", + "--triplet", + "x64-linux", + "--no-clean", + ) + assert proc.returncode == 0, proc.stderr + # Must have toolchain flags for the manifest install to be picked up. + assert "-DCMAKE_TOOLCHAIN_FILE=" in proc.stdout + assert "-DVCPKG_TARGET_TRIPLET=x64-linux" in proc.stdout + def test_test_lang_cpp_no_vcpkg_install_skips_install(self): proc = run_make( "--dry-run", From c421b9d8a6bc6cf8dd4c4756ad2330afdbfd10df Mon Sep 17 00:00:00 2001 From: wenchy Date: Thu, 4 Jun 2026 20:06:36 +0800 Subject: [PATCH 43/66] fix(vcpkg): use commit SHA, not tag SHA, for VCPKG_BASELINE_COMMIT The previous value `dc8d75cfc3281b8e2a4ed8ee4163c891190df932` was the SHA of an *annotated tag object* (release/2026.04.27), not a commit. Symptoms: - github.com/microsoft/vcpkg/commit/ 404s. - `git log` doesn't list it (tag objects aren't commits). - `git cat-file -t` reports "tag", not "commit". Functionally everything worked (git/vcpkg auto-peel tag SHAs to commits), but it's unbrowsable on GitHub and confuses anyone trying to verify the pin. Switched to the underlying commit SHA so the GitHub /commit/ view resolves; the resolved port catalog is identical. Also fixes a pre-existing bug uncovered while validating the SHA change: classic-mode `make.py test --lang cpp` (no --protobuf-version) didn't remove a stale vcpkg.json left over from a previous manifest-mode run. cmake's vcpkg toolchain auto-detected the manifest and built the wrong libprotobuf into `build/vcpkg_installed/`, mismatching the `buf generate`-produced .pb.h files (`Cannot open include file: google/protobuf/runtime_version.h`). Now the cpp handler unlinks any leftover vcpkg.json before configure when not in manifest mode. Workflow YAMLs in this commit are pure formatter normalization (quote style + indent), no semantic change. Verified: 74/74 unit tests pass; full C++ build + 13/13 ctest cases green on Windows against the new SHA. Co-Authored-By: Claude Opus 4.6 --- .devcontainer/versions.env | 8 +- .github/workflows/devcontainer-smoke.yml | 6 +- .github/workflows/testing-cpp.yml | 222 +++++++++++------------ .github/workflows/testing-go.yml | 2 +- .github/workflows/testing-make.yml | 18 +- make.py | 24 +++ 6 files changed, 155 insertions(+), 125 deletions(-) diff --git a/.devcontainer/versions.env b/.devcontainer/versions.env index 94369df..b982ebd 100644 --- a/.devcontainer/versions.env +++ b/.devcontainer/versions.env @@ -35,7 +35,13 @@ PROTOBUF_VERSION=6.33.4 # - .github/workflows/testing-cpp.yml (env: VCPKG_COMMIT) # This commit MUST know about every PROTOBUF_VERSION in the testing-cpp.yml # matrix; bump it forward (never sideways) when adding a new protobuf entry. -VCPKG_BASELINE_COMMIT=dc8d75cfc3281b8e2a4ed8ee4163c891190df932 +# +# Use the *commit* SHA, not the tag-object SHA: GitHub's /commit/ view +# 404s on tag objects, and the resolved manifest baseline is the commit +# anyway. To bump: pick the commit pointed to by a recent quarterly tag at +# https://github.com/microsoft/vcpkg/tags (e.g. `git rev-parse 2026.04.27^{commit}`). +# Currently pins the tip of the `2026.04.27` quarterly release. +VCPKG_BASELINE_COMMIT=56bb2411609227288b70117ead2c47585ba07713 # .NET SDK major.minor. apt installs `dotnet-sdk-${DOTNET_VERSION}` on Linux. # CI uses `${DOTNET_VERSION}.x` with actions/setup-dotnet. diff --git a/.github/workflows/devcontainer-smoke.yml b/.github/workflows/devcontainer-smoke.yml index 3d0ad4e..947c38d 100644 --- a/.github/workflows/devcontainer-smoke.yml +++ b/.github/workflows/devcontainer-smoke.yml @@ -23,12 +23,12 @@ name: Devcontainer Smoke on: pull_request: paths: - - '.devcontainer/**' - - '.github/workflows/devcontainer-smoke.yml' + - ".devcontainer/**" + - ".github/workflows/devcontainer-smoke.yml" push: branches: [master, main] paths: - - '.devcontainer/**' + - ".devcontainer/**" workflow_dispatch: permissions: diff --git a/.github/workflows/testing-cpp.yml b/.github/workflows/testing-cpp.yml index 83db81b..1c5be43 100644 --- a/.github/workflows/testing-cpp.yml +++ b/.github/workflows/testing-cpp.yml @@ -2,118 +2,118 @@ name: Testing C++ # Trigger on pushes, PRs (excluding documentation changes), and nightly. on: - push: - branches: [master, main] - pull_request: - schedule: - - cron: 0 0 * * * # daily at 00:00 - workflow_dispatch: + push: + branches: [master, main] + pull_request: + schedule: + - cron: 0 0 * * * # daily at 00:00 + workflow_dispatch: permissions: - contents: read + contents: read jobs: - test: - strategy: - fail-fast: false - matrix: - os: [ubuntu-latest, windows-latest] - config: - - label: modern - protobuf-version: "6.33.4" - - label: legacy-v3 - protobuf-version: "3.21.12" - include: - - os: ubuntu-latest - triplet: x64-linux - - os: windows-latest - triplet: x64-windows-static - - name: test (${{ matrix.os }}, ${{ matrix.config.label }}) - runs-on: ${{ matrix.os }} - timeout-minutes: 45 - - env: - VCPKG_INSTALLED_DIR: ${{ github.workspace }}/vcpkg_installed - VCPKG_DEFAULT_TRIPLET: ${{ matrix.triplet }} - - steps: - - name: Checkout Code - uses: actions/checkout@v6 - - - name: Read pinned versions - uses: ./.github/actions/load-versions - - - name: Install Go - uses: actions/setup-go@v6 - with: - go-version-file: go.mod - cache-dependency-path: go.sum - cache: true - - - name: Install Ninja (Ubuntu) - if: runner.os == 'Linux' - run: sudo apt-get update && sudo apt-get install -y ninja-build - - - name: Setup MSVC (Windows) - if: runner.os == 'Windows' - uses: ilammy/msvc-dev-cmd@v1 - - - name: Setup Python - uses: actions/setup-python@v6 - with: - python-version: "3.12" - - - name: Render vcpkg.json - working-directory: test/cpp-tableau-loader - shell: bash - run: | - cat > vcpkg.json <> "$GITHUB_PATH" - - - name: Add vcpkg-installed protoc to PATH (Windows) - if: runner.os == 'Windows' - shell: pwsh - run: Add-Content -Path $env:GITHUB_PATH -Value "$env:VCPKG_INSTALLED_DIR\${{ matrix.triplet }}\tools\protobuf" - - - name: Install Buf - uses: bufbuild/buf-action@v1 - with: - version: ${{ env.BUF_VERSION }} - setup_only: true - github_token: ${{ secrets.GITHUB_TOKEN }} - - - name: Test - run: > - python make.py test --lang cpp - --protobuf-version ${{ matrix.config.protobuf-version }} - --triplet ${{ matrix.triplet }} - --no-clean - --no-vcpkg-install + test: + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, windows-latest] + config: + - label: modern + protobuf-version: "6.33.4" + - label: legacy-v3 + protobuf-version: "3.21.12" + include: + - os: ubuntu-latest + triplet: x64-linux + - os: windows-latest + triplet: x64-windows-static + + name: test (${{ matrix.os }}, ${{ matrix.config.label }}) + runs-on: ${{ matrix.os }} + timeout-minutes: 45 + + env: + VCPKG_INSTALLED_DIR: ${{ github.workspace }}/vcpkg_installed + VCPKG_DEFAULT_TRIPLET: ${{ matrix.triplet }} + + steps: + - name: Checkout Code + uses: actions/checkout@v6 + + - name: Read pinned versions + uses: ./.github/actions/load-versions + + - name: Install Go + uses: actions/setup-go@v6 + with: + go-version-file: go.mod + cache-dependency-path: go.sum + cache: true + + - name: Install Ninja (Ubuntu) + if: runner.os == 'Linux' + run: sudo apt-get update && sudo apt-get install -y ninja-build + + - name: Setup MSVC (Windows) + if: runner.os == 'Windows' + uses: ilammy/msvc-dev-cmd@v1 + + - name: Setup Python + uses: actions/setup-python@v6 + with: + python-version: "3.12" + + - name: Render vcpkg.json + working-directory: test/cpp-tableau-loader + shell: bash + run: | + cat > vcpkg.json <> "$GITHUB_PATH" + + - name: Add vcpkg-installed protoc to PATH (Windows) + if: runner.os == 'Windows' + shell: pwsh + run: Add-Content -Path $env:GITHUB_PATH -Value "$env:VCPKG_INSTALLED_DIR\${{ matrix.triplet }}\tools\protobuf" + + - name: Install Buf + uses: bufbuild/buf-action@v1 + with: + version: ${{ env.BUF_VERSION }} + setup_only: true + github_token: ${{ secrets.GITHUB_TOKEN }} + + - name: Test + run: > + python make.py test --lang cpp + --protobuf-version ${{ matrix.config.protobuf-version }} + --triplet ${{ matrix.triplet }} + --no-clean + --no-vcpkg-install diff --git a/.github/workflows/testing-go.yml b/.github/workflows/testing-go.yml index dd57ea1..4b65363 100644 --- a/.github/workflows/testing-go.yml +++ b/.github/workflows/testing-go.yml @@ -49,7 +49,7 @@ jobs: - name: Setup Python uses: actions/setup-python@v6 with: - python-version: '3.12' + python-version: "3.12" - name: Test # `make.py test --lang go` runs `buf generate` then `go test`, which diff --git a/.github/workflows/testing-make.yml b/.github/workflows/testing-make.yml index 6fb5029..d89f48d 100644 --- a/.github/workflows/testing-make.yml +++ b/.github/workflows/testing-make.yml @@ -8,16 +8,16 @@ on: push: branches: [master, main] paths: - - 'make.py' - - 'test_make.py' - - '.devcontainer/versions.env' - - '.github/workflows/testing-make.yml' + - "make.py" + - "test_make.py" + - ".devcontainer/versions.env" + - ".github/workflows/testing-make.yml" pull_request: paths: - - 'make.py' - - 'test_make.py' - - '.devcontainer/versions.env' - - '.github/workflows/testing-make.yml' + - "make.py" + - "test_make.py" + - ".devcontainer/versions.env" + - ".github/workflows/testing-make.yml" workflow_dispatch: permissions: @@ -41,7 +41,7 @@ jobs: - name: Setup Python uses: actions/setup-python@v6 with: - python-version: '3.12' + python-version: "3.12" - name: Install pytest run: python -m pip install --upgrade pip pytest diff --git a/make.py b/make.py index 0e124ce..2250280 100644 --- a/make.py +++ b/make.py @@ -968,6 +968,30 @@ def _cpp_build_or_test(args, ctx: "Context", run_tests: bool) -> int: ctx.runner.rmtree(cwd / "src" / "tableau") ctx.runner.rmtree(cwd / "src" / "protoconf") + # Classic mode: a stale vcpkg.json from a previous --protobuf-version run + # would silently switch cmake's vcpkg toolchain into manifest mode and + # build the wrong libprotobuf into build/vcpkg_installed/. Always remove + # it here unless we're about to render a fresh one below. + if not protobuf_version: + manifest_path = cwd / "vcpkg.json" + if manifest_path.is_file(): + if ctx.runner.dry_run: + print(f"[dry-run] rm {manifest_path}") + else: + manifest_path.unlink() + + # Classic mode: a stale vcpkg.json from a previous --protobuf-version run + # would silently switch cmake's vcpkg toolchain into manifest mode and + # build the wrong libprotobuf into build/vcpkg_installed/. Always remove + # it here unless we're about to render a fresh one below. + if not protobuf_version: + manifest_path = cwd / "vcpkg.json" + if manifest_path.is_file(): + if ctx.runner.dry_run: + print(f"[dry-run] rm {manifest_path}") + else: + manifest_path.unlink() + # Manifest mode: render vcpkg.json pinning the requested protobuf-version, # then run `vcpkg install` to populate vcpkg_installed/. This matches CI's # testing-cpp.yml flow (which uses lukka/run-vcpkg with runVcpkgInstall: From 8f704ce4fcda883c66b2157c8d9f3b890d9b9596 Mon Sep 17 00:00:00 2001 From: wenchy Date: Thu, 4 Jun 2026 20:25:02 +0800 Subject: [PATCH 44/66] feat(make.py): pin protobuf via vcpkg on macOS/Linux too (vcpkg-everywhere) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Until now `make.py setup --lang cpp` on macOS/Linux installed whatever `brew` / `apt` / `dnf` shipped — meaning a fresh dev machine in 2027 might pick up protobuf 7.x while CI is still pinned to 6.33.4. Now mirrors the devcontainer Dockerfile's strategy across every native host: - Go: official tarball from go.dev to ~/.local/go/ (Linux/macOS). - buf: GitHub release binary at BUF_VERSION. - protobuf: vcpkg at VCPKG_BASELINE_COMMIT, classic mode by default. Manifest mode (--protobuf-version) still works as before. - cmake / ninja / build-essential: distro/brew package (version-tolerant). - .NET: Homebrew dotnet@N (macOS), Microsoft apt repo (Linux). - Node: Homebrew node@N (macOS), NodeSource (Linux). Internal refactor: - _setup_vcpkg_windows lifted to cross-platform _setup_vcpkg (.bat vs .sh bootstrap, MSVC wrap on Windows only, exe vs binary). - Platform.cmake_toolchain_args() emits toolchain flags whenever vcpkg_root is known, regardless of OS (was Windows-only). Devcontainer still returns [] (CMAKE_PREFIX_PATH preset). - hydrate_platform_from_env reads ~/.loader-env.json on every OS now (was Windows-only), so `make.py test` after `make.py setup` works without VCPKG_ROOT being on shell PATH. - _ensure_buf_linux generalized to _ensure_buf_unix (Linux/Darwin). - New _ensure_go_tarball matches devcontainer's Go install. - force_vcpkg parameter on cmake_toolchain_args is no longer needed in practice (default behaviour already does the right thing) but kept as an explicit override for callers. 8 new regression tests (TestCrossPlatformPinning + expanded TestPlatform.cmake_toolchain_args*). 82/82 pass. Real Windows C++ build + 13/13 ctest cases still green against the refactored code. Co-Authored-By: Claude Opus 4.6 --- CLAUDE.md | 4 +- README.md | 9 +- make.py | 267 +++++++++++++++++++++++++++++++++------------------ test_make.py | 96 +++++++++++++++++- 4 files changed, 279 insertions(+), 97 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 5327662..c937f0e 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -30,10 +30,12 @@ python make.py env # diagnostic JSON python make.py --version ``` -C++ wipes `test/cpp-tableau-loader/{build,src/tableau,src/protoconf}` before regenerating (gitignored `*.pb.*` shadows fresh codegen). `--no-clean` skips it. +C++ wipes `test/cpp-tableau-loader/{build,src/tableau,src/protoconf}` before regenerating (gitignored `*.pb.*` shadows fresh codegen). `--no-clean` skips it. A leftover `vcpkg.json` from a previous `--protobuf-version` run is auto-removed in classic mode so cmake doesn't accidentally re-enter manifest mode. On Windows, `make.py` wraps every C++ subprocess in `cmd /c "call vcvarsall.bat x64 >nul && "` so MSVC env lives per-subprocess; the shell PATH is never mutated. +`make.py setup` pins every toolchain dimension (matches CI + devcontainer): Go via official go.dev tarball to `~/.local/go/`, buf via GitHub release binary, protobuf via **vcpkg at `VCPKG_BASELINE_COMMIT` on every host (macOS/Linux/Windows)**, .NET / Node via Microsoft+NodeSource (Linux) or Homebrew (macOS) or winget (Windows), cmake/ninja via the host's package manager. Resolved paths cached in `~/.loader-env.json` so subsequent `make.py test --lang cpp` invocations pick them up without re-running setup. + ### Dev container - `.devcontainer/` → **Dev Containers: Reopen in Container**. Ubuntu 24.04 + all toolchains pinned. First build ~25 min; reopens instant. diff --git a/README.md b/README.md index 510ff97..3caf5e7 100644 --- a/README.md +++ b/README.md @@ -22,7 +22,14 @@ python make.py test --lang ts # TypeScript (experimental) Recommended environment: [devcontainer](./.devcontainer/) (open in VS Code → **Dev Containers: Reopen in Container**). Inside the container, `setup` is a no-op. -Native hosts: `python make.py setup` installs everything via `brew` (macOS), `apt`/`dnf` (Linux), or Chocolatey + MSVC + vcpkg (Windows). On Windows it must be run from **cmd as Administrator** the first time; subsequent runs work from any shell because each subprocess sources `vcvarsall.bat` itself — your shell PATH/INCLUDE/LIB are never mutated. +Native hosts: `python make.py setup` installs everything pinned to [`./.devcontainer/versions.env`](./.devcontainer/versions.env) — the same versions CI and the devcontainer use. Toolchain layout per host: + +- **Go** — official tarball from go.dev to `~/.local/go/` (Linux/macOS) or winget (Windows). +- **buf** — pinned binary from GitHub releases to `~/.local/bin/` (Linux/macOS) or `%LOCALAPPDATA%\buf\bin\` (Windows). +- **protobuf** — vcpkg at `VCPKG_BASELINE_COMMIT` on every native host (Linux/macOS/Windows). Switch versions per-test with `--protobuf-version`. +- **.NET / Node / cmake / ninja** — Homebrew (macOS), Microsoft+NodeSource apt repos (Linux), winget+Chocolatey (Windows). + +On Windows, run setup from **cmd as Administrator** the first time. Subsequent commands work from any shell because each subprocess sources `vcvarsall.bat` itself — your shell PATH/INCLUDE/LIB are never mutated. ## Commands diff --git a/make.py b/make.py index 2250280..1fdb0ef 100644 --- a/make.py +++ b/make.py @@ -184,34 +184,31 @@ def cmake_toolchain_args( ) -> list[str]: """Extra cmake -D flags to pick up vcpkg's protobuf, when applicable. - Default behaviour (force_vcpkg=False): - - devcontainer: [] (Dockerfile presets CMAKE_PREFIX_PATH) - - macOS/Linux: [] (system protobuf via brew/apt) - - Windows: toolchain + triplet flags (always uses vcpkg) - - With force_vcpkg=True, every host (incl. Linux/macOS) gets the - toolchain flags. Used when --protobuf-version is set, because - manifest-mode means we ARE using vcpkg regardless of host. + Default behaviour: + - devcontainer: [] (Dockerfile presets CMAKE_PREFIX_PATH). + - any host with VCPKG_ROOT: toolchain + triplet flags. + - host without vcpkg installed (Linux/macOS, no setup run yet): + [] (cmake will fall back to system + protobuf via find_package). + + force_vcpkg=True forces the toolchain flags even inside the + devcontainer (used by manifest-mode invocations that point cmake + at a custom vcpkg_installed/ dir). """ if self.in_devcontainer and not force_vcpkg: return [] # Dockerfile presets CMAKE_PREFIX_PATH=/opt/vcpkg/active. - if not self.is_windows and not force_vcpkg: - # macOS/Linux native: system protobuf or homebrew/apt resolves via - # find_package(Protobuf) without a toolchain file. - return [] - # Locate VCPKG_ROOT (cached attr or env). Required on Windows always, - # and on every OS when force_vcpkg=True (manifest mode). + # Locate VCPKG_ROOT (cached attr or env). If absent on macOS/Linux, + # we silently fall through to system protobuf (apt/brew). On Windows + # / manifest mode the cpp handler errors with an actionable message + # before we get here. vcpkg_root = self.vcpkg_root or _env_path("VCPKG_ROOT") if vcpkg_root is None: - # Best-effort fallback so cmake configure fails with a useful - # message rather than us emitting an empty -D flag. return [] toolchain = vcpkg_root / "scripts" / "buildsystems" / "vcpkg.cmake" - args = [ + return [ f"-DCMAKE_TOOLCHAIN_FILE={toolchain}", f"-DVCPKG_TARGET_TRIPLET={triplet or self.vcpkg_triplet}", ] - return args def windows_msvc_wrap(self, cmd: list[str]) -> list[str]: """Wrap a command so it runs inside an MSVC-environment subshell. @@ -463,22 +460,27 @@ def save_loader_env(data: dict, runner: Runner) -> None: def hydrate_platform_from_env(plat: Platform) -> None: - """Populate Platform from $VCPKG_ROOT and ~/.loader-env.json (Windows).""" - if not plat.is_windows: - return + """Populate Platform from $VCPKG_ROOT and ~/.loader-env.json. + + Cross-platform: macOS / Linux / Windows. Cache file is shared between + `make.py setup` (which writes it) and subsequent `make.py test` + invocations (which read it). + """ if plat.vcpkg_root is None: plat.vcpkg_root = _env_path("VCPKG_ROOT") cache = load_loader_env() if plat.vcpkg_root is None and cache.get("vcpkg_root"): plat.vcpkg_root = Path(cache["vcpkg_root"]) - if plat.vcvarsall_path is None and cache.get("vcvarsall_path"): - candidate = Path(cache["vcvarsall_path"]) - if candidate.is_file(): - plat.vcvarsall_path = candidate if plat.protoc_tools_dir is None and cache.get("protoc_tools_dir"): plat.protoc_tools_dir = Path(cache["protoc_tools_dir"]) if plat.vcpkg_installed_dir is None and cache.get("vcpkg_installed_dir"): plat.vcpkg_installed_dir = Path(cache["vcpkg_installed_dir"]) + # vcvarsall is Windows-only. + if plat.is_windows: + if plat.vcvarsall_path is None and cache.get("vcvarsall_path"): + candidate = Path(cache["vcvarsall_path"]) + if candidate.is_file(): + plat.vcvarsall_path = candidate # --------------------------------------------------------------------------- @@ -529,20 +531,37 @@ def _setup_macos(langs: list[str], ctx: "Context") -> int: file=sys.stderr, ) return 1 + cache = load_loader_env() + + # Brew packages: only the version-tolerant pieces (cmake, ninja, build + # essentials). Go and protobuf are pinned via tarball / vcpkg below; + # buf is pinned via direct download. dotnet@N and node@N are pinnable + # via brew's versioned formulae. pkgs: list[str] = [] - if "go" in langs: - pkgs.append("go") if "cpp" in langs: - pkgs.extend(["protobuf", "cmake", "ninja"]) + pkgs.extend(["cmake", "ninja"]) if "csharp" in langs: - # Homebrew dotnet@8 cask covers .NET 8. - pkgs.append(f"dotnet@{ctx.versions.dotnet_version or '8'}") + # `dotnet@8` (cask) covers .NET 8.x. Use the major. + major = (ctx.versions.dotnet_version or "8.0").split(".")[0] + pkgs.append(f"dotnet@{major}") if "ts" in langs: pkgs.append(f"node@{ctx.versions.node_version or '20'}") - pkgs.append("buf") - pkgs = list(dict.fromkeys(pkgs)) # de-dup, preserving order - ctx.runner.run(["brew", "update"], check=False) - ctx.runner.run(["brew", "install", *pkgs], check=False) + if pkgs: + ctx.runner.run(["brew", "update"], check=False) + ctx.runner.run(["brew", "install", *pkgs], check=False) + + # Pinned Go via official tarball (matches devcontainer). + if "go" in langs or "cpp" in langs or "csharp" in langs: + _ensure_go_tarball(ctx, "darwin") + + # Pinned buf via GitHub release. + _ensure_buf_unix(ctx, os_label="Darwin") + + # Pinned protobuf via vcpkg (matches devcontainer + Windows). + if "cpp" in langs: + _setup_vcpkg(ctx, cache) + + save_loader_env(cache, ctx.runner) print("[info] macOS toolchain ready.") return 0 @@ -556,63 +575,60 @@ def _setup_linux(langs: list[str], ctx: "Context") -> int: file=sys.stderr, ) return 1 + cache = load_loader_env() + # Distro packages: only the version-tolerant pieces (cmake, ninja, build + # essentials, git). Go is pinned via tarball; buf via GitHub release; + # protobuf via vcpkg; .NET via Microsoft repo helper; Node via NodeSource. base_pkgs: list[str] = [] if "cpp" in langs: if apt: base_pkgs.extend( - [ - "protobuf-compiler", - "libprotobuf-dev", - "cmake", - "ninja-build", - "build-essential", - "git", - ] + ["cmake", "ninja-build", "build-essential", "git", "curl", "zip", "unzip", "tar", "pkg-config"] ) else: base_pkgs.extend( - [ - "protobuf-compiler", - "protobuf-devel", - "cmake", - "ninja-build", - "gcc-c++", - "git", - ] + ["cmake", "ninja-build", "gcc-c++", "git", "curl", "zip", "unzip", "tar", "pkgconf-pkg-config"] ) - if "go" in langs and apt: - base_pkgs.append("golang") - if "go" in langs and dnf: - base_pkgs.append("golang") if base_pkgs: + sudo_prefix = ["sudo"] if os.geteuid() != 0 else [] if apt: - sudo_prefix = ["sudo"] if os.geteuid() != 0 else [] ctx.runner.run([*sudo_prefix, "apt-get", "update"], check=False) ctx.runner.run( [*sudo_prefix, "apt-get", "install", "-y", *base_pkgs], check=False ) else: - sudo_prefix = ["sudo"] if os.geteuid() != 0 else [] ctx.runner.run( [*sudo_prefix, "dnf", "install", "-y", *base_pkgs], check=False ) - # buf, .NET, Node are not always packaged at our pinned version; install - # via direct download / vendor scripts to ~/.local/. - if "go" in langs or "cpp" in langs or "csharp" in langs or "ts" in langs: - _ensure_buf_linux(ctx) + # Pinned Go via official tarball. + if "go" in langs or "cpp" in langs or "csharp" in langs: + _ensure_go_tarball(ctx, "linux") + + # Pinned buf via GitHub release. + _ensure_buf_unix(ctx, os_label="Linux") + if "csharp" in langs: _ensure_dotnet_linux(ctx) if "ts" in langs: _ensure_node_linux(ctx) + # Pinned protobuf via vcpkg (matches devcontainer + Windows). + if "cpp" in langs: + _setup_vcpkg(ctx, cache) + + save_loader_env(cache, ctx.runner) print("[info] Linux toolchain ready.") return 0 -def _ensure_buf_linux(ctx: "Context") -> None: +def _ensure_buf_unix(ctx: "Context", os_label: str) -> None: + """Download the pinned buf release to ~/.local/bin/buf. + + os_label: "Linux" or "Darwin" (matches the GitHub release naming). + """ if _which("buf") is not None: return ver = ctx.versions.buf_version @@ -623,11 +639,11 @@ def _ensure_buf_linux(ctx: "Context") -> None: if _stdlib_platform.machine().lower() in ("x86_64", "amd64") else "aarch64" ) - url = f"https://github.com/bufbuild/buf/releases/download/v{ver}/buf-Linux-{arch}" + url = f"https://github.com/bufbuild/buf/releases/download/v{ver}/buf-{os_label}-{arch}" target_dir = Path.home() / ".local" / "bin" ctx.runner.mkdirp(target_dir) target = target_dir / "buf" - print(f"[info] Downloading buf {ver} -> {target}") + print(f"[info] Downloading buf {ver} ({os_label}/{arch}) -> {target}") if not ctx.runner.dry_run: urllib.request.urlretrieve(url, str(target)) target.chmod(0o755) @@ -650,6 +666,47 @@ def _ensure_dotnet_linux(ctx: "Context") -> None: ) +def _go_arch_macos() -> str: + return "arm64" if _stdlib_platform.machine().lower() in ("arm64", "aarch64") else "amd64" + + +def _go_arch_linux() -> str: + return "arm64" if _stdlib_platform.machine().lower() in ("arm64", "aarch64") else "amd64" + + +def _ensure_go_tarball(ctx: "Context", os_label: str) -> None: + """Install the official Go tarball at GO_VERSION into ~/.local/go/. + + Mirrors the devcontainer Dockerfile's Go layer. `go` already on PATH is + accepted as-is — bumping versions.env's GO_VERSION on a machine with an + older Go installed is the user's call (we don't auto-replace). + """ + if _which("go") is not None: + return + ver = ctx.versions.go_version + if not ver: + return + arch = _go_arch_macos() if os_label == "darwin" else _go_arch_linux() + url = f"https://go.dev/dl/go{ver}.{os_label}-{arch}.tar.gz" + target_root = Path.home() / ".local" + ctx.runner.mkdirp(target_root) + print(f"[info] Downloading Go {ver} ({os_label}/{arch}) -> {target_root}/go") + if ctx.runner.dry_run: + print(f"[dry-run] curl -L {url} | tar -C {target_root} -xz") + return + import tempfile + import tarfile + with tempfile.NamedTemporaryFile(suffix=".tar.gz", delete=False) as tmp: + urllib.request.urlretrieve(url, tmp.name) + tarball = tmp.name + with tarfile.open(tarball, "r:gz") as tf: + tf.extractall(target_root) + os.unlink(tarball) + bin_dir = target_root / "go" / "bin" + print(f"[info] Go {ver} installed. Add to your shell profile:") + print(f" export PATH={bin_dir}:$PATH") + + def _ensure_node_linux(ctx: "Context") -> None: if _which("node") is not None: return @@ -753,7 +810,7 @@ def _setup_windows(langs: list[str], ctx: "Context", skip_vcpkg: bool) -> int: # Step 5: vcpkg + protobuf if "cpp" in langs and not skip_vcpkg: - _setup_vcpkg_windows(ctx, cache) + _setup_vcpkg(ctx, cache) # Optional: Go / .NET / Node if "go" in langs and _which("go") is None: @@ -777,26 +834,42 @@ def _setup_windows(langs: list[str], ctx: "Context", skip_vcpkg: bool) -> int: return 0 -def _setup_vcpkg_windows(ctx: "Context", cache: dict) -> None: +def _setup_vcpkg(ctx: "Context", cache: dict) -> None: + """Cross-platform vcpkg + protobuf installer (Windows / macOS / Linux). + + Mirrors the devcontainer Dockerfile's vcpkg layer so native dev gets the + same protobuf version pin as CI and the container. Idempotent — second + runs detect existing checkout and skip. + """ triplet = ctx.platform.vcpkg_triplet baseline = ctx.versions.vcpkg_baseline_commit + is_windows = ctx.platform.is_windows + bootstrap_name = "bootstrap-vcpkg.bat" if is_windows else "bootstrap-vcpkg.sh" + vcpkg_bin_name = "vcpkg.exe" if is_windows else "vcpkg" + protoc_bin_name = "protoc.exe" if is_windows else "protoc" + # Resolve a usable vcpkg root: existing $VCPKG_ROOT, then cache, then a + # standard location under the user's home dir. vcpkg_root = ctx.platform.vcpkg_root or _env_path("VCPKG_ROOT") if vcpkg_root is None and cache.get("vcpkg_root"): vcpkg_root = Path(cache["vcpkg_root"]) - # Reject manifest-only vcpkg under VS install dir (no bootstrap-vcpkg.bat). - if vcpkg_root is not None and not (vcpkg_root / "bootstrap-vcpkg.bat").is_file(): + # Reject manifest-only vcpkg (no bootstrap script). On Windows this filters + # the VS-bundled vcpkg under `\VC\vcpkg\` which refuses classic mode. + if vcpkg_root is not None and not (vcpkg_root / bootstrap_name).is_file(): print(f"[warn] {vcpkg_root} looks like a manifest-only vcpkg; ignoring.") vcpkg_root = None - if vcpkg_root is None: - candidate = Path(os.environ.get("USERPROFILE", str(Path.home()))) / "vcpkg" - if (candidate / "bootstrap-vcpkg.bat").is_file(): - vcpkg_root = candidate + home_default = ( + Path(os.environ.get("USERPROFILE", str(Path.home()))) / "vcpkg" + if is_windows + else Path.home() / "vcpkg" + ) + if vcpkg_root is None and (home_default / bootstrap_name).is_file(): + vcpkg_root = home_default if vcpkg_root is None: - vcpkg_root = Path(os.environ.get("USERPROFILE", str(Path.home()))) / "vcpkg" + vcpkg_root = home_default print(f"[info] Cloning vcpkg into {vcpkg_root}...") ctx.runner.run( ["git", "clone", "https://github.com/microsoft/vcpkg.git", str(vcpkg_root)], @@ -811,28 +884,41 @@ def _setup_vcpkg_windows(ctx: "Context", cache: dict) -> None: ["git", "-C", str(vcpkg_root), "checkout", "--quiet", baseline], check=False, ) - # Bootstrap requires MSVC env on Windows. - bootstrap = vcpkg_root / "bootstrap-vcpkg.bat" - ctx.runner.run( - ctx.platform.windows_msvc_wrap([str(bootstrap), "-disableMetrics"]), - check=False, - ) + # Bootstrap. On Windows it shells out to MSVC's link.exe so wrap in + # vcvarsall; Unix bootstrap is self-contained (downloads prebuilt + # vcpkg binary or builds from source via cc). + bootstrap_path = vcpkg_root / bootstrap_name + bootstrap_cmd: list[str] = [str(bootstrap_path), "-disableMetrics"] + if is_windows: + ctx.runner.run( + ctx.platform.windows_msvc_wrap(bootstrap_cmd), + check=False, + ) + else: + # Make sure bootstrap.sh is executable (clone preserves permissions + # but a fresh git config sometimes drops the +x bit on Windows). + if not ctx.runner.dry_run and bootstrap_path.is_file(): + bootstrap_path.chmod(0o755) + ctx.runner.run(bootstrap_cmd, check=False) cache["vcpkg_root"] = str(vcpkg_root) ctx.platform.vcpkg_root = vcpkg_root - vcpkg_exe = vcpkg_root / "vcpkg.exe" + vcpkg_exe = vcpkg_root / vcpkg_bin_name print(f"[info] vcpkg at: {vcpkg_root}") print(f"[info] Installing protobuf:{triplet} via vcpkg (classic mode)...") - ctx.runner.run( - ctx.platform.windows_msvc_wrap( - [str(vcpkg_exe), "install", f"protobuf:{triplet}"] - ), - check=False, - ) + install_cmd = [str(vcpkg_exe), "install", f"protobuf:{triplet}"] + if is_windows: + ctx.runner.run( + ctx.platform.windows_msvc_wrap(install_cmd), + check=False, + ) + else: + ctx.runner.run(install_cmd, check=False) + protoc_dir = vcpkg_root / "installed" / triplet / "tools" / "protobuf" - if (protoc_dir / "protoc.exe").is_file(): + if (protoc_dir / protoc_bin_name).is_file() or ctx.runner.dry_run: cache["protoc_tools_dir"] = str(protoc_dir) ctx.platform.protoc_tools_dir = protoc_dir @@ -864,13 +950,10 @@ def _buf_generate( # - protoc_dir_override (manifest-mode protoc from this build's # vcpkg_installed/) wins — required when --protobuf-version is set, # so codegen matches the libprotobuf headers cmake will use. - # - Else the classic-mode protoc cached in ~/.loader-env.json. + # - Else the classic-mode protoc cached in ~/.loader-env.json + # (populated on every host by `make.py setup`). protoc_dir = protoc_dir_override or ctx.platform.protoc_tools_dir - if protoc_dir is not None and ctx.platform.is_windows: - env["PATH"] = f"{protoc_dir}{os.pathsep}{env.get('PATH', '')}" - elif protoc_dir is not None and not ctx.platform.is_windows: - # On Linux/macOS the manifest-mode protoc dir matters for vcpkg - # manifest builds too (CI testing-cpp.yml does this on Linux). + if protoc_dir is not None: env["PATH"] = f"{protoc_dir}{os.pathsep}{env.get('PATH', '')}" ctx.runner.run(cmd, cwd=cwd, env=env) diff --git a/test_make.py b/test_make.py index dff5485..2993c05 100644 --- a/test_make.py +++ b/test_make.py @@ -141,14 +141,42 @@ def test_cmake_toolchain_args_devcontainer_is_empty(self): p = make.Platform(sys_platform="linux", machine="x86_64", in_devcontainer=True) assert p.cmake_toolchain_args() == [] - def test_cmake_toolchain_args_linux_is_empty(self): + def test_cmake_toolchain_args_linux_no_vcpkg_is_empty(self, monkeypatch): + # Without vcpkg installed (no VCPKG_ROOT), cmake falls back to + # system protobuf via find_package — no -D flags needed. + monkeypatch.delenv("VCPKG_ROOT", raising=False) p = make.Platform(sys_platform="linux", machine="x86_64", in_devcontainer=False) assert p.cmake_toolchain_args() == [] - def test_cmake_toolchain_args_macos_is_empty(self): + def test_cmake_toolchain_args_linux_with_vcpkg_emits_flags(self, tmp_path): + # After `make.py setup --lang cpp` populates ~/vcpkg, native Linux + # builds also route through vcpkg. Same protobuf pin as Windows. + p = make.Platform( + sys_platform="linux", + machine="x86_64", + in_devcontainer=False, + vcpkg_root=tmp_path, + ) + args = p.cmake_toolchain_args() + assert any("CMAKE_TOOLCHAIN_FILE" in a for a in args) + assert any("VCPKG_TARGET_TRIPLET=x64-linux" in a for a in args) + + def test_cmake_toolchain_args_macos_no_vcpkg_is_empty(self, monkeypatch): + monkeypatch.delenv("VCPKG_ROOT", raising=False) p = make.Platform(sys_platform="darwin", machine="arm64", in_devcontainer=False) assert p.cmake_toolchain_args() == [] + def test_cmake_toolchain_args_macos_with_vcpkg_emits_flags(self, tmp_path): + p = make.Platform( + sys_platform="darwin", + machine="arm64", + in_devcontainer=False, + vcpkg_root=tmp_path, + ) + args = p.cmake_toolchain_args() + assert any("CMAKE_TOOLCHAIN_FILE" in a for a in args) + assert any("VCPKG_TARGET_TRIPLET=arm64-osx" in a for a in args) + def test_cmake_toolchain_args_windows_with_vcpkg_root(self, tmp_path): p = make.Platform( sys_platform="win32", @@ -751,7 +779,69 @@ def test_clean_all(self): # --------------------------------------------------------------------------- -class TestSetupDevcontainer: +class TestCrossPlatformPinning: + """Unit tests for the macOS/Linux pinning helpers (Go tarball, buf + binary, vcpkg). Network calls are guarded by --dry-run.""" + + def test_go_arch_macos_intel(self, monkeypatch): + monkeypatch.setattr(make._stdlib_platform, "machine", lambda: "x86_64") + assert make._go_arch_macos() == "amd64" + + def test_go_arch_macos_apple_silicon(self, monkeypatch): + monkeypatch.setattr(make._stdlib_platform, "machine", lambda: "arm64") + assert make._go_arch_macos() == "arm64" + + def test_go_arch_linux_x64(self, monkeypatch): + monkeypatch.setattr(make._stdlib_platform, "machine", lambda: "x86_64") + assert make._go_arch_linux() == "amd64" + + def test_go_arch_linux_arm64(self, monkeypatch): + monkeypatch.setattr(make._stdlib_platform, "machine", lambda: "aarch64") + assert make._go_arch_linux() == "arm64" + + def test_setup_macos_dispatches_to_vcpkg(self, monkeypatch, capsys): + # Simulate macOS host with brew available. Dry-run so no real install. + # The handler's _setup_vcpkg call should print the cloning hint. + monkeypatch.setattr(make.Platform, "detect", classmethod( + lambda cls: make.Platform(sys_platform="darwin", machine="x86_64", + in_devcontainer=False))) + monkeypatch.setattr(make, "_which", lambda name: "/usr/local/bin/brew" if name == "brew" else None) + ctx = make.Context( + repo_root=REPO_ROOT, + versions=make.Versions.load(REPO_ROOT), + platform=make.Platform.detect(), + runner=make.Runner(verbose=False, dry_run=True), + ) + args = type("Args", (), {"lang": "cpp", "skip_vcpkg": False})() + rc = make.cmd_setup(args, ctx) + assert rc == 0 + out = capsys.readouterr().out + # Must mention vcpkg (cross-platform install path). + assert "vcpkg" in out.lower() + + def test_setup_linux_dispatches_to_vcpkg(self, monkeypatch, capsys): + monkeypatch.setattr(make.Platform, "detect", classmethod( + lambda cls: make.Platform(sys_platform="linux", machine="x86_64", + in_devcontainer=False))) + # Stub _which to make apt-get appear available, brew/dnf absent. + which_table = {"apt-get": "/usr/bin/apt-get"} + monkeypatch.setattr(make, "_which", lambda name: which_table.get(name)) + # Stub os.geteuid (not present on Windows test host). + monkeypatch.setattr(make.os, "geteuid", lambda: 1000, raising=False) + ctx = make.Context( + repo_root=REPO_ROOT, + versions=make.Versions.load(REPO_ROOT), + platform=make.Platform.detect(), + runner=make.Runner(verbose=False, dry_run=True), + ) + args = type("Args", (), {"lang": "cpp", "skip_vcpkg": False})() + rc = make.cmd_setup(args, ctx) + assert rc == 0 + out = capsys.readouterr().out + assert "vcpkg" in out.lower() + + + def test_setup_skips_inside_devcontainer(self, monkeypatch, tmp_path): # Simulate devcontainer detection by patching Platform.detect. original_detect = make.Platform.detect From 45479175072123bfe6d13a51a1e48b91d991b5f1 Mon Sep 17 00:00:00 2001 From: Kybxd <627940450@qq.com> Date: Fri, 5 Jun 2026 17:37:04 +0800 Subject: [PATCH 45/66] feat(vcpkg): pair vcpkg builtin-baseline with protobuf version MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Pinning protobuf via 'overrides' alone leaves builtin-baseline on a newer vcpkg commit, which lets transitive deps (abseil/utf8-range/re2/...) drift forward and breaks ABI (e.g. missing absl::if_constexpr). Pin the baseline itself to the vcpkg commit whose versions/baseline.json had the target protobuf — that snapshot is by construction self-consistent. make.py: - Add _resolve_vcpkg_baseline_for_protobuf: pickaxe-search vcpkg's git history (git log -S '"X.Y.Z"' -- versions/baseline.json), validate each candidate via baseline.json (default.protobuf.baseline == X.Y.Z) to defeat false positives, cache result in ~/.loader-env.json. - Add --vcpkg-baseline override; auto-resolve from $VCPKG_ROOT when only --protobuf-version is given. Dry-run returns a placeholder so snapshot tests stay deterministic. - Drop 'overrides' from rendered vcpkg.json — baseline alone now carries protobuf + transitive deps as one consistent set. CI (.github/workflows/testing-cpp.yml): - Move vcpkg-commit into matrix.config alongside protobuf-version so each row pins its own self-consistent snapshot. - Pass --vcpkg-baseline explicitly to make.py: lukka/run-vcpkg checks out vcpkg shallowly, so the runtime resolver can't pickaxe locally. - Render vcpkg.json baseline-only (no 'overrides') with the same SHA we hand to lukka/run-vcpkg, keeping cache key and final build aligned. --- .github/workflows/testing-cpp.yml | 35 ++++- make.py | 206 +++++++++++++++++++++++++----- 2 files changed, 205 insertions(+), 36 deletions(-) diff --git a/.github/workflows/testing-cpp.yml b/.github/workflows/testing-cpp.yml index 1c5be43..8de8d14 100644 --- a/.github/workflows/testing-cpp.yml +++ b/.github/workflows/testing-cpp.yml @@ -19,10 +19,28 @@ jobs: matrix: os: [ubuntu-latest, windows-latest] config: + # vcpkg-commit pins the `builtin-baseline` for each protobuf + # version. We use the baseline (not `overrides`) so that abseil / + # utf8-range / re2 are pulled from the same self-consistent vcpkg + # snapshot as protobuf — `overrides` alone would let those + # transitive deps drift forward and break ABI. + # + # CI passes the SHA explicitly via --vcpkg-baseline so the run is + # fully reproducible and independent of vcpkg's git history (which + # `lukka/run-vcpkg` checks out shallowly). For ad-hoc local runs + # `make.py` can auto-resolve the SHA from $VCPKG_ROOT instead — see + # `_resolve_vcpkg_baseline_for_protobuf`. + # + # To pin a new protobuf version, run on a full vcpkg checkout: + # git -C $VCPKG_ROOT log -S '"X.Y.Z"' -- versions/baseline.json + # then pick the newest commit whose baseline.json has + # default.protobuf.baseline == "X.Y.Z". - label: modern protobuf-version: "6.33.4" + vcpkg-commit: "1f6bbba3da511773189e5075b6781222402d10fa" - label: legacy-v3 protobuf-version: "3.21.12" + vcpkg-commit: "6245ce44a03f04d19be125ab1bbab578d0933e85" include: - os: ubuntu-latest triplet: x64-linux @@ -65,6 +83,10 @@ jobs: python-version: "3.12" - name: Render vcpkg.json + # `lukka/run-vcpkg` runs `vcpkg install` before `make.py test`, so + # the manifest must exist beforehand. We render the same shape + # `make.py` would (baseline-only, no `overrides`) keyed on the same + # SHA, so the cache key and the eventual build see one manifest. working-directory: test/cpp-tableau-loader shell: bash run: | @@ -73,10 +95,7 @@ jobs: "name": "loader-cpp-test", "version": "0.1.0", "dependencies": ["protobuf"], - "overrides": [ - { "name": "protobuf", "version": "${{ matrix.config.protobuf-version }}" } - ], - "builtin-baseline": "${{ env.VCPKG_COMMIT }}" + "builtin-baseline": "${{ matrix.config.vcpkg-commit }}" } EOF @@ -85,12 +104,12 @@ jobs: uses: actions/cache@v4 with: path: ${{ env.VCPKG_INSTALLED_DIR }} - key: vcpkg-installed-${{ runner.os }}-${{ matrix.triplet }}-${{ env.VCPKG_COMMIT }}-${{ hashFiles('test/cpp-tableau-loader/vcpkg.json') }} + key: vcpkg-installed-${{ runner.os }}-${{ matrix.triplet }}-${{ matrix.config.vcpkg-commit }}-${{ hashFiles('test/cpp-tableau-loader/vcpkg.json') }} - name: Setup vcpkg & install protobuf uses: lukka/run-vcpkg@v11 with: - vcpkgGitCommitId: ${{ env.VCPKG_COMMIT }} + vcpkgGitCommitId: ${{ matrix.config.vcpkg-commit }} vcpkgJsonGlob: "test/cpp-tableau-loader/vcpkg.json" runVcpkgInstall: true @@ -111,9 +130,13 @@ jobs: github_token: ${{ secrets.GITHUB_TOKEN }} - name: Test + # Pass --vcpkg-baseline explicitly so make.py reuses the SHA we + # already rendered into vcpkg.json above, instead of auto-resolving + # from the vcpkg checkout (which is shallow on the CI runner). run: > python make.py test --lang cpp --protobuf-version ${{ matrix.config.protobuf-version }} + --vcpkg-baseline ${{ matrix.config.vcpkg-commit }} --triplet ${{ matrix.triplet }} --no-clean --no-vcpkg-install diff --git a/make.py b/make.py index 1fdb0ef..f933b5d 100644 --- a/make.py +++ b/make.py @@ -459,6 +459,136 @@ def save_loader_env(data: dict, runner: Runner) -> None: LOADER_ENV_PATH.write_text(text, encoding="utf-8") +# --------------------------------------------------------------------------- +# vcpkg baseline lookup +# --------------------------------------------------------------------------- +# +# Pinning protobuf via `overrides` while leaving `builtin-baseline` on a newer +# vcpkg commit lets transitive deps (abseil/utf8-range/re2/...) drift forward +# and breaks ABI (e.g. missing `absl::if_constexpr`). Instead we pin the +# baseline itself to the vcpkg commit whose `versions/baseline.json` had our +# target protobuf — that whole snapshot is by construction self-consistent. +# +# Resolution: `git log -S '"X.Y.Z"' -- versions/baseline.json` → for each hit, +# verify `default.protobuf.baseline == X.Y.Z` at that commit (guards against +# the literal appearing in unrelated ports). Results cached in +# ~/.loader-env.json under "vcpkg_baseline_for_protobuf". + + +def _resolve_vcpkg_baseline_for_protobuf( + protobuf_version: str, + vcpkg_root: Path, + runner: Runner, +) -> str: + """Find the vcpkg commit whose baseline.json has protobuf == protobuf_version. + + Raises RuntimeError if no such commit can be found in the local vcpkg + checkout — caller should surface a clear error and suggest `git fetch` + or a different --protobuf-version. + """ + # Cache hit? + cache = load_loader_env() + cached_map = cache.get("vcpkg_baseline_for_protobuf", {}) or {} + cached = cached_map.get(protobuf_version) + if cached: + # Validate the cached commit still has the expected protobuf — the + # user might have re-cloned vcpkg or rewritten history. + if _vcpkg_baseline_has_protobuf(vcpkg_root, cached, protobuf_version): + return cached + # Cache is stale; fall through to re-resolve. + cached_map.pop(protobuf_version, None) + + if runner.dry_run: + # Dry-run: don't shell out to git, return a placeholder so snapshot + # tests can still verify the command sequence. + return f"" + + if not (vcpkg_root / ".git").exists(): + raise RuntimeError( + f"{vcpkg_root} is not a git checkout; cannot resolve a vcpkg " + f"baseline commit for protobuf {protobuf_version}." + ) + + # Step 1: candidate commits via pickaxe search on the literal version string. + try: + out = subprocess.check_output( + [ + "git", + "-C", + str(vcpkg_root), + "log", + "-S", + f'"{protobuf_version}"', + "--reverse", + "--format=%H", + "--", + "versions/baseline.json", + ], + text=True, + ) + except subprocess.CalledProcessError as e: + raise RuntimeError( + f"git log failed while searching vcpkg history for protobuf " + f"{protobuf_version}: {e}" + ) from e + + candidates = [line.strip() for line in out.splitlines() if line.strip()] + if not candidates: + raise RuntimeError( + f"No commit in {vcpkg_root} touches versions/baseline.json with " + f'the literal "{protobuf_version}". Possible causes:\n' + f" - your local vcpkg checkout is shallow / out-of-date " + f"(`git -C {vcpkg_root} fetch --unshallow origin master` may help)\n" + f" - protobuf {protobuf_version} was never the vcpkg baseline " + f"(use --vcpkg-baseline= to point at a custom snapshot)" + ) + + # Step 2: validate each candidate by reading baseline.json at that commit. + for sha in candidates: + if _vcpkg_baseline_has_protobuf(vcpkg_root, sha, protobuf_version): + cached_map[protobuf_version] = sha + cache["vcpkg_baseline_for_protobuf"] = cached_map + save_loader_env(cache, runner) + print( + f"[info] resolved vcpkg baseline for protobuf " + f"{protobuf_version} -> {sha[:12]}", + file=sys.stderr, + ) + return sha + + raise RuntimeError( + f'Found {len(candidates)} commits mentioning "{protobuf_version}" in ' + f"versions/baseline.json but none had default.protobuf.baseline set " + f"to that exact version. Pass --vcpkg-baseline= manually." + ) + + +def _vcpkg_baseline_has_protobuf( + vcpkg_root: Path, commit_sha: str, expected_version: str +) -> bool: + """Read versions/baseline.json at commit_sha; return True iff protobuf matches.""" + try: + blob = subprocess.check_output( + [ + "git", + "-C", + str(vcpkg_root), + "show", + f"{commit_sha}:versions/baseline.json", + ], + text=True, + stderr=subprocess.DEVNULL, + ) + except subprocess.CalledProcessError: + return False + try: + data = json.loads(blob) + except ValueError: + return False + entry = data.get("default", {}).get("protobuf", {}) + return entry.get("baseline") == expected_version + + def hydrate_platform_from_env(plat: Platform) -> None: """Populate Platform from $VCPKG_ROOT and ~/.loader-env.json. @@ -1063,44 +1193,19 @@ def _cpp_build_or_test(args, ctx: "Context", run_tests: bool) -> int: else: manifest_path.unlink() - # Classic mode: a stale vcpkg.json from a previous --protobuf-version run - # would silently switch cmake's vcpkg toolchain into manifest mode and - # build the wrong libprotobuf into build/vcpkg_installed/. Always remove - # it here unless we're about to render a fresh one below. - if not protobuf_version: - manifest_path = cwd / "vcpkg.json" - if manifest_path.is_file(): - if ctx.runner.dry_run: - print(f"[dry-run] rm {manifest_path}") - else: - manifest_path.unlink() - # Manifest mode: render vcpkg.json pinning the requested protobuf-version, # then run `vcpkg install` to populate vcpkg_installed/. This matches CI's # testing-cpp.yml flow (which uses lukka/run-vcpkg with runVcpkgInstall: # true) and means switching --protobuf-version Just Works without # re-running `make.py setup`. Idempotent: vcpkg detects already-installed # packages and skips them. + # + # Baseline resolution order (no `overrides` — see module comment above): + # 1. --vcpkg-baseline= (explicit override) + # 2. _resolve_vcpkg_baseline_for_protobuf(...) (auto: git-search vcpkg) cmake_extra: list[str] = [] if protobuf_version: - baseline = ctx.versions.vcpkg_baseline_commit or "" - manifest = { - "name": "loader-cpp-test", - "version": "0.1.0", - "dependencies": ["protobuf"], - "overrides": [{"name": "protobuf", "version": protobuf_version}], - "builtin-baseline": baseline, - } - manifest_path = cwd / "vcpkg.json" - if not ctx.runner.dry_run: - manifest_path.write_text(json.dumps(manifest, indent=2), encoding="utf-8") - installed_dir = Path( - os.environ.get("VCPKG_INSTALLED_DIR", str(cwd / "vcpkg_installed")) - ) - - # Locate vcpkg.exe / vcpkg. On Windows we hydrated VCPKG_ROOT from - # ~/.loader-env.json; on CI it's set by lukka/run-vcpkg; on Linux - # it's the system or devcontainer vcpkg. + # Need a vcpkg checkout to resolve the baseline by git history. vcpkg_root = ctx.platform.vcpkg_root or _env_path("VCPKG_ROOT") if vcpkg_root is None: if ctx.runner.dry_run: @@ -1118,6 +1223,38 @@ def _cpp_build_or_test(args, ctx: "Context", run_tests: bool) -> int: # Surface the resolved root on the platform so cmake_toolchain_args # picks it up for the configure command. ctx.platform.vcpkg_root = vcpkg_root + + explicit_baseline = getattr(args, "vcpkg_baseline", None) + if explicit_baseline: + baseline = explicit_baseline + print(f"[info] using --vcpkg-baseline={baseline[:12]}", file=sys.stderr) + else: + try: + baseline = _resolve_vcpkg_baseline_for_protobuf( + protobuf_version, vcpkg_root, ctx.runner + ) + except RuntimeError as e: + print(f"[error] {e}", file=sys.stderr) + print( + "[hint] If you know a vcpkg commit whose baseline matches " + "your protobuf, pass --vcpkg-baseline= to skip auto-resolution.", + file=sys.stderr, + ) + return 1 + + manifest = { + "name": "loader-cpp-test", + "version": "0.1.0", + "dependencies": ["protobuf"], + "builtin-baseline": baseline, + } + manifest_path = cwd / "vcpkg.json" + if not ctx.runner.dry_run: + manifest_path.write_text(json.dumps(manifest, indent=2), encoding="utf-8") + installed_dir = Path( + os.environ.get("VCPKG_INSTALLED_DIR", str(cwd / "vcpkg_installed")) + ) + vcpkg_exe = vcpkg_root / ("vcpkg.exe" if ctx.platform.is_windows else "vcpkg") # Install the manifest: must `cd` into the manifest dir for vcpkg to @@ -1380,6 +1517,15 @@ def _add_build_flags(sp: argparse.ArgumentParser) -> None: default=None, help="(cpp) pin vcpkg protobuf port to this version (manifest mode)", ) + sp.add_argument( + "--vcpkg-baseline", + type=str, + default=None, + help=( + "(cpp manifest mode) explicit vcpkg builtin-baseline commit; " + "skips auto-resolution from --protobuf-version" + ), + ) sp.add_argument( "--triplet", type=str, default=None, help="(cpp) vcpkg triplet override" ) From aeb6ae13b9aaaa8a2ed79699cdb1f399b4cbbacf Mon Sep 17 00:00:00 2001 From: Kybxd <627940450@qq.com> Date: Fri, 5 Jun 2026 19:11:39 +0800 Subject: [PATCH 46/66] fix(ci): disable x-gha binary cache for legacy vcpkg snapshot MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit lukka/run-vcpkg@v11 auto-injects VCPKG_BINARY_SOURCES=clear;x-gha,readwrite, but the 'x-gha' provider only exists in vcpkg ≳ 2023. The legacy-v3 matrix row pins a 2022-era vcpkg snapshot (6245ce44..., for protobuf 3.21.12) whose vcpkg binary errors out with: unknown binary provider type: ... on expression: clear;x-gha,readwrite lukka/run-vcpkg honours a pre-set VCPKG_BINARY_SOURCES, so set it to 'clear' on the step. We don't actually lose caching: actions/cache@v4 above already keys vcpkg_installed/ on (os, triplet, vcpkg-commit, vcpkg.json hash) and restores it before lukka/run-vcpkg runs. --- .github/workflows/testing-cpp.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.github/workflows/testing-cpp.yml b/.github/workflows/testing-cpp.yml index 8de8d14..eb4abdf 100644 --- a/.github/workflows/testing-cpp.yml +++ b/.github/workflows/testing-cpp.yml @@ -107,6 +107,14 @@ jobs: key: vcpkg-installed-${{ runner.os }}-${{ matrix.triplet }}-${{ matrix.config.vcpkg-commit }}-${{ hashFiles('test/cpp-tableau-loader/vcpkg.json') }} - name: Setup vcpkg & install protobuf + # Disable lukka/run-vcpkg's auto-injected `x-gha` binary cache: + # the `x-gha` provider only exists in vcpkg ≳ 2023, while our + # legacy matrix row pins a 2022-era vcpkg snapshot that errors + # out with "unknown binary provider type". `actions/cache` above + # already caches vcpkg_installed/, so we lose nothing here. + # `lukka/run-vcpkg` honours a pre-set VCPKG_BINARY_SOURCES. + env: + VCPKG_BINARY_SOURCES: clear uses: lukka/run-vcpkg@v11 with: vcpkgGitCommitId: ${{ matrix.config.vcpkg-commit }} From 23ad629c0e396b5b0d8bdd21dffd5f9cd2a2f41f Mon Sep 17 00:00:00 2001 From: Kybxd <627940450@qq.com> Date: Fri, 5 Jun 2026 19:20:03 +0800 Subject: [PATCH 47/66] fix(ci): exclude legacy-v3 on Windows (dead MSYS2 distfiles) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The 2023-01-15 vcpkg snapshot pinned by legacy-v3 still references MSYS2 packages whose URLs and SHA512s have since been pruned from repo.msys2.org (MSYS2 doesn't keep older versions). protobuf:x64-windows pulls them in via vcpkg_acquire_msys → vcpkg_fixup_pkgconfig, so the install fails: Failed to download file with error: 1 ... vcpkg-scripts version: 6245ce44a0 2023-01-15 (3 years, 5 months ago) Linux is unaffected because it uses the system pkg-config. The modern matrix row still exercises Windows on a current vcpkg snapshot, so we keep legacy-v3 as a Linux-only smoke test for the protobuf-3 ABI. --- .github/workflows/testing-cpp.yml | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/.github/workflows/testing-cpp.yml b/.github/workflows/testing-cpp.yml index eb4abdf..1b7f051 100644 --- a/.github/workflows/testing-cpp.yml +++ b/.github/workflows/testing-cpp.yml @@ -46,6 +46,19 @@ jobs: triplet: x64-linux - os: windows-latest triplet: x64-windows-static + exclude: + # The 2023-01-15 vcpkg snapshot pinned by legacy-v3 references + # MSYS2 distfiles that have since been pruned from repo.msys2.org; + # the protobuf:x64-windows port pulls them in via vcpkg_acquire_msys + # → vcpkg_fixup_pkgconfig and fails the download. Linux uses the + # system pkg-config so it isn't affected. Keep legacy-v3 as a + # Linux-only smoke test for the protobuf-3 ABI; modern still + # exercises the Windows build path. + - os: windows-latest + config: + label: legacy-v3 + protobuf-version: "3.21.12" + vcpkg-commit: "6245ce44a03f04d19be125ab1bbab578d0933e85" name: test (${{ matrix.os }}, ${{ matrix.config.label }}) runs-on: ${{ matrix.os }} From 0ee7c3b747059034f4b012910271cdcd792f8c24 Mon Sep 17 00:00:00 2001 From: wenchy Date: Fri, 5 Jun 2026 14:58:03 +0800 Subject: [PATCH 48/66] docs(spec): remove TS from make.py + fix /.dockerenv heuristic Captures the design for two cleanups surfaced while testing make.py setup --lang all on a fresh ubuntu:24.04: drop the TS language axis (CLAUDE.md already calls it experimental, no CI references it) and tighten the devcontainer detection to /opt/vcpkg/active only. Co-Authored-By: Claude Opus 4.7 (1M context) --- ...6-05-remove-ts-and-fix-dockerenv-design.md | 103 ++++++++++++++++++ 1 file changed, 103 insertions(+) create mode 100644 docs/superpowers/specs/2026-06-05-remove-ts-and-fix-dockerenv-design.md diff --git a/docs/superpowers/specs/2026-06-05-remove-ts-and-fix-dockerenv-design.md b/docs/superpowers/specs/2026-06-05-remove-ts-and-fix-dockerenv-design.md new file mode 100644 index 0000000..50fec58 --- /dev/null +++ b/docs/superpowers/specs/2026-06-05-remove-ts-and-fix-dockerenv-design.md @@ -0,0 +1,103 @@ +# Remove TypeScript support from `make.py` and fix `/.dockerenv` heuristic + +**Date:** 2026-06-05 +**Status:** Approved + +## Background + +Two real bugs surfaced while testing `python make.py setup --lang all` on a fresh `ubuntu:24.04` container: + +1. **`/.dockerenv` triggers a silent devcontainer short-circuit.** `Platform.detect()` (`make.py` lines 146-148) treats *any* container as the devcontainer: + + ```python + in_devcontainer = ( + Path("/opt/vcpkg/active").exists() or Path("/.dockerenv").exists() + ) + ``` + + `/.dockerenv` is created by Docker for **every** container, so `make.py setup` is a no-op in any non-devcontainer Docker image. The intended marker is `/opt/vcpkg/active`, which the devcontainer Dockerfile actually sets. + +2. **`_ensure_node_linux` is a no-op.** `make.py` lines 710-713 only print + `[info] Node not found; install via your distro or NodeSource manually.` + and return. Setup for `--lang ts` therefore silently leaves the host without Node, contradicting CLAUDE.md's claim that Node is installed via "Microsoft+NodeSource (Linux)". + +Two further bugs were found inside the TS lab itself (`_lab/ts/package.json` is missing a `build` script before its `node build/index.js` test command, and `_lab/ts/src/index.ts` references `../../test/testdata/ThemeConf.json` instead of `../../test/testdata/conf/ThemeConf.json`). CLAUDE.md already documents the TS pipeline as "experimental, not in CI", and grep confirms **no CI workflow references TS**. Rather than maintain a broken experimental path, this work removes TS from `make.py`'s tool surface entirely. The `_lab/ts/` scratchpad stays in the repo for anyone who wants to experiment manually; the lab's two bugs are intentionally left unfixed. + +## Goals + +- Drop TypeScript from the `make.py` tool surface (`setup` / `build` / `test` / `clean` / `env`). +- Fix the `/.dockerenv` heuristic so `make.py setup` actually runs in arbitrary Docker containers. +- Keep `_lab/ts/` in the tree as an untooled scratchpad. +- Keep the C++/Go/.NET workflows green; `python -m pytest test_make.py -v` must pass. + +## Non-goals + +- Fixing the two `_lab/ts/` bugs (missing `build` script, stale data path) — out of scope, the lab is no longer tooled. +- Touching plugin source under `cmd/protoc-gen-{go,cpp,csharp}-tableau-loader/`. +- CI workflow changes — confirmed no workflow references TS. + +## Architecture + +`make.py` keeps its current shape. Three platform installers (`_setup_linux` / `_setup_macos` / `_setup_windows`), per-language helpers (`_lang_dir`, `_build_or_test`, `cmd_clean`, `cmd_env`), and `LANGS_ALL` all stay. The only structural change is shrinking the language axis from 4 → 3 by removing `"ts"`. + +## Changes by file + +### `make.py` + +1. **Module docstring (lines 1-15):** scrub the `npm test` mention on line 6, and remove `|ts` from each of the `--lang go|cpp|csharp|ts[|all]` Usage examples (lines 10-13). +2. **`Versions` (lines 105-106):** delete the `node_version` property. +3. **`Platform.detect` (lines 146-148):** the `/.dockerenv` clause is dropped; only `Path("/opt/vcpkg/active").exists()` remains as the devcontainer signal. +4. **`LANGS_ALL` (line 491):** drop `"ts"`. Resulting tuple: `("go", "cpp", "csharp")`. +5. **`_setup_macos` (lines 547-549):** delete the `if "ts" in langs:` branch that appends `node@N` to the brew package list. +6. **`_setup_linux` (lines 615-616):** delete the `if "ts" in langs: _ensure_node_linux(ctx)` call. +7. **`_ensure_node_linux` (lines 710-713):** delete the function body entirely. +8. **`_setup_windows` (lines 824-827):** delete the `if "ts" in langs and _which("node") is None:` winget block. +9. **`_lang_dir` (lines 932-933):** delete the `if lang == "ts": return repo_root / "_lab" / "ts"` branch. +10. **`generate` dispatch (lines 941-944):** delete the `if lang == "ts":` arm that runs `npm run generate`. +11. **`_build_or_test` (lines 982-983):** delete the `if lang == "ts":` dispatch to `_ts_build_or_test`. +12. **`_ts_build_or_test` (lines 1217-1227):** delete the function entirely. +13. **`cmd_clean` (lines 1251-1252):** delete the `elif lang == "ts": rmtree(node_modules)` branch. +14. **`cmd_env` (lines 1281-1282):** delete the `"node"` and `"npm"` entries from the `tools` dict. + +### `test_make.py` + +- Remove `"NODE_VERSION"` from the two parser-test key lists (lines 49, 479). +- Remove the `assert v.node_version == v.raw["NODE_VERSION"]` line (line 62). +- Delete `test_ts_lives_under_lab` (lines 429-431). +- Audit dry-run snapshot tests for any `--lang ts` cases and remove them. (Inventory step during implementation: grep for `"ts"` and `_lang_dir.*ts` across `test_make.py`.) + +### `.devcontainer/versions.env` + +- Delete the `# Node.js LTS major. NodeSource apt repo is `setup_${NODE_VERSION}.x`.` comment block and the `NODE_VERSION=20` line. + +### `.devcontainer/Dockerfile` + +- Remove the NodeSource setup curl + `nodejs` apt install (lines around 196-199). +- Trim comment headers (lines 5, 9, 185, 186) that mention Node so they accurately describe what the image installs. + +### `CLAUDE.md` + +- Line 19: rewrite to drop `(TS lives under _lab/ts/)`. +- Lines 80-81: delete the `# TypeScript (experimental, not in CI)` block. + +## What we explicitly do NOT touch + +- `_lab/ts/` directory contents (per the brainstorming decision — broken scratchpad, documented as experimental). +- CI workflows (`grep -lriE 'typescript|_lab/ts|--lang ts|"ts"|\bts\b' .github/workflows/` returns empty — no references to TS in any workflow). +- The `_lab/ts/package.json` build-script bug (finding 3) and the stale `ThemeConf.json` path (finding 4). + +## Testing strategy + +1. **Unit + dry-run regression:** `python -m pytest test_make.py -v` — must be green. +2. **Argparse rejection:** `python make.py setup --lang ts` must exit non-zero with `argparse: error: argument --lang: invalid choice: 'ts' (choose from ...)`. +3. **End-to-end on fresh ubuntu:24.04:** repeat the original Docker test from the previous session. Pass criteria: + - `/.dockerenv` does **not** short-circuit setup (the workaround `rm /.dockerenv` should no longer be needed). + - `python make.py setup --lang all` exits 0 in a fresh container. + - `~/.loader-env.json` is populated. + - `python make.py test --lang go`, `--lang cpp`, `--lang csharp` all pass. + +## Risks + +- **Hidden TS references** — possible we miss a TS reference in `make.py` or `test_make.py`. Mitigation: final `grep -nE '"ts"|_ts_|node|npm|NODE_VERSION|_lab/ts|_ensure_node|TypeScript' make.py test_make.py` after the edits, expecting empty (or at most comments unrelated to the lab). +- **Devcontainer image rebuild** — removing Node from the Dockerfile invalidates cached layers, causing a one-time slow rebuild for anyone using the devcontainer. Acceptable; documented in CLAUDE.md. +- **Loose end with `_lab/ts/`** — directory remains, but no tooling path drives it. Anyone who tries `cd _lab/ts && npm install && npm test` will hit findings 3 and 4. Acceptable per user decision; CLAUDE.md is updated to no longer claim make.py supports it. From 6dbf469ad687c2bdefa4679722ea4a639858f0c6 Mon Sep 17 00:00:00 2001 From: wenchy Date: Fri, 5 Jun 2026 15:57:08 +0800 Subject: [PATCH 49/66] fix(make.py): tighten devcontainer detection to /opt/vcpkg/active /.dockerenv is created by Docker for every container, so the previous `exists() or exists()` heuristic silently no-op'd `make.py setup` in any plain Docker image (e.g. ubuntu:24.04). Use only the marker the devcontainer's Dockerfile actually sets. Adds two TestPlatform cases that monkeypatch Path.exists to lock in both the negative (/.dockerenv alone -> False) and positive (/opt/vcpkg/active -> True) signals. Co-Authored-By: Claude Opus 4.7 (1M context) --- make.py | 8 +++++--- test_make.py | 23 +++++++++++++++++++++++ 2 files changed, 28 insertions(+), 3 deletions(-) diff --git a/make.py b/make.py index f933b5d..aaf97cd 100644 --- a/make.py +++ b/make.py @@ -143,9 +143,11 @@ class Platform: def detect(cls) -> "Platform": sys_platform = sys.platform machine = _stdlib_platform.machine().lower() - in_devcontainer = ( - Path("/opt/vcpkg/active").exists() or Path("/.dockerenv").exists() - ) + # Devcontainer signal: only the marker the .devcontainer/Dockerfile + # actually sets. /.dockerenv is created by Docker for EVERY container + # and is too broad — using it would silently no-op `setup` in any + # plain Docker container. + in_devcontainer = Path("/opt/vcpkg/active").exists() return cls( sys_platform=sys_platform, machine=machine, diff --git a/test_make.py b/test_make.py index 2993c05..c1ba944 100644 --- a/test_make.py +++ b/test_make.py @@ -212,6 +212,29 @@ def test_cmake_toolchain_args_windows_no_vcpkg_root_returns_empty( assert p.cmake_toolchain_args() == [] + def test_detect_ignores_dockerenv_marker(self, monkeypatch): + """Bug fix: /.dockerenv is created by Docker for *every* container, + not just the loader devcontainer. The detect() heuristic must NOT + treat its presence as a devcontainer signal.""" + # Simulate: only /.dockerenv exists; /opt/vcpkg/active does NOT. + def fake_exists(self): + return str(self) == "/.dockerenv" + monkeypatch.setattr(make.Path, "exists", fake_exists) + p = make.Platform.detect() + assert p.in_devcontainer is False, ( + "/.dockerenv alone must NOT be treated as devcontainer" + ) + + def test_detect_recognizes_vcpkg_active_marker(self, monkeypatch): + """Positive: /opt/vcpkg/active is the marker the devcontainer's + Dockerfile actually sets. Its presence is the sole devcontainer signal.""" + def fake_exists(self): + return str(self) == "/opt/vcpkg/active" + monkeypatch.setattr(make.Path, "exists", fake_exists) + p = make.Platform.detect() + assert p.in_devcontainer is True + + # --------------------------------------------------------------------------- # Unit: Platform.windows_msvc_wrap # --------------------------------------------------------------------------- From c8bfe19f36ed71e1592974d6013902d1c4a6c58c Mon Sep 17 00:00:00 2001 From: wenchy Date: Fri, 5 Jun 2026 16:42:08 +0800 Subject: [PATCH 50/66] refactor(make.py): drop TypeScript support from the tool surface CLAUDE.md already documents TS as experimental and not in CI; no workflow references it. Rather than maintain a broken pipeline (_ensure_node_linux was a no-op on Linux, _lab/ts/package.json has no build script), shrink the language axis to (go, cpp, csharp). Removes: - LANGS_ALL "ts" entry (argparse choices update transitively) - Versions.node_version property - _ensure_node_linux (silent no-op) - per-platform Node/npm install branches in _setup_{linux,macos,windows} - _ts_build_or_test and its _build_or_test dispatch - _lang_dir, generate, cmd_clean, cmd_env "ts"/node/npm arms - module docstring `npm test` mention and --lang ts examples The _lab/ts/ scratchpad stays in the tree for manual experimentation but is no longer driven by make.py. test_make.py loses NODE_VERSION/node_version assertions and the test_ts_lives_under_lab case. Co-Authored-By: Claude Opus 4.7 (1M context) --- make.py | 65 +++++++--------------------------------------------- test_make.py | 9 +------- 2 files changed, 9 insertions(+), 65 deletions(-) diff --git a/make.py b/make.py index aaf97cd..c518198 100644 --- a/make.py +++ b/make.py @@ -3,14 +3,14 @@ make.py — single cross-platform entrypoint for the tableauio/loader repo. Consolidates per-language `buf generate` / `cmake` / `go test` / `dotnet test` -/ `npm test` recipes into one Python tool that works identically on +recipes into one Python tool that works identically on native Windows, macOS, Linux, and inside the devcontainer. Usage (high level): - python make.py setup [--lang go|cpp|csharp|ts|all] [--dry-run] - python make.py generate --lang go|cpp|csharp|ts - python make.py build --lang go|cpp|csharp|ts [build flags] - python make.py test --lang go|cpp|csharp|ts [build flags] [-k FILTER] [--smoke] + python make.py setup [--lang go|cpp|csharp|all] [--dry-run] + python make.py generate --lang go|cpp|csharp + python make.py build --lang go|cpp|csharp [build flags] + python make.py test --lang go|cpp|csharp [build flags] [-k FILTER] [--smoke] python make.py clean [--lang ...] [--all] python make.py env python make.py --version @@ -101,10 +101,6 @@ def vcpkg_baseline_commit(self) -> Optional[str]: def dotnet_version(self) -> Optional[str]: return self.raw.get("DOTNET_VERSION") - @property - def node_version(self) -> Optional[str]: - return self.raw.get("NODE_VERSION") - @property def cmake_version(self) -> Optional[str]: return self.raw.get("CMAKE_VERSION") @@ -620,7 +616,7 @@ def hydrate_platform_from_env(plat: Platform) -> None: # --------------------------------------------------------------------------- -LANGS_ALL = ("go", "cpp", "csharp", "ts") +LANGS_ALL = ("go", "cpp", "csharp") def _which(name: str) -> Optional[str]: @@ -667,8 +663,8 @@ def _setup_macos(langs: list[str], ctx: "Context") -> int: # Brew packages: only the version-tolerant pieces (cmake, ninja, build # essentials). Go and protobuf are pinned via tarball / vcpkg below; - # buf is pinned via direct download. dotnet@N and node@N are pinnable - # via brew's versioned formulae. + # buf is pinned via direct download. dotnet@N is pinnable via brew's + # versioned formulae. pkgs: list[str] = [] if "cpp" in langs: pkgs.extend(["cmake", "ninja"]) @@ -676,8 +672,6 @@ def _setup_macos(langs: list[str], ctx: "Context") -> int: # `dotnet@8` (cask) covers .NET 8.x. Use the major. major = (ctx.versions.dotnet_version or "8.0").split(".")[0] pkgs.append(f"dotnet@{major}") - if "ts" in langs: - pkgs.append(f"node@{ctx.versions.node_version or '20'}") if pkgs: ctx.runner.run(["brew", "update"], check=False) ctx.runner.run(["brew", "install", *pkgs], check=False) @@ -744,8 +738,6 @@ def _setup_linux(langs: list[str], ctx: "Context") -> int: if "csharp" in langs: _ensure_dotnet_linux(ctx) - if "ts" in langs: - _ensure_node_linux(ctx) # Pinned protobuf via vcpkg (matches devcontainer + Windows). if "cpp" in langs: @@ -839,12 +831,6 @@ def _ensure_go_tarball(ctx: "Context", os_label: str) -> None: print(f" export PATH={bin_dir}:$PATH") -def _ensure_node_linux(ctx: "Context") -> None: - if _which("node") is not None: - return - print("[info] Node not found; install via your distro or NodeSource manually.") - - def _setup_windows(langs: list[str], ctx: "Context", skip_vcpkg: bool) -> int: """Windows host setup.""" cache = load_loader_env() @@ -953,10 +939,6 @@ def _setup_windows(langs: list[str], ctx: "Context", skip_vcpkg: bool) -> int: ctx.runner.run( ["winget", "install", "--id", "Microsoft.DotNet.SDK.8", "-e"], check=False ) - if "ts" in langs and _which("node") is None: - ctx.runner.run( - ["winget", "install", "--id", "OpenJS.NodeJS.LTS", "-e"], check=False - ) save_loader_env(cache, ctx.runner) print("[info] Windows toolchain ready.") @@ -1061,8 +1043,6 @@ def _setup_vcpkg(ctx: "Context", cache: dict) -> None: def _lang_dir(repo_root: Path, lang: str) -> Path: - if lang == "ts": - return repo_root / "_lab" / "ts" return repo_root / "test" / f"{lang}-tableau-loader" @@ -1070,12 +1050,6 @@ def _buf_generate( ctx: "Context", lang: str, protoc_dir_override: Optional[Path] = None ) -> None: cwd = _lang_dir(ctx.repo_root, lang) - if lang == "ts": - # The TypeScript scratchpad has its own `npm run generate` script. - ctx.runner.run( - ["npm", "run", "generate"], cwd=cwd, shell=ctx.platform.is_windows - ) - return cmd = ctx.platform.windows_msvc_wrap(["buf", "generate", ".."]) env = os.environ.copy() # Pick the protoc to put on PATH: @@ -1111,8 +1085,6 @@ def _build_or_test(args, ctx: "Context", run_tests: bool) -> int: return _cpp_build_or_test(args, ctx, run_tests) if lang == "csharp": return _csharp_build_or_test(args, ctx, run_tests) - if lang == "ts": - return _ts_build_or_test(args, ctx, run_tests) print(f"[error] unknown --lang {lang}", file=sys.stderr) return 2 @@ -1350,22 +1322,6 @@ def _csharp_build_or_test(args, ctx: "Context", run_tests: bool) -> int: return 0 -# ----- TypeScript ----- - - -def _ts_build_or_test(args, ctx: "Context", run_tests: bool) -> int: - cwd = _lang_dir(ctx.repo_root, "ts") - if not (cwd / "node_modules").is_dir(): - ctx.runner.run(["npm", "install"], cwd=cwd, shell=ctx.platform.is_windows) - if not getattr(args, "no_generate", False): - ctx.runner.run( - ["npm", "run", "generate"], cwd=cwd, shell=ctx.platform.is_windows - ) - if run_tests: - ctx.runner.run(["npm", "run", "test"], cwd=cwd, shell=ctx.platform.is_windows) - return 0 - - # ----- clean / env ----- @@ -1387,9 +1343,6 @@ def cmd_clean(args, ctx: "Context") -> int: ctx.runner.rmtree(cwd / "protoconf") elif lang == "go": ctx.runner.rmtree(cwd / "protoconf") - elif lang == "ts": - ctx.runner.rmtree(cwd / "node_modules") - ctx.runner.rmtree(cwd / "dist") return 0 @@ -1417,8 +1370,6 @@ def cmd_env(args, ctx: "Context") -> int: "cmake": _which("cmake"), "ninja": _which("ninja"), "dotnet": _which("dotnet"), - "node": _which("node"), - "npm": _which("npm"), }, "versions_env": ctx.versions.raw, } diff --git a/test_make.py b/test_make.py index c1ba944..e3a971c 100644 --- a/test_make.py +++ b/test_make.py @@ -46,7 +46,6 @@ def test_loads_real_versions_env(self): "PROTOBUF_VERSION", "VCPKG_BASELINE_COMMIT", "DOTNET_VERSION", - "NODE_VERSION", "CMAKE_VERSION", ): assert key in v.raw, f"Expected {key} in versions.env" @@ -59,7 +58,6 @@ def test_typed_accessors(self): assert v.protobuf_version == v.raw["PROTOBUF_VERSION"] assert v.vcpkg_baseline_commit == v.raw["VCPKG_BASELINE_COMMIT"] assert v.dotnet_version == v.raw["DOTNET_VERSION"] - assert v.node_version == v.raw["NODE_VERSION"] assert v.cmake_version == v.raw["CMAKE_VERSION"] def test_protobuf_version_looks_like_semver(self): @@ -449,10 +447,6 @@ def test_csharp(self): == REPO_ROOT / "test" / "csharp-tableau-loader" ) - def test_ts_lives_under_lab(self): - # TS is special-cased to _lab/ts/ per CLAUDE.md. - assert make._lang_dir(REPO_ROOT, "ts") == REPO_ROOT / "_lab" / "ts" - # --------------------------------------------------------------------------- # Subprocess helpers @@ -499,7 +493,6 @@ def test_version_prints_versions_env(self): "PROTOBUF_VERSION", "VCPKG_BASELINE_COMMIT", "DOTNET_VERSION", - "NODE_VERSION", "CMAKE_VERSION", ): assert key in proc.stdout, f"--version missing {key}" @@ -791,7 +784,7 @@ def test_clean_all(self): proc = run_make("--dry-run", "clean", "--all") assert proc.returncode == 0 out = proc.stdout - # Should mention dirs from at least cpp, csharp, go, ts. + # Should mention dirs from at least cpp, csharp, go. assert "cpp-tableau-loader" in out assert "csharp-tableau-loader" in out assert "go-tableau-loader" in out From 25a2931f8f7b467f98e777c96a59f9d2e5edee24 Mon Sep 17 00:00:00 2001 From: wenchy Date: Fri, 5 Jun 2026 19:56:30 +0800 Subject: [PATCH 51/66] chore(versions): drop NODE_VERSION pin MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Follow-up to dropping TS support in make.py — NODE_VERSION is no longer consumed by any tooling. Co-Authored-By: Claude Opus 4.7 (1M context) --- .devcontainer/versions.env | 3 --- 1 file changed, 3 deletions(-) diff --git a/.devcontainer/versions.env b/.devcontainer/versions.env index b982ebd..835a2c6 100644 --- a/.devcontainer/versions.env +++ b/.devcontainer/versions.env @@ -47,9 +47,6 @@ VCPKG_BASELINE_COMMIT=56bb2411609227288b70117ead2c47585ba07713 # CI uses `${DOTNET_VERSION}.x` with actions/setup-dotnet. DOTNET_VERSION=8.0 -# Node.js LTS major. NodeSource apt repo is `setup_${NODE_VERSION}.x`. -NODE_VERSION=20 - # CMake version installed by `make.py setup` on Windows (the devcontainer # base image already ships a recent cmake; macOS/Linux use system cmake). CMAKE_VERSION=3.31.8 From a78ef172280afae228bbe4f07543d958de2d8a2d Mon Sep 17 00:00:00 2001 From: wenchy Date: Fri, 5 Jun 2026 20:01:40 +0800 Subject: [PATCH 52/66] chore(devcontainer): drop Node.js install make.py no longer drives any TS pipeline (see prior commit), so the devcontainer image no longer needs Node. Also trims comment headers that named Node. Triggers a one-time devcontainer rebuild for users with the cached image. Co-Authored-By: Claude Opus 4.7 (1M context) --- .devcontainer/Dockerfile | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index 7459f84..4cf3091 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -2,11 +2,11 @@ # tableauio/loader devcontainer # # Single-stage, multi-arch (amd64 + arm64) image bringing the full -# C++/Go/.NET/Node toolchain plus protobuf at the exact versions CI uses. +# C++/Go/.NET toolchain plus protobuf at the exact versions CI uses. # # All version pins are read from ./versions.env (the single source of # truth shared with make.py and the CI workflows). To bump Go / buf / -# protobuf / .NET / Node / vcpkg-baseline, edit that file — not this one. +# protobuf / .NET / vcpkg-baseline, edit that file — not this one. # # Build context is .devcontainer/ (the directory containing this file), # so `COPY versions.env ...` resolves directly. @@ -182,8 +182,8 @@ ln -s /opt/vcpkg/active/tools/protobuf/protoc /usr/local/bin/protoc EOF # --------------------------------------------------------------------------- -# .NET SDK + Node.js — apt-based installs from the official Microsoft and -# NodeSource repositories. Versions are read from /opt/versions.env. +# .NET SDK — apt-based install from the official Microsoft repository. +# Version is read from /opt/versions.env. # apt-get clean + rm /var/lib/apt/lists at the end keeps the layer small. # --------------------------------------------------------------------------- RUN < Date: Fri, 5 Jun 2026 20:14:10 +0800 Subject: [PATCH 53/66] chore(devcontainer): drop node from postcreate banner MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Follow-up to dropping Node from the Dockerfile — banner script called `node --version` which would fail at container startup. Co-Authored-By: Claude Opus 4.7 (1M context) --- .devcontainer/postcreate-banner.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.devcontainer/postcreate-banner.sh b/.devcontainer/postcreate-banner.sh index 16a6794..ec43649 100644 --- a/.devcontainer/postcreate-banner.sh +++ b/.devcontainer/postcreate-banner.sh @@ -1,7 +1,7 @@ #!/bin/sh # Post-create banner for the Linux devcontainer. # Pure echo — no installs, no version-pinning at runtime, no surprises. -# Five-line summary that prints when the container becomes ready, so the +# Four-line summary that prints when the container becomes ready, so the # developer can confirm at a glance which toolchain versions landed. set -e printf 'tableauio/loader devcontainer ready (linux).\n' @@ -9,4 +9,3 @@ printf ' go: %s\n' "$(go version | cut -d' ' -f3)" printf ' buf: %s\n' "$(buf --version 2>&1)" printf ' protoc: %s\n' "$(protoc --version)" printf ' dotnet: %s\n' "$(dotnet --version)" -printf ' node: %s\n' "$(node --version)" From d460045c49def50877f3f01ba9349e26a38c03fc Mon Sep 17 00:00:00 2001 From: wenchy Date: Fri, 5 Jun 2026 20:15:07 +0800 Subject: [PATCH 54/66] docs(CLAUDE.md): drop TS commands and _lab/ts pointer make.py no longer supports --lang ts; the TS scratchpad at _lab/ts remains in-tree but is unmaintained and not surfaced in the docs. Co-Authored-By: Claude Opus 4.7 (1M context) --- CLAUDE.md | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index c937f0e..e63d7c9 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -16,7 +16,7 @@ Generated code is opinionated: every worksheet message becomes a `Messager` with ## Common commands -Build/test happens **per language** under `test/-tableau-loader/` (TS lives under `_lab/ts/`). The repo root only hosts the Go module + plugin sources; `go test ./...` from root only exercises shared packages (`internal/index`, `internal/loadutil`, `pkg/treemap`, `pkg/udiff`). +Build/test happens **per language** under `test/-tableau-loader/`. The repo root only hosts the Go module + plugin sources; `go test ./...` from root only exercises shared packages (`internal/index`, `internal/loadutil`, `pkg/treemap`, `pkg/udiff`). The single cross-platform driver is **`make.py`** (Python 3.10+, stdlib only). It works on Windows, macOS, Linux, and inside the devcontainer, and is what CI calls. @@ -76,9 +76,6 @@ python make.py test --lang cpp --protobuf-version 3.21.12 # legacy v3 # C# python make.py test --lang csharp # full python make.py test --lang csharp -k HubTest.Load # FullyQualifiedName~HubTest.Load - -# TypeScript (experimental, not in CI) -python make.py test --lang ts # npm install + generate + test ``` GoogleTest is fetched via CMake `FetchContent` — no manual install. From 69ae0cafe91b5535697eaf655d8b2dfd81602ff0 Mon Sep 17 00:00:00 2001 From: wenchy Date: Fri, 5 Jun 2026 20:35:36 +0800 Subject: [PATCH 55/66] chore: remove docs --- ...6-05-remove-ts-and-fix-dockerenv-design.md | 103 ------------------ 1 file changed, 103 deletions(-) delete mode 100644 docs/superpowers/specs/2026-06-05-remove-ts-and-fix-dockerenv-design.md diff --git a/docs/superpowers/specs/2026-06-05-remove-ts-and-fix-dockerenv-design.md b/docs/superpowers/specs/2026-06-05-remove-ts-and-fix-dockerenv-design.md deleted file mode 100644 index 50fec58..0000000 --- a/docs/superpowers/specs/2026-06-05-remove-ts-and-fix-dockerenv-design.md +++ /dev/null @@ -1,103 +0,0 @@ -# Remove TypeScript support from `make.py` and fix `/.dockerenv` heuristic - -**Date:** 2026-06-05 -**Status:** Approved - -## Background - -Two real bugs surfaced while testing `python make.py setup --lang all` on a fresh `ubuntu:24.04` container: - -1. **`/.dockerenv` triggers a silent devcontainer short-circuit.** `Platform.detect()` (`make.py` lines 146-148) treats *any* container as the devcontainer: - - ```python - in_devcontainer = ( - Path("/opt/vcpkg/active").exists() or Path("/.dockerenv").exists() - ) - ``` - - `/.dockerenv` is created by Docker for **every** container, so `make.py setup` is a no-op in any non-devcontainer Docker image. The intended marker is `/opt/vcpkg/active`, which the devcontainer Dockerfile actually sets. - -2. **`_ensure_node_linux` is a no-op.** `make.py` lines 710-713 only print - `[info] Node not found; install via your distro or NodeSource manually.` - and return. Setup for `--lang ts` therefore silently leaves the host without Node, contradicting CLAUDE.md's claim that Node is installed via "Microsoft+NodeSource (Linux)". - -Two further bugs were found inside the TS lab itself (`_lab/ts/package.json` is missing a `build` script before its `node build/index.js` test command, and `_lab/ts/src/index.ts` references `../../test/testdata/ThemeConf.json` instead of `../../test/testdata/conf/ThemeConf.json`). CLAUDE.md already documents the TS pipeline as "experimental, not in CI", and grep confirms **no CI workflow references TS**. Rather than maintain a broken experimental path, this work removes TS from `make.py`'s tool surface entirely. The `_lab/ts/` scratchpad stays in the repo for anyone who wants to experiment manually; the lab's two bugs are intentionally left unfixed. - -## Goals - -- Drop TypeScript from the `make.py` tool surface (`setup` / `build` / `test` / `clean` / `env`). -- Fix the `/.dockerenv` heuristic so `make.py setup` actually runs in arbitrary Docker containers. -- Keep `_lab/ts/` in the tree as an untooled scratchpad. -- Keep the C++/Go/.NET workflows green; `python -m pytest test_make.py -v` must pass. - -## Non-goals - -- Fixing the two `_lab/ts/` bugs (missing `build` script, stale data path) — out of scope, the lab is no longer tooled. -- Touching plugin source under `cmd/protoc-gen-{go,cpp,csharp}-tableau-loader/`. -- CI workflow changes — confirmed no workflow references TS. - -## Architecture - -`make.py` keeps its current shape. Three platform installers (`_setup_linux` / `_setup_macos` / `_setup_windows`), per-language helpers (`_lang_dir`, `_build_or_test`, `cmd_clean`, `cmd_env`), and `LANGS_ALL` all stay. The only structural change is shrinking the language axis from 4 → 3 by removing `"ts"`. - -## Changes by file - -### `make.py` - -1. **Module docstring (lines 1-15):** scrub the `npm test` mention on line 6, and remove `|ts` from each of the `--lang go|cpp|csharp|ts[|all]` Usage examples (lines 10-13). -2. **`Versions` (lines 105-106):** delete the `node_version` property. -3. **`Platform.detect` (lines 146-148):** the `/.dockerenv` clause is dropped; only `Path("/opt/vcpkg/active").exists()` remains as the devcontainer signal. -4. **`LANGS_ALL` (line 491):** drop `"ts"`. Resulting tuple: `("go", "cpp", "csharp")`. -5. **`_setup_macos` (lines 547-549):** delete the `if "ts" in langs:` branch that appends `node@N` to the brew package list. -6. **`_setup_linux` (lines 615-616):** delete the `if "ts" in langs: _ensure_node_linux(ctx)` call. -7. **`_ensure_node_linux` (lines 710-713):** delete the function body entirely. -8. **`_setup_windows` (lines 824-827):** delete the `if "ts" in langs and _which("node") is None:` winget block. -9. **`_lang_dir` (lines 932-933):** delete the `if lang == "ts": return repo_root / "_lab" / "ts"` branch. -10. **`generate` dispatch (lines 941-944):** delete the `if lang == "ts":` arm that runs `npm run generate`. -11. **`_build_or_test` (lines 982-983):** delete the `if lang == "ts":` dispatch to `_ts_build_or_test`. -12. **`_ts_build_or_test` (lines 1217-1227):** delete the function entirely. -13. **`cmd_clean` (lines 1251-1252):** delete the `elif lang == "ts": rmtree(node_modules)` branch. -14. **`cmd_env` (lines 1281-1282):** delete the `"node"` and `"npm"` entries from the `tools` dict. - -### `test_make.py` - -- Remove `"NODE_VERSION"` from the two parser-test key lists (lines 49, 479). -- Remove the `assert v.node_version == v.raw["NODE_VERSION"]` line (line 62). -- Delete `test_ts_lives_under_lab` (lines 429-431). -- Audit dry-run snapshot tests for any `--lang ts` cases and remove them. (Inventory step during implementation: grep for `"ts"` and `_lang_dir.*ts` across `test_make.py`.) - -### `.devcontainer/versions.env` - -- Delete the `# Node.js LTS major. NodeSource apt repo is `setup_${NODE_VERSION}.x`.` comment block and the `NODE_VERSION=20` line. - -### `.devcontainer/Dockerfile` - -- Remove the NodeSource setup curl + `nodejs` apt install (lines around 196-199). -- Trim comment headers (lines 5, 9, 185, 186) that mention Node so they accurately describe what the image installs. - -### `CLAUDE.md` - -- Line 19: rewrite to drop `(TS lives under _lab/ts/)`. -- Lines 80-81: delete the `# TypeScript (experimental, not in CI)` block. - -## What we explicitly do NOT touch - -- `_lab/ts/` directory contents (per the brainstorming decision — broken scratchpad, documented as experimental). -- CI workflows (`grep -lriE 'typescript|_lab/ts|--lang ts|"ts"|\bts\b' .github/workflows/` returns empty — no references to TS in any workflow). -- The `_lab/ts/package.json` build-script bug (finding 3) and the stale `ThemeConf.json` path (finding 4). - -## Testing strategy - -1. **Unit + dry-run regression:** `python -m pytest test_make.py -v` — must be green. -2. **Argparse rejection:** `python make.py setup --lang ts` must exit non-zero with `argparse: error: argument --lang: invalid choice: 'ts' (choose from ...)`. -3. **End-to-end on fresh ubuntu:24.04:** repeat the original Docker test from the previous session. Pass criteria: - - `/.dockerenv` does **not** short-circuit setup (the workaround `rm /.dockerenv` should no longer be needed). - - `python make.py setup --lang all` exits 0 in a fresh container. - - `~/.loader-env.json` is populated. - - `python make.py test --lang go`, `--lang cpp`, `--lang csharp` all pass. - -## Risks - -- **Hidden TS references** — possible we miss a TS reference in `make.py` or `test_make.py`. Mitigation: final `grep -nE '"ts"|_ts_|node|npm|NODE_VERSION|_lab/ts|_ensure_node|TypeScript' make.py test_make.py` after the edits, expecting empty (or at most comments unrelated to the lab). -- **Devcontainer image rebuild** — removing Node from the Dockerfile invalidates cached layers, causing a one-time slow rebuild for anyone using the devcontainer. Acceptable; documented in CLAUDE.md. -- **Loose end with `_lab/ts/`** — directory remains, but no tooling path drives it. Anyone who tries `cd _lab/ts && npm install && npm test` will hit findings 3 and 4. Acceptable per user decision; CLAUDE.md is updated to no longer claim make.py supports it. From a190a9158a47fc688418092ba58ebf2a1f30fd2d Mon Sep 17 00:00:00 2001 From: wenchy Date: Fri, 5 Jun 2026 20:43:43 +0800 Subject: [PATCH 56/66] fix(test_make): use as_posix() for cross-platform path comparison On Windows `Path("/opt/vcpkg/active")` renders as `\opt\vcpkg\active` via str(), so the mock never matched the forward-slash literal and `test_detect_recognizes_vcpkg_active_marker` failed on windows-latest. `Path.as_posix()` returns the forward-slash form on every platform. Also fixes `test_detect_ignores_dockerenv_marker` which was passing on Windows for the wrong reason (mock never returned True; assertion expected False, so it accidentally passed). Co-Authored-By: Claude Opus 4.7 (1M context) --- test_make.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/test_make.py b/test_make.py index e3a971c..a8fcfd4 100644 --- a/test_make.py +++ b/test_make.py @@ -215,8 +215,10 @@ def test_detect_ignores_dockerenv_marker(self, monkeypatch): not just the loader devcontainer. The detect() heuristic must NOT treat its presence as a devcontainer signal.""" # Simulate: only /.dockerenv exists; /opt/vcpkg/active does NOT. + # Use as_posix() so the literal compares correctly on Windows where + # str(WindowsPath('/.dockerenv')) renders with backslashes. def fake_exists(self): - return str(self) == "/.dockerenv" + return self.as_posix() == "/.dockerenv" monkeypatch.setattr(make.Path, "exists", fake_exists) p = make.Platform.detect() assert p.in_devcontainer is False, ( @@ -227,7 +229,7 @@ def test_detect_recognizes_vcpkg_active_marker(self, monkeypatch): """Positive: /opt/vcpkg/active is the marker the devcontainer's Dockerfile actually sets. Its presence is the sole devcontainer signal.""" def fake_exists(self): - return str(self) == "/opt/vcpkg/active" + return self.as_posix() == "/opt/vcpkg/active" monkeypatch.setattr(make.Path, "exists", fake_exists) p = make.Platform.detect() assert p.in_devcontainer is True From 7607fdf03dbccf6badd09d38d80914982975b769 Mon Sep 17 00:00:00 2001 From: wenchy Date: Fri, 5 Jun 2026 20:57:31 +0800 Subject: [PATCH 57/66] fix(devcontainer): apt-get update after adding Microsoft repo Regression from dropping the NodeSource curl: that script ran its own `apt-get update` as a side effect, which refreshed the cache after the preceding `dpkg -i packages-microsoft-prod.deb` added Microsoft's apt source. Removing it left no update between adding the repo and installing dotnet-sdk-${DOTNET_VERSION}, so apt couldn't find the package and the build failed with exit 100. Co-Authored-By: Claude Opus 4.7 (1M context) --- .devcontainer/Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index 4cf3091..f92f877 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -193,6 +193,7 @@ curl -fsSL https://packages.microsoft.com/config/ubuntu/24.04/packages-microsoft -o /tmp/ms.deb dpkg -i /tmp/ms.deb rm /tmp/ms.deb +apt-get update apt-get install -y --no-install-recommends \ "dotnet-sdk-${DOTNET_VERSION}" apt-get clean From b5b985e171a016208b347a3c988eb18b2892b492 Mon Sep 17 00:00:00 2001 From: wenchy Date: Fri, 5 Jun 2026 21:55:18 +0800 Subject: [PATCH 58/66] feat(versions): variant-prefixed protobuf+vcpkg matrix in versions.env MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit versions.env now declares one (protobuf, vcpkg-baseline) row per variant (MODERN_*, LEGACY_V3_*) plus a DEFAULT_VARIANT pointer, replacing the flat single-row PROTOBUF_VERSION + VCPKG_BASELINE_COMMIT keys. This: - eliminates the drift between versions.env's modern vcpkg SHA (was 56bb2411) and testing-cpp.yml's modern SHA (was 1f6bbba3) — they now both pin 56bb2411 (tip of vcpkg's 2026.04.27 quarterly) - brings the legacy-v3 pair (protobuf 3.21.12 / vcpkg 6245ce44) into versions.env so it's discoverable outside the CI YAML - adds a clean knob to switch the entire row at once via LOADER_DEFAULT_VARIANT host env -> DEFAULT_VARIANT build-arg -> Dockerfile case statement Consumers all learn the resolver: * make.py Versions class: new default_variant + variants() accessors; existing protobuf_version / vcpkg_baseline_commit properties transparently resolve through DEFAULT_VARIANT (with fallback to unprefixed keys for backward compat) * .devcontainer/Dockerfile: case statement after sourcing versions.env re-exports PROTOBUF_VERSION / VCPKG_BASELINE_COMMIT for the rest of the build script * .devcontainer/devcontainer.json: new LOADER_DEFAULT_VARIANT host env mapping (alongside the existing LOADER_PROTOBUF_VERSION surgical override) * .github/actions/load-versions: same case logic, exports the resolved unprefixed names to GITHUB_ENV so existing workflow references keep working * .github/workflows/testing-cpp.yml: modern row's vcpkg-commit synced to versions.env, comment now points readers at versions.env test_make.py: +4 new TestVersions cases covering default_variant fallback, variants() enumeration, dash-vs-underscore label parsing, and unprefixed-key backward compat. All 87 tests pass. Co-Authored-By: Claude Opus 4.7 (1M context) --- .devcontainer/Dockerfile | 32 +++++++++++++- .devcontainer/devcontainer.json | 12 ++++-- .devcontainer/versions.env | 54 +++++++++++++++--------- .github/actions/load-versions/action.yml | 24 ++++++++++- .github/workflows/testing-cpp.yml | 8 +++- CLAUDE.md | 4 +- make.py | 45 +++++++++++++++++++- test_make.py | 54 +++++++++++++++++++++--- 8 files changed, 195 insertions(+), 38 deletions(-) diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index f92f877..4c00b6e 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -84,6 +84,8 @@ EOF # post-install assertion catches the case where vcpkg's resolution still picks # a different port revision than requested. # --------------------------------------------------------------------------- +ARG DEFAULT_VARIANT= +ARG DEFAULT_VARIANT= ARG PROTOBUF_VERSION= ENV VCPKG_ROOT=/opt/vcpkg # Tell vcpkg to share its compiled binaries via /vcpkg-binarycache. The @@ -119,12 +121,38 @@ ENV VCPKG_DEFAULT_BINARY_CACHE=/vcpkg-binarycache # (matters only on multi-builder hosts; safe default). RUN --mount=type=cache,target=/vcpkg-binarycache,sharing=locked <&2 + exit 1 + ;; +esac + +# Surgical override: when LOADER_PROTOBUF_VERSION is set, use it for the +# protobuf pin only — keep the variant's vcpkg baseline. Mostly useful for +# bisecting protobuf releases against a known-good vcpkg snapshot. if [ -n "${_ARG_PROTOBUF_VERSION:-}" ]; then PROTOBUF_VERSION="${_ARG_PROTOBUF_VERSION}" fi diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index c2f3fc8..02b1c7d 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -8,10 +8,15 @@ // See ./versions.env for all toolchain pins. "name": "tableauio/loader", // Build args wire host env to Dockerfile ARGs: + // LOADER_DEFAULT_VARIANT on the host -> DEFAULT_VARIANT inside. + // Switches both protobuf AND vcpkg baseline atomically; values come + // from versions.env's MODERN_*/LEGACY_V3_* rows. // LOADER_PROTOBUF_VERSION on the host -> PROTOBUF_VERSION inside. - // Default falls through to versions.env's PROTOBUF_VERSION (CI's modern - // matrix entry). To rebuild against the legacy v3 line: - // LOADER_PROTOBUF_VERSION=3.21.12 code . # then Reopen in Container. + // Surgical: overrides ONLY the protobuf version, vcpkg baseline + // stays at the variant's pin. For a clean variant switch, prefer + // LOADER_DEFAULT_VARIANT. + // Both default empty (versions.env's DEFAULT_VARIANT row wins). Example: + // LOADER_DEFAULT_VARIANT=legacy-v3 code . # then Reopen in Container. // // "context" defaults to the directory holding this devcontainer.json // (.devcontainer/), so the Dockerfile's `COPY versions.env ...` @@ -19,6 +24,7 @@ "build": { "dockerfile": "Dockerfile", "args": { + "DEFAULT_VARIANT": "${localEnv:LOADER_DEFAULT_VARIANT:}", "PROTOBUF_VERSION": "${localEnv:LOADER_PROTOBUF_VERSION:}" } }, diff --git a/.devcontainer/versions.env b/.devcontainer/versions.env index 835a2c6..893983e 100644 --- a/.devcontainer/versions.env +++ b/.devcontainer/versions.env @@ -10,10 +10,6 @@ # - Comments start with `#` at column 0 only (not after a value). # - Blank lines are ignored. # - Values are bare strings — no shell expansion, no $VAR references. -# -# Bumping any of these is a one-line change. The matching post-install -# assertion in the Dockerfile / make.py catches the case where the -# vcpkg baseline doesn't yet know about the requested PROTOBUF_VERSION. # Go SDK shipped in the devcontainer. Must be ≥ the `go` directive in go.mod; # CI workflows resolve their Go version from go.mod via setup-go's @@ -24,24 +20,42 @@ GO_VERSION=1.24.0 # testing-*.yml workflows. Bump everywhere together. BUF_VERSION=1.67.0 -# Default protobuf version (CI's "modern" matrix entry). devcontainer.json -# wires this to the LOADER_PROTOBUF_VERSION host env var so users can rebuild -# against the legacy v3 line (3.21.12) without editing this file. -PROTOBUF_VERSION=6.33.4 - -# vcpkg checkout commit. Same value used by: -# - .devcontainer/Dockerfile (ARG VCPKG_BASELINE_COMMIT) -# - make.py (Versions.vcpkg_baseline_commit) -# - .github/workflows/testing-cpp.yml (env: VCPKG_COMMIT) -# This commit MUST know about every PROTOBUF_VERSION in the testing-cpp.yml -# matrix; bump it forward (never sideways) when adding a new protobuf entry. +# --------------------------------------------------------------------------- +# protobuf / vcpkg matrix. # -# Use the *commit* SHA, not the tag-object SHA: GitHub's /commit/ view -# 404s on tag objects, and the resolved manifest baseline is the commit -# anyway. To bump: pick the commit pointed to by a recent quarterly tag at +# Each variant pins a (protobuf, vcpkg-snapshot) pair where the snapshot's +# baseline.json knows about that protobuf version (so abseil / utf8-range / +# re2 transitive deps come from one self-consistent vcpkg cut). DEFAULT_VARIANT +# selects which row the devcontainer image and `make.py setup` install. CI's +# .github/workflows/testing-cpp.yml mirrors these labels in its matrix. +# +# Override at devcontainer build time: set DEFAULT_VARIANT in the host env +# before "Rebuild Container". +# +# Override per-invocation in `make.py`: pass --protobuf-version / --vcpkg-baseline. +# +# Bumping a row: pick the new commit pointed to by a recent quarterly tag at # https://github.com/microsoft/vcpkg/tags (e.g. `git rev-parse 2026.04.27^{commit}`). -# Currently pins the tip of the `2026.04.27` quarterly release. -VCPKG_BASELINE_COMMIT=56bb2411609227288b70117ead2c47585ba07713 +# Use the *commit* SHA, not the tag-object SHA: GitHub's /commit/ view +# 404s on tag objects, and the resolved manifest baseline is the commit anyway. +# Verify the chosen commit's versions/baseline.json has +# default.protobuf.baseline matching the requested protobuf version. +# --------------------------------------------------------------------------- + +# Active variant — Dockerfile, load-versions action, and make.py read this +# to resolve PROTOBUF_VERSION / VCPKG_BASELINE_COMMIT to the matching row. +DEFAULT_VARIANT=modern + +# Modern variant: protobuf 6.x line. Currently pins the tip of the +# `2026.04.27` quarterly vcpkg release. +MODERN_PROTOBUF_VERSION=6.33.4 +MODERN_VCPKG_BASELINE_COMMIT=56bb2411609227288b70117ead2c47585ba07713 + +# Legacy v3 variant: protobuf 3.x ABI smoke test. Snapshot from 2023-01-15. +# Linux-only in CI — the windows port pulls MSYS2 distfiles that have since +# been pruned from repo.msys2.org (see testing-cpp.yml's matrix exclude). +LEGACY_V3_PROTOBUF_VERSION=3.21.12 +LEGACY_V3_VCPKG_BASELINE_COMMIT=6245ce44a03f04d19be125ab1bbab578d0933e85 # .NET SDK major.minor. apt installs `dotnet-sdk-${DOTNET_VERSION}` on Linux. # CI uses `${DOTNET_VERSION}.x` with actions/setup-dotnet. diff --git a/.github/actions/load-versions/action.yml b/.github/actions/load-versions/action.yml index eebb9d6..ff6d0ef 100644 --- a/.github/actions/load-versions/action.yml +++ b/.github/actions/load-versions/action.yml @@ -5,6 +5,10 @@ description: > GITHUB_ENV so subsequent steps can reference them as env-context values (e.g. env.BUF_VERSION). + Also resolves the active (DEFAULT_VARIANT) row's MODERN_/LEGACY_V3_-prefixed + protobuf+vcpkg pins back to the unprefixed PROTOBUF_VERSION / + VCPKG_BASELINE_COMMIT names so existing workflow references keep working. + Format rules of versions.env (kept simple so every consumer can parse it with a builtin): - One KEY=VALUE per line, no quotes, no spaces around `=`. @@ -23,12 +27,28 @@ runs: echo "::error::Missing $file; cannot resolve pinned tool versions." exit 1 fi + # Pass 1: every KEY=VALUE → $GITHUB_ENV verbatim. + declare -A kv while IFS='=' read -r k v; do [ -z "$k" ] && continue case "$k" in \#*) continue ;; esac printf '%s=%s\n' "$k" "$v" >> "$GITHUB_ENV" + kv[$k]=$v done < "$file" + + # Pass 2: resolve the active variant's protobuf + vcpkg pins back to + # the unprefixed names so workflows can keep referencing + # ${{ env.PROTOBUF_VERSION }} / ${{ env.VCPKG_BASELINE_COMMIT }}. + variant="${kv[DEFAULT_VARIANT]:-modern}" + prefix=$(printf '%s' "$variant" | tr '[:lower:]-' '[:upper:]_') + protobuf_key="${prefix}_PROTOBUF_VERSION" + vcpkg_key="${prefix}_VCPKG_BASELINE_COMMIT" + if [ -z "${kv[$protobuf_key]:-}" ] || [ -z "${kv[$vcpkg_key]:-}" ]; then + echo "::error::variant '$variant' missing $protobuf_key and/or $vcpkg_key in $file" + exit 1 + fi + printf 'PROTOBUF_VERSION=%s\n' "${kv[$protobuf_key]}" >> "$GITHUB_ENV" + printf 'VCPKG_BASELINE_COMMIT=%s\n' "${kv[$vcpkg_key]}" >> "$GITHUB_ENV" # Mirror VCPKG_BASELINE_COMMIT under VCPKG_COMMIT for backward # compat with testing-cpp.yml's pre-existing variable name. - grep '^VCPKG_BASELINE_COMMIT=' "$file" \ - | sed 's/^VCPKG_BASELINE_COMMIT=/VCPKG_COMMIT=/' >> "$GITHUB_ENV" + printf 'VCPKG_COMMIT=%s\n' "${kv[$vcpkg_key]}" >> "$GITHUB_ENV" diff --git a/.github/workflows/testing-cpp.yml b/.github/workflows/testing-cpp.yml index 1b7f051..91800ff 100644 --- a/.github/workflows/testing-cpp.yml +++ b/.github/workflows/testing-cpp.yml @@ -25,6 +25,12 @@ jobs: # snapshot as protobuf — `overrides` alone would let those # transitive deps drift forward and break ABI. # + # The labels and SHAs below mirror .devcontainer/versions.env's + # MODERN_*/LEGACY_V3_* entries — keep them in sync when bumping. + # GitHub Actions matrices are evaluated at workflow parse time, + # so we can't read them from versions.env via the load-versions + # composite action; the duplication is intentional. + # # CI passes the SHA explicitly via --vcpkg-baseline so the run is # fully reproducible and independent of vcpkg's git history (which # `lukka/run-vcpkg` checks out shallowly). For ad-hoc local runs @@ -37,7 +43,7 @@ jobs: # default.protobuf.baseline == "X.Y.Z". - label: modern protobuf-version: "6.33.4" - vcpkg-commit: "1f6bbba3da511773189e5075b6781222402d10fa" + vcpkg-commit: "56bb2411609227288b70117ead2c47585ba07713" - label: legacy-v3 protobuf-version: "3.21.12" vcpkg-commit: "6245ce44a03f04d19be125ab1bbab578d0933e85" diff --git a/CLAUDE.md b/CLAUDE.md index e63d7c9..21b406e 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -34,13 +34,13 @@ C++ wipes `test/cpp-tableau-loader/{build,src/tableau,src/protoconf}` before reg On Windows, `make.py` wraps every C++ subprocess in `cmd /c "call vcvarsall.bat x64 >nul && "` so MSVC env lives per-subprocess; the shell PATH is never mutated. -`make.py setup` pins every toolchain dimension (matches CI + devcontainer): Go via official go.dev tarball to `~/.local/go/`, buf via GitHub release binary, protobuf via **vcpkg at `VCPKG_BASELINE_COMMIT` on every host (macOS/Linux/Windows)**, .NET / Node via Microsoft+NodeSource (Linux) or Homebrew (macOS) or winget (Windows), cmake/ninja via the host's package manager. Resolved paths cached in `~/.loader-env.json` so subsequent `make.py test --lang cpp` invocations pick them up without re-running setup. +`make.py setup` pins every toolchain dimension (matches CI + devcontainer): Go via official go.dev tarball to `~/.local/go/`, buf via GitHub release binary, protobuf via **vcpkg at the active variant's baseline commit on every host (macOS/Linux/Windows)**, .NET via Microsoft repo (Linux) or Homebrew (macOS) or winget (Windows), cmake/ninja via the host's package manager. Resolved paths cached in `~/.loader-env.json` so subsequent `make.py test --lang cpp` invocations pick them up without re-running setup. ### Dev container - `.devcontainer/` → **Dev Containers: Reopen in Container**. Ubuntu 24.04 + all toolchains pinned. First build ~25 min; reopens instant. - Inside: `python make.py setup` is a no-op. `python make.py test --lang ` works for all languages — Dockerfile presets `CMAKE_PREFIX_PATH=/opt/vcpkg/active`. -- Override protobuf version: `LOADER_PROTOBUF_VERSION=3.21.12 code .` then **Rebuild Container**. +- Override protobuf version: `LOADER_DEFAULT_VARIANT=legacy-v3 code .` then **Rebuild Container** (switches both protobuf and vcpkg baseline atomically). Variants are declared as `_PROTOBUF_VERSION` + `_VCPKG_BASELINE_COMMIT` pairs in `.devcontainer/versions.env`. For a surgical override of just the protobuf version, `LOADER_PROTOBUF_VERSION=X.Y.Z` still works. - Single source of truth for all toolchain versions: **`.devcontainer/versions.env`**. CI primary tests (`testing-{cpp,go,csharp}.yml`) use `lukka/run-vcpkg` for cached vcpkg installs + `python make.py test --lang ` for build/test. `devcontainer-smoke.yml` builds the image on `.devcontainer/**` PRs (amd64 + arm64). `testing-make.yml` runs the make.py unit + dry-run regression suite on every push. diff --git a/make.py b/make.py index c518198..593c28f 100644 --- a/make.py +++ b/make.py @@ -91,11 +91,52 @@ def buf_version(self) -> Optional[str]: @property def protobuf_version(self) -> Optional[str]: - return self.raw.get("PROTOBUF_VERSION") + """Resolved protobuf version for the active (DEFAULT_VARIANT) row.""" + return self._variant_value("PROTOBUF_VERSION") @property def vcpkg_baseline_commit(self) -> Optional[str]: - return self.raw.get("VCPKG_BASELINE_COMMIT") + """Resolved vcpkg baseline SHA for the active (DEFAULT_VARIANT) row.""" + return self._variant_value("VCPKG_BASELINE_COMMIT") + + @property + def default_variant(self) -> str: + """Active variant label (e.g. ``modern`` / ``legacy_v3``). + + Matched case-insensitively against variant-prefixed keys: a value of + ``modern`` resolves keys with prefix ``MODERN_``; ``legacy-v3`` / + ``legacy_v3`` both resolve ``LEGACY_V3_``. Defaults to ``modern`` if + the key is absent (preserves behaviour of older versions.env files). + """ + return (self.raw.get("DEFAULT_VARIANT") or "modern").strip().lower() + + def variants(self) -> dict[str, dict[str, str]]: + """Enumerate every (protobuf, vcpkg-baseline) variant defined. + + Returns ``{variant_label: {"protobuf_version": ..., "vcpkg_baseline_commit": ...}}`` + keyed by lowercase label. Labels that are missing one of the two keys + are still reported with the key they have, so callers can detect a + half-defined variant. + """ + suffixes = { + "_PROTOBUF_VERSION": "protobuf_version", + "_VCPKG_BASELINE_COMMIT": "vcpkg_baseline_commit", + } + out: dict[str, dict[str, str]] = {} + for k, v in self.raw.items(): + for sfx, field_name in suffixes.items(): + if k.endswith(sfx): + label = k[: -len(sfx)].lower() + out.setdefault(label, {})[field_name] = v + break + return out + + def _variant_value(self, suffix: str) -> Optional[str]: + """Resolve ``_`` (uppercase). Falls back to + the unprefixed key for backward compat with old versions.env files + that used the flat ``PROTOBUF_VERSION`` / ``VCPKG_BASELINE_COMMIT``.""" + prefix = self.default_variant.upper().replace("-", "_") + return self.raw.get(f"{prefix}_{suffix}") or self.raw.get(suffix) @property def dotnet_version(self) -> Optional[str]: diff --git a/test_make.py b/test_make.py index a8fcfd4..531bb6a 100644 --- a/test_make.py +++ b/test_make.py @@ -43,8 +43,11 @@ def test_loads_real_versions_env(self): for key in ( "GO_VERSION", "BUF_VERSION", - "PROTOBUF_VERSION", - "VCPKG_BASELINE_COMMIT", + "DEFAULT_VARIANT", + "MODERN_PROTOBUF_VERSION", + "MODERN_VCPKG_BASELINE_COMMIT", + "LEGACY_V3_PROTOBUF_VERSION", + "LEGACY_V3_VCPKG_BASELINE_COMMIT", "DOTNET_VERSION", "CMAKE_VERSION", ): @@ -55,10 +58,46 @@ def test_typed_accessors(self): v = make.Versions.load(REPO_ROOT) assert v.go_version == v.raw["GO_VERSION"] assert v.buf_version == v.raw["BUF_VERSION"] - assert v.protobuf_version == v.raw["PROTOBUF_VERSION"] - assert v.vcpkg_baseline_commit == v.raw["VCPKG_BASELINE_COMMIT"] assert v.dotnet_version == v.raw["DOTNET_VERSION"] assert v.cmake_version == v.raw["CMAKE_VERSION"] + # default_variant is lowercased. + assert v.default_variant == v.raw["DEFAULT_VARIANT"].lower() + # protobuf_version / vcpkg_baseline_commit resolve through DEFAULT_VARIANT + # to the corresponding MODERN_/LEGACY_V3_-prefixed key. + prefix = v.default_variant.upper().replace("-", "_") + assert v.protobuf_version == v.raw[f"{prefix}_PROTOBUF_VERSION"] + assert v.vcpkg_baseline_commit == v.raw[f"{prefix}_VCPKG_BASELINE_COMMIT"] + + def test_variants_enumerates_all_rows(self): + v = make.Versions.load(REPO_ROOT) + variants = v.variants() + # Both rows declared in versions.env must be present and complete. + assert "modern" in variants + assert "legacy_v3" in variants + for label, row in variants.items(): + assert row.get("protobuf_version"), f"{label} missing protobuf_version" + assert row.get("vcpkg_baseline_commit"), f"{label} missing vcpkg_baseline_commit" + + def test_default_variant_falls_back_to_modern(self): + # Old versions.env files without DEFAULT_VARIANT default to 'modern'. + v = make.Versions(raw={}) + assert v.default_variant == "modern" + + def test_variant_value_falls_back_to_unprefixed_key(self): + # Backward compat: a flat versions.env (legacy schema) still resolves. + v = make.Versions(raw={"PROTOBUF_VERSION": "9.9.9", "VCPKG_BASELINE_COMMIT": "f" * 40}) + assert v.protobuf_version == "9.9.9" + assert v.vcpkg_baseline_commit == "f" * 40 + + def test_default_variant_label_with_dash_resolves(self): + # 'legacy-v3' (CI label form) and 'legacy_v3' (env-key form) both work. + v = make.Versions(raw={ + "DEFAULT_VARIANT": "legacy-v3", + "LEGACY_V3_PROTOBUF_VERSION": "3.21.12", + "LEGACY_V3_VCPKG_BASELINE_COMMIT": "a" * 40, + }) + assert v.protobuf_version == "3.21.12" + assert v.vcpkg_baseline_commit == "a" * 40 def test_protobuf_version_looks_like_semver(self): v = make.Versions.load(REPO_ROOT) @@ -492,8 +531,11 @@ def test_version_prints_versions_env(self): for key in ( "GO_VERSION", "BUF_VERSION", - "PROTOBUF_VERSION", - "VCPKG_BASELINE_COMMIT", + "DEFAULT_VARIANT", + "MODERN_PROTOBUF_VERSION", + "MODERN_VCPKG_BASELINE_COMMIT", + "LEGACY_V3_PROTOBUF_VERSION", + "LEGACY_V3_VCPKG_BASELINE_COMMIT", "DOTNET_VERSION", "CMAKE_VERSION", ): From 273e64848078c63de775857bfd393ffc8657418a Mon Sep 17 00:00:00 2001 From: wenchy Date: Fri, 5 Jun 2026 22:08:04 +0800 Subject: [PATCH 59/66] chore: clean code and docs --- .devcontainer/Dockerfile | 1 - .gitignore | 1 + README.md | 2 +- 3 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index 4c00b6e..829380c 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -85,7 +85,6 @@ EOF # a different port revision than requested. # --------------------------------------------------------------------------- ARG DEFAULT_VARIANT= -ARG DEFAULT_VARIANT= ARG PROTOBUF_VERSION= ENV VCPKG_ROOT=/opt/vcpkg # Tell vcpkg to share its compiled binaries via /vcpkg-binarycache. The diff --git a/.gitignore b/.gitignore index f059865..58e3329 100644 --- a/.gitignore +++ b/.gitignore @@ -37,6 +37,7 @@ node_modules *.pb.* __pycache__/ +.pytest_cache cmd/protoc-gen-cpp-tableau-loader/protoc-gen-cpp-tableau-loader cmd/protoc-gen-go-tableau-loader/protoc-gen-go-tableau-loader diff --git a/README.md b/README.md index 3caf5e7..ec4ba6b 100644 --- a/README.md +++ b/README.md @@ -72,6 +72,6 @@ CI: [`.github/workflows/testing-make.yml`](.github/workflows/testing-make.yml). - [Protocol Buffers C++ Reference](https://protobuf.dev/reference/cpp/) - [Protocol Buffers Go Reference](https://protobuf.dev/reference/go/) +- [protobuf-es](https://github.com/bufbuild/protobuf-es) - [vcpkg](https://github.com/microsoft/vcpkg) - [buf CLI](https://buf.build/docs/cli/) -- [proto3-json-serializer](https://github.com/googleapis/proto3-json-serializer-nodejs) From 698dc59f806e8da0a4ba2774545fee612b59696a Mon Sep 17 00:00:00 2001 From: Kybxd <627940450@qq.com> Date: Mon, 8 Jun 2026 16:30:15 +0800 Subject: [PATCH 60/66] feat(cpp-loader): add util::GetExtension wrapper for protobuf v3.15.0 compat - embed/util.pc.h: introduce util::GetExtension() wrapper that works around the protobuf v3.15.0 GetExtension reflection bug by falling back to reparse-via-FieldDescriptor when GOOGLE_PROTOBUF_VERSION < 3015000. - embed/util.pc.cc, embed/load.pc.cc: switch the two extension lookups (tableau::field on field options, tableau::worksheet on message options) to use util::GetExtension() instead of calling .GetExtension() directly. - test/cpp-tableau-loader/src/protoconf/*: regenerate the mirrored sources to stay in sync with the embed templates. - make.py: vcpkg-based protobuf integration for cpp-loader testing. --- .../embed/load.pc.cc | 2 +- .../embed/util.pc.cc | 2 +- .../embed/util.pc.h | 43 +++++ make.py | 161 +++++++++++++++--- .../src/protoconf/load.pc.cc | 2 +- .../src/protoconf/util.pc.cc | 2 +- .../src/protoconf/util.pc.h | 43 +++++ 7 files changed, 224 insertions(+), 31 deletions(-) diff --git a/cmd/protoc-gen-cpp-tableau-loader/embed/load.pc.cc b/cmd/protoc-gen-cpp-tableau-loader/embed/load.pc.cc index dc8d06d..b586f9a 100644 --- a/cmd/protoc-gen-cpp-tableau-loader/embed/load.pc.cc +++ b/cmd/protoc-gen-cpp-tableau-loader/embed/load.pc.cc @@ -63,7 +63,7 @@ bool LoadMessagerInDir(google::protobuf::Message& msg, const std::filesystem::pa const google::protobuf::Descriptor* descriptor = msg.GetDescriptor(); // access the extension directly using the generated identifier - const tableau::WorksheetOptions& worksheet_options = descriptor->options().GetExtension(tableau::worksheet); + const tableau::WorksheetOptions& worksheet_options = util::GetExtension(descriptor->options(), tableau::worksheet); if (worksheet_options.patch() != tableau::PATCH_NONE) { return LoadMessagerWithPatch(msg, path, fmt, worksheet_options.patch(), options); } diff --git a/cmd/protoc-gen-cpp-tableau-loader/embed/util.pc.cc b/cmd/protoc-gen-cpp-tableau-loader/embed/util.pc.cc index 40821ae..7088f12 100644 --- a/cmd/protoc-gen-cpp-tableau-loader/embed/util.pc.cc +++ b/cmd/protoc-gen-cpp-tableau-loader/embed/util.pc.cc @@ -105,7 +105,7 @@ bool PatchMessage(google::protobuf::Message& dst, const google::protobuf::Messag // Iterates over every populated field. for (auto fd : fields) { - const tableau::FieldOptions& opts = fd->options().GetExtension(tableau::field); + const tableau::FieldOptions& opts = util::GetExtension(fd->options(), tableau::field); tableau::Patch patch = opts.prop().patch(); if (patch == tableau::PATCH_REPLACE) { dst_reflection->ClearField(&dst, fd); diff --git a/cmd/protoc-gen-cpp-tableau-loader/embed/util.pc.h b/cmd/protoc-gen-cpp-tableau-loader/embed/util.pc.h index 574848c..d39a27a 100644 --- a/cmd/protoc-gen-cpp-tableau-loader/embed/util.pc.h +++ b/cmd/protoc-gen-cpp-tableau-loader/embed/util.pc.h @@ -5,6 +5,7 @@ #include #include #include +#include // Protobuf versions before v4.22.0 (GOOGLE_PROTOBUF_VERSION < 4022000) use the legacy // logging interface (LogLevel, SetLogHandler). Newer versions removed it in favor of Abseil logging. @@ -68,6 +69,48 @@ const std::string& Format2Ext(Format fmt); // PatchMessage patches src into dst, which must be a message with the same descriptor. bool PatchMessage(google::protobuf::Message& dst, const google::protobuf::Message& src); +// GetExtension reads a typed custom-option extension off any *Options message +// (e.g. MessageOptions, FieldOptions). It transparently works around a +// protobuf static-initialization-order quirk present on runtimes < v3.15.0: +// when a .pb.cc that *defines* an extension (e.g. tableau.pb.cc -> +// `tableau::worksheet`) is initialized after the descriptors that *carry* it +// have already been built, the custom-option payload ends up parked in the +// options message's unknown_fields, and `options.GetExtension(id)` then +// returns the default instance. +// +// The fix is to round-trip the options bytes through a fresh OptionsT instance +// at call time: by then all static initializers (including the extension's +// generated registration via `descriptor_table_*` / `dynamic_init_dummy_*`) +// have run, so the second parse resolves the extension correctly. +// +// Verified by checkout-build-and-test against protoc/libprotobuf at the same +// tag: v3.14.0 reproduces (PatchTest fails when this fallback is bypassed), +// v3.15.0 is clean. The v3.15.0 release notes don't call out this specific +// symptom, but two C++ entries plausibly cover it by reshaping static +// initialization: "Constant initialize the global message instances" and +// "Use init_seg in MSVC to push initialization to an earlier phase". On +// runtimes >= 3.15.0 we just forward to the native call with no overhead. +// +// Reference: https://github.com/protocolbuffers/protobuf/releases/tag/v3.15.0 +template +#if GOOGLE_PROTOBUF_VERSION < 3015000 +// Returns by value (a freshly reparsed OptionsT extension copy). +inline auto GetExtension(const OptionsT& options, const ExtT& id) + -> typename std::decay::type { + OptionsT reparsed; + std::string buf; + options.SerializeToString(&buf); + reparsed.ParseFromString(buf); + return reparsed.GetExtension(id); +} +#else +// Returns by const-reference (zero-copy passthrough). +inline auto GetExtension(const OptionsT& options, const ExtT& id) + -> decltype(options.GetExtension(id)) { + return options.GetExtension(id); +} +#endif + #if TABLEAU_PB_LOG_LEGACY // ProtobufLogHandler redirects protobuf internal logs to tableau logger. // Only available for protobuf < v4.22.0, as newer versions removed the old logging interface. diff --git a/make.py b/make.py index 593c28f..eabc752 100644 --- a/make.py +++ b/make.py @@ -580,6 +580,7 @@ def _resolve_vcpkg_baseline_for_protobuf( f"(`git -C {vcpkg_root} fetch --unshallow origin master` may help)\n" f" - protobuf {protobuf_version} was never the vcpkg baseline " f"(use --vcpkg-baseline= to point at a custom snapshot)" + f"{_format_known_protobuf_versions_hint(vcpkg_root)}" ) # Step 2: validate each candidate by reading baseline.json at that commit. @@ -598,7 +599,101 @@ def _resolve_vcpkg_baseline_for_protobuf( raise RuntimeError( f'Found {len(candidates)} commits mentioning "{protobuf_version}" in ' f"versions/baseline.json but none had default.protobuf.baseline set " - f"to that exact version. Pass --vcpkg-baseline= manually." + f"to that exact version (the literal likely appears in unrelated " + f"ports). Pass --vcpkg-baseline= manually." + f"{_format_known_protobuf_versions_hint(vcpkg_root)}" + ) + + +# Earliest protobuf version usable as a vcpkg `builtin-baseline`. +# vcpkg's versions/baseline.json was introduced 2021-01-21, and its very +# first commit already pinned `default.protobuf.baseline = "3.14.0"`. Port +# versions older than this (3.0.2, 3.2.0, ..., 3.13.0) exist in +# versions/p-/protobuf.json but were never the global baseline and so can't +# serve as builtin-baseline. The user-facing error message uses a rounder +# "~3.18" wording on purpose; the precise floor lives only here. +_VCPKG_PROTOBUF_BASELINE_FLOOR = (3, 14, 0) + + +def _parse_protobuf_version_tuple(v: str) -> Optional[tuple[int, ...]]: + """Parse "3.14.0" / "3.21.12" / "5.29.5" into a numeric tuple for ordering. + + Returns None for anything that isn't pure dotted-numeric (defensive: the + registry has historically been numeric-only for protobuf, but we'd + rather drop a weird entry than crash a hint formatter). + """ + parts = v.split(".") + if not all(p.isdigit() for p in parts) or not parts: + return None + return tuple(int(p) for p in parts) + + +def _list_known_protobuf_versions(vcpkg_root: Path) -> list[str]: + """Return protobuf versions usable as a vcpkg `builtin-baseline`. + + Reads `versions/p-/protobuf.json` at HEAD (cheap: one `git show`) — this + is vcpkg's authoritative list of protobuf versions the registry has ever + known. Returned in registry order (newest first), de-duplicated, and + filtered to `>= _VCPKG_PROTOBUF_BASELINE_FLOOR` so the caller can blindly + surface every entry as something the user can actually pass to + `--protobuf-version`. On any failure returns [] so callers can degrade + gracefully. + """ + try: + blob = subprocess.check_output( + [ + "git", + "-C", + str(vcpkg_root), + "show", + "HEAD:versions/p-/protobuf.json", + ], + text=True, + stderr=subprocess.DEVNULL, + ) + except (subprocess.CalledProcessError, FileNotFoundError): + return [] + try: + data = json.loads(blob) + except ValueError: + return [] + seen: set[str] = set() + versions: list[str] = [] + for entry in data.get("versions", []): + v = ( + entry.get("version") + or entry.get("version-semver") + or entry.get("version-string") + ) + if not v or v in seen: + continue + tup = _parse_protobuf_version_tuple(v) + if tup is None or tup < _VCPKG_PROTOBUF_BASELINE_FLOOR: + continue + seen.add(v) + versions.append(v) + return versions + + +def _format_known_protobuf_versions_hint(vcpkg_root: Path) -> str: + """Render a human-readable bullet listing known protobuf versions. + + Returns "" (caller can append unconditionally) when the version DB + can't be read. Otherwise returns a leading "\n - known versions: ..." + suitable for tacking onto the end of a RuntimeError message. + """ + versions = _list_known_protobuf_versions(vcpkg_root) + if not versions: + return "" + # Newest first (registry order). The list is already filtered to versions + # that can serve as a builtin-baseline. + head = ", ".join(versions[:20]) + tail = f" (+{len(versions) - 20} older)" if len(versions) > 20 else "" + return ( + f"\n - known protobuf versions in {vcpkg_root}/versions/p-/protobuf.json " + f"(newest first): {head}{tail}\n" + f" - note: versions older than 3.14.0 predate vcpkg's baseline.json " + f"and cannot be used as builtin-baseline" ) @@ -1189,36 +1284,16 @@ def _cpp_build_or_test(args, ctx: "Context", run_tests: bool) -> int: cxx_std = getattr(args, "cxx_std", "17") cxx_compiler = getattr(args, "cxx_compiler", None) - # Stale-codegen wipe (gitignored *.pb.* files left over from a previous - # protoc version shadow fresh codegen). Skip with --no-clean. - if not getattr(args, "no_clean", False): - ctx.runner.rmtree(cwd / "build") - ctx.runner.rmtree(cwd / "src" / "tableau") - ctx.runner.rmtree(cwd / "src" / "protoconf") - - # Classic mode: a stale vcpkg.json from a previous --protobuf-version run - # would silently switch cmake's vcpkg toolchain into manifest mode and - # build the wrong libprotobuf into build/vcpkg_installed/. Always remove - # it here unless we're about to render a fresh one below. - if not protobuf_version: - manifest_path = cwd / "vcpkg.json" - if manifest_path.is_file(): - if ctx.runner.dry_run: - print(f"[dry-run] rm {manifest_path}") - else: - manifest_path.unlink() - - # Manifest mode: render vcpkg.json pinning the requested protobuf-version, - # then run `vcpkg install` to populate vcpkg_installed/. This matches CI's - # testing-cpp.yml flow (which uses lukka/run-vcpkg with runVcpkgInstall: - # true) and means switching --protobuf-version Just Works without - # re-running `make.py setup`. Idempotent: vcpkg detects already-installed - # packages and skips them. + # Resolve the vcpkg baseline up-front when --protobuf-version is given. + # This MUST happen before the stale-codegen rmtree below: an invalid + # --protobuf-version that would later raise should not have already + # wiped the user's src/protoconf, src/tableau, build/ trees. # # Baseline resolution order (no `overrides` — see module comment above): # 1. --vcpkg-baseline= (explicit override) # 2. _resolve_vcpkg_baseline_for_protobuf(...) (auto: git-search vcpkg) - cmake_extra: list[str] = [] + vcpkg_root: Optional[Path] = None + baseline: Optional[str] = None if protobuf_version: # Need a vcpkg checkout to resolve the baseline by git history. vcpkg_root = ctx.platform.vcpkg_root or _env_path("VCPKG_ROOT") @@ -1249,6 +1324,7 @@ def _cpp_build_or_test(args, ctx: "Context", run_tests: bool) -> int: protobuf_version, vcpkg_root, ctx.runner ) except RuntimeError as e: + # Fail fast — user trees on disk are still untouched. print(f"[error] {e}", file=sys.stderr) print( "[hint] If you know a vcpkg commit whose baseline matches " @@ -1257,6 +1333,37 @@ def _cpp_build_or_test(args, ctx: "Context", run_tests: bool) -> int: ) return 1 + # Stale-codegen wipe (gitignored *.pb.* files left over from a previous + # protoc version shadow fresh codegen). Skip with --no-clean. + if not getattr(args, "no_clean", False): + ctx.runner.rmtree(cwd / "build") + ctx.runner.rmtree(cwd / "src" / "tableau") + ctx.runner.rmtree(cwd / "src" / "protoconf") + + # Classic mode: a stale vcpkg.json from a previous --protobuf-version run + # would silently switch cmake's vcpkg toolchain into manifest mode and + # build the wrong libprotobuf into build/vcpkg_installed/. Always remove + # it here unless we're about to render a fresh one below. + if not protobuf_version: + manifest_path = cwd / "vcpkg.json" + if manifest_path.is_file(): + if ctx.runner.dry_run: + print(f"[dry-run] rm {manifest_path}") + else: + manifest_path.unlink() + + # Manifest mode: render vcpkg.json pinning the requested protobuf-version, + # then run `vcpkg install` to populate vcpkg_installed/. This matches CI's + # testing-cpp.yml flow (which uses lukka/run-vcpkg with runVcpkgInstall: + # true) and means switching --protobuf-version Just Works without + # re-running `make.py setup`. Idempotent: vcpkg detects already-installed + # packages and skips them. + cmake_extra: list[str] = [] + if protobuf_version: + # vcpkg_root and baseline were resolved above; assert for the type + # checker (both are guaranteed non-None inside this branch). + assert vcpkg_root is not None and baseline is not None + manifest = { "name": "loader-cpp-test", "version": "0.1.0", diff --git a/test/cpp-tableau-loader/src/protoconf/load.pc.cc b/test/cpp-tableau-loader/src/protoconf/load.pc.cc index f7018a1..e39634a 100644 --- a/test/cpp-tableau-loader/src/protoconf/load.pc.cc +++ b/test/cpp-tableau-loader/src/protoconf/load.pc.cc @@ -69,7 +69,7 @@ bool LoadMessagerInDir(google::protobuf::Message& msg, const std::filesystem::pa const google::protobuf::Descriptor* descriptor = msg.GetDescriptor(); // access the extension directly using the generated identifier - const tableau::WorksheetOptions& worksheet_options = descriptor->options().GetExtension(tableau::worksheet); + const tableau::WorksheetOptions& worksheet_options = util::GetExtension(descriptor->options(), tableau::worksheet); if (worksheet_options.patch() != tableau::PATCH_NONE) { return LoadMessagerWithPatch(msg, path, fmt, worksheet_options.patch(), options); } diff --git a/test/cpp-tableau-loader/src/protoconf/util.pc.cc b/test/cpp-tableau-loader/src/protoconf/util.pc.cc index 87cdbfb..46487b2 100644 --- a/test/cpp-tableau-loader/src/protoconf/util.pc.cc +++ b/test/cpp-tableau-loader/src/protoconf/util.pc.cc @@ -111,7 +111,7 @@ bool PatchMessage(google::protobuf::Message& dst, const google::protobuf::Messag // Iterates over every populated field. for (auto fd : fields) { - const tableau::FieldOptions& opts = fd->options().GetExtension(tableau::field); + const tableau::FieldOptions& opts = util::GetExtension(fd->options(), tableau::field); tableau::Patch patch = opts.prop().patch(); if (patch == tableau::PATCH_REPLACE) { dst_reflection->ClearField(&dst, fd); diff --git a/test/cpp-tableau-loader/src/protoconf/util.pc.h b/test/cpp-tableau-loader/src/protoconf/util.pc.h index 3937627..dc55d69 100644 --- a/test/cpp-tableau-loader/src/protoconf/util.pc.h +++ b/test/cpp-tableau-loader/src/protoconf/util.pc.h @@ -11,6 +11,7 @@ #include #include #include +#include // Protobuf versions before v4.22.0 (GOOGLE_PROTOBUF_VERSION < 4022000) use the legacy // logging interface (LogLevel, SetLogHandler). Newer versions removed it in favor of Abseil logging. @@ -74,6 +75,48 @@ const std::string& Format2Ext(Format fmt); // PatchMessage patches src into dst, which must be a message with the same descriptor. bool PatchMessage(google::protobuf::Message& dst, const google::protobuf::Message& src); +// GetExtension reads a typed custom-option extension off any *Options message +// (e.g. MessageOptions, FieldOptions). It transparently works around a +// protobuf static-initialization-order quirk present on runtimes < v3.15.0: +// when a .pb.cc that *defines* an extension (e.g. tableau.pb.cc -> +// `tableau::worksheet`) is initialized after the descriptors that *carry* it +// have already been built, the custom-option payload ends up parked in the +// options message's unknown_fields, and `options.GetExtension(id)` then +// returns the default instance. +// +// The fix is to round-trip the options bytes through a fresh OptionsT instance +// at call time: by then all static initializers (including the extension's +// generated registration via `descriptor_table_*` / `dynamic_init_dummy_*`) +// have run, so the second parse resolves the extension correctly. +// +// Verified by checkout-build-and-test against protoc/libprotobuf at the same +// tag: v3.14.0 reproduces (PatchTest fails when this fallback is bypassed), +// v3.15.0 is clean. The v3.15.0 release notes don't call out this specific +// symptom, but two C++ entries plausibly cover it by reshaping static +// initialization: "Constant initialize the global message instances" and +// "Use init_seg in MSVC to push initialization to an earlier phase". On +// runtimes >= 3.15.0 we just forward to the native call with no overhead. +// +// Reference: https://github.com/protocolbuffers/protobuf/releases/tag/v3.15.0 +template +#if GOOGLE_PROTOBUF_VERSION < 3015000 +// Returns by value (a freshly reparsed OptionsT extension copy). +inline auto GetExtension(const OptionsT& options, const ExtT& id) + -> typename std::decay::type { + OptionsT reparsed; + std::string buf; + options.SerializeToString(&buf); + reparsed.ParseFromString(buf); + return reparsed.GetExtension(id); +} +#else +// Returns by const-reference (zero-copy passthrough). +inline auto GetExtension(const OptionsT& options, const ExtT& id) + -> decltype(options.GetExtension(id)) { + return options.GetExtension(id); +} +#endif + #if TABLEAU_PB_LOG_LEGACY // ProtobufLogHandler redirects protobuf internal logs to tableau logger. // Only available for protobuf < v4.22.0, as newer versions removed the old logging interface. From 1d71db8b67c33f569b53ac1a73d3cf750f2a61bb Mon Sep 17 00:00:00 2001 From: Kybxd <627940450@qq.com> Date: Mon, 8 Jun 2026 17:35:01 +0800 Subject: [PATCH 61/66] docs: specify language for command code block --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index ec4ba6b..0857ad8 100644 --- a/README.md +++ b/README.md @@ -33,7 +33,7 @@ On Windows, run setup from **cmd as Administrator** the first time. Subsequent c ## Commands -``` +```sh python make.py setup [--lang go|cpp|csharp|ts|all] python make.py generate --lang go|cpp|csharp|ts python make.py build --lang go|cpp|csharp|ts [--cxx-std 17|20] [--cxx-compiler msvc|clang|gcc] From 6d6541b76b0995822270e5837270324649a0a595 Mon Sep 17 00:00:00 2001 From: Kybxd <627940450@qq.com> Date: Mon, 8 Jun 2026 18:04:47 +0800 Subject: [PATCH 62/66] fix(setup): prepend buf dir to PATH after install --- make.py | 64 +++++++++++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 56 insertions(+), 8 deletions(-) diff --git a/make.py b/make.py index eabc752..61e691b 100644 --- a/make.py +++ b/make.py @@ -759,6 +759,25 @@ def _which(name: str) -> Optional[str]: return shutil.which(name) +def _prepend_path(directory: Path) -> None: + """Prepend ``directory`` to ``os.environ['PATH']`` (idempotent, in-process). + + This makes a freshly-installed tool visible to (a) subsequent ``_which`` + probes inside the same ``setup`` run and (b) every child process spawned + by ``Runner`` afterwards (we never override ``env=``, so they inherit + ``os.environ``). No-op if the directory is already on PATH. + """ + p = str(directory) + sep = os.pathsep + current = os.environ.get("PATH", "") + parts = current.split(sep) if current else [] + # Case-insensitive comparison on Windows; exact match elsewhere. + norm = (lambda s: s.lower()) if os.name == "nt" else (lambda s: s) + if any(norm(x) == norm(p) for x in parts if x): + return + os.environ["PATH"] = p + (sep + current if current else "") + + def cmd_setup(args, ctx: "Context") -> int: """Install host toolchains. OS-dispatched. Idempotent.""" if ctx.platform.in_devcontainer: @@ -888,25 +907,39 @@ def _ensure_buf_unix(ctx: "Context", os_label: str) -> None: """Download the pinned buf release to ~/.local/bin/buf. os_label: "Linux" or "Darwin" (matches the GitHub release naming). + + On a fresh host ``~/.local/bin`` is rarely on PATH (distros add it lazily + via ``~/.profile``, which a non-login shell never sources). After we drop + ``buf`` there we eagerly prepend the directory to ``os.environ['PATH']`` + so ``generate``/``test`` invoked later in the same ``setup`` run — and + every child process Runner spawns — can resolve ``buf`` without the user + having to relog or edit shell rc files. """ if _which("buf") is not None: return ver = ctx.versions.buf_version if not ver: return + target_dir = Path.home() / ".local" / "bin" + target = target_dir / "buf" + # Previously-installed binary that's just not on this shell's PATH — + # don't redownload, just expose it. + if target.exists(): + print(f"[info] buf already present at {target}; reusing.") + _prepend_path(target_dir) + return arch = ( "x86_64" if _stdlib_platform.machine().lower() in ("x86_64", "amd64") else "aarch64" ) url = f"https://github.com/bufbuild/buf/releases/download/v{ver}/buf-{os_label}-{arch}" - target_dir = Path.home() / ".local" / "bin" ctx.runner.mkdirp(target_dir) - target = target_dir / "buf" print(f"[info] Downloading buf {ver} ({os_label}/{arch}) -> {target}") if not ctx.runner.dry_run: urllib.request.urlretrieve(url, str(target)) target.chmod(0o755) + _prepend_path(target_dir) def _ensure_dotnet_linux(ctx: "Context") -> None: @@ -1052,13 +1085,28 @@ def _setup_windows(langs: list[str], ctx: "Context", skip_vcpkg: bool) -> int: buf_dir = ( Path(os.environ.get("LOCALAPPDATA", str(Path.home()))) / "buf" / "bin" ) - ctx.runner.mkdirp(buf_dir) buf_exe = buf_dir / "buf.exe" - url = f"https://github.com/bufbuild/buf/releases/download/v{ver}/buf-Windows-x86_64.exe" - print(f"[info] Downloading buf {ver} -> {buf_exe}") - if not ctx.runner.dry_run: - urllib.request.urlretrieve(url, str(buf_exe)) - print(f"[info] buf installed at {buf_exe}; add to PATH manually if needed.") + if buf_exe.exists(): + # Previously-installed binary that just isn't on this shell's + # PATH — reuse it instead of redownloading. + print(f"[info] buf already present at {buf_exe}; reusing.") + _prepend_path(buf_dir) + else: + ctx.runner.mkdirp(buf_dir) + url = f"https://github.com/bufbuild/buf/releases/download/v{ver}/buf-Windows-x86_64.exe" + print(f"[info] Downloading buf {ver} -> {buf_exe}") + if not ctx.runner.dry_run: + urllib.request.urlretrieve(url, str(buf_exe)) + # Make buf usable for the rest of this `setup` run and any + # child process Runner spawns afterwards. Persisting the + # entry in the user's permanent PATH (registry / shell rc) + # is intentionally left to the user — modifying global env + # vars from a build script is too invasive. + _prepend_path(buf_dir) + print( + f"[info] buf installed at {buf_exe} (added to this session's PATH; " + f"add {buf_dir} to your user PATH to make it permanent)." + ) else: print("[info] buf already on PATH.") From a286f3dcfb2f2333de1c692c280d861476bc0d85 Mon Sep 17 00:00:00 2001 From: Kybxd <627940450@qq.com> Date: Mon, 8 Jun 2026 20:37:29 +0800 Subject: [PATCH 63/66] feat(cpp): support older protobuf runtimes in tableau loader util - Add MapKeyFd helper: falls back to field(0) on protobuf < v3.12.0 where Descriptor::map_key() is unavailable (map-entry key is always field 0 per proto3 wire-format contract). - Rework GetExtension to work around a static-initialization-order quirk on runtimes < v3.15.0 by serializing and reparsing into a fresh OptionsT; unify the signature with a single return type across branches. - Drop redundant util:: qualifier on GetExtension call inside tableau::util::PatchMessage. --- .../embed/util.pc.cc | 19 +++++++++- .../embed/util.pc.h | 37 +++++-------------- .../src/protoconf/util.pc.cc | 19 +++++++++- .../src/protoconf/util.pc.h | 37 +++++-------------- 4 files changed, 52 insertions(+), 60 deletions(-) diff --git a/cmd/protoc-gen-cpp-tableau-loader/embed/util.pc.cc b/cmd/protoc-gen-cpp-tableau-loader/embed/util.pc.cc index 7088f12..c570d17 100644 --- a/cmd/protoc-gen-cpp-tableau-loader/embed/util.pc.cc +++ b/cmd/protoc-gen-cpp-tableau-loader/embed/util.pc.cc @@ -63,6 +63,21 @@ const std::string& Format2Ext(Format fmt) { #undef GetMessage #endif +// MapKeyFd returns the key FieldDescriptor of a map-entry message. +// +// ``Descriptor::map_key()`` is a convenience wrapper added in protobuf +// v3.12.0; on older runtimes we fall back to ``field(0)``, which is +// equivalent because map-entry messages always synthesize the key as +// field 0 (and value as field 1) per the proto3 wire-format contract. +inline const google::protobuf::FieldDescriptor* MapKeyFd( + const google::protobuf::Descriptor* map_entry) { +#if GOOGLE_PROTOBUF_VERSION < 3012000 + return map_entry->field(0); +#else + return map_entry->map_key(); +#endif +} + // PatchMessage patches src into dst, which must be a message with the same descriptor. // // # Default PatchMessage mechanism @@ -105,7 +120,7 @@ bool PatchMessage(google::protobuf::Message& dst, const google::protobuf::Messag // Iterates over every populated field. for (auto fd : fields) { - const tableau::FieldOptions& opts = util::GetExtension(fd->options(), tableau::field); + const tableau::FieldOptions& opts = GetExtension(fd->options(), tableau::field); tableau::Patch patch = opts.prop().patch(); if (patch == tableau::PATCH_REPLACE) { dst_reflection->ClearField(&dst, fd); @@ -113,7 +128,7 @@ bool PatchMessage(google::protobuf::Message& dst, const google::protobuf::Messag if (fd->is_map()) { // Reference: // https://github.com/protocolbuffers/protobuf/blob/95ef4134d3f65237b7adfb66e5e7aa10fcfa1fa3/src/google/protobuf/map_field.cc#L500 - auto key_fd = fd->message_type()->map_key(); + auto key_fd = MapKeyFd(fd->message_type()); int src_count = src_reflection->FieldSize(src, fd); int dst_count = dst_reflection->FieldSize(dst, fd); switch (key_fd->cpp_type()) { diff --git a/cmd/protoc-gen-cpp-tableau-loader/embed/util.pc.h b/cmd/protoc-gen-cpp-tableau-loader/embed/util.pc.h index d39a27a..8616839 100644 --- a/cmd/protoc-gen-cpp-tableau-loader/embed/util.pc.h +++ b/cmd/protoc-gen-cpp-tableau-loader/embed/util.pc.h @@ -69,47 +69,28 @@ const std::string& Format2Ext(Format fmt); // PatchMessage patches src into dst, which must be a message with the same descriptor. bool PatchMessage(google::protobuf::Message& dst, const google::protobuf::Message& src); -// GetExtension reads a typed custom-option extension off any *Options message -// (e.g. MessageOptions, FieldOptions). It transparently works around a -// protobuf static-initialization-order quirk present on runtimes < v3.15.0: -// when a .pb.cc that *defines* an extension (e.g. tableau.pb.cc -> -// `tableau::worksheet`) is initialized after the descriptors that *carry* it -// have already been built, the custom-option payload ends up parked in the -// options message's unknown_fields, and `options.GetExtension(id)` then -// returns the default instance. -// -// The fix is to round-trip the options bytes through a fresh OptionsT instance -// at call time: by then all static initializers (including the extension's -// generated registration via `descriptor_table_*` / `dynamic_init_dummy_*`) -// have run, so the second parse resolves the extension correctly. -// -// Verified by checkout-build-and-test against protoc/libprotobuf at the same -// tag: v3.14.0 reproduces (PatchTest fails when this fallback is bypassed), -// v3.15.0 is clean. The v3.15.0 release notes don't call out this specific -// symptom, but two C++ entries plausibly cover it by reshaping static -// initialization: "Constant initialize the global message instances" and -// "Use init_seg in MSVC to push initialization to an earlier phase". On -// runtimes >= 3.15.0 we just forward to the native call with no overhead. +// GetExtension reads a typed custom-option extension off any *Options +// message (e.g. MessageOptions, FieldOptions). // +// On protobuf runtimes < v3.15.0 a static-initialization-order quirk can +// leave the extension payload parked in the options' unknown_fields, so +// `options.GetExtension(id)` returns the default instance. We work around +// it by serializing and reparsing into a fresh OptionsT at call time, by +// which point all extension registrations have run. // Reference: https://github.com/protocolbuffers/protobuf/releases/tag/v3.15.0 template -#if GOOGLE_PROTOBUF_VERSION < 3015000 -// Returns by value (a freshly reparsed OptionsT extension copy). inline auto GetExtension(const OptionsT& options, const ExtT& id) -> typename std::decay::type { +#if GOOGLE_PROTOBUF_VERSION < 3015000 OptionsT reparsed; std::string buf; options.SerializeToString(&buf); reparsed.ParseFromString(buf); return reparsed.GetExtension(id); -} #else -// Returns by const-reference (zero-copy passthrough). -inline auto GetExtension(const OptionsT& options, const ExtT& id) - -> decltype(options.GetExtension(id)) { return options.GetExtension(id); -} #endif +} #if TABLEAU_PB_LOG_LEGACY // ProtobufLogHandler redirects protobuf internal logs to tableau logger. diff --git a/test/cpp-tableau-loader/src/protoconf/util.pc.cc b/test/cpp-tableau-loader/src/protoconf/util.pc.cc index 46487b2..7146119 100644 --- a/test/cpp-tableau-loader/src/protoconf/util.pc.cc +++ b/test/cpp-tableau-loader/src/protoconf/util.pc.cc @@ -69,6 +69,21 @@ const std::string& Format2Ext(Format fmt) { #undef GetMessage #endif +// MapKeyFd returns the key FieldDescriptor of a map-entry message. +// +// ``Descriptor::map_key()`` is a convenience wrapper added in protobuf +// v3.12.0; on older runtimes we fall back to ``field(0)``, which is +// equivalent because map-entry messages always synthesize the key as +// field 0 (and value as field 1) per the proto3 wire-format contract. +inline const google::protobuf::FieldDescriptor* MapKeyFd( + const google::protobuf::Descriptor* map_entry) { +#if GOOGLE_PROTOBUF_VERSION < 3012000 + return map_entry->field(0); +#else + return map_entry->map_key(); +#endif +} + // PatchMessage patches src into dst, which must be a message with the same descriptor. // // # Default PatchMessage mechanism @@ -111,7 +126,7 @@ bool PatchMessage(google::protobuf::Message& dst, const google::protobuf::Messag // Iterates over every populated field. for (auto fd : fields) { - const tableau::FieldOptions& opts = util::GetExtension(fd->options(), tableau::field); + const tableau::FieldOptions& opts = GetExtension(fd->options(), tableau::field); tableau::Patch patch = opts.prop().patch(); if (patch == tableau::PATCH_REPLACE) { dst_reflection->ClearField(&dst, fd); @@ -119,7 +134,7 @@ bool PatchMessage(google::protobuf::Message& dst, const google::protobuf::Messag if (fd->is_map()) { // Reference: // https://github.com/protocolbuffers/protobuf/blob/95ef4134d3f65237b7adfb66e5e7aa10fcfa1fa3/src/google/protobuf/map_field.cc#L500 - auto key_fd = fd->message_type()->map_key(); + auto key_fd = MapKeyFd(fd->message_type()); int src_count = src_reflection->FieldSize(src, fd); int dst_count = dst_reflection->FieldSize(dst, fd); switch (key_fd->cpp_type()) { diff --git a/test/cpp-tableau-loader/src/protoconf/util.pc.h b/test/cpp-tableau-loader/src/protoconf/util.pc.h index dc55d69..8ef011c 100644 --- a/test/cpp-tableau-loader/src/protoconf/util.pc.h +++ b/test/cpp-tableau-loader/src/protoconf/util.pc.h @@ -75,47 +75,28 @@ const std::string& Format2Ext(Format fmt); // PatchMessage patches src into dst, which must be a message with the same descriptor. bool PatchMessage(google::protobuf::Message& dst, const google::protobuf::Message& src); -// GetExtension reads a typed custom-option extension off any *Options message -// (e.g. MessageOptions, FieldOptions). It transparently works around a -// protobuf static-initialization-order quirk present on runtimes < v3.15.0: -// when a .pb.cc that *defines* an extension (e.g. tableau.pb.cc -> -// `tableau::worksheet`) is initialized after the descriptors that *carry* it -// have already been built, the custom-option payload ends up parked in the -// options message's unknown_fields, and `options.GetExtension(id)` then -// returns the default instance. -// -// The fix is to round-trip the options bytes through a fresh OptionsT instance -// at call time: by then all static initializers (including the extension's -// generated registration via `descriptor_table_*` / `dynamic_init_dummy_*`) -// have run, so the second parse resolves the extension correctly. -// -// Verified by checkout-build-and-test against protoc/libprotobuf at the same -// tag: v3.14.0 reproduces (PatchTest fails when this fallback is bypassed), -// v3.15.0 is clean. The v3.15.0 release notes don't call out this specific -// symptom, but two C++ entries plausibly cover it by reshaping static -// initialization: "Constant initialize the global message instances" and -// "Use init_seg in MSVC to push initialization to an earlier phase". On -// runtimes >= 3.15.0 we just forward to the native call with no overhead. +// GetExtension reads a typed custom-option extension off any *Options +// message (e.g. MessageOptions, FieldOptions). // +// On protobuf runtimes < v3.15.0 a static-initialization-order quirk can +// leave the extension payload parked in the options' unknown_fields, so +// `options.GetExtension(id)` returns the default instance. We work around +// it by serializing and reparsing into a fresh OptionsT at call time, by +// which point all extension registrations have run. // Reference: https://github.com/protocolbuffers/protobuf/releases/tag/v3.15.0 template -#if GOOGLE_PROTOBUF_VERSION < 3015000 -// Returns by value (a freshly reparsed OptionsT extension copy). inline auto GetExtension(const OptionsT& options, const ExtT& id) -> typename std::decay::type { +#if GOOGLE_PROTOBUF_VERSION < 3015000 OptionsT reparsed; std::string buf; options.SerializeToString(&buf); reparsed.ParseFromString(buf); return reparsed.GetExtension(id); -} #else -// Returns by const-reference (zero-copy passthrough). -inline auto GetExtension(const OptionsT& options, const ExtT& id) - -> decltype(options.GetExtension(id)) { return options.GetExtension(id); -} #endif +} #if TABLEAU_PB_LOG_LEGACY // ProtobufLogHandler redirects protobuf internal logs to tableau logger. From 65b2f144c4964c358450df6b22297997ffd90127 Mon Sep 17 00:00:00 2001 From: Kybxd <627940450@qq.com> Date: Tue, 9 Jun 2026 11:38:20 +0800 Subject: [PATCH 64/66] refactor(helper): align identifier keyword escaping with protobuf sources - go: drop the hand-maintained golangKeywords map and detect Go keywords via go/token (token.Lookup(...).IsKeyword()), matching protoc-gen-go's GoSanitized; keep the loader-specific "x" receiver-name guard. Also removes a stale bogus "pi" entry. - cpp: align cppKeywords with protoc's kKeywordList and point the reference at protobuf's compiler/cpp/helpers.cc. - csharp: clarify that protobuf's C# generator keeps no language-keyword list (PascalCase + reserved member names), so the loader maintains the C# keyword set itself; reference csharp_helpers.cc GetPropertyName and the C# spec. --- .../helper/keyword.go | 34 ++++++----- .../helper/keyword.go | 14 ++++- .../helper/keyword.go | 60 +++++-------------- 3 files changed, 47 insertions(+), 61 deletions(-) diff --git a/cmd/protoc-gen-cpp-tableau-loader/helper/keyword.go b/cmd/protoc-gen-cpp-tableau-loader/helper/keyword.go index 772b6f8..4119dbe 100644 --- a/cmd/protoc-gen-cpp-tableau-loader/helper/keyword.go +++ b/cmd/protoc-gen-cpp-tableau-loader/helper/keyword.go @@ -29,19 +29,23 @@ func escapeIdentifier(str string) string { return str } +// cppKeywords mirrors protoc's C++ codegen reserved-identifier list +// (kKeywordList). Keeping them in sync ensures we escape identifiers exactly +// the way protoc does (append a trailing underscore on collision), so the +// names we emit match the generated *.pb.h symbols. +// // Ref: // -// https://en.cppreference.com/w/cpp/keyword +// https://github.com/protocolbuffers/protobuf/blob/main/src/google/protobuf/compiler/cpp/helpers.cc func init() { cppKeywords = map[string]bool{ + "NULL": true, "alignas": true, "alignof": true, "and": true, "and_eq": true, "asm": true, - "atomic_cancel": true, - "atomic_commit": true, - "atomic_noexcept": true, + "assert": true, "auto": true, "bitand": true, "bitor": true, @@ -50,21 +54,12 @@ func init() { "case": true, "catch": true, "char": true, - "char8_t": true, - "char16_t": true, - "char32_t": true, "class": true, "compl": true, - "concept": true, "const": true, - "consteval": true, "constexpr": true, - "constinit": true, "const_cast": true, "continue": true, - "co_await": true, - "co_return": true, - "co_yield": true, "decltype": true, "default": true, "delete": true, @@ -99,9 +94,7 @@ func init() { "protected": true, "public": true, "register": true, - "reflexpr": true, "reinterpret_cast": true, - "requires": true, "return": true, "short": true, "signed": true, @@ -111,7 +104,6 @@ func init() { "static_cast": true, "struct": true, "switch": true, - "synchronized": true, "template": true, "this": true, "thread_local": true, @@ -131,5 +123,15 @@ func init() { "while": true, "xor": true, "xor_eq": true, + "char8_t": true, + "char16_t": true, + "char32_t": true, + "concept": true, + "consteval": true, + "constinit": true, + "co_await": true, + "co_return": true, + "co_yield": true, + "requires": true, } } diff --git a/cmd/protoc-gen-csharp-tableau-loader/helper/keyword.go b/cmd/protoc-gen-csharp-tableau-loader/helper/keyword.go index d69e738..807579d 100644 --- a/cmd/protoc-gen-csharp-tableau-loader/helper/keyword.go +++ b/cmd/protoc-gen-csharp-tableau-loader/helper/keyword.go @@ -43,7 +43,19 @@ func escapeIdentifier(str string) string { // csharpKeywords is the set of C# reserved keywords used by escapeIdentifier // to detect and escape naming conflicts. // -// Ref: https://learn.microsoft.com/en-us/dotnet/csharp/language-reference/keywords +// Note on protobuf parity: protobuf's own C# generator does NOT keep a C# +// language-keyword list, because it emits PascalCase property names that never +// collide with the (lower-case) C# keywords; it only guards against its own +// generated member names (e.g. Types/Descriptor/Equals/Parser...) and the +// containing type name, appending "_" on collision (see GetPropertyName in +// csharp_helpers.cc). This loader instead emits lowerCamelCase parameter +// identifiers, which CAN collide with C# keywords, so it maintains the language +// keyword set itself. The authoritative source for that set is the C# spec. +// +// Ref: +// +// https://github.com/protocolbuffers/protobuf/blob/main/src/google/protobuf/compiler/csharp/csharp_helpers.cc (GetPropertyName) +// https://learn.microsoft.com/en-us/dotnet/csharp/language-reference/keywords func init() { csharpKeywords = map[string]bool{ "abstract": true, diff --git a/cmd/protoc-gen-go-tableau-loader/helper/keyword.go b/cmd/protoc-gen-go-tableau-loader/helper/keyword.go index e71d22f..e0d7377 100644 --- a/cmd/protoc-gen-go-tableau-loader/helper/keyword.go +++ b/cmd/protoc-gen-go-tableau-loader/helper/keyword.go @@ -1,14 +1,22 @@ package helper import ( + "go/token" "strings" "unicode" "github.com/iancoleman/strcase" ) -var golangKeywords map[string]bool - +// escapeIdentifier converts a raw string into a valid Go lowerCamelCase +// identifier, escaping Go reserved words the same way protoc-gen-go's +// GoSanitized does: it consults go/token's keyword table via +// token.Lookup(...).IsKeyword(). On collision a trailing "_" is appended. +// +// Ref: +// +// https://github.com/protocolbuffers/protobuf-go/blob/master/internal/strs/strings.go (GoSanitized) +// https://pkg.go.dev/go/token#Lookup func escapeIdentifier(str string) string { // Filter invalid runes var result strings.Builder @@ -24,49 +32,13 @@ func escapeIdentifier(str string) string { if len(str) != 0 && unicode.IsDigit(rune(str[0])) { str = "_" + str } - // Avoid go keywords - if _, ok := golangKeywords[str]; ok { + // Avoid Go keywords, plus the loader-specific reserved name "x": the + // generated Go code uses "x" as the method receiver name (e.g., + // `func (x *FooConf) FindIndex1(...)`). If a proto field named "X" is used as + // an index key, escapeIdentifier converts it to "x" (lowerCamelCase), which + // would shadow the receiver and cause a compile error, so it is escaped too. + if token.Lookup(str).IsKeyword() || str == "x" { return str + "_" } return str } - -// Ref: -// -// https://go.dev/ref/spec#Keywords -func init() { - golangKeywords = map[string]bool{ - "break": true, - "case": true, - "chan": true, - "const": true, - "continue": true, - "default": true, - "defer": true, - "pi": true, - "else": true, - "fallthrough": true, - "for": true, - "func": true, - "go": true, - "goto": true, - "if": true, - "import": true, - "interface": true, - "map": true, - "package": true, - "range": true, - "return": true, - "select": true, - "struct": true, - "switch": true, - "type": true, - "var": true, - // "x" is treated as a keyword because the generated Go code uses "x" as the - // method receiver name (e.g., `func (x *FooConf) FindIndex1(...)`). If a proto - // field named "X" is used as an index key, escapeIdentifier converts it to "x" - // (lowerCamelCase), which would shadow the receiver and cause a compile error. - // By treating "x" as a keyword, it gets escaped to "x_" to avoid the conflict. - "x": true, - } -} From 4d07b6ebdb918e051b2a436f0d55068373d9bed8 Mon Sep 17 00:00:00 2001 From: wenchy Date: Tue, 9 Jun 2026 11:12:06 +0800 Subject: [PATCH 65/66] chore: prefer python3 over python in commands and docs Update every `python make.py ...` example across docs, CI workflows, and make.py's own usage docstring + error messages to use `python3`, matching the file's `#!/usr/bin/env python3` shebang and the Python >= 3.10 requirement called out in the module docstring. Co-Authored-By: Claude Opus 4.7 (1M context) --- .devcontainer/README.md | 8 +++--- .devcontainer/devcontainer.json | 4 +-- .github/workflows/testing-cpp.yml | 2 +- .github/workflows/testing-csharp.yml | 2 +- .github/workflows/testing-go.yml | 2 +- CLAUDE.md | 42 ++++++++++++++-------------- README.md | 36 ++++++++++++------------ make.py | 16 +++++------ test_make.py | 4 +-- 9 files changed, 58 insertions(+), 58 deletions(-) diff --git a/.devcontainer/README.md b/.devcontainer/README.md index 871c905..568d4a2 100644 --- a/.devcontainer/README.md +++ b/.devcontainer/README.md @@ -17,7 +17,7 @@ code . Then **Dev Containers: Reopen in Container** from the command palette. First build is one-time ~25 min (vcpkg compiles protobuf); reopens are near-instant. -Inside the container, `python make.py setup` is a no-op. Use `python make.py test --lang ` for any language. +Inside the container, `python3 make.py setup` is a no-op. Use `python3 make.py test --lang ` for any language. ## Pin a different protobuf version @@ -55,7 +55,7 @@ Build context is `.devcontainer/`, so `COPY versions.env …` resolves directly. Consumers: Dockerfile (sourced as a shell file), [`make.py`](../make.py) (`Versions.load()`), and `.github/actions/load-versions` (exports to `$GITHUB_ENV`). -Use `python make.py env` for a JSON dump of resolved values. +Use `python3 make.py env` for a JSON dump of resolved values. ## Troubleshooting @@ -65,8 +65,8 @@ Symptoms: - C++: `fatal error: google/protobuf/generated_message_table_driven.h: No such file or directory` - C#: hundreds of `error CS0101: The namespace already contains a definition for ...` -The host workspace has gitignored `*.pb.*` from a previous protobuf version that `git pull` didn't remove. Wipe with `python make.py clean --lang cpp` (or `--lang csharp`), then rerun the test. +The host workspace has gitignored `*.pb.*` from a previous protobuf version that `git pull` didn't remove. Wipe with `python3 make.py clean --lang cpp` (or `--lang csharp`), then rerun the test. ## Falling back -No Docker? Use [`make.py`](../make.py) directly: `python make.py setup --lang all` then `python make.py test --lang `. Works on macOS / Linux / Windows native. +No Docker? Use [`make.py`](../make.py) directly: `python3 make.py setup --lang all` then `python3 make.py test --lang `. Works on macOS / Linux / Windows native. diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 02b1c7d..40c609e 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -2,7 +2,7 @@ // The loader's only devcontainer. Multi-arch: builds linux/amd64 on // x64 hosts, linux/arm64 on Apple Silicon and Windows-on-ARM. Use // this on every host that can run Docker Desktop or Docker Engine. - // Native users (no Docker) bootstrap via `python make.py setup` at + // Native users (no Docker) bootstrap via `python3 make.py setup` at // the repo root. // // See ./versions.env for all toolchain pins. @@ -64,4 +64,4 @@ // (e.g. gopls auto-install from the Go extension) can write into GOPATH. // Idempotent and ~50ms on an already-correct tree. "postStartCommand": "sudo chown -R vscode:vscode /home/vscode/go" -} +} \ No newline at end of file diff --git a/.github/workflows/testing-cpp.yml b/.github/workflows/testing-cpp.yml index 91800ff..afc1ab3 100644 --- a/.github/workflows/testing-cpp.yml +++ b/.github/workflows/testing-cpp.yml @@ -161,7 +161,7 @@ jobs: # already rendered into vcpkg.json above, instead of auto-resolving # from the vcpkg checkout (which is shallow on the CI runner). run: > - python make.py test --lang cpp + python3 make.py test --lang cpp --protobuf-version ${{ matrix.config.protobuf-version }} --vcpkg-baseline ${{ matrix.config.vcpkg-commit }} --triplet ${{ matrix.triplet }} diff --git a/.github/workflows/testing-csharp.yml b/.github/workflows/testing-csharp.yml index db8bd22..eef6e88 100644 --- a/.github/workflows/testing-csharp.yml +++ b/.github/workflows/testing-csharp.yml @@ -65,4 +65,4 @@ jobs: python-version: '3.12' - name: Test - run: python make.py test --lang csharp + run: python3 make.py test --lang csharp diff --git a/.github/workflows/testing-go.yml b/.github/workflows/testing-go.yml index 4b65363..4c6df0f 100644 --- a/.github/workflows/testing-go.yml +++ b/.github/workflows/testing-go.yml @@ -60,7 +60,7 @@ jobs: # MSVC available; -race needs cgo + a C compiler). On a fresh # Windows dev machine without MSVC, make.py's default is --no-race; # CI overrides that. - run: python make.py test --lang go --race --coverage + run: python3 make.py test --lang go --race --coverage - name: Upload coverage reports to Codecov uses: codecov/codecov-action@v5 diff --git a/CLAUDE.md b/CLAUDE.md index 21b406e..25bd1e3 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -21,13 +21,13 @@ Build/test happens **per language** under `test/-tableau-loader/`. The rep The single cross-platform driver is **`make.py`** (Python 3.10+, stdlib only). It works on Windows, macOS, Linux, and inside the devcontainer, and is what CI calls. ```sh -python make.py setup --lang all # one-time host toolchain install (no-op in container) -python make.py generate --lang go # buf generate .. -python make.py build --lang cpp -python make.py test --lang go -python make.py test --lang cpp -k HubTest.Load -python make.py env # diagnostic JSON -python make.py --version +python3 make.py setup --lang all # one-time host toolchain install (no-op in container) +python3 make.py generate --lang go # buf generate .. +python3 make.py build --lang cpp +python3 make.py test --lang go +python3 make.py test --lang cpp -k HubTest.Load +python3 make.py env # diagnostic JSON +python3 make.py --version ``` C++ wipes `test/cpp-tableau-loader/{build,src/tableau,src/protoconf}` before regenerating (gitignored `*.pb.*` shadows fresh codegen). `--no-clean` skips it. A leftover `vcpkg.json` from a previous `--protobuf-version` run is auto-removed in classic mode so cmake doesn't accidentally re-enter manifest mode. @@ -39,11 +39,11 @@ On Windows, `make.py` wraps every C++ subprocess in `cmd /c "call vcvarsall.bat ### Dev container - `.devcontainer/` → **Dev Containers: Reopen in Container**. Ubuntu 24.04 + all toolchains pinned. First build ~25 min; reopens instant. -- Inside: `python make.py setup` is a no-op. `python make.py test --lang ` works for all languages — Dockerfile presets `CMAKE_PREFIX_PATH=/opt/vcpkg/active`. +- Inside: `python3 make.py setup` is a no-op. `python3 make.py test --lang ` works for all languages — Dockerfile presets `CMAKE_PREFIX_PATH=/opt/vcpkg/active`. - Override protobuf version: `LOADER_DEFAULT_VARIANT=legacy-v3 code .` then **Rebuild Container** (switches both protobuf and vcpkg baseline atomically). Variants are declared as `_PROTOBUF_VERSION` + `_VCPKG_BASELINE_COMMIT` pairs in `.devcontainer/versions.env`. For a surgical override of just the protobuf version, `LOADER_PROTOBUF_VERSION=X.Y.Z` still works. - Single source of truth for all toolchain versions: **`.devcontainer/versions.env`**. -CI primary tests (`testing-{cpp,go,csharp}.yml`) use `lukka/run-vcpkg` for cached vcpkg installs + `python make.py test --lang ` for build/test. `devcontainer-smoke.yml` builds the image on `.devcontainer/**` PRs (amd64 + arm64). `testing-make.yml` runs the make.py unit + dry-run regression suite on every push. +CI primary tests (`testing-{cpp,go,csharp}.yml`) use `lukka/run-vcpkg` for cached vcpkg installs + `python3 make.py test --lang ` for build/test. `devcontainer-smoke.yml` builds the image on `.devcontainer/**` PRs (amd64 + arm64). `testing-make.yml` runs the make.py unit + dry-run regression suite on every push. ### Plugin development (Go module at repo root) @@ -60,22 +60,22 @@ Plugins are invoked through `buf generate` from a test directory (`buf.gen.yaml` ```sh # Go -python make.py test --lang go # full -python make.py test --lang go -k Test_ActivityConf_OrderedMap # filter -python make.py test --lang go --smoke # plugin-only `go vet` (devcontainer-smoke) -python make.py test --lang go --coverage # CI: -coverprofile=coverage.txt -covermode=atomic -python make.py test --lang go --race # opt in to -race (default off on Windows; needs cgo+MSVC) +python3 make.py test --lang go # full +python3 make.py test --lang go -k Test_ActivityConf_OrderedMap # filter +python3 make.py test --lang go --smoke # plugin-only `go vet` (devcontainer-smoke) +python3 make.py test --lang go --coverage # CI: -coverprofile=coverage.txt -covermode=atomic +python3 make.py test --lang go --race # opt in to -race (default off on Windows; needs cgo+MSVC) # C++ (requires matching protoc + libprotobuf — protobuf v22+ enforces gencode/runtime check) -python make.py test --lang cpp # full -python make.py test --lang cpp -k HubTest.Load # filter -python make.py test --lang cpp --cxx-std 20 # C++20 -python make.py test --lang cpp --cxx-compiler clang # clang++ -python make.py test --lang cpp --protobuf-version 3.21.12 # legacy v3 (vcpkg manifest mode) +python3 make.py test --lang cpp # full +python3 make.py test --lang cpp -k HubTest.Load # filter +python3 make.py test --lang cpp --cxx-std 20 # C++20 +python3 make.py test --lang cpp --cxx-compiler clang # clang++ +python3 make.py test --lang cpp --protobuf-version 3.21.12 # legacy v3 (vcpkg manifest mode) # C# -python make.py test --lang csharp # full -python make.py test --lang csharp -k HubTest.Load # FullyQualifiedName~HubTest.Load +python3 make.py test --lang csharp # full +python3 make.py test --lang csharp -k HubTest.Load # FullyQualifiedName~HubTest.Load ``` GoogleTest is fetched via CMake `FetchContent` — no manual install. diff --git a/README.md b/README.md index 0857ad8..ade5820 100644 --- a/README.md +++ b/README.md @@ -13,16 +13,16 @@ The official config loader for [Tableau](https://github.com/tableauio/tableau). Use [`make.py`](./make.py) (Python 3.10+, stdlib only): ```sh -python make.py setup --lang all # one-time host toolchain install -python make.py test --lang go # Go -python make.py test --lang cpp # C++ -python make.py test --lang csharp # C# -python make.py test --lang ts # TypeScript (experimental) +python3 make.py setup --lang all # one-time host toolchain install +python3 make.py test --lang go # Go +python3 make.py test --lang cpp # C++ +python3 make.py test --lang csharp # C# +python3 make.py test --lang ts # TypeScript (experimental) ``` Recommended environment: [devcontainer](./.devcontainer/) (open in VS Code → **Dev Containers: Reopen in Container**). Inside the container, `setup` is a no-op. -Native hosts: `python make.py setup` installs everything pinned to [`./.devcontainer/versions.env`](./.devcontainer/versions.env) — the same versions CI and the devcontainer use. Toolchain layout per host: +Native hosts: `python3 make.py setup` installs everything pinned to [`./.devcontainer/versions.env`](./.devcontainer/versions.env) — the same versions CI and the devcontainer use. Toolchain layout per host: - **Go** — official tarball from go.dev to `~/.local/go/` (Linux/macOS) or winget (Windows). - **buf** — pinned binary from GitHub releases to `~/.local/bin/` (Linux/macOS) or `%LOCALAPPDATA%\buf\bin\` (Windows). @@ -34,15 +34,15 @@ On Windows, run setup from **cmd as Administrator** the first time. Subsequent c ## Commands ```sh -python make.py setup [--lang go|cpp|csharp|ts|all] -python make.py generate --lang go|cpp|csharp|ts -python make.py build --lang go|cpp|csharp|ts [--cxx-std 17|20] [--cxx-compiler msvc|clang|gcc] +python3 make.py setup [--lang go|cpp|csharp|ts|all] +python3 make.py generate --lang go|cpp|csharp|ts +python3 make.py build --lang go|cpp|csharp|ts [--cxx-std 17|20] [--cxx-compiler msvc|clang|gcc] [--protobuf-version ] [--triplet ] -python make.py test --lang go|cpp|csharp|ts [-k ] [--smoke] [--coverage] [--no-race] +python3 make.py test --lang go|cpp|csharp|ts [-k ] [--smoke] [--coverage] [--no-race] (+ all build flags) -python make.py clean [--lang ...] [--all] -python make.py env # diagnostic JSON -python make.py --version +python3 make.py clean [--lang ...] [--all] +python3 make.py env # diagnostic JSON +python3 make.py --version ``` Global flags: `--verbose / -v`, `--dry-run`, `--cwd `. @@ -50,11 +50,11 @@ Global flags: `--verbose / -v`, `--dry-run`, `--cwd `. Examples: ```sh -python make.py test --lang go -k Test_ActivityConf_OrderedMap -python make.py test --lang go --race # opt in (Windows default is off; needs cgo+MSVC) -python make.py test --lang cpp --protobuf-version 3.21.12 -python make.py test --lang csharp -k HubTest.Load -python make.py test --lang cpp --no-clean # skip pre-build wipe +python3 make.py test --lang go -k Test_ActivityConf_OrderedMap +python3 make.py test --lang go --race # opt in (Windows default is off; needs cgo+MSVC) +python3 make.py test --lang cpp --protobuf-version 3.21.12 +python3 make.py test --lang csharp -k HubTest.Load +python3 make.py test --lang cpp --no-clean # skip pre-build wipe ``` The C++ flow wipes `test/cpp-tableau-loader/{build,src/tableau,src/protoconf}` by default (gitignored `*.pb.*` from a prior protobuf version shadows fresh codegen). diff --git a/make.py b/make.py index 61e691b..548941e 100644 --- a/make.py +++ b/make.py @@ -7,13 +7,13 @@ native Windows, macOS, Linux, and inside the devcontainer. Usage (high level): - python make.py setup [--lang go|cpp|csharp|all] [--dry-run] - python make.py generate --lang go|cpp|csharp - python make.py build --lang go|cpp|csharp [build flags] - python make.py test --lang go|cpp|csharp [build flags] [-k FILTER] [--smoke] - python make.py clean [--lang ...] [--all] - python make.py env - python make.py --version + python3 make.py setup [--lang go|cpp|csharp|all] [--dry-run] + python3 make.py generate --lang go|cpp|csharp + python3 make.py build --lang go|cpp|csharp [build flags] + python3 make.py test --lang go|cpp|csharp [build flags] [-k FILTER] [--smoke] + python3 make.py clean [--lang ...] [--all] + python3 make.py env + python3 make.py --version Standard flags (apply to every subcommand): --verbose / -v echo every subprocess @@ -1353,7 +1353,7 @@ def _cpp_build_or_test(args, ctx: "Context", run_tests: bool) -> int: else: print( "[error] --protobuf-version requires VCPKG_ROOT to be set " - "(run `python make.py setup --lang cpp` first, or set " + "(run `python3 make.py setup --lang cpp` first, or set " "VCPKG_ROOT in your environment).", file=sys.stderr, ) diff --git a/test_make.py b/test_make.py index 531bb6a..3f1d4d0 100644 --- a/test_make.py +++ b/test_make.py @@ -6,7 +6,7 @@ (Versions, Platform, _winquote, Runner, etc.). No subprocess, no network, no filesystem mutation outside pytest's tmp_path. -2. **Dry-run snapshot tests** — spawn `python make.py --dry-run ` +2. **Dry-run snapshot tests** — spawn `python3 make.py --dry-run ` and assert the printed command sequence. Canonical contract test for "the orchestrator still emits the right cmake/ctest/buf calls." @@ -495,7 +495,7 @@ def test_csharp(self): def run_make(*args: str, cwd: Path = REPO_ROOT) -> subprocess.CompletedProcess: - """Spawn `python make.py ` and capture stdout+stderr.""" + """Spawn `python3 make.py ` and capture stdout+stderr.""" return subprocess.run( [sys.executable, str(MAKE_PY), *args], cwd=str(cwd), From 11e04299fa390875a855bbd55037e2a80b9bfd0d Mon Sep 17 00:00:00 2001 From: wenchy Date: Tue, 9 Jun 2026 12:06:27 +0800 Subject: [PATCH 66/66] fix(setup): always install Go on Windows, pin Go and .NET from versions.env Windows setup previously gated the Go install on `"go" in langs` and hardcoded the winget ids `GoLang.Go.1.24` / `Microsoft.DotNet.SDK.8`. Two issues: 1. Every `buf generate` invokes the Go protoc plugins via `go run ../../cmd/protoc-gen-...` (per `buf.gen.yaml`), so Go is required regardless of which language is being built. With the gate in place, `setup --lang cpp` and `setup --lang csharp` left a Windows host without Go and every codegen step failed. 2. The hardcoded winget ids drifted from versions.env on bumps, breaking the "single source of truth" contract macOS/Linux already satisfy (their tarball/winget paths read ctx.versions.go_version / .dotnet_version). Drop the gate and derive both ids from versions.env: GoLang.Go.. from GO_VERSION Microsoft.DotNet.SDK. from DOTNET_VERSION Add `TestSetupWindows` covering both invariants. Tests use synthetic GO_VERSION=1.99.7 / DOTNET_VERSION=9.0 so they catch a regression even though the real versions.env (1.24.0 / 8.0) happens to match the old hardcoded values. Co-Authored-By: Claude Opus 4.7 (1M context) --- make.py | 30 +++++++++++++++++--- test_make.py | 79 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 105 insertions(+), 4 deletions(-) diff --git a/make.py b/make.py index 548941e..ac8989b 100644 --- a/make.py +++ b/make.py @@ -1114,14 +1114,36 @@ def _setup_windows(langs: list[str], ctx: "Context", skip_vcpkg: bool) -> int: if "cpp" in langs and not skip_vcpkg: _setup_vcpkg(ctx, cache) - # Optional: Go / .NET / Node - if "go" in langs and _which("go") is None: + # Step 6: Go (always required — every buf-generate invokes the Go + # protoc plugins via `go run ../../cmd/protoc-gen-...`, so Go must be + # on PATH regardless of which language is being built. Mirrors the + # macOS/Linux setup paths which install Go for any --lang.). + if _which("go") is None: + go_ver = ctx.versions.go_version or "1.24.0" + # winget package id format is `GoLang.Go..`; the patch + # level isn't part of the id (the package itself tracks it). + go_major_minor = ".".join(go_ver.split(".")[:2]) ctx.runner.run( - ["winget", "install", "--id", "GoLang.Go.1.24", "-e"], check=False + ["winget", "install", "--id", f"GoLang.Go.{go_major_minor}", "-e"], + check=False, ) + else: + print("[info] go already on PATH.") + + # Step 7: .NET SDK (only when csharp tests are requested). if "csharp" in langs and _which("dotnet") is None: + dotnet_ver = ctx.versions.dotnet_version or "8.0" + # winget package id format is `Microsoft.DotNet.SDK.`. + dotnet_major = dotnet_ver.split(".")[0] ctx.runner.run( - ["winget", "install", "--id", "Microsoft.DotNet.SDK.8", "-e"], check=False + [ + "winget", + "install", + "--id", + f"Microsoft.DotNet.SDK.{dotnet_major}", + "-e", + ], + check=False, ) save_loader_env(cache, ctx.runner) diff --git a/test_make.py b/test_make.py index 3f1d4d0..dfaad5c 100644 --- a/test_make.py +++ b/test_make.py @@ -928,3 +928,82 @@ def fake_detect(cls): assert rc == 0 finally: make.Platform.detect = original_detect + + +# --------------------------------------------------------------------------- +# Unit: Windows setup +# --------------------------------------------------------------------------- + + +class TestSetupWindows: + """Windows-specific setup invariants. Runs on every host because + --dry-run intercepts the subprocess; we only assert against the printed + command sequence.""" + + def _windows_ctx(self, monkeypatch, versions=None): + """Pretend we're a clean Windows host: nothing on PATH, no MSVC.""" + monkeypatch.setattr( + make.Platform, + "detect", + classmethod( + lambda cls: make.Platform( + sys_platform="win32", machine="amd64", in_devcontainer=False + ) + ), + ) + monkeypatch.setattr(make, "_which", lambda name: None) + # locate_vcvarsall already returns None off-Windows; explicit for clarity. + monkeypatch.setattr(make, "locate_vcvarsall", lambda: None) + return make.Context( + repo_root=REPO_ROOT, + versions=versions or make.Versions.load(REPO_ROOT), + platform=make.Platform.detect(), + runner=make.Runner(verbose=False, dry_run=True), + ) + + def test_installs_go_for_cpp_lang(self, monkeypatch, capsys): + """Regression: --lang cpp on Windows must still install Go. + buf-generate runs the Go protoc plugins via `go run`, so without + Go on PATH every codegen step fails — for every target language.""" + ctx = self._windows_ctx(monkeypatch) + args = type("Args", (), {"lang": "cpp", "skip_vcpkg": True})() + rc = make.cmd_setup(args, ctx) + assert rc == 0 + out = capsys.readouterr().out + assert "GoLang.Go" in out, "--lang cpp on Windows must still install Go" + + def test_installs_go_for_csharp_lang(self, monkeypatch, capsys): + """Same as above but for --lang csharp.""" + ctx = self._windows_ctx(monkeypatch) + args = type("Args", (), {"lang": "csharp", "skip_vcpkg": True})() + rc = make.cmd_setup(args, ctx) + assert rc == 0 + out = capsys.readouterr().out + assert "GoLang.Go" in out, "--lang csharp on Windows must still install Go" + + def test_go_winget_id_uses_versions_env_major_minor(self, monkeypatch, capsys): + """winget id must derive from GO_VERSION (not hardcoded) so bumping + versions.env alone is enough to bump the host pin — matches the + macOS/Linux tarball flow which already reads ctx.versions.go_version.""" + versions = make.Versions(raw={"GO_VERSION": "1.99.7", "DOTNET_VERSION": "8.0"}) + ctx = self._windows_ctx(monkeypatch, versions=versions) + args = type("Args", (), {"lang": "go", "skip_vcpkg": True})() + rc = make.cmd_setup(args, ctx) + assert rc == 0 + out = capsys.readouterr().out + # GoLang.Go winget id is `.` — patch level is dropped. + assert "GoLang.Go.1.99" in out, f"Expected pinned 1.99 winget id; got: {out}" + # Hardcoded 1.24 from the old code path must NOT leak through. + assert "GoLang.Go.1.24" not in out + + def test_dotnet_winget_id_uses_versions_env_major(self, monkeypatch, capsys): + """winget .NET SDK id format is `Microsoft.DotNet.SDK.` — + derive from DOTNET_VERSION instead of hardcoding `.8`.""" + versions = make.Versions(raw={"GO_VERSION": "1.24.0", "DOTNET_VERSION": "9.0"}) + ctx = self._windows_ctx(monkeypatch, versions=versions) + args = type("Args", (), {"lang": "csharp", "skip_vcpkg": True})() + rc = make.cmd_setup(args, ctx) + assert rc == 0 + out = capsys.readouterr().out + assert "Microsoft.DotNet.SDK.9" in out + assert "Microsoft.DotNet.SDK.8" not in out