From 3bac2362b777843ea9d1ac24c9b53f4de6a75cc8 Mon Sep 17 00:00:00 2001 From: Felipe Fujiy Pessoto Date: Sat, 22 Aug 2026 01:28:36 +0000 Subject: [PATCH 1/4] [GLUTEN-12850][VL] Support multiple Dev Container configurations - Keep the dynamic-link environment as the default for daily development. - Add a static-link configuration for portable jars and vcpkg reproduction. - Share post-create setup while printing variant-specific build instructions. - Persist separate ccache, Maven and vcpkg binary-cache volumes. - Select the arm64 vcpkg triplet automatically in the static environment. - Document configuration selection, limitations and safe switching. - Make the JDK 17 switch explicit for Spark 4.1 builds so Maven does not run the Java 17 release target under JDK 8. Generated-by: GitHub Copilot CLI gpt-5.6-sol Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .devcontainer/devcontainer.json | 3 +- .devcontainer/post-create.sh | 95 +++++++- .devcontainer/velox-static/devcontainer.json | 64 ++++++ docs/developers/NewToGluten.md | 6 +- docs/developers/dev-container.md | 229 +++++++++++-------- 5 files changed, 289 insertions(+), 108 deletions(-) create mode 100644 .devcontainer/velox-static/devcontainer.json diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 81b75291cdb..a709ed556d4 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -1,4 +1,4 @@ -// Dev Container for the Gluten Velox backend. +// Default Dev Container for daily Gluten Velox development. // See docs/developers/dev-container.md for the rationale behind these settings // and for the build commands to run once the container is up. { @@ -19,6 +19,7 @@ "containerEnv": { "CCACHE_DIR": "/root/.ccache", "CCACHE_MAXSIZE": "10G", + "GLUTEN_DEV_CONTAINER_VARIANT": "velox-dynamic", // folly, gflags and glog are shared libraries in this image; a Velox // dependency build that disagrees fails to link. "VELOX_BUILD_SHARED": "ON" diff --git a/.devcontainer/post-create.sh b/.devcontainer/post-create.sh index 6ca6215872e..e5c9166d95b 100755 --- a/.devcontainer/post-create.sh +++ b/.devcontainer/post-create.sh @@ -26,13 +26,27 @@ set -uo pipefail NUM_THREADS_MARKER='# >>> gluten dev container num_threads >>>' +STATIC_ARM_MARKER='# >>> gluten static dev container arm64 >>>' +DEV_CONTAINER_VARIANT=${GLUTEN_DEV_CONTAINER_VARIANT:-velox-dynamic} warn() { echo "WARNING: $*" >&2; } echo "Preparing the Gluten dev container..." -# Spark 4.0/4.1 and the UDF tests need JDK 17, which this JDK 8 image lacks. -# Both JDKs can coexist: JAVA_HOME still points at JDK 8 for the default build. +# vcpkg otherwise defaults to the x64 triplet on arm64. +if [ "$DEV_CONTAINER_VARIANT" = "velox-static" ] && + [ "$(uname -m)" = "aarch64" ] && + ! grep -qF "$STATIC_ARM_MARKER" "$HOME/.bashrc" 2>/dev/null; then + cat >>"$HOME/.bashrc" </dev/null || @@ -108,10 +122,22 @@ NUM_THREADS=$(/usr/local/bin/gluten-num-threads) CPU_THREADS=$(nproc --ignore=2) MEM_GB=$(awk '/^MemTotal:/ {printf "%d", $2 / 1048576}' /proc/meminfo 2>/dev/null) -cat <&2 + exit 1 + ;; +esac diff --git a/.devcontainer/velox-static/devcontainer.json b/.devcontainer/velox-static/devcontainer.json new file mode 100644 index 00000000000..e35bc6047c7 --- /dev/null +++ b/.devcontainer/velox-static/devcontainer.json @@ -0,0 +1,64 @@ +// Static-link environment for packaging and vcpkg issue reproduction. +// See docs/developers/dev-container.md for build instructions and limitations. +{ + "name": "Gluten Velox Backend (centos-9, static link)", + + "image": "apache/gluten:vcpkg-centos-9", + + // Static linking needs substantially more memory than daily development. + "hostRequirements": { + "cpus": 8, + "memory": "64gb", + "storage": "64gb" + }, + + "containerEnv": { + "CCACHE_DIR": "/root/.ccache", + "CCACHE_MAXSIZE": "10G", + "GLUTEN_DEV_CONTAINER_VARIANT": "velox-static" + }, + + // Keep the compiler aligned with the toolchain used to build the vcpkg cache. + "remoteEnv": { + "PATH": "/opt/rh/gcc-toolset-12/root/usr/bin:${containerEnv:PATH}", + "LD_LIBRARY_PATH": "/opt/rh/gcc-toolset-12/root/usr/lib64:/opt/rh/gcc-toolset-12/root/usr/lib", + "PKG_CONFIG_PATH": "/opt/rh/gcc-toolset-12/root/usr/lib64/pkgconfig" + }, + + "mounts": [ + "source=gluten-vcpkg-ccache,target=/root/.ccache,type=volume", + "source=gluten-vcpkg-m2,target=/root/.m2,type=volume", + "source=gluten-vcpkg-binary-cache,target=/var/cache/vcpkg,type=volume" + ], + + "postCreateCommand": "bash .devcontainer/post-create.sh", + + "customizations": { + "vscode": { + "extensions": [ + "scalameta.metals", + "ms-vscode.cpptools", + "ms-python.python" + ], + "settings": { + "files.watcherExclude": { + "**/target/**": true, + "**/cpp/build/**": true, + "**/ep/_ep/**": true, + "**/ep/build-velox/build/**": true, + "**/dev/vcpkg/.vcpkg/**": true, + "**/dev/vcpkg/vcpkg_installed/**": true + }, + "search.exclude": { + "**/target/**": true, + "**/cpp/build/**": true, + "**/ep/_ep/**": true, + "**/ep/build-velox/build/**": true, + "**/dev/vcpkg/.vcpkg/**": true, + "**/dev/vcpkg/vcpkg_installed/**": true + }, + "C_Cpp.default.compileCommands": "${workspaceFolder}/cpp/build/compile_commands.json" + } + } + } +} diff --git a/docs/developers/NewToGluten.md b/docs/developers/NewToGluten.md index 27620395cf5..16f0463247d 100644 --- a/docs/developers/NewToGluten.md +++ b/docs/developers/NewToGluten.md @@ -44,9 +44,9 @@ Gluten requires GCC 11 or above. ### Dev Container To skip the manual environment setup, you can develop Gluten inside a pre-built Docker -image using the [Dev Container](https://containers.dev/) configuration shipped at -`.devcontainer/devcontainer.json`. See [Dev Container](./dev-container.md) for a brief -guide and the list of available Gluten Docker images. +image using the [Dev Container](https://containers.dev/) configurations under +`.devcontainer/`. See [Dev Containers](./dev-container.md) for configuration, +build and test instructions. ## Development diff --git a/docs/developers/dev-container.md b/docs/developers/dev-container.md index d4346533d98..bc8c0974d2e 100644 --- a/docs/developers/dev-container.md +++ b/docs/developers/dev-container.md @@ -1,16 +1,15 @@ --- layout: page -title: Dev Container +title: Dev Containers nav_order: 18 parent: Developer Overview --- # Develop Gluten in a Dev Container -Gluten ships a [Dev Container](https://containers.dev/) configuration at -[`.devcontainer/devcontainer.json`](https://github.com/apache/gluten/blob/main/.devcontainer/devcontainer.json), -so you can develop inside a pre-built image that already has the JDK, Maven, GCC -toolset and the whole Velox native dependency stack installed. +Gluten provides Dev Container configurations for daily Velox development and +static-link packaging. Both use pre-built images with the native dependency +stack installed, and neither builds Gluten during container creation. ## Prerequisites @@ -19,25 +18,34 @@ toolset and the whole Velox native dependency stack installed. [Dev Containers extension](https://code.visualstudio.com/docs/devcontainers/containers), or a [Codespaces](https://docs.github.com/en/codespaces)-enabled account. -## Open the workspace +## Choose a configuration -In VS Code, run **Dev Containers: Reopen in Container** from the Command Palette -(`F1`). In Codespaces, create a Codespace for the repository. +| Configuration | Path | Use case | Spark unit tests | +|---|---|---|---| +| **Velox dynamic link** (default) | `.devcontainer/devcontainer.json` | Daily development | Yes | +| **Velox static link** | `.devcontainer/velox-static/devcontainer.json` | Portable jars and vcpkg issue reproduction | No (`/opt/shims` is not installed) | -## What the configuration does +In VS Code, run **Dev Containers: Reopen in Container** from the Command +Palette (`F1`) and select a configuration. In Codespaces, open **Create +codespace with options** and select the configuration before creating the +codespace. -It opens the workspace in `apache/gluten:centos-9-jdk8` (CentOS Stream 9, JDK 8, -**dynamically linked** dependencies), puts GCC 12 on `PATH` through `remoteEnv` -because the default GCC 11 cannot compile Velox's C++20 sources, and runs -[`.devcontainer/post-create.sh`](https://github.com/apache/gluten/blob/main/.devcontainer/post-create.sh), -which installs JDK 17, `clang-format` 15 and the `regex` module, sizes `NUM_THREADS` -for the machine and prints the build commands. +The shared +[post-create script](https://github.com/apache/gluten/blob/main/.devcontainer/post-create.sh) +installs missing development tools, sizes `NUM_THREADS` for the machine and +prints commands for the selected configuration. -**The native build is not run automatically.** It takes tens of minutes to several -hours, which would stall container creation and leave a half-built tree behind -whenever the editor disconnects or a Codespace times out. +**The native build is not run automatically.** It takes tens of minutes to +several hours, which would stall container creation and leave a half-built tree +behind if the editor disconnects or a Codespace times out. -## Build Gluten +## Velox dynamic-link development + +The default configuration uses `apache/gluten:centos-9-jdk8`. It includes +dynamically linked dependencies, Arrow under `/usr/local`, a pre-warmed Maven +repository and Spark distributions under `/opt/shims`. + +Build the native backend and the Spark 3.5 jars: ```bash ./dev/buildbundle-veloxbe.sh --run_setup_script=OFF --build_arrow=OFF \ @@ -46,12 +54,37 @@ whenever the editor disconnects or a Codespace times out. | Flag | Why | |---|---| -| `--run_setup_script=OFF` | Velox's third-party libraries are already installed in the image; `ON` rebuilds them all from source into `/usr/local`. | +| `--run_setup_script=OFF` | Velox's third-party libraries are already installed; `ON` rebuilds them from source. | | `--build_arrow=OFF` | Arrow is already installed under `/usr/local` and its jars are in `~/.m2`. | | `--build_tests=ON` | Also builds the C++ unit tests. Drop it if you only need the jars. | -| `--spark_version=3.5` | The default, `ALL`, runs five full Maven builds (Spark 3.3 to 4.1). | +| `--spark_version=3.5` | The default, `ALL`, runs five Maven builds (Spark 3.3 to 4.1). | + +### Build for Spark 4.1 -To rebuild only the native side after a C++ change: +Spark 4.0/4.1 require JDK 17 and Scala 2.13. The dynamic image defaults to JDK +8, while `post-create.sh` installs JDK 17 alongside it. Switch the running JDK +before starting the build: + +```bash +export JAVA_HOME=/usr/lib/jvm/java-17-openjdk +export PATH="$JAVA_HOME/bin:$PATH" +java -version # must report 17 + +./dev/buildbundle-veloxbe.sh --run_setup_script=OFF --build_arrow=OFF \ + --build_tests=ON --spark_version=4.1 +``` + +`buildbundle-veloxbe.sh` adds `-Pjava-17`, `-Pscala-2.13` and the Java 17 +release target for Spark 4.x, but Maven profiles cannot switch the JDK that is +already running. If JDK 8 remains active, Scala fails with: + +```text +scalac error: '17' is not a valid choice for '-release' +``` + +### Rebuild native code + +After changing C++ code: ```bash ./dev/builddeps-veloxbe.sh --run_setup_script=OFF --build_arrow=OFF \ @@ -59,108 +92,112 @@ To rebuild only the native side after a C++ change: ``` Drop `build_velox` when only Gluten's own C++ under `cpp/` changed. Keep -`--build_tests` matched with the flag you built with: `build_gluten_cpp` wipes -`cpp/build` on every run, so omitting it silently drops the C++ test binaries. +`--build_tests` matched with the original build: `build_gluten_cpp` wipes +`cpp/build`, so omitting it also removes the C++ test binaries. -Spark 4.0/4.1 need JDK 17 and Scala 2.13. `buildbundle-veloxbe.sh` adds the Maven -profiles, but `JAVA_HOME` is yours to set: +### Run tests + +Run a Spark 3.5 suite on JDK 17: ```bash export JAVA_HOME=/usr/lib/jvm/java-17-openjdk -./dev/buildbundle-veloxbe.sh --run_setup_script=OFF --build_arrow=OFF --spark_version=4.0 +export PATH="$JAVA_HOME/bin:$PATH" + +./build/mvn test -Pspark-ut -Pbackends-velox -Pspark-3.5 -Pjava-17 \ + -DargLine="-Dspark.test.home=/opt/shims/spark35/spark_home/" \ + -DwildcardSuites=org.apache.spark.sql.GlutenSQLQuerySuite ``` -### Build parallelism and the OOM killer +`-DwildcardSuites` takes a fully qualified class name and keeps a run to +minutes. Do not add `-pl gluten-ut`: it selects only the aggregator POM and no +test suite runs. See [HowTo](./HowTo.md#3-how-to-debug-javascala) for more. -`builddeps-veloxbe.sh` sizes `NUM_THREADS` as `nproc --ignore=2`, ignoring memory, -while Velox's heavier translation units peak at ~3.5 GB resident each. On a 32-core, -62 GB container that is 30 jobs asking for roughly 100 GB, and the OOM killer takes -down the build or the container. `post-create.sh` therefore exports a value allowing -~4 GB per job — 13 jobs on that machine, measured at a 41 GB peak. +Run the C++ unit tests after building with `--build_tests=ON`: -An explicit `export NUM_THREADS=` still wins. VS Code tasks do not read -`~/.bashrc`, so pass `--num_threads=` there. +```bash +cd cpp/build && ctest -V +``` + +## Velox static-link packaging -## Run the tests +The static configuration uses `apache/gluten:vcpkg-centos-9`. It opens without +building Gluten and persists the image's vcpkg binary cache across container +rebuilds. -The image unpacks a Spark distribution for every supported version under `/opt/shims`, -which is what `spark.test.home` needs. CI runs the Spark 3.3/3.4/3.5 unit tests on -JDK 17: +Build a portable Spark 3.5 jar: ```bash -export JAVA_HOME=/usr/lib/jvm/java-17-openjdk -./build/mvn test -Pspark-ut -Pbackends-velox -Pspark-3.5 -Pjava-17 \ - -DargLine="-Dspark.test.home=/opt/shims/spark35/spark_home/" \ - -DwildcardSuites=org.apache.spark.sql.GlutenSQLQuerySuite +./dev/buildbundle-veloxbe.sh --enable_vcpkg=ON --build_arrow=OFF \ + --spark_version=3.5 +``` + +S3, GCS, HDFS and ABFS are disabled by default. Enable only what the jar needs: + +```bash +./dev/buildbundle-veloxbe.sh --enable_vcpkg=ON --build_arrow=OFF \ + --spark_version=3.5 --enable_s3=ON ``` -`-DwildcardSuites` takes a fully qualified class name and keeps a run to minutes; -without it the whole suite runs for hours. Do not add `-pl gluten-ut`: it is an -aggregator POM, `-pl` does not pull in its children, and the build finishes in seconds -having run nothing. See [HowTo](./HowTo.md#3-how-to-debug-javascala) for more. +Each enabled feature may restore or build additional vcpkg ports. vcpkg caches +ports by ABI hash; changing the compiler, triplet or relevant port inputs can +invalidate that cache. Native test binaries are disabled in this packaging +workflow. -C++ unit tests (requires `--build_tests=ON`): +The static image uses JDK 17 by default. For Spark 4.1: ```bash -cd cpp/build && ctest -V +java -version # must report 17 + +./dev/buildbundle-veloxbe.sh --enable_vcpkg=ON --build_arrow=OFF \ + --spark_version=4.1 ``` -## Static or dynamic link? - -"Dynamic" and "static" describe **how third-party dependencies are linked**. Velox -itself is archived into `libvelox.a` and linked into Gluten's shared libraries either -way, so `VELOX_BUILD_SHARED` only controls whether folly, gflags and glog are built -shared; `devcontainer.json` sets it to `ON` to match the image. - -| | Dynamic (`--enable_vcpkg=OFF`, the default) | Static (`--enable_vcpkg=ON`) | -|---|---|---| -| Image | `apache/gluten:centos-9-jdk8` | `apache/gluten:vcpkg-centos-9` | -| Dependencies | Pre-installed system libraries | vcpkg ports | -| Spark distributions | `/opt/shims`, so `gluten-ut` runs | Not installed | -| JDK | 8, with 17 added by `post-create.sh` | 17 only | -| Resulting jar | Needs the same shared libraries | Runs on any x86-64 Linux | - -Static linking exists to produce **portable release jars**, which is worth paying for -in CI and releases but buys a developer nothing, since the jar never leaves the -container. The vcpkg image also lacks `/opt/shims`, so the Spark unit tests cannot -run in it at all. - -vcpkg itself caches well: `apache/gluten:vcpkg-centos-9` bakes in -`VCPKG_BINARY_SOURCES=clear;files,/var/cache/vcpkg,readwrite`, and because the ABI -hash is computed per port, a checkout that has moved rebuilds only the ports it -actually changed plus their dependents. Be aware that the ABI hash also covers the -toolchain, so a change of compiler or triplet invalidates every port at once, and -vcpkg does not explain why it started rebuilding. - -To reproduce a static-link problem, switch `image` to `apache/gluten:vcpkg-centos-9` -and build with `./dev/ci-velox-buildstatic-centos-9.sh` (it exports `NUM_THREADS=2` -for CI runners; raise it first). Build trees are tied to the image that produced -them — `cpp/build/CMakeCache.txt` records the vcpkg toolchain and -`ep/build-velox/build/velox_ep/_build/` records `VELOX_BUILD_SHARED` — so wipe them -when switching: +The static image does not include `/opt/shims`; use the dynamic configuration +for Spark unit tests. On arm64, `post-create.sh` also selects the arm64 vcpkg +triplet and enables the required system binaries automatically. + +## Build parallelism + +`builddeps-veloxbe.sh` defaults `NUM_THREADS` to `nproc --ignore=2`, which +ignores memory. Velox's heavier translation units use about 3.5 GB each, so a +core-rich machine can invoke the OOM killer. `post-create.sh` exports a value +allowing about 4 GB per job and recomputes it whenever a shell opens, so it also +follows a resized Codespace. + +An explicit `export NUM_THREADS=` still wins. VS Code tasks do not read +`~/.bashrc`, so pass `--num_threads=` in a task. + +## Switch between configurations + +Static and dynamic build trees are not interchangeable: +`cpp/build/CMakeCache.txt` records the vcpkg toolchain, and +`ep/build-velox/build/velox_ep/_build/` records dependency linkage. Remove the +image-specific state after switching: ```bash rm -rf cpp/build ep/build-velox/build/velox_ep/_build ep/_ep \ dev/vcpkg/.vcpkg dev/vcpkg/vcpkg_installed ``` +The configurations use separate ccache, Maven and vcpkg-cache volumes, so +rebuilding one environment does not contaminate the other. + ## Machine sizing -`hostRequirements` asks for 4 CPUs, 16 GB of memory and 64 GB of storage. Storage is -the binding constraint: the image plus the Velox build tree does not fit the 32 GB -disk of the smaller Codespaces machine types, so those are marked "Below dev container -requirements". Only Codespaces honours `hostRequirements`; other runtimes ignore it, -so size the Docker VM yourself. +The dynamic configuration requests 4 CPUs, 16 GB of memory and 64 GB of +storage. The static configuration requests 8 CPUs, 64 GB of memory and 64 GB +of storage because static linking needs more memory. + +Only Codespaces honors `hostRequirements`; for local development, size the +Docker VM yourself. -## Use another image +## Images -Other pre-built images are published to -[Docker Hub](https://hub.docker.com/r/apache/gluten/tags); their Dockerfiles live in -[`dev/docker/`](https://github.com/apache/gluten/tree/main/dev/docker) and are -described in [Velox Backend CI](./velox-backend-CI.md#docker-build). -`apache/gluten:centos-9-jdk17` is the same dynamic-link image with JDK 17 as the -default; `centos-8-*` variants build against an older glibc. To switch, edit the -`image` field in `.devcontainer/devcontainer.json`. +Published images are available on +[Docker Hub](https://hub.docker.com/r/apache/gluten/tags); their Dockerfiles +live in [`dev/docker/`](https://github.com/apache/gluten/tree/main/dev/docker) +and are described in +[Velox Backend CI](./velox-backend-CI.md#docker-build). -To use these images outside a Dev Container, see +To use the images outside a Dev Container, see [Build Gluten Velox backend in docker](./velox-backend-build-in-docker.md). From a1b30899994f2026ee6836a3955a26e70751feb8 Mon Sep 17 00:00:00 2001 From: Felipe Fujiy Pessoto Date: Sat, 22 Aug 2026 23:32:02 +0000 Subject: [PATCH 2/4] [MINOR] Warn on stale build tree when switching Dev Container variants /workspaces persists across Dev Container rebuilds and switches between the velox-dynamic and velox-static configs, but ep/build-velox/build/velox_ep/_build and cpp/build bake in the vcpkg toolchain choice (or its absence) at first CMake configure and CMake never re-evaluates it. Reusing a build tree from the other variant does not fail clearly -- it silently resolves dependencies like zlib/zstd from the wrong place and fails much later, e.g. "could not find SnappyConfig.cmake", deep into a build that can take hours. Add a check to post-create.sh that detects this mismatch and warns with the exact cleanup command up front, instead of letting it surface as a cryptic CMake error after a long build. Generated-by: Claude claude-sonnet-5 Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .devcontainer/post-create.sh | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/.devcontainer/post-create.sh b/.devcontainer/post-create.sh index e5c9166d95b..38ae47c92bf 100755 --- a/.devcontainer/post-create.sh +++ b/.devcontainer/post-create.sh @@ -33,6 +33,31 @@ warn() { echo "WARNING: $*" >&2; } echo "Preparing the Gluten dev container..." +# /workspaces persists across "Reopen in Container"/"Rebuild Container" and +# across switching between the velox-dynamic and velox-static configs, but +# ep/build-velox/build/velox_ep/_build and cpp/build bake in the vcpkg +# toolchain choice (or its absence) at first CMake configure and CMake never +# re-evaluates it. Reusing a build tree from the other variant does not error +# clearly -- it silently resolves dependencies like zlib/zstd from the wrong +# place and fails much later, e.g. "could not find SnappyConfig.cmake", deep +# into a build that can take hours. Catch the mismatch up front instead. +check_stale_build_tree() { + local cache="$1" + [ -f "$cache" ] || return 0 + local has_toolchain=false + grep -q '^CMAKE_TOOLCHAIN_FILE:' "$cache" 2>/dev/null && has_toolchain=true + + if [ "$DEV_CONTAINER_VARIANT" = "velox-static" ] && [ "$has_toolchain" = false ]; then + warn "$cache was configured without the vcpkg toolchain (looks like it came from the velox-dynamic container, or a build before --enable_vcpkg=ON). Remove stale build trees before building here: rm -rf ep/build-velox/build/velox_ep/_build cpp/build" + elif [ "$DEV_CONTAINER_VARIANT" = "velox-dynamic" ] && [ "$has_toolchain" = true ]; then + warn "$cache was configured with the vcpkg toolchain (looks like it came from the velox-static container). Remove stale build trees before building here: rm -rf ep/build-velox/build/velox_ep/_build cpp/build" + fi +} + +for cache in ep/build-velox/build/velox_ep/_build/*/CMakeCache.txt cpp/build/CMakeCache.txt; do + check_stale_build_tree "$cache" +done + # vcpkg otherwise defaults to the x64 triplet on arm64. if [ "$DEV_CONTAINER_VARIANT" = "velox-static" ] && [ "$(uname -m)" = "aarch64" ] && From 5efeab7974cd27922c6a2b692cf31888657b400d Mon Sep 17 00:00:00 2001 From: Felipe Fujiy Pessoto Date: Sun, 23 Aug 2026 02:34:15 +0000 Subject: [PATCH 3/4] [MINOR] Repeat post-create.sh warnings in yellow at the end warn() output goes to a script that also installs JDK 17, clang-format, and prints long per-variant build instructions. A warning printed early (e.g. the stale build tree check) can easily scroll past unnoticed before the final banner. Collect warnings as they happen and reprint them in yellow after everything else, so they can't be missed. Generated-by: Claude claude-sonnet-5 Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .devcontainer/post-create.sh | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/.devcontainer/post-create.sh b/.devcontainer/post-create.sh index 38ae47c92bf..ba6043488d9 100755 --- a/.devcontainer/post-create.sh +++ b/.devcontainer/post-create.sh @@ -29,7 +29,11 @@ NUM_THREADS_MARKER='# >>> gluten dev container num_threads >>>' STATIC_ARM_MARKER='# >>> gluten static dev container arm64 >>>' DEV_CONTAINER_VARIANT=${GLUTEN_DEV_CONTAINER_VARIANT:-velox-dynamic} -warn() { echo "WARNING: $*" >&2; } +WARNINGS=() +warn() { + echo "WARNING: $*" >&2 + WARNINGS+=("$*") +} echo "Preparing the Gluten dev container..." @@ -250,3 +254,16 @@ EOF exit 1 ;; esac + +# Warnings logged with warn() above can easily scroll past unnoticed among all +# the setup output, so repeat them here, in yellow, after everything else. +if [ "${#WARNINGS[@]}" -gt 0 ]; then + YELLOW='\033[1;33m' + NO_COLOR='\033[0m' + echo -e "${YELLOW}============================================================================${NO_COLOR}" >&2 + echo -e "${YELLOW}WARNINGS:${NO_COLOR}" >&2 + for w in "${WARNINGS[@]}"; do + echo -e "${YELLOW} - $w${NO_COLOR}" >&2 + done + echo -e "${YELLOW}============================================================================${NO_COLOR}" >&2 +fi From b7bb74517464626374574743b7933c75468fd284 Mon Sep 17 00:00:00 2001 From: Felipe Fujiy Pessoto Date: Sun, 23 Aug 2026 02:45:16 +0000 Subject: [PATCH 4/4] Fix Git safe directory setup in dev container --- .devcontainer/post-create.sh | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/.devcontainer/post-create.sh b/.devcontainer/post-create.sh index ba6043488d9..c1ae9f9af68 100755 --- a/.devcontainer/post-create.sh +++ b/.devcontainer/post-create.sh @@ -37,6 +37,15 @@ warn() { echo "Preparing the Gluten dev container..." +# The container runs as root while the bind-mounted workspace retains the host +# user's ownership. Register only this repository so Git accepts that mismatch. +WORKSPACE_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd -P) +if ! git config --global --get-all safe.directory 2>/dev/null | + grep -Fqx "$WORKSPACE_DIR"; then + git config --global --add safe.directory "$WORKSPACE_DIR" || + warn "could not mark $WORKSPACE_DIR as a safe Git directory." +fi + # /workspaces persists across "Reopen in Container"/"Rebuild Container" and # across switching between the velox-dynamic and velox-static configs, but # ep/build-velox/build/velox_ep/_build and cpp/build bake in the vcpkg