diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 81b75291cd..a709ed556d 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -1,4 +1,4 @@ -// Dev Container for the Gluten Velox backend. +// Default Dev Container for daily Gluten Velox development. // See docs/developers/dev-container.md for the rationale behind these settings // and for the build commands to run once the container is up. { @@ -19,6 +19,7 @@ "containerEnv": { "CCACHE_DIR": "/root/.ccache", "CCACHE_MAXSIZE": "10G", + "GLUTEN_DEV_CONTAINER_VARIANT": "velox-dynamic", // folly, gflags and glog are shared libraries in this image; a Velox // dependency build that disagrees fails to link. "VELOX_BUILD_SHARED": "ON" diff --git a/.devcontainer/post-create.sh b/.devcontainer/post-create.sh index 6ca6215872..c1ae9f9af6 100755 --- a/.devcontainer/post-create.sh +++ b/.devcontainer/post-create.sh @@ -26,13 +26,65 @@ set -uo pipefail NUM_THREADS_MARKER='# >>> gluten dev container num_threads >>>' +STATIC_ARM_MARKER='# >>> gluten static dev container arm64 >>>' +DEV_CONTAINER_VARIANT=${GLUTEN_DEV_CONTAINER_VARIANT:-velox-dynamic} -warn() { echo "WARNING: $*" >&2; } +WARNINGS=() +warn() { + echo "WARNING: $*" >&2 + WARNINGS+=("$*") +} echo "Preparing the Gluten dev container..." -# Spark 4.0/4.1 and the UDF tests need JDK 17, which this JDK 8 image lacks. -# Both JDKs can coexist: JAVA_HOME still points at JDK 8 for the default build. +# The container runs as root while the bind-mounted workspace retains the host +# user's ownership. Register only this repository so Git accepts that mismatch. +WORKSPACE_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd -P) +if ! git config --global --get-all safe.directory 2>/dev/null | + grep -Fqx "$WORKSPACE_DIR"; then + git config --global --add safe.directory "$WORKSPACE_DIR" || + warn "could not mark $WORKSPACE_DIR as a safe Git directory." +fi + +# /workspaces persists across "Reopen in Container"/"Rebuild Container" and +# across switching between the velox-dynamic and velox-static configs, but +# ep/build-velox/build/velox_ep/_build and cpp/build bake in the vcpkg +# toolchain choice (or its absence) at first CMake configure and CMake never +# re-evaluates it. Reusing a build tree from the other variant does not error +# clearly -- it silently resolves dependencies like zlib/zstd from the wrong +# place and fails much later, e.g. "could not find SnappyConfig.cmake", deep +# into a build that can take hours. Catch the mismatch up front instead. +check_stale_build_tree() { + local cache="$1" + [ -f "$cache" ] || return 0 + local has_toolchain=false + grep -q '^CMAKE_TOOLCHAIN_FILE:' "$cache" 2>/dev/null && has_toolchain=true + + if [ "$DEV_CONTAINER_VARIANT" = "velox-static" ] && [ "$has_toolchain" = false ]; then + warn "$cache was configured without the vcpkg toolchain (looks like it came from the velox-dynamic container, or a build before --enable_vcpkg=ON). Remove stale build trees before building here: rm -rf ep/build-velox/build/velox_ep/_build cpp/build" + elif [ "$DEV_CONTAINER_VARIANT" = "velox-dynamic" ] && [ "$has_toolchain" = true ]; then + warn "$cache was configured with the vcpkg toolchain (looks like it came from the velox-static container). Remove stale build trees before building here: rm -rf ep/build-velox/build/velox_ep/_build cpp/build" + fi +} + +for cache in ep/build-velox/build/velox_ep/_build/*/CMakeCache.txt cpp/build/CMakeCache.txt; do + check_stale_build_tree "$cache" +done + +# vcpkg otherwise defaults to the x64 triplet on arm64. +if [ "$DEV_CONTAINER_VARIANT" = "velox-static" ] && + [ "$(uname -m)" = "aarch64" ] && + ! grep -qF "$STATIC_ARM_MARKER" "$HOME/.bashrc" 2>/dev/null; then + cat >>"$HOME/.bashrc" </dev/null || @@ -108,10 +160,22 @@ NUM_THREADS=$(/usr/local/bin/gluten-num-threads) CPU_THREADS=$(nproc --ignore=2) MEM_GB=$(awk '/^MemTotal:/ {printf "%d", $2 / 1048576}' /proc/meminfo 2>/dev/null) -cat <&2 + exit 1 + ;; +esac + +# Warnings logged with warn() above can easily scroll past unnoticed among all +# the setup output, so repeat them here, in yellow, after everything else. +if [ "${#WARNINGS[@]}" -gt 0 ]; then + YELLOW='\033[1;33m' + NO_COLOR='\033[0m' + echo -e "${YELLOW}============================================================================${NO_COLOR}" >&2 + echo -e "${YELLOW}WARNINGS:${NO_COLOR}" >&2 + for w in "${WARNINGS[@]}"; do + echo -e "${YELLOW} - $w${NO_COLOR}" >&2 + done + echo -e "${YELLOW}============================================================================${NO_COLOR}" >&2 +fi diff --git a/.devcontainer/velox-static/devcontainer.json b/.devcontainer/velox-static/devcontainer.json new file mode 100644 index 0000000000..e35bc6047c --- /dev/null +++ b/.devcontainer/velox-static/devcontainer.json @@ -0,0 +1,64 @@ +// Static-link environment for packaging and vcpkg issue reproduction. +// See docs/developers/dev-container.md for build instructions and limitations. +{ + "name": "Gluten Velox Backend (centos-9, static link)", + + "image": "apache/gluten:vcpkg-centos-9", + + // Static linking needs substantially more memory than daily development. + "hostRequirements": { + "cpus": 8, + "memory": "64gb", + "storage": "64gb" + }, + + "containerEnv": { + "CCACHE_DIR": "/root/.ccache", + "CCACHE_MAXSIZE": "10G", + "GLUTEN_DEV_CONTAINER_VARIANT": "velox-static" + }, + + // Keep the compiler aligned with the toolchain used to build the vcpkg cache. + "remoteEnv": { + "PATH": "/opt/rh/gcc-toolset-12/root/usr/bin:${containerEnv:PATH}", + "LD_LIBRARY_PATH": "/opt/rh/gcc-toolset-12/root/usr/lib64:/opt/rh/gcc-toolset-12/root/usr/lib", + "PKG_CONFIG_PATH": "/opt/rh/gcc-toolset-12/root/usr/lib64/pkgconfig" + }, + + "mounts": [ + "source=gluten-vcpkg-ccache,target=/root/.ccache,type=volume", + "source=gluten-vcpkg-m2,target=/root/.m2,type=volume", + "source=gluten-vcpkg-binary-cache,target=/var/cache/vcpkg,type=volume" + ], + + "postCreateCommand": "bash .devcontainer/post-create.sh", + + "customizations": { + "vscode": { + "extensions": [ + "scalameta.metals", + "ms-vscode.cpptools", + "ms-python.python" + ], + "settings": { + "files.watcherExclude": { + "**/target/**": true, + "**/cpp/build/**": true, + "**/ep/_ep/**": true, + "**/ep/build-velox/build/**": true, + "**/dev/vcpkg/.vcpkg/**": true, + "**/dev/vcpkg/vcpkg_installed/**": true + }, + "search.exclude": { + "**/target/**": true, + "**/cpp/build/**": true, + "**/ep/_ep/**": true, + "**/ep/build-velox/build/**": true, + "**/dev/vcpkg/.vcpkg/**": true, + "**/dev/vcpkg/vcpkg_installed/**": true + }, + "C_Cpp.default.compileCommands": "${workspaceFolder}/cpp/build/compile_commands.json" + } + } + } +} diff --git a/docs/developers/NewToGluten.md b/docs/developers/NewToGluten.md index 27620395cf..16f0463247 100644 --- a/docs/developers/NewToGluten.md +++ b/docs/developers/NewToGluten.md @@ -44,9 +44,9 @@ Gluten requires GCC 11 or above. ### Dev Container To skip the manual environment setup, you can develop Gluten inside a pre-built Docker -image using the [Dev Container](https://containers.dev/) configuration shipped at -`.devcontainer/devcontainer.json`. See [Dev Container](./dev-container.md) for a brief -guide and the list of available Gluten Docker images. +image using the [Dev Container](https://containers.dev/) configurations under +`.devcontainer/`. See [Dev Containers](./dev-container.md) for configuration, +build and test instructions. ## Development diff --git a/docs/developers/dev-container.md b/docs/developers/dev-container.md index d4346533d9..bc8c0974d2 100644 --- a/docs/developers/dev-container.md +++ b/docs/developers/dev-container.md @@ -1,16 +1,15 @@ --- layout: page -title: Dev Container +title: Dev Containers nav_order: 18 parent: Developer Overview --- # Develop Gluten in a Dev Container -Gluten ships a [Dev Container](https://containers.dev/) configuration at -[`.devcontainer/devcontainer.json`](https://github.com/apache/gluten/blob/main/.devcontainer/devcontainer.json), -so you can develop inside a pre-built image that already has the JDK, Maven, GCC -toolset and the whole Velox native dependency stack installed. +Gluten provides Dev Container configurations for daily Velox development and +static-link packaging. Both use pre-built images with the native dependency +stack installed, and neither builds Gluten during container creation. ## Prerequisites @@ -19,25 +18,34 @@ toolset and the whole Velox native dependency stack installed. [Dev Containers extension](https://code.visualstudio.com/docs/devcontainers/containers), or a [Codespaces](https://docs.github.com/en/codespaces)-enabled account. -## Open the workspace +## Choose a configuration -In VS Code, run **Dev Containers: Reopen in Container** from the Command Palette -(`F1`). In Codespaces, create a Codespace for the repository. +| Configuration | Path | Use case | Spark unit tests | +|---|---|---|---| +| **Velox dynamic link** (default) | `.devcontainer/devcontainer.json` | Daily development | Yes | +| **Velox static link** | `.devcontainer/velox-static/devcontainer.json` | Portable jars and vcpkg issue reproduction | No (`/opt/shims` is not installed) | -## What the configuration does +In VS Code, run **Dev Containers: Reopen in Container** from the Command +Palette (`F1`) and select a configuration. In Codespaces, open **Create +codespace with options** and select the configuration before creating the +codespace. -It opens the workspace in `apache/gluten:centos-9-jdk8` (CentOS Stream 9, JDK 8, -**dynamically linked** dependencies), puts GCC 12 on `PATH` through `remoteEnv` -because the default GCC 11 cannot compile Velox's C++20 sources, and runs -[`.devcontainer/post-create.sh`](https://github.com/apache/gluten/blob/main/.devcontainer/post-create.sh), -which installs JDK 17, `clang-format` 15 and the `regex` module, sizes `NUM_THREADS` -for the machine and prints the build commands. +The shared +[post-create script](https://github.com/apache/gluten/blob/main/.devcontainer/post-create.sh) +installs missing development tools, sizes `NUM_THREADS` for the machine and +prints commands for the selected configuration. -**The native build is not run automatically.** It takes tens of minutes to several -hours, which would stall container creation and leave a half-built tree behind -whenever the editor disconnects or a Codespace times out. +**The native build is not run automatically.** It takes tens of minutes to +several hours, which would stall container creation and leave a half-built tree +behind if the editor disconnects or a Codespace times out. -## Build Gluten +## Velox dynamic-link development + +The default configuration uses `apache/gluten:centos-9-jdk8`. It includes +dynamically linked dependencies, Arrow under `/usr/local`, a pre-warmed Maven +repository and Spark distributions under `/opt/shims`. + +Build the native backend and the Spark 3.5 jars: ```bash ./dev/buildbundle-veloxbe.sh --run_setup_script=OFF --build_arrow=OFF \ @@ -46,12 +54,37 @@ whenever the editor disconnects or a Codespace times out. | Flag | Why | |---|---| -| `--run_setup_script=OFF` | Velox's third-party libraries are already installed in the image; `ON` rebuilds them all from source into `/usr/local`. | +| `--run_setup_script=OFF` | Velox's third-party libraries are already installed; `ON` rebuilds them from source. | | `--build_arrow=OFF` | Arrow is already installed under `/usr/local` and its jars are in `~/.m2`. | | `--build_tests=ON` | Also builds the C++ unit tests. Drop it if you only need the jars. | -| `--spark_version=3.5` | The default, `ALL`, runs five full Maven builds (Spark 3.3 to 4.1). | +| `--spark_version=3.5` | The default, `ALL`, runs five Maven builds (Spark 3.3 to 4.1). | + +### Build for Spark 4.1 -To rebuild only the native side after a C++ change: +Spark 4.0/4.1 require JDK 17 and Scala 2.13. The dynamic image defaults to JDK +8, while `post-create.sh` installs JDK 17 alongside it. Switch the running JDK +before starting the build: + +```bash +export JAVA_HOME=/usr/lib/jvm/java-17-openjdk +export PATH="$JAVA_HOME/bin:$PATH" +java -version # must report 17 + +./dev/buildbundle-veloxbe.sh --run_setup_script=OFF --build_arrow=OFF \ + --build_tests=ON --spark_version=4.1 +``` + +`buildbundle-veloxbe.sh` adds `-Pjava-17`, `-Pscala-2.13` and the Java 17 +release target for Spark 4.x, but Maven profiles cannot switch the JDK that is +already running. If JDK 8 remains active, Scala fails with: + +```text +scalac error: '17' is not a valid choice for '-release' +``` + +### Rebuild native code + +After changing C++ code: ```bash ./dev/builddeps-veloxbe.sh --run_setup_script=OFF --build_arrow=OFF \ @@ -59,108 +92,112 @@ To rebuild only the native side after a C++ change: ``` Drop `build_velox` when only Gluten's own C++ under `cpp/` changed. Keep -`--build_tests` matched with the flag you built with: `build_gluten_cpp` wipes -`cpp/build` on every run, so omitting it silently drops the C++ test binaries. +`--build_tests` matched with the original build: `build_gluten_cpp` wipes +`cpp/build`, so omitting it also removes the C++ test binaries. -Spark 4.0/4.1 need JDK 17 and Scala 2.13. `buildbundle-veloxbe.sh` adds the Maven -profiles, but `JAVA_HOME` is yours to set: +### Run tests + +Run a Spark 3.5 suite on JDK 17: ```bash export JAVA_HOME=/usr/lib/jvm/java-17-openjdk -./dev/buildbundle-veloxbe.sh --run_setup_script=OFF --build_arrow=OFF --spark_version=4.0 +export PATH="$JAVA_HOME/bin:$PATH" + +./build/mvn test -Pspark-ut -Pbackends-velox -Pspark-3.5 -Pjava-17 \ + -DargLine="-Dspark.test.home=/opt/shims/spark35/spark_home/" \ + -DwildcardSuites=org.apache.spark.sql.GlutenSQLQuerySuite ``` -### Build parallelism and the OOM killer +`-DwildcardSuites` takes a fully qualified class name and keeps a run to +minutes. Do not add `-pl gluten-ut`: it selects only the aggregator POM and no +test suite runs. See [HowTo](./HowTo.md#3-how-to-debug-javascala) for more. -`builddeps-veloxbe.sh` sizes `NUM_THREADS` as `nproc --ignore=2`, ignoring memory, -while Velox's heavier translation units peak at ~3.5 GB resident each. On a 32-core, -62 GB container that is 30 jobs asking for roughly 100 GB, and the OOM killer takes -down the build or the container. `post-create.sh` therefore exports a value allowing -~4 GB per job — 13 jobs on that machine, measured at a 41 GB peak. +Run the C++ unit tests after building with `--build_tests=ON`: -An explicit `export NUM_THREADS=` still wins. VS Code tasks do not read -`~/.bashrc`, so pass `--num_threads=` there. +```bash +cd cpp/build && ctest -V +``` + +## Velox static-link packaging -## Run the tests +The static configuration uses `apache/gluten:vcpkg-centos-9`. It opens without +building Gluten and persists the image's vcpkg binary cache across container +rebuilds. -The image unpacks a Spark distribution for every supported version under `/opt/shims`, -which is what `spark.test.home` needs. CI runs the Spark 3.3/3.4/3.5 unit tests on -JDK 17: +Build a portable Spark 3.5 jar: ```bash -export JAVA_HOME=/usr/lib/jvm/java-17-openjdk -./build/mvn test -Pspark-ut -Pbackends-velox -Pspark-3.5 -Pjava-17 \ - -DargLine="-Dspark.test.home=/opt/shims/spark35/spark_home/" \ - -DwildcardSuites=org.apache.spark.sql.GlutenSQLQuerySuite +./dev/buildbundle-veloxbe.sh --enable_vcpkg=ON --build_arrow=OFF \ + --spark_version=3.5 +``` + +S3, GCS, HDFS and ABFS are disabled by default. Enable only what the jar needs: + +```bash +./dev/buildbundle-veloxbe.sh --enable_vcpkg=ON --build_arrow=OFF \ + --spark_version=3.5 --enable_s3=ON ``` -`-DwildcardSuites` takes a fully qualified class name and keeps a run to minutes; -without it the whole suite runs for hours. Do not add `-pl gluten-ut`: it is an -aggregator POM, `-pl` does not pull in its children, and the build finishes in seconds -having run nothing. See [HowTo](./HowTo.md#3-how-to-debug-javascala) for more. +Each enabled feature may restore or build additional vcpkg ports. vcpkg caches +ports by ABI hash; changing the compiler, triplet or relevant port inputs can +invalidate that cache. Native test binaries are disabled in this packaging +workflow. -C++ unit tests (requires `--build_tests=ON`): +The static image uses JDK 17 by default. For Spark 4.1: ```bash -cd cpp/build && ctest -V +java -version # must report 17 + +./dev/buildbundle-veloxbe.sh --enable_vcpkg=ON --build_arrow=OFF \ + --spark_version=4.1 ``` -## Static or dynamic link? - -"Dynamic" and "static" describe **how third-party dependencies are linked**. Velox -itself is archived into `libvelox.a` and linked into Gluten's shared libraries either -way, so `VELOX_BUILD_SHARED` only controls whether folly, gflags and glog are built -shared; `devcontainer.json` sets it to `ON` to match the image. - -| | Dynamic (`--enable_vcpkg=OFF`, the default) | Static (`--enable_vcpkg=ON`) | -|---|---|---| -| Image | `apache/gluten:centos-9-jdk8` | `apache/gluten:vcpkg-centos-9` | -| Dependencies | Pre-installed system libraries | vcpkg ports | -| Spark distributions | `/opt/shims`, so `gluten-ut` runs | Not installed | -| JDK | 8, with 17 added by `post-create.sh` | 17 only | -| Resulting jar | Needs the same shared libraries | Runs on any x86-64 Linux | - -Static linking exists to produce **portable release jars**, which is worth paying for -in CI and releases but buys a developer nothing, since the jar never leaves the -container. The vcpkg image also lacks `/opt/shims`, so the Spark unit tests cannot -run in it at all. - -vcpkg itself caches well: `apache/gluten:vcpkg-centos-9` bakes in -`VCPKG_BINARY_SOURCES=clear;files,/var/cache/vcpkg,readwrite`, and because the ABI -hash is computed per port, a checkout that has moved rebuilds only the ports it -actually changed plus their dependents. Be aware that the ABI hash also covers the -toolchain, so a change of compiler or triplet invalidates every port at once, and -vcpkg does not explain why it started rebuilding. - -To reproduce a static-link problem, switch `image` to `apache/gluten:vcpkg-centos-9` -and build with `./dev/ci-velox-buildstatic-centos-9.sh` (it exports `NUM_THREADS=2` -for CI runners; raise it first). Build trees are tied to the image that produced -them — `cpp/build/CMakeCache.txt` records the vcpkg toolchain and -`ep/build-velox/build/velox_ep/_build/` records `VELOX_BUILD_SHARED` — so wipe them -when switching: +The static image does not include `/opt/shims`; use the dynamic configuration +for Spark unit tests. On arm64, `post-create.sh` also selects the arm64 vcpkg +triplet and enables the required system binaries automatically. + +## Build parallelism + +`builddeps-veloxbe.sh` defaults `NUM_THREADS` to `nproc --ignore=2`, which +ignores memory. Velox's heavier translation units use about 3.5 GB each, so a +core-rich machine can invoke the OOM killer. `post-create.sh` exports a value +allowing about 4 GB per job and recomputes it whenever a shell opens, so it also +follows a resized Codespace. + +An explicit `export NUM_THREADS=` still wins. VS Code tasks do not read +`~/.bashrc`, so pass `--num_threads=` in a task. + +## Switch between configurations + +Static and dynamic build trees are not interchangeable: +`cpp/build/CMakeCache.txt` records the vcpkg toolchain, and +`ep/build-velox/build/velox_ep/_build/` records dependency linkage. Remove the +image-specific state after switching: ```bash rm -rf cpp/build ep/build-velox/build/velox_ep/_build ep/_ep \ dev/vcpkg/.vcpkg dev/vcpkg/vcpkg_installed ``` +The configurations use separate ccache, Maven and vcpkg-cache volumes, so +rebuilding one environment does not contaminate the other. + ## Machine sizing -`hostRequirements` asks for 4 CPUs, 16 GB of memory and 64 GB of storage. Storage is -the binding constraint: the image plus the Velox build tree does not fit the 32 GB -disk of the smaller Codespaces machine types, so those are marked "Below dev container -requirements". Only Codespaces honours `hostRequirements`; other runtimes ignore it, -so size the Docker VM yourself. +The dynamic configuration requests 4 CPUs, 16 GB of memory and 64 GB of +storage. The static configuration requests 8 CPUs, 64 GB of memory and 64 GB +of storage because static linking needs more memory. + +Only Codespaces honors `hostRequirements`; for local development, size the +Docker VM yourself. -## Use another image +## Images -Other pre-built images are published to -[Docker Hub](https://hub.docker.com/r/apache/gluten/tags); their Dockerfiles live in -[`dev/docker/`](https://github.com/apache/gluten/tree/main/dev/docker) and are -described in [Velox Backend CI](./velox-backend-CI.md#docker-build). -`apache/gluten:centos-9-jdk17` is the same dynamic-link image with JDK 17 as the -default; `centos-8-*` variants build against an older glibc. To switch, edit the -`image` field in `.devcontainer/devcontainer.json`. +Published images are available on +[Docker Hub](https://hub.docker.com/r/apache/gluten/tags); their Dockerfiles +live in [`dev/docker/`](https://github.com/apache/gluten/tree/main/dev/docker) +and are described in +[Velox Backend CI](./velox-backend-CI.md#docker-build). -To use these images outside a Dev Container, see +To use the images outside a Dev Container, see [Build Gluten Velox backend in docker](./velox-backend-build-in-docker.md).