[GLUTEN-12850][VL] Support multiple Dev Container configurations

.devcontainer/devcontainer.json

@@ -1,4 +1,4 @@
-// Dev Container for the Gluten Velox backend.
+// Default Dev Container for daily Gluten Velox development.
 // See docs/developers/dev-container.md for the rationale behind these settings
 // and for the build commands to run once the container is up.
 {
@@ -19,6 +19,7 @@
     "containerEnv": {
         "CCACHE_DIR": "/root/.ccache",
         "CCACHE_MAXSIZE": "10G",
+        "GLUTEN_DEV_CONTAINER_VARIANT": "velox-dynamic",
         // folly, gflags and glog are shared libraries in this image; a Velox
         // dependency build that disagrees fails to link.
         "VELOX_BUILD_SHARED": "ON"

.devcontainer/post-create.sh

@@ -26,13 +26,65 @@
 set -uo pipefail
 
 NUM_THREADS_MARKER='# >>> gluten dev container num_threads >>>'
+STATIC_ARM_MARKER='# >>> gluten static dev container arm64 >>>'
+DEV_CONTAINER_VARIANT=${GLUTEN_DEV_CONTAINER_VARIANT:-velox-dynamic}
 
-warn() { echo "WARNING: $*" >&2; }
+WARNINGS=()
+warn() {
+    echo "WARNING: $*" >&2
+    WARNINGS+=("$*")
+}
 
 echo "Preparing the Gluten dev container..."
 
-# Spark 4.0/4.1 and the UDF tests need JDK 17, which this JDK 8 image lacks.
-# Both JDKs can coexist: JAVA_HOME still points at JDK 8 for the default build.
+# The container runs as root while the bind-mounted workspace retains the host
+# user's ownership. Register only this repository so Git accepts that mismatch.
+WORKSPACE_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd -P)
+if ! git config --global --get-all safe.directory 2>/dev/null |
+    grep -Fqx "$WORKSPACE_DIR"; then
+    git config --global --add safe.directory "$WORKSPACE_DIR" ||
+        warn "could not mark $WORKSPACE_DIR as a safe Git directory."
+fi
+
+# /workspaces persists across "Reopen in Container"/"Rebuild Container" and
+# across switching between the velox-dynamic and velox-static configs, but
+# ep/build-velox/build/velox_ep/_build and cpp/build bake in the vcpkg
+# toolchain choice (or its absence) at first CMake configure and CMake never
+# re-evaluates it. Reusing a build tree from the other variant does not error
+# clearly -- it silently resolves dependencies like zlib/zstd from the wrong
+# place and fails much later, e.g. "could not find SnappyConfig.cmake", deep
+# into a build that can take hours. Catch the mismatch up front instead.
+check_stale_build_tree() {
+    local cache="$1"
+    [ -f "$cache" ] || return 0
+    local has_toolchain=false
+    grep -q '^CMAKE_TOOLCHAIN_FILE:' "$cache" 2>/dev/null && has_toolchain=true
+
+    if [ "$DEV_CONTAINER_VARIANT" = "velox-static" ] && [ "$has_toolchain" = false ]; then
+        warn "$cache was configured without the vcpkg toolchain (looks like it came from the velox-dynamic container, or a build before --enable_vcpkg=ON). Remove stale build trees before building here: rm -rf ep/build-velox/build/velox_ep/_build cpp/build"
+    elif [ "$DEV_CONTAINER_VARIANT" = "velox-dynamic" ] && [ "$has_toolchain" = true ]; then
+        warn "$cache was configured with the vcpkg toolchain (looks like it came from the velox-static container). Remove stale build trees before building here: rm -rf ep/build-velox/build/velox_ep/_build cpp/build"
+    fi
+}
+
+for cache in ep/build-velox/build/velox_ep/_build/*/CMakeCache.txt cpp/build/CMakeCache.txt; do
+    check_stale_build_tree "$cache"
+done
+
+# vcpkg otherwise defaults to the x64 triplet on arm64.
+if [ "$DEV_CONTAINER_VARIANT" = "velox-static" ] &&
+    [ "$(uname -m)" = "aarch64" ] &&
+    ! grep -qF "$STATIC_ARM_MARKER" "$HOME/.bashrc" 2>/dev/null; then
+    cat >>"$HOME/.bashrc" <<EOF
+
+$STATIC_ARM_MARKER
+export CPU_TARGET=aarch64
+export VCPKG_FORCE_SYSTEM_BINARIES=1
+# <<< gluten static dev container arm64 <<<
+EOF
+fi
+
+# The dynamic image defaults to JDK 8; the static image already has JDK 17.
 if [ ! -d /usr/lib/jvm/java-17-openjdk ]; then
     echo "Installing JDK 17 alongside JDK 8 (needed for Spark 4.x)..."
     dnf install -y --setopt=install_weak_deps=False java-17-openjdk-devel >/dev/null ||
@@ -108,10 +160,22 @@ NUM_THREADS=$(/usr/local/bin/gluten-num-threads)
 CPU_THREADS=$(nproc --ignore=2)
 MEM_GB=$(awk '/^MemTotal:/ {printf "%d", $2 / 1048576}' /proc/meminfo 2>/dev/null)
 
-cat <<EOF
+print_parallelism() {
+    cat <<EOF
 
+NUM_THREADS=${NUM_THREADS} is exported for you, sized from this machine's ${MEM_GB:-?} GB at
+~4 GB per compile job. The build scripts would take $CPU_THREADS from the core count
+alone, which invites the OOM killer. VS Code tasks do not read ~/.bashrc, so pass
+--num_threads=${NUM_THREADS} there.
+
+EOF
+}
+
+case "$DEV_CONTAINER_VARIANT" in
+velox-dynamic)
+    cat <<EOF
 ============================================================================
-Gluten dev container is ready. The native build has NOT been run.
+Gluten Velox dynamic-link container is ready. The native build has NOT been run.
 
 Build the Velox backend and install the Gluten jars:
 
@@ -122,27 +186,93 @@ Build the Velox backend and install the Gluten jars:
   --build_arrow=OFF       Arrow is already installed under /usr/local
   --build_tests=ON        also build the C++ unit tests (drop it to build faster)
   --spark_version=3.5     build one Spark version instead of all five
+EOF
 
-NUM_THREADS=${NUM_THREADS} is exported for you, sized from this machine's ${MEM_GB:-?} GB at
-~4 GB per compile job. The build scripts would take $CPU_THREADS from the core count
-alone, which invites the OOM killer. VS Code tasks do not read ~/.bashrc, so pass
---num_threads=${NUM_THREADS} there.
+    print_parallelism
 
+    cat <<EOF
 After changing C++ code, rebuild just the native side (drop build_velox when only
 Gluten's own C++ under cpp/ changed):
 
     ./dev/builddeps-veloxbe.sh --run_setup_script=OFF --build_arrow=OFF \\
                                --build_tests=ON build_velox build_gluten_cpp
 
+Build for Spark 4.1 with JDK 17 (the Maven profile does not switch the running JDK):
+
+    export JAVA_HOME=/usr/lib/jvm/java-17-openjdk
+    export PATH="\$JAVA_HOME/bin:\$PATH"
+    java -version  # must report 17
+    ./dev/buildbundle-veloxbe.sh --run_setup_script=OFF --build_arrow=OFF \\
+                                 --build_tests=ON --spark_version=4.1
+
 Run a Spark unit test suite (Spark distributions are pre-installed in this image;
 CI runs these on JDK 17, and without -DwildcardSuites the whole suite runs for
 hours):
 
     export JAVA_HOME=/usr/lib/jvm/java-17-openjdk
+    export PATH="\$JAVA_HOME/bin:\$PATH"
     ./build/mvn test -Pspark-ut -Pbackends-velox -Pspark-3.5 -Pjava-17 \\
         -DargLine="-Dspark.test.home=/opt/shims/spark35/spark_home/" \\
         -DwildcardSuites=org.apache.spark.sql.GlutenSQLQuerySuite
 
 See docs/developers/dev-container.md for details.
 ============================================================================
 EOF
+    ;;
+velox-static)
+    cat <<EOF
+============================================================================
+Gluten Velox static-link container is ready. The native build has NOT been run.
+
+Build a portable static-link jar:
+
+    ./dev/buildbundle-veloxbe.sh --enable_vcpkg=ON --build_arrow=OFF \\
+                                 --spark_version=3.5
+
+  --enable_vcpkg=ON       statically link third-party dependencies
+  --build_arrow=OFF       Arrow is already installed by the image
+  --spark_version=3.5     build one Spark version instead of all five
+
+S3, GCS, HDFS and ABFS are disabled by default. Enable only what you need with
+--enable_s3=ON, --enable_gcs=ON, --enable_hdfs=ON or --enable_abfs=ON.
+EOF
+
+    print_parallelism
+
+    cat <<EOF
+After changing C++ code, rebuild just the native side:
+
+    ./dev/builddeps-veloxbe.sh --enable_vcpkg=ON --build_arrow=OFF \\
+                               build_velox build_gluten_cpp
+
+JDK 17 is already active. To build for Spark 4.1:
+
+    java -version  # must report 17
+    ./dev/buildbundle-veloxbe.sh --enable_vcpkg=ON --build_arrow=OFF \\
+                                 --spark_version=4.1
+
+This image does not include /opt/shims, so it is intended for static packaging
+and reproduction rather than Spark unit tests.
+
+See docs/developers/dev-container.md for details.
+============================================================================
+EOF
+    ;;
+*)
+    echo "ERROR: unknown GLUTEN_DEV_CONTAINER_VARIANT: $DEV_CONTAINER_VARIANT" >&2
+    exit 1
+    ;;
+esac
+
+# Warnings logged with warn() above can easily scroll past unnoticed among all
+# the setup output, so repeat them here, in yellow, after everything else.
+if [ "${#WARNINGS[@]}" -gt 0 ]; then
+    YELLOW='\033[1;33m'
+    NO_COLOR='\033[0m'
+    echo -e "${YELLOW}============================================================================${NO_COLOR}" >&2
+    echo -e "${YELLOW}WARNINGS:${NO_COLOR}" >&2
+    for w in "${WARNINGS[@]}"; do
+        echo -e "${YELLOW}  - $w${NO_COLOR}" >&2
+    done
+    echo -e "${YELLOW}============================================================================${NO_COLOR}" >&2
+fi

.devcontainer/velox-static/devcontainer.json

@@ -0,0 +1,64 @@
+// Static-link environment for packaging and vcpkg issue reproduction.
+// See docs/developers/dev-container.md for build instructions and limitations.
+{
+    "name": "Gluten Velox Backend (centos-9, static link)",
+
+    "image": "apache/gluten:vcpkg-centos-9",
+
+    // Static linking needs substantially more memory than daily development.
+    "hostRequirements": {
+        "cpus": 8,
+        "memory": "64gb",
+        "storage": "64gb"
+    },
+
+    "containerEnv": {
+        "CCACHE_DIR": "/root/.ccache",
+        "CCACHE_MAXSIZE": "10G",
+        "GLUTEN_DEV_CONTAINER_VARIANT": "velox-static"
+    },
+
+    // Keep the compiler aligned with the toolchain used to build the vcpkg cache.
+    "remoteEnv": {
+        "PATH": "/opt/rh/gcc-toolset-12/root/usr/bin:${containerEnv:PATH}",
+        "LD_LIBRARY_PATH": "/opt/rh/gcc-toolset-12/root/usr/lib64:/opt/rh/gcc-toolset-12/root/usr/lib",
+        "PKG_CONFIG_PATH": "/opt/rh/gcc-toolset-12/root/usr/lib64/pkgconfig"
+    },
+
+    "mounts": [
+        "source=gluten-vcpkg-ccache,target=/root/.ccache,type=volume",
+        "source=gluten-vcpkg-m2,target=/root/.m2,type=volume",
+        "source=gluten-vcpkg-binary-cache,target=/var/cache/vcpkg,type=volume"
+    ],
+
+    "postCreateCommand": "bash .devcontainer/post-create.sh",
+
+    "customizations": {
+        "vscode": {
+            "extensions": [
+                "scalameta.metals",
+                "ms-vscode.cpptools",
+                "ms-python.python"
+            ],
+            "settings": {
+                "files.watcherExclude": {
+                    "**/target/**": true,
+                    "**/cpp/build/**": true,
+                    "**/ep/_ep/**": true,
+                    "**/ep/build-velox/build/**": true,
+                    "**/dev/vcpkg/.vcpkg/**": true,
+                    "**/dev/vcpkg/vcpkg_installed/**": true
+                },
+                "search.exclude": {
+                    "**/target/**": true,
+                    "**/cpp/build/**": true,
+                    "**/ep/_ep/**": true,
+                    "**/ep/build-velox/build/**": true,
+                    "**/dev/vcpkg/.vcpkg/**": true,
+                    "**/dev/vcpkg/vcpkg_installed/**": true
+                },
+                "C_Cpp.default.compileCommands": "${workspaceFolder}/cpp/build/compile_commands.json"
+            }
+        }
+    }
+}

docs/developers/NewToGluten.md

@@ -44,9 +44,9 @@ Gluten requires GCC 11 or above.
 ### Dev Container
 
 To skip the manual environment setup, you can develop Gluten inside a pre-built Docker
-image using the [Dev Container](https://containers.dev/) configuration shipped at
-`.devcontainer/devcontainer.json`. See [Dev Container](./dev-container.md) for a brief
-guide and the list of available Gluten Docker images.
+image using the [Dev Container](https://containers.dev/) configurations under
+`.devcontainer/`. See [Dev Containers](./dev-container.md) for configuration,
+build and test instructions.
 
 ## Development