WIP: Initial exploration for migration from minio to seaweedfs

.github/workflows/cpp.yml

@@ -33,7 +33,7 @@ on:
       - 'ci/scripts/cpp_*'
       - 'ci/scripts/install_azurite.sh'
       - 'ci/scripts/install_gcs_testbench.sh'
-      - 'ci/scripts/install_minio.sh'
+      - 'ci/scripts/install_seaweedfs.sh'
       - 'ci/scripts/msys2_*'
       - 'ci/scripts/util_*'
       - 'cpp/**'
@@ -50,7 +50,7 @@ on:
       - 'ci/scripts/cpp_*'
       - 'ci/scripts/install_azurite.sh'
       - 'ci/scripts/install_gcs_testbench.sh'
-      - 'ci/scripts/install_minio.sh'
+      - 'ci/scripts/install_seaweedfs.sh'
       - 'ci/scripts/msys2_*'
       - 'ci/scripts/util_*'
       - 'cpp/**'
@@ -235,10 +235,10 @@ jobs:
           # Remove once the runner ships a newer Homebrew.
           brew update
           brew bundle --file=cpp/Brewfile
-      - name: Install MinIO
+      - name: Install SeaweedFS
         run: |
           $(brew --prefix bash)/bin/bash \
-            ci/scripts/install_minio.sh latest ${ARROW_HOME}
+            ci/scripts/install_seaweedfs.sh 4.31 ${ARROW_HOME}
       - name: Set up Python
         uses: actions/setup-python@v6
         with:
@@ -381,6 +381,7 @@ jobs:
           # https://github.com/apache/arrow/issues/48593
           ci/scripts/download_tz_database.sh
       - name: Download MinIO
+        # TODO: Install seaweedfs
         # Match the version pinned in ci/scripts/install_minio.sh.
         shell: msys2 {0}
         run: |

.github/workflows/cpp_extra.yml

@@ -34,7 +34,7 @@ on:
       - 'ci/scripts/cpp_*'
       - 'ci/scripts/install_azurite.sh'
       - 'ci/scripts/install_gcs_testbench.sh'
-      - 'ci/scripts/install_minio.sh'
+      - 'ci/scripts/install_seaweedfs.sh'
       - 'ci/scripts/msys2_*'
       - 'ci/scripts/util_*'
       - 'cpp/**'
@@ -57,7 +57,7 @@ on:
       - 'ci/scripts/cpp_*'
       - 'ci/scripts/install_azurite.sh'
       - 'ci/scripts/install_gcs_testbench.sh'
-      - 'ci/scripts/install_minio.sh'
+      - 'ci/scripts/install_seaweedfs.sh'
       - 'ci/scripts/msys2_*'
       - 'ci/scripts/util_*'
       - 'cpp/**'

.github/workflows/python.yml

@@ -205,10 +205,10 @@ jobs:
           python -m pip install \
             -r python/requirements-build.txt \
             -r python/requirements-test.txt
-      - name: Install MinIO
+      - name: Install SeaweedFS
         run: |
           $(brew --prefix bash)/bin/bash \
-            ci/scripts/install_minio.sh latest /usr/local
+            ci/scripts/install_seaweedfs.sh 4.31 /usr/local
       - name: Setup ccache
         shell: bash
         run: ci/scripts/ccache_setup.sh

.github/workflows/r.yml

@@ -31,7 +31,7 @@ on:
       - "ci/etc/rprofile"
       - "ci/scripts/PKGBUILD"
       - "ci/scripts/cpp_*.sh"
-      - "ci/scripts/install_minio.sh"
+      - "ci/scripts/install_seaweedfs.sh"
       - "ci/scripts/r_*.sh"
       - "cpp/**"
       - "compose.yaml"
@@ -44,7 +44,7 @@ on:
       - "ci/etc/rprofile"
       - "ci/scripts/PKGBUILD"
       - "ci/scripts/cpp_*.sh"
-      - "ci/scripts/install_minio.sh"
+      - "ci/scripts/install_seaweedfs.sh"
       - "ci/scripts/r_*.sh"
       - "cpp/**"
       - "compose.yaml"
@@ -279,7 +279,7 @@ jobs:
         shell: bash
         run: |
           mkdir -p "$HOME/.local/bin"
-          ci/scripts/install_minio.sh latest "$HOME/.local"
+          ci/scripts/install_seaweedfs.sh 4.31 "$HOME/.local"
           echo "$HOME/.local/bin" >> $GITHUB_PATH
       - name: Download Timezone Database
         shell: bash

.github/workflows/r_extra.yml

@@ -31,7 +31,7 @@ on:
       - 'ci/etc/rprofile'
       - 'ci/scripts/PKGBUILD'
       - 'ci/scripts/cpp_*.sh'
-      - 'ci/scripts/install_minio.sh'
+      - 'ci/scripts/install_seaweedfs.sh'
       - 'ci/scripts/r_*.sh'
       - 'cpp/**'
       - 'compose.yaml'
@@ -49,7 +49,7 @@ on:
       - 'ci/etc/rprofile'
       - 'ci/scripts/PKGBUILD'
       - 'ci/scripts/cpp_*.sh'
-      - 'ci/scripts/install_minio.sh'
+      - 'ci/scripts/install_seaweedfs.sh'
       - 'ci/scripts/r_*.sh'
       - 'cpp/**'
       - 'compose.yaml'

ci/docker/alpine-linux-3.22-cpp.dockerfile

@@ -70,8 +70,8 @@ RUN apk add \
     ln -s /usr/share/zoneinfo/Etc/UTC /etc/localtime && \
     echo "Etc/UTC" > /etc/timezone
 
-COPY ci/scripts/install_minio.sh /arrow/ci/scripts/
-RUN /arrow/ci/scripts/install_minio.sh latest /usr/local
+COPY ci/scripts/install_seaweedfs.sh /arrow/ci/scripts/
+RUN /arrow/ci/scripts/install_seaweedfs.sh 4.31 /usr/local
 
 COPY ci/scripts/install_gcs_testbench.sh /arrow/ci/scripts/
 RUN /arrow/ci/scripts/install_gcs_testbench.sh default

ci/docker/alpine-linux-3.22-r.dockerfile

@@ -67,7 +67,7 @@ RUN echo "MAKEFLAGS=-j$(R -s -e 'cat(parallel::detectCores())')" >> /usr/lib/R/e
 COPY ci/scripts/r_docker_configure.sh /arrow/ci/scripts/
 COPY ci/etc/rprofile /arrow/ci/etc/
 COPY ci/scripts/r_install_system_dependencies.sh /arrow/ci/scripts/
-COPY ci/scripts/install_minio.sh /arrow/ci/scripts/
+COPY ci/scripts/install_seaweedfs.sh /arrow/ci/scripts/
 COPY ci/scripts/install_gcs_testbench.sh /arrow/ci/scripts/
 RUN /arrow/ci/scripts/r_docker_configure.sh
 

ci/docker/conda-cpp.dockerfile

@@ -19,8 +19,8 @@ ARG repo
 ARG arch
 FROM ${repo}:${arch}-conda
 
-COPY ci/scripts/install_minio.sh /arrow/ci/scripts
-RUN /arrow/ci/scripts/install_minio.sh latest /opt/conda
+COPY ci/scripts/install_seaweedfs.sh /arrow/ci/scripts
+RUN /arrow/ci/scripts/install_seaweedfs.sh 4.31 /opt/conda
 
 # Unless overridden use Python 3.10
 # Google GCS fails building with Python 3.11 at the moment.

ci/docker/debian-13-cpp.dockerfile

@@ -95,8 +95,8 @@ RUN apt-get update -y -q && \
     apt-get clean && \
     rm -rf /var/lib/apt/lists/*
 
-COPY ci/scripts/install_minio.sh /arrow/ci/scripts/
-RUN /arrow/ci/scripts/install_minio.sh latest /usr/local
+COPY ci/scripts/install_seaweedfs.sh /arrow/ci/scripts/
+RUN /arrow/ci/scripts/install_seaweedfs.sh 4.31 /usr/local
 
 COPY ci/scripts/install_gcs_testbench.sh /arrow/ci/scripts/
 RUN /arrow/ci/scripts/install_gcs_testbench.sh default

ci/docker/debian-experimental-cpp.dockerfile

@@ -98,8 +98,8 @@ RUN if [ -n "${gcc}" ]; then \
     apt-get clean && \
     rm -rf /var/lib/apt/lists/*
 
-COPY ci/scripts/install_minio.sh /arrow/ci/scripts/
-RUN /arrow/ci/scripts/install_minio.sh latest /usr/local
+COPY ci/scripts/install_seaweedfs.sh /arrow/ci/scripts/
+RUN /arrow/ci/scripts/install_seaweedfs.sh 4.31 /usr/local
 
 COPY ci/scripts/install_gcs_testbench.sh /arrow/ci/scripts/
 RUN /arrow/ci/scripts/install_gcs_testbench.sh default

ci/docker/fedora-42-cpp.dockerfile

@@ -68,8 +68,8 @@ RUN dnf update -y && \
         which \
         zlib-devel
 
-COPY ci/scripts/install_minio.sh /arrow/ci/scripts/
-RUN /arrow/ci/scripts/install_minio.sh latest /usr/local
+COPY ci/scripts/install_seaweedfs.sh /arrow/ci/scripts/
+RUN /arrow/ci/scripts/install_seaweedfs.sh 4.31 /usr/local
 
 COPY ci/scripts/install_gcs_testbench.sh /arrow/ci/scripts/
 RUN /arrow/ci/scripts/install_gcs_testbench.sh default

ci/docker/fedora-42-r-clang.dockerfile

@@ -205,7 +205,7 @@ RUN mkdir -p /root/.R && \
 COPY ci/scripts/r_docker_configure.sh /arrow/ci/scripts/
 COPY ci/etc/rprofile /arrow/ci/etc/
 COPY ci/scripts/r_install_system_dependencies.sh /arrow/ci/scripts/
-COPY ci/scripts/install_minio.sh /arrow/ci/scripts/
+COPY ci/scripts/install_seaweedfs.sh /arrow/ci/scripts/
 COPY ci/scripts/install_gcs_testbench.sh /arrow/ci/scripts/
 RUN /arrow/ci/scripts/r_docker_configure.sh
 

ci/docker/linux-apt-r.dockerfile

@@ -93,8 +93,8 @@ RUN python3 -m venv ${ARROW_PYTHON_VENV} && \
     pip install -U pip setuptools wheel && \
     pip install -r arrow/python/requirements-build.txt
 
-COPY ci/scripts/install_minio.sh /arrow/ci/scripts/
-RUN /arrow/ci/scripts/install_minio.sh latest /usr/local
+COPY ci/scripts/install_seaweedfs.sh /arrow/ci/scripts/
+RUN /arrow/ci/scripts/install_seaweedfs.sh 4.31 /usr/local
 
 COPY ci/scripts/install_gcs_testbench.sh /arrow/ci/scripts/
 RUN /arrow/ci/scripts/install_gcs_testbench.sh default

ci/docker/linux-r.dockerfile

@@ -43,7 +43,7 @@ ENV PATH "${RPREFIX}/bin:${PATH}"
 COPY ci/scripts/r_docker_configure.sh /arrow/ci/scripts/
 COPY ci/etc/rprofile /arrow/ci/etc/
 COPY ci/scripts/r_install_system_dependencies.sh /arrow/ci/scripts/
-COPY ci/scripts/install_minio.sh /arrow/ci/scripts/
+COPY ci/scripts/install_seaweedfs.sh /arrow/ci/scripts/
 COPY ci/scripts/install_gcs_testbench.sh /arrow/ci/scripts/
 RUN /arrow/ci/scripts/r_docker_configure.sh
 

ci/docker/ubuntu-22.04-cpp-minimal.dockerfile

@@ -74,8 +74,8 @@ ARG cmake
 COPY ci/scripts/install_cmake.sh /arrow/ci/scripts/
 RUN /arrow/ci/scripts/install_cmake.sh ${cmake} /usr/local/
 
-COPY ci/scripts/install_minio.sh /arrow/ci/scripts/
-RUN /arrow/ci/scripts/install_minio.sh latest /usr/local
+COPY ci/scripts/install_seaweedfs.sh /arrow/ci/scripts/
+RUN /arrow/ci/scripts/install_seaweedfs.sh 4.31 /usr/local
 
 COPY ci/scripts/install_gcs_testbench.sh /arrow/ci/scripts/
 RUN /arrow/ci/scripts/install_gcs_testbench.sh default

ci/docker/ubuntu-22.04-cpp.dockerfile

@@ -169,8 +169,8 @@ ARG cmake
 COPY ci/scripts/install_cmake.sh /arrow/ci/scripts/
 RUN /arrow/ci/scripts/install_cmake.sh ${cmake} /usr/local/
 
-COPY ci/scripts/install_minio.sh /arrow/ci/scripts/
-RUN /arrow/ci/scripts/install_minio.sh latest /usr/local
+COPY ci/scripts/install_seaweedfs.sh /arrow/ci/scripts/
+RUN /arrow/ci/scripts/install_seaweedfs.sh 4.31 /usr/local
 
 COPY ci/scripts/install_gcs_testbench.sh /arrow/ci/scripts/
 RUN /arrow/ci/scripts/install_gcs_testbench.sh default

ci/docker/ubuntu-24.04-cpp-minimal.dockerfile

@@ -71,8 +71,8 @@ RUN latest_system_llvm=14 && \
     apt-get clean && \
     rm -rf /var/lib/apt/lists*
 
-COPY ci/scripts/install_minio.sh /arrow/ci/scripts/
-RUN /arrow/ci/scripts/install_minio.sh latest /usr/local
+COPY ci/scripts/install_seaweedfs.sh /arrow/ci/scripts/
+RUN /arrow/ci/scripts/install_seaweedfs.sh 4.31 /usr/local
 
 COPY ci/scripts/install_gcs_testbench.sh /arrow/ci/scripts/
 RUN /arrow/ci/scripts/install_gcs_testbench.sh default

ci/docker/ubuntu-24.04-cpp.dockerfile

@@ -155,8 +155,8 @@ RUN if [ "${gcc}" = "" ]; then \
       update-alternatives --set c++ /usr/bin/g++; \
     fi
 
-COPY ci/scripts/install_minio.sh /arrow/ci/scripts/
-RUN /arrow/ci/scripts/install_minio.sh latest /usr/local
+COPY ci/scripts/install_seaweedfs.sh /arrow/ci/scripts/
+RUN /arrow/ci/scripts/install_seaweedfs.sh 4.31 /usr/local
 
 COPY ci/scripts/install_gcs_testbench.sh /arrow/ci/scripts/
 RUN /arrow/ci/scripts/install_gcs_testbench.sh default

ci/scripts/cpp_test.sh

@@ -50,7 +50,7 @@ fi
 if ! type storage-testbench >/dev/null 2>&1; then
   exclude_tests+=("arrow-gcsfs-test")
 fi
-if ! type minio >/dev/null 2>&1; then
+if ! type weed >/dev/null 2>&1; then
   exclude_tests+=("arrow-s3fs-test")
 fi
 case "$(uname)" in

ci/scripts/install_seaweedfs.sh

@@ -0,0 +1,92 @@
+#!/usr/bin/env bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+set -eu
+
+if [ "$#" -ne 2 ]; then
+  echo "Usage: $0 <version> <prefix>"
+  exit 1
+fi
+
+# The version is the SeaweedFS release tag without the leading "v" (e.g. 4.31).
+version=$1
+prefix=$2
+
+declare -A archs
+archs=([x86_64]=amd64
+       [arm64]=arm64
+       [aarch64]=arm64)
+
+arch=$(uname -m)
+if [ -z "${archs[$arch]}" ]; then
+  echo "Unsupported architecture: ${arch}"
+  exit 0
+fi
+arch=${archs[$arch]}
+
+# SeaweedFS ships statically linked Go binaries, so there is no
+# glibc/musl distinction: a single Linux archive works everywhere.
+platform=$(uname)
+extension=tar.gz
+case ${platform} in
+  Linux)
+    platform=linux
+    ;;
+  Darwin)
+    platform=darwin
+    ;;
+  MSYS_NT*|MINGW64_NT*)
+    platform=windows
+    extension=zip
+    ;;
+  *)
+    echo "Unsupported platform: ${platform}"
+    exit 0
+    ;;
+esac
+
+download()
+{
+  local output=$1
+  local url=$2
+
+  mkdir -p "$(dirname "${output}")"
+  if type wget > /dev/null 2>&1; then
+    wget -nv --output-document "${output}" "${url}"
+  else
+    curl --fail --location --output "${output}" "${url}"
+  fi
+}
+
+if [[ ! -x ${prefix}/bin/weed ]]; then
+  url="https://github.com/seaweedfs/seaweedfs/releases/download/${version}/${platform}_${arch}.${extension}"
+  echo "Fetching ${url}..."
+  tmp=$(mktemp -d)
+  archive="${tmp}/seaweedfs.${extension}"
+  download "${archive}" "${url}"
+  if [ "${extension}" = "zip" ]; then
+    unzip -q "${archive}" -d "${tmp}"
+  else
+    tar -xzf "${archive}" -C "${tmp}"
+  fi
+  mkdir -p "${prefix}/bin"
+  mv "${tmp}/weed" "${prefix}/bin/weed"
+  chmod +x "${prefix}/bin/weed"
+  rm -rf "${tmp}"
+fi

ci/scripts/r_install_system_dependencies.sh

@@ -53,8 +53,8 @@ esac
 
 if [ "$ARROW_S3" == "ON" ] || [ "$ARROW_GCS" == "ON" ] || [ "$ARROW_R_DEV" == "TRUE" ]; then
   # The Dockerfile should have put this file here
-  if [ "$ARROW_S3" == "ON" ] && [ -f "${ARROW_SOURCE_HOME}/ci/scripts/install_minio.sh" ] && [ "`which wget`" ]; then
-    "${ARROW_SOURCE_HOME}/ci/scripts/install_minio.sh" latest /usr/local
+  if [ "$ARROW_S3" == "ON" ] && [ -f "${ARROW_SOURCE_HOME}/ci/scripts/install_seaweedfs.sh" ] && [ "`which wget`" ]; then
+    "${ARROW_SOURCE_HOME}/ci/scripts/install_seaweedfs.sh" 4.31 /usr/local
   fi
 
   if [ "$ARROW_GCS" == "ON" ] && [ -f "${ARROW_SOURCE_HOME}/ci/scripts/install_gcs_testbench.sh" ]; then

dev/tasks/python-wheels/github.osx.yml

@@ -123,10 +123,10 @@ jobs:
           PIPX_BASE_PYTHON: {{ '${{ steps.gcs-python-install.outputs.python-path }}' }}
         run: arrow/ci/scripts/install_gcs_testbench.sh default
 
-      - name: Install MinIO
+      - name: Install SeaweedFS
         run: |
           $(brew --prefix bash)/bin/bash \
-            arrow/ci/scripts/install_minio.sh latest /usr/local
+            arrow/ci/scripts/install_seaweedfs.sh 4.31 /usr/local
 
       - name: Test Wheel
         env:

docs/source/developers/continuous_integration/docker.rst

@@ -222,7 +222,7 @@ responsible for. Like:
 - ``docs_build.sh``: build the Sphinx documentation.
 - ``integration_dask.sh``: execute the dask integration tests.
 - ``integration_pandas.sh``: execute the pandas integration tests.
-- ``install_minio.sh``: install minio server for multiple platforms.
+- ``install_seaweedfs.sh``: install SeaweedFS server for multiple platforms.
 - ``install_conda.sh``: install miniconda for multiple platforms.
 - ``install_gcs_testbench.sh``: install the GCS testbench for multiple platforms.