feat(mosaic): support Mosaic file format

Author: lxy-9602

.github/workflows/build_and_test.yaml

@@ -75,7 +75,6 @@ jobs:
           - name: asan-ubsan-x86_64
             build_args: --enable_asan --enable_ubsan
           - name: tsan-x86_64
-            skip_rust: true
             build_args: --enable_tsan
           - name: gcc-debug-aarch64
             runner: ubuntu-24.04-arm
@@ -96,7 +95,6 @@ jobs:
             build_args: --build_type Release
           - name: tsan-aarch64
             runner: ubuntu-24.04-arm
-            skip_rust: true
             build_args: --enable_tsan
     steps:
       - name: Checkout paimon-cpp
@@ -108,8 +106,7 @@ jobs:
         uses: ./.github/actions/setup-ccache
         with:
           cache-key-prefix: ccache-${{ matrix.name }}
-      - name: Install Rust toolchain (tantivy-fts)
-        if: ${{ !matrix.skip_rust }}
+      - name: Install Rust toolchain (Mosaic and tantivy-fts)
         shell: bash
         run: ci/scripts/setup_rust.sh
       - name: Install HTTP and TLS development dependencies

.github/workflows/gcc8_test.yaml

@@ -60,6 +60,9 @@ jobs:
           ls -la
       - name: Checkout paimon-cpp
         uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
+      - name: Install Rust toolchain (Mosaic)
+        shell: bash
+        run: ci/scripts/setup_rust.sh
       - name: Setup ccache
         uses: ./.github/actions/setup-ccache
         with:

.github/workflows/release_candidate.yaml

@@ -131,7 +131,7 @@ jobs:
           name: source-archive
           path: release/ci
 
-      - name: Install Rust toolchain (tantivy-fts)
+      - name: Install Rust toolchain (Mosaic and tantivy-fts)
         shell: bash
         run: ci/scripts/setup_rust.sh
 

CMakeLists.txt

@@ -60,6 +60,7 @@ option(PAIMON_USE_UBSAN "Use Undefined Behavior Sanitizer" OFF)
 option(PAIMON_USE_CXX11_ABI "Use C++11 ABI" ON)
 option(PAIMON_ENABLE_AVRO "Whether to enable avro file format" ON)
 option(PAIMON_ENABLE_ORC "Whether to enable orc file format" ON)
+option(PAIMON_ENABLE_MOSAIC "Whether to enable mosaic file format (Rust FFI)" OFF)
 option(PAIMON_ENABLE_JINDO "Whether to enable jindo file system" OFF)
 option(PAIMON_ENABLE_S3 "Whether to enable S3 file system" OFF)
 option(PAIMON_ENABLE_NETWORK_TESTS
@@ -82,6 +83,9 @@ endif()
 if(PAIMON_ENABLE_AVRO)
     add_definitions(-DPAIMON_ENABLE_AVRO)
 endif()
+if(PAIMON_ENABLE_MOSAIC)
+    add_definitions(-DPAIMON_ENABLE_MOSAIC)
+endif()
 if(PAIMON_ENABLE_JINDO)
     add_definitions(-DPAIMON_ENABLE_JINDO)
 endif()
@@ -384,6 +388,11 @@ if(PAIMON_BUILD_TESTS OR PAIMON_BUILD_BENCHMARKS)
     paimon_link_libraries_whole_archive(PAIMON_PARQUET_FILE_FORMAT_STATIC_LINK_LIBS
                                         paimon_parquet_file_format_static)
 
+    if(PAIMON_ENABLE_MOSAIC)
+        paimon_link_libraries_whole_archive(PAIMON_MOSAIC_FILE_FORMAT_STATIC_LINK_LIBS
+                                            paimon_mosaic_file_format_static)
+    endif()
+
     if(PAIMON_ENABLE_ORC)
         paimon_link_libraries_whole_archive(PAIMON_ORC_FILE_FORMAT_STATIC_LINK_LIBS
                                             paimon_orc_file_format_static)
@@ -435,6 +444,14 @@ if(PAIMON_BUILD_TESTS)
     paimon_link_libraries_whole_archive(PAIMON_PARQUET_FILE_FORMAT_STATIC_LINK_LIBS
                                         paimon_parquet_file_format_static)
 
+    if(PAIMON_ENABLE_MOSAIC)
+        paimon_link_libraries_no_as_needed(TEST_PLUGIN_LINK_LIBS
+                                           paimon_mosaic_file_format_shared)
+        list(APPEND TEST_STATIC_LINK_LIBS ${TEST_PLUGIN_LINK_LIBS})
+        paimon_link_libraries_whole_archive(PAIMON_MOSAIC_FILE_FORMAT_STATIC_LINK_LIBS
+                                            paimon_mosaic_file_format_static)
+    endif()
+
     if(PAIMON_ENABLE_ORC)
         paimon_link_libraries_no_as_needed(TEST_PLUGIN_LINK_LIBS
                                            paimon_orc_file_format_shared)
@@ -516,6 +533,9 @@ add_subdirectory(src/paimon/format/blob)
 add_subdirectory(src/paimon/format/orc)
 add_subdirectory(src/paimon/format/parquet)
 add_subdirectory(src/paimon/format/avro)
+if(PAIMON_ENABLE_MOSAIC)
+    add_subdirectory(src/paimon/format/mosaic)
+endif()
 if(PAIMON_ENABLE_LUMINA)
     add_subdirectory(src/paimon/global_index/lumina)
 endif()

ci/scripts/build_paimon.sh

@@ -147,6 +147,7 @@ CMAKE_ARGS=(
     "-G Ninja"
     "-DCMAKE_BUILD_TYPE=${build_type}"
     "-DPAIMON_BUILD_TESTS=ON"
+    "-DPAIMON_ENABLE_MOSAIC=ON"
     "-DPAIMON_ENABLE_JINDO=ON"
     "-DPAIMON_ENABLE_S3=ON"
     "-DPAIMON_ENABLE_LUMINA=${ENABLE_LUMINA}"

ci/scripts/setup_rust.sh

@@ -15,12 +15,11 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
-# Install the Rust toolchain + cbindgen required to build the
-# tantivy-fts FFI crate (crates/tantivy_ffi) from CI.
+# Install the Rust toolchain used by the Mosaic and tantivy-fts FFI builds, plus cbindgen required
+# by tantivy-fts.
 #
 # The dev container (see .devcontainer/) already has these preinstalled;
-# this script is for the GitHub Actions runners. Called by
-# .github/workflows/build_and_test.yaml before ci/scripts/build_paimon.sh.
+# this script is for the GitHub Actions runners and is called before ci/scripts/build_paimon.sh.
 #
 # Idempotent: a second invocation is a no-op when the tools already exist.
 

cmake_modules/ThirdpartyToolchain.cmake

@@ -91,6 +91,18 @@ else()
     endif()
 endif()
 
+if(DEFINED ENV{PAIMON_MOSAIC_URL})
+    set(MOSAIC_SOURCE_URL "$ENV{PAIMON_MOSAIC_URL}")
+else()
+    if(EXISTS "${THIRDPARTY_DIR}/${PAIMON_MOSAIC_PKG_NAME}")
+        set_urls(MOSAIC_SOURCE_URL "${THIRDPARTY_DIR}/${PAIMON_MOSAIC_PKG_NAME}")
+    else()
+        set_urls(MOSAIC_SOURCE_URL
+                 "https://downloads.apache.org/paimon/paimon-mosaic-${PAIMON_MOSAIC_BUILD_VERSION}/${PAIMON_MOSAIC_PKG_NAME}"
+        )
+    endif()
+endif()
+
 if(DEFINED ENV{PAIMON_RAPIDJSON_URL})
     set(RAPIDJSON_SOURCE_URL "$ENV{PAIMON_RAPIDJSON_URL}")
 else()
@@ -1314,6 +1326,51 @@ macro(build_lumina)
     install(FILES "${LUMINA_DYNAMIC_LIB}" DESTINATION ${CMAKE_INSTALL_LIBDIR})
 endmacro()
 
+macro(build_mosaic)
+    message(STATUS "Building Apache Paimon Mosaic Rust FFI from source")
+    find_program(PAIMON_CARGO_EXECUTABLE cargo REQUIRED)
+
+    set(MOSAIC_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/mosaic_ep-install")
+    set(MOSAIC_INCLUDE_DIR "${MOSAIC_PREFIX}/include")
+    set(MOSAIC_LIB_DIR "${MOSAIC_PREFIX}/${CMAKE_INSTALL_LIBDIR}")
+    set(MOSAIC_DYNAMIC_LIB
+        "${MOSAIC_LIB_DIR}/${CMAKE_SHARED_LIBRARY_PREFIX}paimon_mosaic_ffi${CMAKE_SHARED_LIBRARY_SUFFIX}"
+    )
+    set(MOSAIC_CARGO_TARGET_DIR "${CMAKE_CURRENT_BINARY_DIR}/mosaic_ep-cargo")
+    set(MOSAIC_CARGO_DYNAMIC_LIB
+        "${MOSAIC_CARGO_TARGET_DIR}/release/${CMAKE_SHARED_LIBRARY_PREFIX}paimon_mosaic_ffi${CMAKE_SHARED_LIBRARY_SUFFIX}"
+    )
+
+    file(MAKE_DIRECTORY "${MOSAIC_INCLUDE_DIR}")
+    file(MAKE_DIRECTORY "${MOSAIC_LIB_DIR}")
+
+    externalproject_add(
+        mosaic_ep
+        URL ${MOSAIC_SOURCE_URL}
+        URL_HASH "SHA256=${PAIMON_MOSAIC_BUILD_SHA256_CHECKSUM}"
+        ${THIRDPARTY_LOG_OPTIONS}
+        CONFIGURE_COMMAND ""
+        BUILD_COMMAND
+            ${CMAKE_COMMAND} -E env CARGO_TARGET_DIR=${MOSAIC_CARGO_TARGET_DIR}
+            ${PAIMON_CARGO_EXECUTABLE} build --release --manifest-path
+            <SOURCE_DIR>/ffi/Cargo.toml
+        COMMAND ${CMAKE_COMMAND} -E copy_if_different ${MOSAIC_CARGO_DYNAMIC_LIB}
+                ${MOSAIC_DYNAMIC_LIB}
+        INSTALL_COMMAND ${CMAKE_COMMAND} -E copy_directory <SOURCE_DIR>/include
+                        ${MOSAIC_INCLUDE_DIR}
+        BUILD_BYPRODUCTS "${MOSAIC_DYNAMIC_LIB}")
+
+    add_library(paimon_mosaic_ffi SHARED IMPORTED GLOBAL)
+    set_target_properties(
+        paimon_mosaic_ffi
+        PROPERTIES IMPORTED_LOCATION "${MOSAIC_DYNAMIC_LIB}"
+                   IMPORTED_NO_SONAME TRUE
+                   INTERFACE_INCLUDE_DIRECTORIES "${MOSAIC_INCLUDE_DIR}")
+    add_dependencies(paimon_mosaic_ffi mosaic_ep)
+
+    install(FILES "${MOSAIC_DYNAMIC_LIB}" DESTINATION ${CMAKE_INSTALL_LIBDIR})
+endmacro()
+
 macro(build_jindosdk_nextarch)
     message(STATUS "Building jindosdk-nextarch from local source")
 
@@ -1953,6 +2010,9 @@ paimon_warn_if_mixed_arrow_dependencies()
 resolve_dependency(TBB)
 resolve_dependency(glog)
 
+if(PAIMON_ENABLE_MOSAIC)
+    build_mosaic()
+endif()
 if(PAIMON_ENABLE_AVRO)
     resolve_dependency(Avro)
 endif()

include/paimon/defs.h

@@ -156,6 +156,10 @@ struct PAIMON_EXPORT Options {
     /// 9, but the read and write speed will significantly decrease. Default value is 1.
     static const char FILE_COMPRESSION_ZSTD_LEVEL[];
 
+    /// "file.block-size" - File block size of format, default value of orc stripe is 64 MB, and
+    /// parquet row group is 128 MB.
+    static const char FILE_BLOCK_SIZE[];
+
     /// "manifest.target-file-size" - Suggested file size of a manifest file.
     /// Default value is 8MB.
     static const char MANIFEST_TARGET_FILE_SIZE[];

src/paimon/common/defs.cpp

@@ -46,6 +46,7 @@ const char Options::PAGE_SIZE[] = "page-size";
 const char Options::PARTITION_DEFAULT_NAME[] = "partition.default-name";
 const char Options::FILE_COMPRESSION[] = "file.compression";
 const char Options::FILE_COMPRESSION_ZSTD_LEVEL[] = "file.compression.zstd-level";
+const char Options::FILE_BLOCK_SIZE[] = "file.block-size";
 const char Options::MANIFEST_TARGET_FILE_SIZE[] = "manifest.target-file-size";
 const char Options::MANIFEST_FORMAT[] = "manifest.format";
 const char Options::MANIFEST_COMPRESSION[] = "manifest.compression";

src/paimon/format/mosaic/CMakeLists.txt

@@ -0,0 +1,56 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+set(PAIMON_MOSAIC_FILE_FORMAT
+    mosaic_file_batch_reader.cpp
+    mosaic_file_format.cpp
+    mosaic_file_format_factory.cpp
+    mosaic_format_writer.cpp
+    mosaic_stream.cpp
+    mosaic_writer_builder.cpp)
+
+add_paimon_lib(paimon_mosaic_file_format
+               SOURCES
+               ${PAIMON_MOSAIC_FILE_FORMAT}
+               DEPENDENCIES
+               paimon_shared
+               paimon_mosaic_ffi
+               STATIC_LINK_LIBS
+               arrow
+               fmt
+               paimon_mosaic_ffi
+               Threads::Threads
+               SHARED_LINK_LIBS
+               paimon_shared
+               paimon_mosaic_ffi
+               SHARED_LINK_FLAGS
+               ${PAIMON_VERSION_SCRIPT_FLAGS})
+
+target_link_libraries(paimon_mosaic_file_format_objlib PUBLIC paimon_mosaic_ffi)
+
+if(PAIMON_BUILD_TESTS)
+    add_paimon_test(mosaic_format_test
+                    SOURCES
+                    mosaic_file_format_test.cpp
+                    STATIC_LINK_LIBS
+                    paimon_shared
+                    test_utils_static
+                    ${PAIMON_MOSAIC_FILE_FORMAT_STATIC_LINK_LIBS}
+                    ${PAIMON_LOCAL_FILE_SYSTEM_STATIC_LINK_LIBS}
+                    paimon_mosaic_ffi
+                    ${GTEST_LINK_TOOLCHAIN})
+endif()