diff --git a/be/cmake/thirdparty.cmake b/be/cmake/thirdparty.cmake index 1e18bf739a1b80..7f518a8cb5e2cc 100644 --- a/be/cmake/thirdparty.cmake +++ b/be/cmake/thirdparty.cmake @@ -161,6 +161,7 @@ endif() add_thirdparty(minizip LIB64) add_thirdparty(simdjson LIB64) +add_thirdparty(simdutf LIB64) add_thirdparty(idn LIB64) add_thirdparty(xml2 LIB64) add_thirdparty(lzma LIB64) diff --git a/dist/LICENSE-dist.txt b/dist/LICENSE-dist.txt index 1b522d2df14fed..74ebcec7dc9191 100644 --- a/dist/LICENSE-dist.txt +++ b/dist/LICENSE-dist.txt @@ -1472,6 +1472,7 @@ The Apache Software License, Version 2.0 * aws sdk: 1.9.211 * benchmark: 1.8.0 * simdjson: 3.0.1 + * simdutf: 9.2.0 * libhdfs3: 2.3.8 * opentelemetry-proto: 0.18.0 * opentelemetry-cpp: 1.4.0 diff --git a/thirdparty/build-thirdparty.sh b/thirdparty/build-thirdparty.sh index e4c83186df97a1..893825d7aa75c2 100755 --- a/thirdparty/build-thirdparty.sh +++ b/thirdparty/build-thirdparty.sh @@ -1758,6 +1758,26 @@ build_simdjson() { cp -r "${TP_SOURCE_DIR}/${SIMDJSON_SOURCE}/include"/* "${TP_INCLUDE_DIR}/" } +# simdutf +build_simdutf() { + check_if_source_exist "${SIMDUTF_SOURCE}" + cd "${TP_SOURCE_DIR}/${SIMDUTF_SOURCE}" + + "${CMAKE_CMD}" -G "${GENERATOR}" -S . -B "${BUILD_DIR}" \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_INSTALL_PREFIX="${TP_INSTALL_DIR}" \ + -DCMAKE_INSTALL_LIBDIR=lib64 \ + -DCMAKE_POSITION_INDEPENDENT_CODE=ON \ + -DBUILD_SHARED_LIBS=OFF \ + -DSIMDUTF_CXX_STANDARD="${TP_CXX_STANDARD}" \ + -DSIMDUTF_TESTS=OFF \ + -DSIMDUTF_TOOLS=OFF \ + -DSIMDUTF_BENCHMARKS=OFF + + "${CMAKE_CMD}" --build "${BUILD_DIR}" --target simdutf -j "${PARALLEL}" + "${CMAKE_CMD}" --install "${BUILD_DIR}" +} + # nlohmann_json build_nlohmann_json() { check_if_source_exist "${NLOHMANN_JSON_SOURCE}" @@ -2229,6 +2249,7 @@ if [[ "${#packages[@]}" -eq 0 ]]; then hdfs3 benchmark simdjson + simdutf nlohmann_json google_cloud_cpp libbacktrace @@ -2314,6 +2335,7 @@ cleanup_package_source() { libunwind) src_var="LIBUNWIND_SOURCE" ;; benchmark) src_var="BENCHMARK_SOURCE" ;; simdjson) src_var="SIMDJSON_SOURCE" ;; + simdutf) src_var="SIMDUTF_SOURCE" ;; nlohmann_json) src_var="NLOHMANN_JSON_SOURCE" ;; google_cloud_cpp) src_var="GOOGLE_CLOUD_CPP_SOURCE" ;; libbacktrace) src_var="LIBBACKTRACE_SOURCE" ;; diff --git a/thirdparty/vars.sh b/thirdparty/vars.sh index 16570caacb15c3..08bbf06681e66a 100644 --- a/thirdparty/vars.sh +++ b/thirdparty/vars.sh @@ -438,6 +438,12 @@ SIMDJSON_NAME=simdjson-3.11.6.tar.gz SIMDJSON_SOURCE=simdjson-3.11.6 SIMDJSON_MD5SUM="e7d9c814a4fdd6e47119ce5cf4240f4e" +# simdutf +SIMDUTF_DOWNLOAD="https://github.com/simdutf/simdutf/archive/refs/tags/v9.2.0.tar.gz" +SIMDUTF_NAME=simdutf-9.2.0.tar.gz +SIMDUTF_SOURCE=simdutf-9.2.0 +SIMDUTF_MD5SUM="abeae9267c1c1caf183b8d24a39c0147" + # nlohmann_json NLOHMANN_JSON_DOWNLOAD="https://github.com/nlohmann/json/archive/refs/tags/v3.10.1.tar.gz" NLOHMANN_JSON_NAME=json-3.10.1.tar.gz @@ -642,6 +648,7 @@ export TP_ARCHIVES=( 'BENCHMARK' 'XSIMD' 'SIMDJSON' + 'SIMDUTF' 'NLOHMANN_JSON' 'GOOGLE_CLOUD_CPP' 'LIBBACKTRACE'