From 1bb9171a00d8304f9cbee9061b4e6b6a843cfbc2 Mon Sep 17 00:00:00 2001 From: zhangstar333 Date: Mon, 7 Sep 2026 20:58:04 +0800 Subject: [PATCH 1/2] test --- thirdparty/download-thirdparty.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/thirdparty/download-thirdparty.sh b/thirdparty/download-thirdparty.sh index 23f3be88f18e34..7a3a52819b50ec 100755 --- a/thirdparty/download-thirdparty.sh +++ b/thirdparty/download-thirdparty.sh @@ -732,6 +732,7 @@ if [[ " ${TP_ARCHIVES[*]} " =~ " LANCE_C " ]]; then cd - fi echo "Finished patching ${LANCE_C_SOURCE}" + echo "Finished patching test build ${LANCE_C_SOURCE}" fi if [[ " ${TP_ARCHIVES[*]} " =~ " CCTZ " ]] ; then From 19b79a769da3a6bdcc78c7a546aaf91e11f47d7c Mon Sep 17 00:00:00 2001 From: zhangstar333 Date: Fri, 11 Sep 2026 17:00:32 +0800 Subject: [PATCH 2/2] [fix](lance) Fix Lance JNI compatibility with glibc 2.17 --- build.sh | 2 +- docker/thirdparties/lance-jni-helpers.sh | 117 +++++++++++++++++++++++ docs/lance-jni-packaging.md | 42 ++++++++ fe/pom.xml | 2 +- post-build.sh | 10 +- thirdparty/download-thirdparty.sh | 1 - 6 files changed, 169 insertions(+), 5 deletions(-) create mode 100644 docker/thirdparties/lance-jni-helpers.sh create mode 100644 docs/lance-jni-packaging.md diff --git a/build.sh b/build.sh index 199cf09729e31d..b20175e0c0d0e7 100755 --- a/build.sh +++ b/build.sh @@ -1000,7 +1000,7 @@ if [[ "${BUILD_FE}" -eq 1 ]]; then #cp -r -p "${DORIS_HOME}/docs/build/help-resource.zip" "${DORIS_OUTPUT}/fe/lib"/ - # Third-party filesystem jars (JuiceFS, JindoFS) are packaged by post-build.sh + # Third-party filesystem JARs and the Lance JNI library are packaged by post-build.sh "${DORIS_HOME}/post-build.sh" --fe --output "${DORIS_OUTPUT}" cp -r -p "${DORIS_HOME}/webroot/static" "${DORIS_OUTPUT}/fe/webroot"/ diff --git a/docker/thirdparties/lance-jni-helpers.sh b/docker/thirdparties/lance-jni-helpers.sh new file mode 100644 index 00000000000000..39a909b600bc30 --- /dev/null +++ b/docker/thirdparties/lance-jni-helpers.sh @@ -0,0 +1,117 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Keep these pins in sync with lance.version in fe/pom.xml. +LANCE_JNI_VERSION="11.0.0" +LANCE_JNI_ASSET="liblance_jni-11.0.0-linux-x86_64-glibc2.17-r1.so.gz" +LANCE_JNI_URL="https://github.com/apache/doris-thirdparty/releases/download/lance-jni-11.0.0-glibc2.17-r1/${LANCE_JNI_ASSET}" +LANCE_JNI_ARCHIVE_SHA256="f9dc713269632e26c06ea2ea20637ca16ec9d76772dc20db0bead11e5cd34a6b" +LANCE_JNI_LIBRARY_SHA256="b6540edd3bdcd76b04f96a9e78fe804a2f1bffbd75eb35abe84de9091ba432ac" + +lance_jni_download() ( + set -eo pipefail + local cache_dir="$1" + local archive="${cache_dir}/${LANCE_JNI_ASSET}" + local work_dir url digest + local -a urls=() + + mkdir -p "${cache_dir}" + if [[ -f "${archive}" ]]; then + digest=$(sha256sum "${archive}" | awk '{print $1}') + if [[ "${digest}" == "${LANCE_JNI_ARCHIVE_SHA256}" ]]; then + printf '%s\n' "${archive}" + exit 0 + fi + echo "Lance JNI cache checksum mismatch; downloading again: ${archive}" >&2 + fi + + work_dir=$(mktemp -d "${cache_dir}/.lance-jni-download.XXXXXX") + trap 'rm -rf -- "${work_dir}"' EXIT + if [[ -n "${REPOSITORY_URL:-}" ]]; then + urls+=("${REPOSITORY_URL%/}/${LANCE_JNI_ASSET}") + fi + urls+=("${LANCE_JNI_URL}") + for url in "${urls[@]}"; do + echo "Downloading Lance JNI from ${url}" >&2 + if ! curl -fL --retry 3 --retry-delay 2 --connect-timeout 10 \ + -o "${work_dir}/library.gz" "${url}"; then + continue + fi + digest=$(sha256sum "${work_dir}/library.gz" | awk '{print $1}') + if [[ "${digest}" != "${LANCE_JNI_ARCHIVE_SHA256}" ]]; then + echo "ERROR: Lance JNI archive SHA256 mismatch from ${url}: expected ${LANCE_JNI_ARCHIVE_SHA256}, got ${digest}" >&2 + continue + fi + # Concurrent builds only publish complete, verified cache files. + mv -f "${work_dir}/library.gz" "${archive}" + printf '%s\n' "${archive}" + exit 0 + done + echo "ERROR: failed to download and verify ${LANCE_JNI_ASSET}" >&2 + exit 1 +) + +lance_jni_replace() ( + set -eo pipefail + local output_dir="$1" + local thirdparty_dir="$2" + local target_system="$3" + local target_arch="$4" + if [[ "${target_system}" != "Linux" || "${target_arch}" != "x86_64" ]]; then + exit 0 + fi + + local entry="nativelib/linux-x86-64/liblance_jni.so" + local target_jar="${output_dir}/fe/lib/lance-core-${LANCE_JNI_VERSION}.jar" + local archive work_dir source_hash packaged_hash + local -a lance_jars + shopt -s nullglob + lance_jars=("${output_dir}/fe/lib/"lance-core-*.jar) + if [[ ${#lance_jars[@]} -ne 1 || "${lance_jars[0]}" != "${target_jar}" ]]; then + echo "ERROR: expected exactly one lance-core-${LANCE_JNI_VERSION}.jar in ${output_dir}/fe/lib" >&2 + exit 1 + fi + if ! unzip -Z1 "${target_jar}" | grep -Fx "${entry}" >/dev/null; then + echo "ERROR: missing JNI entry ${entry} in ${target_jar}" >&2 + exit 1 + fi + + archive=$(lance_jni_download "${thirdparty_dir}/installed/lance-jni") + work_dir=$(mktemp -d "${output_dir}/fe/lib/.lance-jni.XXXXXX") + trap 'rm -rf -- "${work_dir}"' EXIT + mkdir -p "${work_dir}/$(dirname "${entry}")" + gzip -dc "${archive}" > "${work_dir}/${entry}" + source_hash=$(sha256sum "${work_dir}/${entry}" | awk '{print $1}') + if [[ "${source_hash}" != "${LANCE_JNI_LIBRARY_SHA256}" ]]; then + echo "ERROR: Lance JNI library SHA256 mismatch: expected ${LANCE_JNI_LIBRARY_SHA256}, got ${source_hash}" >&2 + exit 1 + fi + # Preserve the Maven cache and only replace the output JAR after verification. + cp -p "${target_jar}" "${work_dir}/lance-core.jar" + ( + cd "${work_dir}" + zip -q lance-core.jar "${entry}" + ) + packaged_hash=$(unzip -p "${work_dir}/lance-core.jar" "${entry}" | sha256sum | awk '{print $1}') + if [[ "${source_hash}" != "${packaged_hash}" ]]; then + echo "ERROR: JNI checksum mismatch in ${target_jar}" >&2 + exit 1 + fi + mv -f "${work_dir}/lance-core.jar" "${target_jar}" + echo "Replaced ${target_jar}!/${entry} (SHA256: ${packaged_hash})" +) diff --git a/docs/lance-jni-packaging.md b/docs/lance-jni-packaging.md new file mode 100644 index 00000000000000..b6941bdff2eb60 --- /dev/null +++ b/docs/lance-jni-packaging.md @@ -0,0 +1,42 @@ +# Lance JNI packaging for Linux x86_64 + +`bash build.sh --fe` automatically replaces the Linux x86_64 JNI entry in +`lance-core-11.0.0.jar` with the glibc 2.17 rebuild. No local Lance checkout, +Rust toolchain, or `LANCE_JNI_SO` variable is required on the Doris build host. + +The artifact is pinned in `docker/thirdparties/lance-jni-helpers.sh` and published +in [apache/doris-thirdparty](https://github.com/apache/doris-thirdparty/releases/tag/lance-jni-11.0.0-glibc2.17-r1). +Both the compressed archive and the extracted library have fixed SHA256 values. + +The archive is cached under: + +```text +thirdparty/installed/lance-jni/liblance_jni-11.0.0-linux-x86_64-glibc2.17-r1.so.gz +``` + +With a custom `DORIS_THIRDPARTY`, the cache is under that directory's +`installed/lance-jni/`. A valid cache is reused without network access. A corrupt +cache is downloaded again. The existing `REPOSITORY_URL` mirror is tried first +when configured, followed by the pinned GitHub Release URL. Downloads use curl; +packaging also needs gzip, zip, unzip, and sha256sum. + +To update an existing FE output without compiling Doris: + +```bash +bash post-build.sh --fe +# Or use a custom output directory: +bash post-build.sh --fe --output /path/to/output +``` + +Only the JAR entry `nativelib/linux-x86-64/liblance_jni.so` is changed. The Maven +cache, Java classes, and other native entries are preserved. Download, checksum, +or JAR version failures stop packaging and leave the original output JAR intact. +ARM64, macOS, and BE-only packaging do not download or replace this library. + +The rebuild retains Lance's Haswell CPU baseline (AVX2/FMA/F16C). Its GLIBC symbol +requirements were checked statically; a full CentOS 7 FE runtime test is still +required. An already running FE must restart to load a newly packaged JNI library. + +When upgrading Lance, update the release URL, filename, version, and both hashes +in the helper together with `lance.version` in `fe/pom.xml`. Offline packaging +tests are available via `python3 thirdparty/test/lance-jni-packaging-test.py`. diff --git a/fe/pom.xml b/fe/pom.xml index 8349b523c9605a..5802ef1a6a8b26 100644 --- a/fe/pom.xml +++ b/fe/pom.xml @@ -349,7 +349,7 @@ under the License. 1.11.0 - 9.1.0-beta.3 + 11.0.0 0.40.0 0.53.2-public diff --git a/post-build.sh b/post-build.sh index b69cd7f02ad881..a0605359162d63 100755 --- a/post-build.sh +++ b/post-build.sh @@ -18,7 +18,7 @@ ############################################################## # Post-build script for packaging third-party filesystem JARs -# (JuiceFS, JindoFS) into Doris FE/BE output directories. +# (JuiceFS, JindoFS) and the Lance JNI library into Doris output directories. # # This script is independent of the main build and can be # executed standalone or called from build.sh. @@ -47,6 +47,7 @@ TARGET_ARCH="${TARGET_ARCH:-$(uname -m)}" # --- Source helper scripts --- . "${DORIS_HOME}/docker/thirdparties/juicefs-helpers.sh" . "${DORIS_HOME}/docker/thirdparties/jindofs-helpers.sh" +. "${DORIS_HOME}/docker/thirdparties/lance-jni-helpers.sh" # --- JuiceFS wrapper functions (migrated from build.sh) --- @@ -133,7 +134,8 @@ usage() { Usage: $0 [--fe] [--be] [--output ] [--juicefs] [--jindofs] Package third-party filesystem JARs (JuiceFS, JindoFS) into Doris output directories. -By default, no third-party JARs are packaged. Use --juicefs/--jindofs or environment variables to opt in. +Linux x86_64 FE packaging automatically installs the glibc 2.17 Lance JNI library. +Use --juicefs/--jindofs or environment variables to opt into filesystem JAR packaging. Options: --fe Process FE output directory @@ -222,6 +224,10 @@ for target_type in fe be; do continue fi + if [[ "${target_type}" == "fe" ]]; then + lance_jni_replace "${OUTPUT_DIR}" "${DORIS_THIRDPARTY}" "${TARGET_SYSTEM}" "${TARGET_ARCH}" + fi + if [[ "${BUILD_JINDOFS}" == "ON" ]]; then install_jindofs "${target_type}" "${OUTPUT_DIR}" fi diff --git a/thirdparty/download-thirdparty.sh b/thirdparty/download-thirdparty.sh index 7a3a52819b50ec..23f3be88f18e34 100755 --- a/thirdparty/download-thirdparty.sh +++ b/thirdparty/download-thirdparty.sh @@ -732,7 +732,6 @@ if [[ " ${TP_ARCHIVES[*]} " =~ " LANCE_C " ]]; then cd - fi echo "Finished patching ${LANCE_C_SOURCE}" - echo "Finished patching test build ${LANCE_C_SOURCE}" fi if [[ " ${TP_ARCHIVES[*]} " =~ " CCTZ " ]] ; then