From 5f8c8cbcc6d17c7c08785275a93bb4939e541bd8 Mon Sep 17 00:00:00 2001 From: Gabriel Date: Thu, 1 Oct 2026 07:03:54 +0800 Subject: [PATCH 1/4] [improvement](lance) Update lance-c to merged upstream main ### What problem does this PR solve? Related PR: #68687 Pin lance-c to merged upstream main commit `cd63420bfbe27f6f0a1edcc873b9191af7d52852`, including the distance-bounded search (#90), independent batch nearest search (#92), and boolean scalar-segment predicates (#93) changes. Update the archive checksum and refresh the existing Foyer patch against that base with an unchanged implementation payload. Foyer PR [lance-format/lance-c#73](https://github.com/lance-format/lance-c/pull/73) is still open. Its patch must remain temporarily because Doris already calls its cache APIs; it can be removed after upstream merge. This PR changes only `thirdparty/vars.sh` and `thirdparty/patches/lance-c-foyer.patch`. The FE predicate converter/planner changes, fixtures, and SQL tests in #68687 are not included. ### Validation - Verified the downloaded archive checksum and exact patch application. - Third-party extraction, cached-source reuse/refresh, and invalid-patch rejection tests passed on master. - The updated dependency plus the unchanged Foyer patch passed 75 Rust unit tests, 376 C API tests, and all 3 native C/C++/static OSS consumer tests. - The same dependency passed 27 FE-generated Substrait integration scenarios as part of #68687 validation. - Full Doris compilation and BE UT for this revision are pending CI. ### Release note Update the Lance C dependency to merged upstream search and scalar-predicate improvements. ### Check List (For Author) - Test: Dependency unit and integration tests described above. - Behavior changed: Yes; incorporates upstream lance-c fixes and additive APIs. - Does this need documentation: No new Doris user interface in this dependency-only update. --- thirdparty/patches/lance-c-foyer.patch | 31 +++++++++++++------------- thirdparty/vars.sh | 8 +++---- 2 files changed, 19 insertions(+), 20 deletions(-) diff --git a/thirdparty/patches/lance-c-foyer.patch b/thirdparty/patches/lance-c-foyer.patch index 73f244c49fd57d..bb2104ab32d66d 100644 --- a/thirdparty/patches/lance-c-foyer.patch +++ b/thirdparty/patches/lance-c-foyer.patch @@ -1,8 +1,7 @@ # Foyer data-cache integration for lance-format/lance-c#73. -# Base: 9bd730add2ac70316c1d642b8459011e2dd92022 +# Base: cd63420bfbe27f6f0a1edcc873b9191af7d52852 (lance-format/lance-c main, including #93) # Source: https://github.com/Gabriel39/lance-c/commit/24c7ca4bcb9422c113b0d3e07e4efe1173b0bc9f -# Regenerate the payload in lance-c; do not maintain separate downstream edits: -# git diff --full-index --binary 9bd730add2ac70316c1d642b8459011e2dd92022 24c7ca4bcb9422c113b0d3e07e4efe1173b0bc9f -- . | sed 's/^ $//' +# The unchanged Foyer payload is reapplied to the newer upstream base with git apply --3way. diff --git a/Cargo.lock b/Cargo.lock index 78812f1330e146db295a14276f90f9e28654f399..daf9f85daf3e0ea59bb906e8e3c32470519b084b 100644 --- a/Cargo.lock @@ -199,7 +198,7 @@ index 18654b862440a90aa6bf0dd846e8f1ad6036fd7c..fd134a7151e1df61da201d867a642e9b prost = "0.14" snafu = "0.9" diff --git a/README.md b/README.md -index 9ceccdc4f3f6b5d13dc271be3d2659c7be400306..94de4d1c26d8ab869d1726e2b416d21f21661d6d 100644 +index 9f0871b17b13ebfd96916947b4bee0dd06f6a468..441719c9f3904058127e55d1cd29ea1afc878935 100644 --- a/README.md +++ b/README.md @@ -68,6 +68,7 @@ Based on the [liblance RFC](https://github.com/lance-format/lance/discussions/60 @@ -208,9 +207,9 @@ index 9ceccdc4f3f6b5d13dc271be3d2659c7be400306..94de4d1c26d8ab869d1726e2b416d21f | [x] | Filter pushdown | `lance_scanner_set_substrait_filter()` accepts a serialized Substrait `ExtendedExpression`; `lance_scanner_additional_sql_filter()` adds SQL predicates with AND before scanning starts | +| [x] | Data-file cache | Optional Foyer memory/disk cache for immutable `data/*.lance` reads | - ## Multi-vector search + ## Segment-scoped array label filters -@@ -231,6 +232,35 @@ auto ds = lance::Dataset::open_with_session(session, "data.lance"); +@@ -362,6 +363,35 @@ auto ds = lance::Dataset::open_with_session(session, "data.lance"); auto stats = session.cache_stats(); ``` @@ -247,7 +246,7 @@ index 9ceccdc4f3f6b5d13dc271be3d2659c7be400306..94de4d1c26d8ab869d1726e2b416d21f `lance_dataset_open` takes a `version` argument — `0` means the latest, any diff --git a/include/lance/lance.h b/include/lance/lance.h -index 7630913fd7849d0bff2cb0f31e1ab1199cb9f0fb..1c1d0752efdda5023b29eafe5aa9a87e03dc5303 100644 +index 94772e2a14bf9a8a950dfb2d39112136e27197c0..fe48eb0d9afd1c7157a3bd2230cce7e769120bac 100644 --- a/include/lance/lance.h +++ b/include/lance/lance.h @@ -214,6 +214,36 @@ typedef struct LanceSessionCacheStats { @@ -335,7 +334,7 @@ index 7630913fd7849d0bff2cb0f31e1ab1199cb9f0fb..1c1d0752efdda5023b29eafe5aa9a87e void lance_dataset_close(LanceDataset* dataset); diff --git a/include/lance/lance.hpp b/include/lance/lance.hpp -index 286724e96736e354785e046f2fcfe5ae7c65147b..5070d423453a5d60d31d03ce758ad7dbe83a6ea2 100644 +index edcd39fc3caba387ab490fa84139f24e9376130a..17b71957d0f5c419fe0b3392ce11faa757ddbfe1 100644 --- a/include/lance/lance.hpp +++ b/include/lance/lance.hpp @@ -176,6 +176,13 @@ struct SqlColumn { @@ -2244,7 +2243,7 @@ index 1971510de4a13ee6f04fc39c159b088e78e21d55..ba51c87a8cebb6897c48fb55509f71da // SAFETY: `out_dataset` is non-NULL (checked above) and the caller // guarantees it points to caller-owned, writable storage of size diff --git a/tests/c_api_test.rs b/tests/c_api_test.rs -index 1a25e34a8ca50623812ffedaa7bb5a8c54f7850e..c341859e9434d4578b2833eca6ec313a28f31f32 100644 +index d3bea44529a842fe3ae28ee3f90cd8d45fe0b394..98e6901a2cfeb2cc98df6606e26b04788d4218b2 100644 --- a/tests/c_api_test.rs +++ b/tests/c_api_test.rs @@ -100,10 +100,83 @@ fn create_large_dataset(num_rows: i32) -> (tempfile::TempDir, String) { @@ -2503,7 +2502,7 @@ index 1a25e34a8ca50623812ffedaa7bb5a8c54f7850e..c341859e9434d4578b2833eca6ec313a fn test_dataset_restore_to_current_latest_writes_new_manifest() { // Restoring to the current latest still writes a new manifest. The diff --git a/tests/cpp/test_c_api.c b/tests/cpp/test_c_api.c -index 5df9cde4d8fa971823b9e6c974a8a0db6c9ec9ca..1444e55161631ac8ad976f427d867ad91d358195 100644 +index efad5ed55987976a3f1820220e42fc58bef42638..050543aebc02454cab09f6e0bd38ec10b1d21720 100644 --- a/tests/cpp/test_c_api.c +++ b/tests/cpp/test_c_api.c @@ -171,6 +171,37 @@ static void test_shared_session(const char *uri) { @@ -2544,16 +2543,16 @@ index 5df9cde4d8fa971823b9e6c974a8a0db6c9ec9ca..1444e55161631ac8ad976f427d867ad9 static void test_scan(const char *uri) { printf(" test_scan... "); -@@ -1323,6 +1354,7 @@ int main(int argc, char **argv) { - +@@ -1471,6 +1502,7 @@ int main(int argc, char **argv) { + test_batch_nearest(uri); test_open_and_metadata(uri); test_shared_session(uri); + test_data_cache_session(uri, write_uri); test_scan(uri); + test_distance_range(uri); test_scan_with_limit(uri); - test_scanner_blob_handling(blob_uri); diff --git a/tests/cpp/test_cpp_api.cpp b/tests/cpp/test_cpp_api.cpp -index 0332d1a2687354984d0551e10716df08344a4013..ef64ab0d02bed2f1750c3d388792e4aa44265e24 100644 +index e2d541e4430889aba89ab85c01cc7e3eacf97baa..ac1c10bccfcf399bbbfcfe908d3188484aff0add 100644 --- a/tests/cpp/test_cpp_api.cpp +++ b/tests/cpp/test_cpp_api.cpp @@ -131,6 +131,28 @@ static void test_shared_session(const std::string& uri) { @@ -2585,11 +2584,11 @@ index 0332d1a2687354984d0551e10716df08344a4013..ef64ab0d02bed2f1750c3d388792e4aa static void test_dataset_schema(const std::string& uri) { TEST(test_dataset_schema); -@@ -1211,6 +1233,7 @@ int main(int argc, char** argv) { +@@ -1353,6 +1375,7 @@ int main(int argc, char** argv) { test_dataset_open(uri); test_shared_session(uri); + test_data_cache_session(uri, write_uri); test_dataset_schema(uri); test_scanner_fluent(uri); - test_scanner_async_stream_ownership(uri); + test_distance_range(uri); diff --git a/thirdparty/vars.sh b/thirdparty/vars.sh index ed78c1e0f423cf..556f6bf0bf3b03 100644 --- a/thirdparty/vars.sh +++ b/thirdparty/vars.sh @@ -618,10 +618,10 @@ PUGIXML_MD5SUM="3b894c29455eb33a40b165c6e2de5895" # lance-c # Complete-segment prefilter fixes are supplied by upstream lance-c, not local patches. -LANCE_C_DOWNLOAD="https://codeload.github.com/lance-format/lance-c/tar.gz/9bd730add2ac70316c1d642b8459011e2dd92022" -LANCE_C_NAME="lance-c-9bd730add2ac70316c1d642b8459011e2dd92022.tar.gz" -LANCE_C_SOURCE="lance-c-9bd730add2ac70316c1d642b8459011e2dd92022" -LANCE_C_MD5SUM="63851b09bf1689032579f1a094ff2f37" +LANCE_C_DOWNLOAD="https://codeload.github.com/lance-format/lance-c/tar.gz/cd63420bfbe27f6f0a1edcc873b9191af7d52852" +LANCE_C_NAME="lance-c-cd63420bfbe27f6f0a1edcc873b9191af7d52852.tar.gz" +LANCE_C_SOURCE="lance-c-cd63420bfbe27f6f0a1edcc873b9191af7d52852" +LANCE_C_MD5SUM="37d82907559c4fdb8ed6e5b767b0d309" # paimon-rust PAIMON_RUST_DOWNLOAD="https://github.com/apache/paimon-rust/archive/refs/tags/v0.4.0-rc1.tar.gz" From c2858482cf4c35d67df6e8f6b9eac3f13d6dfd33 Mon Sep 17 00:00:00 2001 From: Gabriel Date: Thu, 1 Oct 2026 11:42:59 +0800 Subject: [PATCH 2/4] [fix](build) Validate installed Lance source and patch revisions ### What problem does this PR solve? Related PR: #68687, #68689 Problem Summary: The archive-presence sentinel reused an old ABI-compatible Lance library after the dependency pin changed. Compare the installed source/archive/patch fingerprint with this checkout, rebuild stale or incomplete installs, and fail if an external builder still supplies mismatched output. Invalidate the fingerprint before header/archive publication and publish it only after both succeed. ### Release note Rebuild stale Lance dependencies instead of silently linking the previous revision. ### Check List (For Author) - Test: Build-gate and publication harness passed on master and branch-4.1; shell syntax checks passed. Full Doris compilation and BE UT are pending CI. - Behavior changed: Yes, outdated compilation images require refreshed dependencies. - Does this need documentation: No new user-facing interface. --- build.sh | 9 +- thirdparty/build-thirdparty.sh | 7 ++ thirdparty/lance-install.sh | 38 +++++++ thirdparty/test/lance-install-test.sh | 148 ++++++++++++++++++++++++++ 4 files changed, 201 insertions(+), 1 deletion(-) create mode 100644 thirdparty/lance-install.sh create mode 100644 thirdparty/test/lance-install-test.sh diff --git a/build.sh b/build.sh index de304892669cd9..e4d07ce37b1856 100755 --- a/build.sh +++ b/build.sh @@ -480,6 +480,7 @@ if [[ "${CLEAN}" -eq 1 && "${BUILD_BE}" -eq 0 && "${BUILD_FE}" -eq 0 && ${BUILD_ fi # build thirdparty libraries if necessary. check last thirdparty lib installation +source "${DORIS_HOME}/thirdparty/lance-install.sh" if [[ "${TARGET_SYSTEM}" == 'Darwin' ]]; then LAST_THIRDPARTY_LIB='libbrotlienc.a' else @@ -495,7 +496,8 @@ if [[ ! -f "${DORIS_THIRDPARTY}/installed/lib/${LAST_THIRDPARTY_LIB}" || ! -f "${DORIS_THIRDPARTY}/installed/include/paimon_rust/paimon.h" || ! -s "${DORIS_THIRDPARTY}/installed/lib64/libpaimon_c.a" || ! -s "${DORIS_THIRDPARTY}/installed/include/paimon_rust/paimon.h" || - -e "${DORIS_THIRDPARTY}/installed/lib64/.paimon-installing" ]]; then + -e "${DORIS_THIRDPARTY}/installed/lib64/.paimon-installing" ]] || + ! lance_c_install_is_current "${DORIS_HOME}/thirdparty" "${DORIS_THIRDPARTY}/installed"; then # Compilation images may contain only installed artifacts; never erase them without a rebuild source. if [[ ! -f "${DORIS_THIRDPARTY}/build-thirdparty.sh" ]]; then echo "Third-party dependencies require a rebuild, but build-thirdparty.sh is missing." >&2 @@ -511,6 +513,11 @@ if [[ ! -f "${DORIS_THIRDPARTY}/installed/lib/${LAST_THIRDPARTY_LIB}" || else bash "${DORIS_THIRDPARTY}/build-thirdparty.sh" -j "${PARALLEL}" --clean fi + # An external build script can itself be stale. Never link its old output silently. + if ! lance_c_install_is_current "${DORIS_HOME}/thirdparty" "${DORIS_THIRDPARTY}/installed"; then + echo "Lance dependency revision does not match this checkout. Refresh the third-party build tree." >&2 + exit 1 + fi fi update_submodule() { diff --git a/thirdparty/build-thirdparty.sh b/thirdparty/build-thirdparty.sh index 52c2d481b2068c..da3e36b1e3f8d7 100755 --- a/thirdparty/build-thirdparty.sh +++ b/thirdparty/build-thirdparty.sh @@ -219,6 +219,8 @@ if [[ ! -f "${TP_DIR}/vars.sh" ]]; then fi . "${TP_DIR}/vars.sh" +. "${TP_DIR}/lance-install.sh" +LANCE_C_INSTALL_FINGERPRINT="$(lance_c_install_fingerprint "${TP_DIR}")" cd "${TP_DIR}" @@ -2507,9 +2509,14 @@ build_lance_c() { env "${cargo_env[@]}" "${cargo_bin}" "${cargo_args[@]}" mkdir -p "${TP_INSTALL_DIR}/include" "${TP_INSTALL_DIR}/lib64" + # Invalidate before publishing either file so interrupted installs cannot reuse + # a matching marker with a partial header/archive pair. + rm -f "${TP_INSTALL_DIR}/lib64/.lance-c-fingerprint" rm -rf "${TP_INSTALL_DIR}/include/lance" cp -av include/lance "${TP_INSTALL_DIR}/include/" install_rust_archive "${BUILD_DIR}/release/liblance_c.a" + printf '%s\n' "${LANCE_C_INSTALL_FINGERPRINT}" > "${TP_INSTALL_DIR}/lib64/.lance-c-fingerprint.tmp" + mv "${TP_INSTALL_DIR}/lib64/.lance-c-fingerprint.tmp" "${TP_INSTALL_DIR}/lib64/.lance-c-fingerprint" } build_paimon_rust() { diff --git a/thirdparty/lance-install.sh b/thirdparty/lance-install.sh new file mode 100644 index 00000000000000..28594d61c79026 --- /dev/null +++ b/thirdparty/lance-install.sh @@ -0,0 +1,38 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Keep the check independent of the installed prefix: an external compilation image +# can carry old vars.sh alongside an ABI-compatible but behaviorally stale archive. +lance_c_install_fingerprint() ( + local definitions="$1" + local TP_DIR="${definitions}" + source "${definitions}/vars.sh" || return 1 + local patch_checksum + patch_checksum="$(cksum < "${definitions}/patches/lance-c-foyer.patch")" || return 1 + printf '%s\n' "${LANCE_C_SOURCE}" "${LANCE_C_MD5SUM}" "${patch_checksum}" +) + +lance_c_install_is_current() { + local definitions="$1" installed="$2" expected + [[ -s "${installed}/lib64/liblance_c.a" && + -s "${installed}/include/lance/lance.h" && + -s "${installed}/include/lance/lance.hpp" && + -s "${installed}/lib64/.lance-c-fingerprint" ]] || return 1 + expected="$(lance_c_install_fingerprint "${definitions}")" || return 1 + [[ "$(cat "${installed}/lib64/.lance-c-fingerprint")" == "${expected}" ]] +} diff --git a/thirdparty/test/lance-install-test.sh b/thirdparty/test/lance-install-test.sh new file mode 100644 index 00000000000000..fa3a015114246b --- /dev/null +++ b/thirdparty/test/lance-install-test.sh @@ -0,0 +1,148 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +set -euo pipefail +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +work="$(mktemp -d)" +trap 'rm -rf "${work}"' EXIT +mkdir -p "${work}/repo/thirdparty/patches" "${work}/external" +cp "${ROOT}/thirdparty/vars.sh" "${work}/repo/thirdparty/" +cp "${ROOT}/thirdparty/patches/lance-c-foyer.patch" "${work}/repo/thirdparty/patches/" +# Extract the real build gate; all destructive operations stay inside this temporary install. +sed -n '/^# build thirdparty libraries if necessary/,/^update_submodule()/p' "${ROOT}/build.sh" \ + | sed '$d' > "${work}/gate.sh" +export DORIS_HOME="${work}/repo" DORIS_THIRDPARTY="${work}/external" +export TARGET_SYSTEM=Linux CLEAN=0 PARALLEL=1 +export TEST_HELPER="${ROOT}/thirdparty/lance-install.sh" +if [[ -f "${TEST_HELPER}" ]]; then + cp "${TEST_HELPER}" "${work}/repo/thirdparty/" +fi +cat > "${DORIS_THIRDPARTY}/build-thirdparty.sh" <<'BUILDER' +set -euo pipefail +source "${TEST_HELPER}" +echo rebuilt >> "${DORIS_THIRDPARTY}/builds" +installed="${DORIS_THIRDPARTY}/installed" +mkdir -p "${installed}/lib/hadoop_hdfs/native" "${installed}/lib/hadoop_hdfs_3_4/native" "${installed}/lib64" \ + "${installed}/include/lance" "${installed}/include/paimon_rust" +for file in lib/hadoop_hdfs/native/libhdfs.a lib/hadoop_hdfs_3_4/native/libhdfs.a \ + lib64/liblance_c.a lib64/libpaimon_c.a \ + include/lance/lance.h include/lance/lance.hpp include/paimon_rust/paimon.h; do + echo artifact > "${installed}/${file}" +done +lance_c_install_fingerprint "${DORIS_HOME}/thirdparty" > "${installed}/lib64/.lance-c-fingerprint" +BUILDER +# A complete legacy image has all old sentinels but no Lance revision marker. +installed="${DORIS_THIRDPARTY}/installed" +mkdir -p "${installed}/lib/hadoop_hdfs/native" "${installed}/lib/hadoop_hdfs_3_4/native" "${installed}/lib64" \ + "${installed}/include/lance" "${installed}/include/paimon_rust" +for file in lib/hadoop_hdfs/native/libhdfs.a lib/hadoop_hdfs_3_4/native/libhdfs.a \ + lib64/liblance_c.a lib64/libpaimon_c.a \ + include/lance/lance.h include/lance/lance.hpp include/paimon_rust/paimon.h; do + echo legacy > "${installed}/${file}" +done +bash "${work}/gate.sh" +[[ -f "${DORIS_THIRDPARTY}/builds" ]] || { echo 'FAIL: reused unversioned Lance archive'; exit 1; } +source "${TEST_HELPER}" +lance_c_install_is_current "${DORIS_HOME}/thirdparty" "${installed}" +check_builds() { + [[ "$(wc -l < "${DORIS_THIRDPARTY}/builds")" -eq "$1" ]] +} +check_builds 1 +bash "${work}/gate.sh" +check_builds 1 +echo 'PASS: legacy image rebuilt; matching install reused' +echo stale > "${installed}/lib64/.lance-c-fingerprint" +bash "${work}/gate.sh" +check_builds 2 +# Pin and patch updates both invalidate installed artifacts, even with identical ABI. +sed 's/^LANCE_C_SOURCE=.*/LANCE_C_SOURCE="lance-c-test-revision"/' "${DORIS_HOME}/thirdparty/vars.sh" \ + > "${work}/new-vars.sh" +mv "${work}/new-vars.sh" "${DORIS_HOME}/thirdparty/vars.sh" +bash "${work}/gate.sh" +check_builds 3 +echo '# test patch update' >> "${DORIS_HOME}/thirdparty/patches/lance-c-foyer.patch" +bash "${work}/gate.sh" +check_builds 4 +echo 'PASS: stale marker, changed pin and changed patch rebuild' +rm "${installed}/include/lance/lance.h" +bash "${work}/gate.sh" +check_builds 5 +: > "${installed}/lib64/liblance_c.a" +bash "${work}/gate.sh" +check_builds 6 +echo 'PASS: incomplete header/archive install rebuilt' +# A legacy external builder may exit successfully without installing the new revision. +cp "${DORIS_THIRDPARTY}/build-thirdparty.sh" "${work}/good-builder.sh" +printf '#!/usr/bin/env bash\nexit 0\n' > "${DORIS_THIRDPARTY}/build-thirdparty.sh" +rm "${installed}/lib64/.lance-c-fingerprint" +if bash "${work}/gate.sh" > "${work}/old-builder.log" 2>&1; then + echo 'FAIL: accepted output from a stale external builder'; exit 1 +fi +grep -q 'Lance dependency revision does not match' "${work}/old-builder.log" +bash "${work}/good-builder.sh" +echo 'PASS: stale external builder cannot silently satisfy the revision gate' +# Images without rebuild sources must fail before deleting installed dependencies. +rm "${DORIS_THIRDPARTY}/build-thirdparty.sh" "${installed}/lib64/.lance-c-fingerprint" +if bash "${work}/gate.sh" > "${work}/missing-source.log" 2>&1; then + echo 'FAIL: accepted stale compilation image without build sources'; exit 1 +fi +[[ -s "${installed}/lib64/liblance_c.a" ]] +echo 'PASS: missing rebuild source fails without deleting installed artifacts' +# Execute the actual publication function with a tiny Cargo stand-in. A failed +# archive copy must invalidate the old marker, and only a complete retry may stamp it. +sed -n '/^build_lance_c()/,/^}/p' "${ROOT}/thirdparty/build-thirdparty.sh" > "${work}/publish-function.sh" +export TP_DIR="${DORIS_HOME}/thirdparty" +source "${TP_DIR}/vars.sh" +export LANCE_C_SOURCE TP_SOURCE_DIR TP_INSTALL_DIR +export LANCE_C_INSTALL_FINGERPRINT="$(lance_c_install_fingerprint "${TP_DIR}")" +mkdir -p "${TP_SOURCE_DIR}/${LANCE_C_SOURCE}/include/lance" "${TP_INSTALL_DIR}/bin" +echo header > "${TP_SOURCE_DIR}/${LANCE_C_SOURCE}/include/lance/lance.h" +echo header > "${TP_SOURCE_DIR}/${LANCE_C_SOURCE}/include/lance/lance.hpp" +printf '#!/bin/sh\nexit 0\n' > "${TP_INSTALL_DIR}/bin/protoc" +chmod +x "${TP_INSTALL_DIR}/bin/protoc" +cat > "${work}/cargo" <<'CARGO' +#!/usr/bin/env bash +set -eu +if [[ "$1" == --version ]]; then + echo 'cargo 1.94.0'; exit 0 +fi +mkdir -p "${CARGO_TARGET_DIR}/release" +echo archive > "${CARGO_TARGET_DIR}/release/liblance_c.a" +CARGO +chmod +x "${work}/cargo" +export LANCE_C_CARGO="${work}/cargo" RUSTUP_TOOLCHAIN=1.94.0 +export BUILD_DIR=build KERNEL=Linux LANCE_C_CARGO_OFFLINE=OFF +cat > "${work}/publish.sh" <<'PUBLISH' +set -eo pipefail +check_if_source_exist() { :; } +install_rust_archive() { + if [[ "${FAIL_INSTALL:-0}" == 1 ]]; then return 1; fi + cp "$1" "${TP_INSTALL_DIR}/lib64/liblance_c.a" +} +source "$1" +build_lance_c +PUBLISH +bash "${work}/publish.sh" "${work}/publish-function.sh" > "${work}/publish.log" 2>&1 +lance_c_install_is_current "${TP_DIR}" "${TP_INSTALL_DIR}" +if FAIL_INSTALL=1 bash "${work}/publish.sh" "${work}/publish-function.sh" >> "${work}/publish.log" 2>&1; then + echo 'FAIL: expected archive publication failure'; exit 1 +fi +[[ ! -e "${TP_INSTALL_DIR}/lib64/.lance-c-fingerprint" ]] +bash "${work}/publish.sh" "${work}/publish-function.sh" >> "${work}/publish.log" 2>&1 +lance_c_install_is_current "${TP_DIR}" "${TP_INSTALL_DIR}" +echo 'PASS: successful install stamped; failed publication invalidated; retry repaired' From 717a851ce9b765a838a0d1294c2626f051774012 Mon Sep 17 00:00:00 2001 From: Gabriel Date: Thu, 1 Oct 2026 12:35:46 +0800 Subject: [PATCH 3/4] [fix](build) Validate Lance rebuild inputs before removing dependencies ### What problem does this PR solve? Problem Summary: An incomplete or outdated external third-party tree could erase installed dependencies before its builder failed. Check required inputs and compare the external Lance fingerprint with the checkout before removal. Make the installation harness use independent external definitions, verify preservation on rejected rebuilds, and run it in the third-party CI script job. ### Release note Preserve installed third-party dependencies when Lance rebuild inputs are incomplete or do not match the checkout. ### Check List (For Author) - Test: Installation harness passed on master and branch-4.1; the new missing-helper regression failed before the fix. Shell syntax, workflow YAML, and diff checks passed. - Behavior changed: Yes. Reject invalid Lance rebuild inputs before deleting installed dependencies. - Does this need documentation: No. --- .github/workflows/build-thirdparty.yml | 4 +++ build.sh | 16 ++++++++-- thirdparty/test/lance-install-test.sh | 44 +++++++++++++++++++++----- 3 files changed, 53 insertions(+), 11 deletions(-) diff --git a/.github/workflows/build-thirdparty.yml b/.github/workflows/build-thirdparty.yml index 5e25d5fdc8706e..95098ba3ea4c10 100644 --- a/.github/workflows/build-thirdparty.yml +++ b/.github/workflows/build-thirdparty.yml @@ -49,6 +49,7 @@ jobs: - 'thirdparty/**' - 'env.sh' focused_test_changes: + - 'build.sh' - 'regression-test/pipeline/external/conf/fe.conf' - 'thirdparty/test/adbc-jni-config-test.sh' - '.github/workflows/build-thirdparty.yml' @@ -82,6 +83,9 @@ jobs: run: | thirdparty/test/adbc-jni-config-test.sh + - name: Test Lance installation + run: bash thirdparty/test/lance-install-test.sh + build_linux: name: Build Third Party Libraries (Linux) needs: changes diff --git a/build.sh b/build.sh index e4d07ce37b1856..917c10005aebbd 100755 --- a/build.sh +++ b/build.sh @@ -498,9 +498,19 @@ if [[ ! -f "${DORIS_THIRDPARTY}/installed/lib/${LAST_THIRDPARTY_LIB}" || ! -s "${DORIS_THIRDPARTY}/installed/include/paimon_rust/paimon.h" || -e "${DORIS_THIRDPARTY}/installed/lib64/.paimon-installing" ]] || ! lance_c_install_is_current "${DORIS_HOME}/thirdparty" "${DORIS_THIRDPARTY}/installed"; then - # Compilation images may contain only installed artifacts; never erase them without a rebuild source. - if [[ ! -f "${DORIS_THIRDPARTY}/build-thirdparty.sh" ]]; then - echo "Third-party dependencies require a rebuild, but build-thirdparty.sh is missing." >&2 + # External trees can be partially updated or pinned to another revision. Preserve + # the existing prefix unless their build inputs can produce the requested Lance version. + for input in build-thirdparty.sh download-thirdparty.sh vars.sh lance-install.sh patches/lance-c-foyer.patch; do + if [[ ! -f "${DORIS_THIRDPARTY}/${input}" || ! -r "${DORIS_THIRDPARTY}/${input}" ]]; then + echo "Third-party dependencies require a rebuild, but ${input} is missing or unreadable." >&2 + echo "Refresh the compilation image or set DORIS_THIRDPARTY to a complete third-party source tree." >&2 + exit 1 + fi + done + if ! expected_lance_fingerprint="$(lance_c_install_fingerprint "${DORIS_HOME}/thirdparty")" || + ! rebuild_lance_fingerprint="$(lance_c_install_fingerprint "${DORIS_THIRDPARTY}")" || + [[ "${rebuild_lance_fingerprint}" != "${expected_lance_fingerprint}" ]]; then + echo "Lance rebuild sources do not match this checkout; installed dependencies have been preserved." >&2 echo "Refresh the compilation image or set DORIS_THIRDPARTY to a complete third-party source tree." >&2 exit 1 fi diff --git a/thirdparty/test/lance-install-test.sh b/thirdparty/test/lance-install-test.sh index fa3a015114246b..db93c355c0f421 100644 --- a/thirdparty/test/lance-install-test.sh +++ b/thirdparty/test/lance-install-test.sh @@ -24,17 +24,20 @@ mkdir -p "${work}/repo/thirdparty/patches" "${work}/external" cp "${ROOT}/thirdparty/vars.sh" "${work}/repo/thirdparty/" cp "${ROOT}/thirdparty/patches/lance-c-foyer.patch" "${work}/repo/thirdparty/patches/" # Extract the real build gate; all destructive operations stay inside this temporary install. +echo 'set -eo pipefail' > "${work}/gate.sh" sed -n '/^# build thirdparty libraries if necessary/,/^update_submodule()/p' "${ROOT}/build.sh" \ - | sed '$d' > "${work}/gate.sh" + | sed '$d' >> "${work}/gate.sh" export DORIS_HOME="${work}/repo" DORIS_THIRDPARTY="${work}/external" export TARGET_SYSTEM=Linux CLEAN=0 PARALLEL=1 export TEST_HELPER="${ROOT}/thirdparty/lance-install.sh" -if [[ -f "${TEST_HELPER}" ]]; then - cp "${TEST_HELPER}" "${work}/repo/thirdparty/" -fi +cp "${TEST_HELPER}" "${work}/repo/thirdparty/" +cp -r "${DORIS_HOME}/thirdparty/." "${DORIS_THIRDPARTY}/" +printf '#!/usr/bin/env bash\nexit 0\n' > "${DORIS_THIRDPARTY}/download-thirdparty.sh" cat > "${DORIS_THIRDPARTY}/build-thirdparty.sh" <<'BUILDER' set -euo pipefail -source "${TEST_HELPER}" +TP_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +source "${TP_DIR}/lance-install.sh" +fingerprint="$(lance_c_install_fingerprint "${TP_DIR}")" echo rebuilt >> "${DORIS_THIRDPARTY}/builds" installed="${DORIS_THIRDPARTY}/installed" mkdir -p "${installed}/lib/hadoop_hdfs/native" "${installed}/lib/hadoop_hdfs_3_4/native" "${installed}/lib64" \ @@ -44,7 +47,7 @@ for file in lib/hadoop_hdfs/native/libhdfs.a lib/hadoop_hdfs_3_4/native/libhdfs. include/lance/lance.h include/lance/lance.hpp include/paimon_rust/paimon.h; do echo artifact > "${installed}/${file}" done -lance_c_install_fingerprint "${DORIS_HOME}/thirdparty" > "${installed}/lib64/.lance-c-fingerprint" +printf '%s\n' "${fingerprint}" > "${installed}/lib64/.lance-c-fingerprint" BUILDER # A complete legacy image has all old sentinels but no Lance revision marker. installed="${DORIS_THIRDPARTY}/installed" @@ -66,6 +69,26 @@ check_builds 1 bash "${work}/gate.sh" check_builds 1 echo 'PASS: legacy image rebuilt; matching install reused' +# Failed preflight must preserve every dependency, not just the Lance archive. +expect_preserved_install() { + cp -a "${installed}" "${work}/saved-install" + cp "${DORIS_THIRDPARTY}/builds" "${work}/saved-builds" + if bash "${work}/gate.sh" > "${work}/preflight.log" 2>&1; then + echo 'FAIL: accepted incomplete or mismatched rebuild sources'; exit 1 + fi + if ! diff -r "${work}/saved-install" "${installed}"; then + echo 'FAIL: rejected rebuild sources after changing installed dependencies'; exit 1 + fi + cmp "${work}/saved-builds" "${DORIS_THIRDPARTY}/builds" + rm -rf "${work}/saved-install" +} +echo stale > "${installed}/lib64/.lance-c-fingerprint" +for file in lance-install.sh vars.sh patches/lance-c-foyer.patch download-thirdparty.sh build-thirdparty.sh; do + mv "${DORIS_THIRDPARTY}/${file}" "${work}/missing-input" + expect_preserved_install + mv "${work}/missing-input" "${DORIS_THIRDPARTY}/${file}" +done +echo 'PASS: incomplete external sources rejected before changing installed dependencies' echo stale > "${installed}/lib64/.lance-c-fingerprint" bash "${work}/gate.sh" check_builds 2 @@ -73,12 +96,16 @@ check_builds 2 sed 's/^LANCE_C_SOURCE=.*/LANCE_C_SOURCE="lance-c-test-revision"/' "${DORIS_HOME}/thirdparty/vars.sh" \ > "${work}/new-vars.sh" mv "${work}/new-vars.sh" "${DORIS_HOME}/thirdparty/vars.sh" +expect_preserved_install +cp "${DORIS_HOME}/thirdparty/vars.sh" "${DORIS_THIRDPARTY}/vars.sh" bash "${work}/gate.sh" check_builds 3 echo '# test patch update' >> "${DORIS_HOME}/thirdparty/patches/lance-c-foyer.patch" +expect_preserved_install +cp "${DORIS_HOME}/thirdparty/patches/lance-c-foyer.patch" "${DORIS_THIRDPARTY}/patches/" bash "${work}/gate.sh" check_builds 4 -echo 'PASS: stale marker, changed pin and changed patch rebuild' +echo 'PASS: mismatched sources rejected; synchronized pin and patch updates rebuild' rm "${installed}/include/lance/lance.h" bash "${work}/gate.sh" check_builds 5 @@ -94,7 +121,8 @@ if bash "${work}/gate.sh" > "${work}/old-builder.log" 2>&1; then echo 'FAIL: accepted output from a stale external builder'; exit 1 fi grep -q 'Lance dependency revision does not match' "${work}/old-builder.log" -bash "${work}/good-builder.sh" +cp "${work}/good-builder.sh" "${DORIS_THIRDPARTY}/build-thirdparty.sh" +bash "${DORIS_THIRDPARTY}/build-thirdparty.sh" echo 'PASS: stale external builder cannot silently satisfy the revision gate' # Images without rebuild sources must fail before deleting installed dependencies. rm "${DORIS_THIRDPARTY}/build-thirdparty.sh" "${installed}/lib64/.lance-c-fingerprint" From 3dcc69710f8bcda8cb99b83d382014c1437af0e6 Mon Sep 17 00:00:00 2001 From: Gabriel Date: Thu, 1 Oct 2026 13:55:30 +0800 Subject: [PATCH 4/4] [chore](build) Keep existing thirdparty workflow path triggers ### What problem does this PR solve? Related PR: #68689 Problem Summary: Remove the added build.sh path triggers while retaining the Lance installation test in the existing thirdparty script job. ### Release note None ### Check List (For Author) - Test: Workflow YAML and trigger assertions, git diff --check, and Lance installation script tests passed. - Behavior changed: Yes; build.sh-only changes no longer select the thirdparty script job. - Does this need documentation: No --- .github/workflows/build-thirdparty.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/build-thirdparty.yml b/.github/workflows/build-thirdparty.yml index 95098ba3ea4c10..b8ac6624bb4ca9 100644 --- a/.github/workflows/build-thirdparty.yml +++ b/.github/workflows/build-thirdparty.yml @@ -49,7 +49,6 @@ jobs: - 'thirdparty/**' - 'env.sh' focused_test_changes: - - 'build.sh' - 'regression-test/pipeline/external/conf/fe.conf' - 'thirdparty/test/adbc-jni-config-test.sh' - '.github/workflows/build-thirdparty.yml'