[improvement](lance) Update lance-c to merged upstream main

.github/workflows/build-thirdparty.yml

@@ -82,6 +82,9 @@ jobs:
         run: |
           thirdparty/test/adbc-jni-config-test.sh
 
+      - name: Test Lance installation
+        run: bash thirdparty/test/lance-install-test.sh
+
   build_linux:
     name: Build Third Party Libraries (Linux)
     needs: changes

build.sh

@@ -480,6 +480,7 @@ if [[ "${CLEAN}" -eq 1 && "${BUILD_BE}" -eq 0 && "${BUILD_FE}" -eq 0 && ${BUILD_
 fi
 
 # build thirdparty libraries if necessary. check last thirdparty lib installation
+source "${DORIS_HOME}/thirdparty/lance-install.sh"
 if [[ "${TARGET_SYSTEM}" == 'Darwin' ]]; then
     LAST_THIRDPARTY_LIB='libbrotlienc.a'
 else
@@ -495,10 +496,21 @@ if [[ ! -f "${DORIS_THIRDPARTY}/installed/lib/${LAST_THIRDPARTY_LIB}" ||
       ! -f "${DORIS_THIRDPARTY}/installed/include/paimon_rust/paimon.h" ||
       ! -s "${DORIS_THIRDPARTY}/installed/lib64/libpaimon_c.a" ||
       ! -s "${DORIS_THIRDPARTY}/installed/include/paimon_rust/paimon.h" ||
-      -e "${DORIS_THIRDPARTY}/installed/lib64/.paimon-installing" ]]; then
-    # Compilation images may contain only installed artifacts; never erase them without a rebuild source.
-    if [[ ! -f "${DORIS_THIRDPARTY}/build-thirdparty.sh" ]]; then
-        echo "Third-party dependencies require a rebuild, but build-thirdparty.sh is missing." >&2
+      -e "${DORIS_THIRDPARTY}/installed/lib64/.paimon-installing" ]] ||
+        ! lance_c_install_is_current "${DORIS_HOME}/thirdparty" "${DORIS_THIRDPARTY}/installed"; then
+    # External trees can be partially updated or pinned to another revision. Preserve
+    # the existing prefix unless their build inputs can produce the requested Lance version.
+    for input in build-thirdparty.sh download-thirdparty.sh vars.sh lance-install.sh patches/lance-c-foyer.patch; do
+        if [[ ! -f "${DORIS_THIRDPARTY}/${input}" || ! -r "${DORIS_THIRDPARTY}/${input}" ]]; then
+            echo "Third-party dependencies require a rebuild, but ${input} is missing or unreadable." >&2
+            echo "Refresh the compilation image or set DORIS_THIRDPARTY to a complete third-party source tree." >&2
+            exit 1
+        fi
+    done
+    if ! expected_lance_fingerprint="$(lance_c_install_fingerprint "${DORIS_HOME}/thirdparty")" ||
+       ! rebuild_lance_fingerprint="$(lance_c_install_fingerprint "${DORIS_THIRDPARTY}")" ||
+       [[ "${rebuild_lance_fingerprint}" != "${expected_lance_fingerprint}" ]]; then
+        echo "Lance rebuild sources do not match this checkout; installed dependencies have been preserved." >&2
         echo "Refresh the compilation image or set DORIS_THIRDPARTY to a complete third-party source tree." >&2
         exit 1
     fi
@@ -511,6 +523,11 @@ if [[ ! -f "${DORIS_THIRDPARTY}/installed/lib/${LAST_THIRDPARTY_LIB}" ||
     else
         bash "${DORIS_THIRDPARTY}/build-thirdparty.sh" -j "${PARALLEL}" --clean
     fi
+    # An external build script can itself be stale. Never link its old output silently.
+    if ! lance_c_install_is_current "${DORIS_HOME}/thirdparty" "${DORIS_THIRDPARTY}/installed"; then
+        echo "Lance dependency revision does not match this checkout. Refresh the third-party build tree." >&2
+        exit 1
+    fi
 fi
 
 update_submodule() {

thirdparty/build-thirdparty.sh

@@ -219,6 +219,8 @@ if [[ ! -f "${TP_DIR}/vars.sh" ]]; then
 fi
 
 . "${TP_DIR}/vars.sh"
+. "${TP_DIR}/lance-install.sh"
+LANCE_C_INSTALL_FINGERPRINT="$(lance_c_install_fingerprint "${TP_DIR}")"
 
 cd "${TP_DIR}"
 
@@ -2507,9 +2509,14 @@ build_lance_c() {
     env "${cargo_env[@]}" "${cargo_bin}" "${cargo_args[@]}"
 
     mkdir -p "${TP_INSTALL_DIR}/include" "${TP_INSTALL_DIR}/lib64"
+    # Invalidate before publishing either file so interrupted installs cannot reuse
+    # a matching marker with a partial header/archive pair.
+    rm -f "${TP_INSTALL_DIR}/lib64/.lance-c-fingerprint"
     rm -rf "${TP_INSTALL_DIR}/include/lance"
     cp -av include/lance "${TP_INSTALL_DIR}/include/"
     install_rust_archive "${BUILD_DIR}/release/liblance_c.a"
+    printf '%s\n' "${LANCE_C_INSTALL_FINGERPRINT}" > "${TP_INSTALL_DIR}/lib64/.lance-c-fingerprint.tmp"
+    mv "${TP_INSTALL_DIR}/lib64/.lance-c-fingerprint.tmp" "${TP_INSTALL_DIR}/lib64/.lance-c-fingerprint"
 }
 
 build_paimon_rust() {

thirdparty/lance-install.sh

@@ -0,0 +1,38 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# Keep the check independent of the installed prefix: an external compilation image
+# can carry old vars.sh alongside an ABI-compatible but behaviorally stale archive.
+lance_c_install_fingerprint() (
+    local definitions="$1"
+    local TP_DIR="${definitions}"
+    source "${definitions}/vars.sh" || return 1
+    local patch_checksum
+    patch_checksum="$(cksum < "${definitions}/patches/lance-c-foyer.patch")" || return 1
+    printf '%s\n' "${LANCE_C_SOURCE}" "${LANCE_C_MD5SUM}" "${patch_checksum}"
+)
+
+lance_c_install_is_current() {
+    local definitions="$1" installed="$2" expected
+    [[ -s "${installed}/lib64/liblance_c.a" &&
+       -s "${installed}/include/lance/lance.h" &&
+       -s "${installed}/include/lance/lance.hpp" &&
+       -s "${installed}/lib64/.lance-c-fingerprint" ]] || return 1
+    expected="$(lance_c_install_fingerprint "${definitions}")" || return 1
+    [[ "$(cat "${installed}/lib64/.lance-c-fingerprint")" == "${expected}" ]]
+}

thirdparty/patches/lance-c-foyer.patch

@@ -1,8 +1,7 @@
 # Foyer data-cache integration for lance-format/lance-c#73.
-# Base: 9bd730add2ac70316c1d642b8459011e2dd92022
+# Base: cd63420bfbe27f6f0a1edcc873b9191af7d52852 (lance-format/lance-c main, including #93)
 # Source: https://github.com/Gabriel39/lance-c/commit/24c7ca4bcb9422c113b0d3e07e4efe1173b0bc9f
-# Regenerate the payload in lance-c; do not maintain separate downstream edits:
-# git diff --full-index --binary 9bd730add2ac70316c1d642b8459011e2dd92022 24c7ca4bcb9422c113b0d3e07e4efe1173b0bc9f -- . | sed 's/^ $//'
+# The unchanged Foyer payload is reapplied to the newer upstream base with git apply --3way.
 diff --git a/Cargo.lock b/Cargo.lock
 index 78812f1330e146db295a14276f90f9e28654f399..daf9f85daf3e0ea59bb906e8e3c32470519b084b 100644
 --- a/Cargo.lock
@@ -199,7 +198,7 @@ index 18654b862440a90aa6bf0dd846e8f1ad6036fd7c..fd134a7151e1df61da201d867a642e9b
  prost = "0.14"
  snafu = "0.9"
 diff --git a/README.md b/README.md
-index 9ceccdc4f3f6b5d13dc271be3d2659c7be400306..94de4d1c26d8ab869d1726e2b416d21f21661d6d 100644
+index 9f0871b17b13ebfd96916947b4bee0dd06f6a468..441719c9f3904058127e55d1cd29ea1afc878935 100644
 --- a/README.md
 +++ b/README.md
 @@ -68,6 +68,7 @@ Based on the [liblance RFC](https://github.com/lance-format/lance/discussions/60
@@ -208,9 +207,9 @@ index 9ceccdc4f3f6b5d13dc271be3d2659c7be400306..94de4d1c26d8ab869d1726e2b416d21f
  | [x] | Filter pushdown | `lance_scanner_set_substrait_filter()` accepts a serialized Substrait `ExtendedExpression`; `lance_scanner_additional_sql_filter()` adds SQL predicates with AND before scanning starts |
 +| [x] | Data-file cache | Optional Foyer memory/disk cache for immutable `data/*.lance` reads |
 
- ## Multi-vector search
+ ## Segment-scoped array label filters
 
-@@ -231,6 +232,35 @@ auto ds = lance::Dataset::open_with_session(session, "data.lance");
+@@ -362,6 +363,35 @@ auto ds = lance::Dataset::open_with_session(session, "data.lance");
  auto stats = session.cache_stats();
  ```
 
@@ -247,7 +246,7 @@ index 9ceccdc4f3f6b5d13dc271be3d2659c7be400306..94de4d1c26d8ab869d1726e2b416d21f
 
  `lance_dataset_open` takes a `version` argument — `0` means the latest, any
 diff --git a/include/lance/lance.h b/include/lance/lance.h
-index 7630913fd7849d0bff2cb0f31e1ab1199cb9f0fb..1c1d0752efdda5023b29eafe5aa9a87e03dc5303 100644
+index 94772e2a14bf9a8a950dfb2d39112136e27197c0..fe48eb0d9afd1c7157a3bd2230cce7e769120bac 100644
 --- a/include/lance/lance.h
 +++ b/include/lance/lance.h
 @@ -214,6 +214,36 @@ typedef struct LanceSessionCacheStats {
@@ -335,7 +334,7 @@ index 7630913fd7849d0bff2cb0f31e1ab1199cb9f0fb..1c1d0752efdda5023b29eafe5aa9a87e
  void lance_dataset_close(LanceDataset* dataset);
 
 diff --git a/include/lance/lance.hpp b/include/lance/lance.hpp
-index 286724e96736e354785e046f2fcfe5ae7c65147b..5070d423453a5d60d31d03ce758ad7dbe83a6ea2 100644
+index edcd39fc3caba387ab490fa84139f24e9376130a..17b71957d0f5c419fe0b3392ce11faa757ddbfe1 100644
 --- a/include/lance/lance.hpp
 +++ b/include/lance/lance.hpp
 @@ -176,6 +176,13 @@ struct SqlColumn {
@@ -2244,7 +2243,7 @@ index 1971510de4a13ee6f04fc39c159b088e78e21d55..ba51c87a8cebb6897c48fb55509f71da
          // SAFETY: `out_dataset` is non-NULL (checked above) and the caller
          // guarantees it points to caller-owned, writable storage of size
 diff --git a/tests/c_api_test.rs b/tests/c_api_test.rs
-index 1a25e34a8ca50623812ffedaa7bb5a8c54f7850e..c341859e9434d4578b2833eca6ec313a28f31f32 100644
+index d3bea44529a842fe3ae28ee3f90cd8d45fe0b394..98e6901a2cfeb2cc98df6606e26b04788d4218b2 100644
 --- a/tests/c_api_test.rs
 +++ b/tests/c_api_test.rs
 @@ -100,10 +100,83 @@ fn create_large_dataset(num_rows: i32) -> (tempfile::TempDir, String) {
@@ -2503,7 +2502,7 @@ index 1a25e34a8ca50623812ffedaa7bb5a8c54f7850e..c341859e9434d4578b2833eca6ec313a
  fn test_dataset_restore_to_current_latest_writes_new_manifest() {
      // Restoring to the current latest still writes a new manifest. The
 diff --git a/tests/cpp/test_c_api.c b/tests/cpp/test_c_api.c
-index 5df9cde4d8fa971823b9e6c974a8a0db6c9ec9ca..1444e55161631ac8ad976f427d867ad91d358195 100644
+index efad5ed55987976a3f1820220e42fc58bef42638..050543aebc02454cab09f6e0bd38ec10b1d21720 100644
 --- a/tests/cpp/test_c_api.c
 +++ b/tests/cpp/test_c_api.c
 @@ -171,6 +171,37 @@ static void test_shared_session(const char *uri) {
@@ -2544,16 +2543,16 @@ index 5df9cde4d8fa971823b9e6c974a8a0db6c9ec9ca..1444e55161631ac8ad976f427d867ad9
  static void test_scan(const char *uri) {
      printf("  test_scan... ");
 
-@@ -1323,6 +1354,7 @@ int main(int argc, char **argv) {
-
+@@ -1471,6 +1502,7 @@ int main(int argc, char **argv) {
+     test_batch_nearest(uri);
      test_open_and_metadata(uri);
      test_shared_session(uri);
 +    test_data_cache_session(uri, write_uri);
      test_scan(uri);
+     test_distance_range(uri);
      test_scan_with_limit(uri);
-     test_scanner_blob_handling(blob_uri);
 diff --git a/tests/cpp/test_cpp_api.cpp b/tests/cpp/test_cpp_api.cpp
-index 0332d1a2687354984d0551e10716df08344a4013..ef64ab0d02bed2f1750c3d388792e4aa44265e24 100644
+index e2d541e4430889aba89ab85c01cc7e3eacf97baa..ac1c10bccfcf399bbbfcfe908d3188484aff0add 100644
 --- a/tests/cpp/test_cpp_api.cpp
 +++ b/tests/cpp/test_cpp_api.cpp
 @@ -131,6 +131,28 @@ static void test_shared_session(const std::string& uri) {
@@ -2585,11 +2584,11 @@ index 0332d1a2687354984d0551e10716df08344a4013..ef64ab0d02bed2f1750c3d388792e4aa
  static void test_dataset_schema(const std::string& uri) {
      TEST(test_dataset_schema);
 
-@@ -1211,6 +1233,7 @@ int main(int argc, char** argv) {
+@@ -1353,6 +1375,7 @@ int main(int argc, char** argv) {
 
      test_dataset_open(uri);
      test_shared_session(uri);
 +    test_data_cache_session(uri, write_uri);
      test_dataset_schema(uri);
      test_scanner_fluent(uri);
-     test_scanner_async_stream_ownership(uri);
+     test_distance_range(uri);