feat: add index build progress callbacks to segment builders

Cargo.toml

@@ -18,6 +18,10 @@ rust-version = "1.91.0"
 crate-type = ["cdylib", "staticlib", "rlib"]
 
 [dependencies]
+# Direct for the `#[async_trait::async_trait]` impl of lance's
+# `lance_index::progress::IndexBuildProgress`; already in the graph
+# transitively via lance-index.
+async-trait = "0.1"
 lance = { git = "https://github.com/lance-format/lance.git", rev = "356acb0d333c96e970f6f84b97314fc5bc4193f7", features = ["substrait"] }
 lance-core = { git = "https://github.com/lance-format/lance.git", rev = "356acb0d333c96e970f6f84b97314fc5bc4193f7" }
 lance-file = { git = "https://github.com/lance-format/lance.git", rev = "356acb0d333c96e970f6f84b97314fc5bc4193f7" }

include/lance/lance.h

@@ -1816,6 +1816,88 @@ int32_t lance_index_segment_builder_execute_uncommitted(
     size_t* out_len
 );
 
+/**
+ * Event codes for LanceIndexBuildProgressCallback, passed as the `event`
+ * argument. Exactly one stage is active at a time: a stage's
+ * LANCE_INDEX_BUILD_PROGRESS_STAGE_COMPLETE is always delivered before the
+ * next stage's LANCE_INDEX_BUILD_PROGRESS_STAGE_START.
+ */
+typedef enum {
+    LANCE_INDEX_BUILD_PROGRESS_STAGE_START = 0,
+    LANCE_INDEX_BUILD_PROGRESS_STAGE_PROGRESS = 1,
+    LANCE_INDEX_BUILD_PROGRESS_STAGE_COMPLETE = 2,
+} LanceIndexBuildProgressEvent;
+
+/**
+ * Receives index build progress events while
+ * lance_index_segment_builder_execute_uncommitted runs.
+ *
+ * `stage` is non-NULL, NUL-terminated, and borrowed: it is valid only for the
+ * duration of this call. `unit` is non-NULL and NUL-terminated, but is the
+ * empty string ("") for LANCE_INDEX_BUILD_PROGRESS_STAGE_PROGRESS and
+ * LANCE_INDEX_BUILD_PROGRESS_STAGE_COMPLETE. The parameter mapping is:
+ *
+ *  - LANCE_INDEX_BUILD_PROGRESS_STAGE_START: `stage` is the stage name,
+ *    `total` is the number of work units (0 = unknown), `unit` describes what
+ *    is being counted (e.g. "partitions", "batches", "rows"; "" = unknown),
+ *    and `completed` is 0.
+ *  - LANCE_INDEX_BUILD_PROGRESS_STAGE_PROGRESS: `total` is 0, `unit` is "",
+ *    and `completed` is the number of units completed so far.
+ *  - LANCE_INDEX_BUILD_PROGRESS_STAGE_COMPLETE: `total` is 0, `unit` is "",
+ *    and `completed` is 0.
+ *
+ * Stage names are index-type-specific (e.g. "train_ivf", "shuffle",
+ * "merge_partitions" for vector indices; "load_data" for scalar indices) and
+ * are diagnostic-only: they are not a stable cross-version contract, so
+ * consumers must treat them as opaque strings.
+ *
+ * The callback is invoked from lance-c's internal tokio runtime worker
+ * threads. Certain stages report progress concurrently from parallel worker
+ * tasks, so the callback MUST be thread-safe and reentrant. It must be
+ * non-blocking and must not call back into any `lance_*` function (no
+ * reentrancy).
+ *
+ * The callback is invoked without a panic guard: it must return normally,
+ * because unwinding or throwing across this boundary can abort the host
+ * process. The callback cannot abort the build; progress reporting is
+ * advisory and diagnostic and cannot affect the build outcome.
+ */
+typedef void (*LanceIndexBuildProgressCallback)(
+    void* callback_ctx,
+    int32_t event,
+    const char* stage,
+    uint64_t total,
+    const char* unit,
+    uint64_t completed
+);
+
+/**
+ * Register the index-build progress callback for a segment builder.
+ *
+ * Must be called before the builder is executed; the builder is single-use,
+ * so calling it after lance_index_segment_builder_execute_uncommitted has
+ * been called (even if that call failed) returns -1. `callback` must not be
+ * NULL. `callback_ctx` may be NULL and is passed through to the callback
+ * opaquely. Setting a callback replaces any previously set callback.
+ *
+ * Invocations occur only while lance_index_segment_builder_execute_uncommitted
+ * is executing, and this is enforced rather than contractual: lance-c
+ * disables the callback and drains in-flight invocations through a retirement
+ * gate before that call returns, including on error, so a worker task
+ * detached by lance core on an error path can never invoke the callback
+ * afterwards. `callback` and `callback_ctx` must therefore remain valid and
+ * safe to invoke until lance_index_segment_builder_execute_uncommitted
+ * returns. See LanceIndexBuildProgressCallback for the full threading and
+ * reentrancy contract.
+ *
+ * @return 0 on success, -1 on error.
+ */
+int32_t lance_index_segment_builder_set_progress_callback(
+    LanceIndexSegmentBuilder* builder,
+    LanceIndexBuildProgressCallback callback,
+    void* callback_ctx
+);
+
 /** Free metadata bytes returned by an uncommitted segment build. NULL-safe. */
 void lance_free_bytes(uint8_t* bytes);
 

include/lance/lance.hpp

@@ -1280,6 +1280,27 @@ class IndexSegmentBuilder {
         return std::vector<uint8_t>(guard.bytes, guard.bytes + len);
     }
 
+    /// Register a non-null index-build progress callback. Must be called
+    /// before execute_uncommitted; the builder is single-use. The callback is
+    /// invoked from internal worker threads and may be called concurrently
+    /// from parallel worker tasks, so it must be thread-safe, non-blocking,
+    /// and must not re-enter any lance_* function. It must return normally;
+    /// unwinding or throwing across this boundary can abort the host process.
+    /// Invocations occur only while execute_uncommitted runs, and this is
+    /// enforced: lance-c disables the callback and drains in-flight
+    /// invocations before that call returns, including on error, so a worker
+    /// task detached by core can never invoke it afterwards. The callback and
+    /// the context (if non-null) must remain valid until execute_uncommitted
+    /// returns. Progress reporting is advisory and cannot affect the build
+    /// outcome.
+    IndexSegmentBuilder& progress_callback(LanceIndexBuildProgressCallback callback,
+                                           void* callback_ctx) {
+        if (lance_index_segment_builder_set_progress_callback(handle_.get(), callback,
+                                                              callback_ctx) != 0)
+            check_error();
+        return *this;
+    }
+
     LanceIndexSegmentBuilder* c_handle() { return handle_.get(); }
 };