From 1a5d51e4a47b12fb8dbbf141b8b733dff0445b60 Mon Sep 17 00:00:00 2001 From: "jan (via bardioc)" Date: Wed, 3 Jun 2026 12:58:08 +0000 Subject: [PATCH] feat(sdk): add Lance backend struct + endpoint helper Companion to PR #35 (which forwarded the kv-lance feature from the SDK to surrealdb-core). PR #35 made the feature reachable at the SDK level; this PR adds the actual surface adopters need to USE the Lance backend from the SDK: - Lance struct in engine::local (mirror of RocksDb, gated by kv-lance feature) - opt/endpoint/lance.rs: endpoint helper that converts paths into lance:// URLs (mirror of rocksdb.rs) - opt/endpoint/mod.rs: register mod lance; + Lance EndpointKind variant + "lance" URL scheme parser arm - engine/any/native.rs: Lance arm in the connect-by-scheme match - engine/any/wasm.rs: same for wasm Pattern is a direct mirror of RocksDb; reviewers can diff the new files against rocksdb.rs / RocksDb's existing wiring to verify the correspondence. Concrete downstream use (AdaWorldAPI/bardioc substrate-b): ```rust use surrealdb::engine::local::Lance; use surrealdb::Surreal; #[tokio::main] async fn main() -> surrealdb::Result<()> { let db = Surreal::new::("path/to/lance/dataset").await?; // ... use db with the Lance backend Ok(()) } ``` Without this PR, declaring features = ["kv-lance"] at the SDK level (PR #35) lets the kv-lance backend compile in via surrealdb-core but the SDK exposes no Lance struct + no lance:// endpoint helper - so adopters cannot instantiate Surreal::new::(...) from SDK-only code. Adopters had to depend directly on surrealdb-core OR construct the Datastore by hand, both of which defeat the SDK's purpose. Tests: the existing rocksdb integration test in surrealdb/tests/api_integration/mod.rs uses the pattern "surrealdb::engine::local::{Db, RocksDb}"; a parallel Lance test can follow once this PR lands (CI bandwidth permitting; not in this PR). Provenance: - bardioc B1-step2 brutal 02 (code-correctness lens) surfaced the gap; PR #35 forwarded the feature; this PR completes the surface - Draft: bardioc/.agent-logs/upstream-contributions/surrealdb-01-kv-lance-feature-on-sdk.md - Companion: AdaWorldAPI/surrealdb#35 (merged) --- surrealdb/src/engine/any/native.rs | 21 +++++++++++++++ surrealdb/src/engine/any/wasm.rs | 17 ++++++++++++ surrealdb/src/engine/local/mod.rs | 41 +++++++++++++++++++++++++++++ surrealdb/src/opt/endpoint/lance.rs | 41 +++++++++++++++++++++++++++++ surrealdb/src/opt/endpoint/mod.rs | 8 ++++-- 5 files changed, 126 insertions(+), 2 deletions(-) create mode 100644 surrealdb/src/opt/endpoint/lance.rs diff --git a/surrealdb/src/engine/any/native.rs b/surrealdb/src/engine/any/native.rs index 472d184a1fdf..6be4b68b07da 100644 --- a/surrealdb/src/engine/any/native.rs +++ b/surrealdb/src/engine/any/native.rs @@ -86,6 +86,27 @@ impl conn::Sealed for Any { )); } + EndpointKind::Lance => { + #[cfg(feature = "kv-lance")] + { + features.insert(ExtraFeatures::Backup); + features.insert(ExtraFeatures::LiveQueries); + tokio::spawn(engine::local::native::run_router( + address, + conn_tx, + route_rx, + session_clone.receiver.clone(), + )); + conn_rx.recv().await.map_err(crate::std_error_to_types_error)?? + } + + #[cfg(not(feature = "kv-lance"))] + return Err(Error::configuration( + "Cannot connect to the `lance` storage engine as it is not enabled in this build of SurrealDB".to_string(), + None, + )); + } + EndpointKind::TiKv => { #[cfg(feature = "kv-tikv")] { diff --git a/surrealdb/src/engine/any/wasm.rs b/surrealdb/src/engine/any/wasm.rs index b7638bf56350..9e9a711efada 100644 --- a/surrealdb/src/engine/any/wasm.rs +++ b/surrealdb/src/engine/any/wasm.rs @@ -92,6 +92,23 @@ impl conn::Sealed for Any { return Err(Error::internal("Cannot connect to the `rocksdb` storage engine as it is not enabled in this build of SurrealDB".to_owned())); } + EndpointKind::Lance => { + #[cfg(feature = "kv-lance")] + { + features.insert(ExtraFeatures::LiveQueries); + spawn_local(engine::local::wasm::run_router( + address, + conn_tx, + route_rx, + session_clone.receiver.clone(), + )); + conn_rx.recv().await.map_err(crate::std_error_to_types_error)??; + } + + #[cfg(not(feature = "kv-lance"))] + return Err(Error::internal("Cannot connect to the `lance` storage engine as it is not enabled in this build of SurrealDB".to_owned())); + } + EndpointKind::SurrealKv => { #[cfg(feature = "kv-surrealkv")] { diff --git a/surrealdb/src/engine/local/mod.rs b/surrealdb/src/engine/local/mod.rs index c1ec5f57be59..3a1b9644bc31 100644 --- a/surrealdb/src/engine/local/mod.rs +++ b/surrealdb/src/engine/local/mod.rs @@ -287,6 +287,47 @@ pub struct Mem; #[derive(Debug)] pub struct RocksDb; +/// Lance columnar database +/// +/// Provides a Lance-backed local storage engine. Lance is a columnar dataset +/// format with built-in vector search, manifest-based versioning, and +/// append-only fragment storage. The Lance backend uses the path argument as +/// the dataset location on the local filesystem. +/// +/// # Examples +/// +/// Instantiating a Lance-backed instance +/// +/// ```no_run +/// # #[tokio::main] +/// # async fn main() -> surrealdb::Result<()> { +/// use surrealdb::Surreal; +/// use surrealdb::engine::local::Lance; +/// +/// let db = Surreal::new::("path/to/lance/dataset").await?; +/// # Ok(()) +/// # } +/// ``` +/// +/// Instantiating a Lance-backed strict instance +/// +/// ```no_run +/// # #[tokio::main] +/// # async fn main() -> surrealdb::Result<()> { +/// use surrealdb::opt::Config; +/// use surrealdb::Surreal; +/// use surrealdb::engine::local::Lance; +/// +/// let config = Config::default().strict(); +/// let db = Surreal::new::(("path/to/lance/dataset", config)).await?; +/// # Ok(()) +/// # } +/// ``` +#[cfg(feature = "kv-lance")] +#[cfg_attr(docsrs, doc(cfg(feature = "kv-lance")))] +#[derive(Debug)] +pub struct Lance; + /// IndxDB database /// /// # Examples diff --git a/surrealdb/src/opt/endpoint/lance.rs b/surrealdb/src/opt/endpoint/lance.rs new file mode 100644 index 000000000000..888da83c1851 --- /dev/null +++ b/surrealdb/src/opt/endpoint/lance.rs @@ -0,0 +1,41 @@ +use std::path::{Path, PathBuf}; + +use url::Url; + +use crate::Result; +use crate::engine::local::{Db, Lance}; +use crate::opt::endpoint::into_endpoint; +use crate::opt::{Config, Endpoint, IntoEndpoint}; + +macro_rules! endpoints { + ($($name:ty),*) => { + $( + impl IntoEndpoint for $name {} + impl into_endpoint::Sealed for $name { + type Client = Db; + + fn into_endpoint(self) -> Result { + let protocol = "lance://"; + let url = Url::parse(protocol) + .unwrap_or_else(|_| unreachable!("`{protocol}` should be static and valid")); + let mut endpoint = Endpoint::new(url); + endpoint.path = super::path_to_string(protocol, self); + Ok(endpoint) + } + } + + impl IntoEndpoint for ($name, Config) {} + impl into_endpoint::Sealed for ($name, Config) { + type Client = Db; + + fn into_endpoint(self) -> Result { + let mut endpoint = into_endpoint::Sealed::::into_endpoint(self.0)?; + endpoint.config = self.1; + Ok(endpoint) + } + } + )* + } +} + +endpoints!(&str, &String, String, &Path, PathBuf); diff --git a/surrealdb/src/opt/endpoint/mod.rs b/surrealdb/src/opt/endpoint/mod.rs index 4b29a1eafa5c..f6f8ffc65b09 100644 --- a/surrealdb/src/opt/endpoint/mod.rs +++ b/surrealdb/src/opt/endpoint/mod.rs @@ -9,12 +9,14 @@ mod indxdb; mod mem; #[cfg(feature = "kv-rocksdb")] mod rocksdb; +#[cfg(feature = "kv-lance")] +mod lance; #[cfg(feature = "kv-surrealkv")] mod surrealkv; #[cfg(feature = "kv-tikv")] mod tikv; -#[cfg(any(feature = "kv-mem", feature = "kv-surrealkv", feature = "kv-rocksdb"))] +#[cfg(any(feature = "kv-mem", feature = "kv-surrealkv", feature = "kv-rocksdb", feature = "kv-lance"))] mod local; use url::Url; @@ -54,7 +56,7 @@ impl Endpoint { /// Append a query parameter to the endpoint path string. /// Only used when a local engine (e.g. `kv-mem`, `kv-rocksdb`) is enabled. #[cfg_attr( - not(any(feature = "kv-mem", feature = "kv-surrealkv", feature = "kv-rocksdb")), + not(any(feature = "kv-mem", feature = "kv-surrealkv", feature = "kv-rocksdb", feature = "kv-lance")), allow(dead_code) )] pub(crate) fn append_query_param(&mut self, key: &str, value: &str) { @@ -151,6 +153,7 @@ pub enum EndpointKind { IndxDb, Memory, RocksDb, + Lance, TiKv, Unsupported(String), SurrealKv, @@ -167,6 +170,7 @@ impl From<&str> for EndpointKind { "indxdb" => Self::IndxDb, "mem" => Self::Memory, "rocksdb" => Self::RocksDb, + "lance" => Self::Lance, "tikv" => Self::TiKv, "surrealkv" => Self::SurrealKv, _ => Self::Unsupported(s.to_owned()),