diff --git a/Cargo.lock b/Cargo.lock index 3193fec..78812f1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -57,56 +57,6 @@ dependencies = [ "libc", ] -[[package]] -name = "anstream" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "824a212faf96e9acacdbd09febd34438f8f711fb84e09a8916013cd7815ca28d" -dependencies = [ - "anstyle", - "anstyle-parse", - "anstyle-query", - "anstyle-wincon", - "colorchoice", - "is_terminal_polyfill", - "utf8parse", -] - -[[package]] -name = "anstyle" -version = "1.0.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000" - -[[package]] -name = "anstyle-parse" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52ce7f38b242319f7cabaa6813055467063ecdc9d355bbb4ce0c68908cd8130e" -dependencies = [ - "utf8parse", -] - -[[package]] -name = "anstyle-query" -version = "1.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" -dependencies = [ - "windows-sys 0.61.2", -] - -[[package]] -name = "anstyle-wincon" -version = "3.0.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" -dependencies = [ - "anstyle", - "once_cell_polyfill", - "windows-sys 0.61.2", -] - [[package]] name = "anyhow" version = "1.0.102" @@ -436,13 +386,9 @@ dependencies = [ [[package]] name = "asyncband" -version = "0.6.7" +version = "0.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94a214ba60d6231afd0e805e3c27c45a1626d9debaa5a5061c45a1ea1b2f1ed0" -dependencies = [ - "hashbrown 0.17.1", - "slab", -] +checksum = "f2d85fd3d291fabcc40c7232c92c280ec1754fd7b5d7ea769f143222143e179a" [[package]] name = "atoi" @@ -1098,46 +1044,6 @@ dependencies = [ "inout", ] -[[package]] -name = "clap" -version = "4.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ddb117e43bbf7dacf0a4190fef4d345b9bad68dfc649cb349e7d17d28428e51" -dependencies = [ - "clap_builder", - "clap_derive", -] - -[[package]] -name = "clap_builder" -version = "4.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "714a53001bf66416adb0e2ef5ac857140e7dc3a0c48fb28b2f10762fc4b5069f" -dependencies = [ - "anstream", - "anstyle", - "clap_lex", - "strsim", -] - -[[package]] -name = "clap_derive" -version = "4.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2ce8604710f6733aa641a2b3731eaa1e8b3d9973d5e3565da11800813f997a9" -dependencies = [ - "heck", - "proc-macro2", - "quote", - "syn 2.0.117", -] - -[[package]] -name = "clap_lex" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9" - [[package]] name = "cmake" version = "0.1.57" @@ -1153,12 +1059,6 @@ version = "0.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0c9ea0ac24bc397ab3c98583a3c9ba74fa56b09a4449bbe172b9b1ddb016027a" -[[package]] -name = "colorchoice" -version = "1.0.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" - [[package]] name = "colored" version = "3.1.1" @@ -1438,16 +1338,6 @@ dependencies = [ "memchr", ] -[[package]] -name = "ctor" -version = "0.6.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "424e0138278faeb2b401f174ad17e715c829512d74f3d1e81eb43365c2e0590e" -dependencies = [ - "ctor-proc-macro", - "dtor", -] - [[package]] name = "ctor" version = "1.0.13" @@ -1458,12 +1348,6 @@ dependencies = [ "linktime-proc-macro", ] -[[package]] -name = "ctor-proc-macro" -version = "0.0.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52560adf09603e58c9a7ee1fe1dcb95a16927b17c127f0ac02d6e768a0e25bc1" - [[package]] name = "ctutils" version = "0.4.2" @@ -1559,7 +1443,7 @@ dependencies = [ "indexmap 2.14.0", "itertools 0.14.0", "log", - "object_store", + "object_store 0.13.2", "parking_lot", "sqlparser", "tempfile", @@ -1588,7 +1472,7 @@ dependencies = [ "futures", "itertools 0.14.0", "log", - "object_store", + "object_store 0.13.2", "parking_lot", "tokio", ] @@ -1613,7 +1497,7 @@ dependencies = [ "futures", "itertools 0.14.0", "log", - "object_store", + "object_store 0.13.2", ] [[package]] @@ -1633,7 +1517,7 @@ dependencies = [ "itertools 0.14.0", "libc", "log", - "object_store", + "object_store 0.13.2", "sqlparser", "tokio", "uuid", @@ -1674,7 +1558,7 @@ dependencies = [ "glob", "itertools 0.14.0", "log", - "object_store", + "object_store 0.13.2", "parking_lot", "rand 0.9.2", "tokio", @@ -1701,7 +1585,7 @@ dependencies = [ "datafusion-session", "futures", "itertools 0.14.0", - "object_store", + "object_store 0.13.2", "tokio", ] @@ -1723,7 +1607,7 @@ dependencies = [ "datafusion-physical-plan", "datafusion-session", "futures", - "object_store", + "object_store 0.13.2", "regex", "tokio", ] @@ -1746,7 +1630,7 @@ dependencies = [ "datafusion-physical-plan", "datafusion-session", "futures", - "object_store", + "object_store 0.13.2", "tokio", "tokio-stream", ] @@ -1772,7 +1656,7 @@ dependencies = [ "datafusion-physical-expr-common", "futures", "log", - "object_store", + "object_store 0.13.2", "parking_lot", "rand 0.9.2", "tempfile", @@ -1836,7 +1720,7 @@ dependencies = [ "hex", "itertools 0.14.0", "log", - "md-5 0.11.0", + "md-5", "memchr", "num-traits", "rand 0.9.2", @@ -2140,7 +2024,7 @@ dependencies = [ "datafusion", "half", "itertools 0.14.0", - "object_store", + "object_store 0.13.2", "pbjson-types", "prost", "substrait", @@ -2148,6 +2032,12 @@ dependencies = [ "url", ] +[[package]] +name = "datasketches" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "46c4cf71a36b46dcfc00e5014c0c20ccad2b1b6a008304d7d57d2749b2d41b3d" + [[package]] name = "defmt" version = "1.1.1" @@ -2265,21 +2155,6 @@ dependencies = [ "const-random", ] -[[package]] -name = "dtor" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "404d02eeb088a82cfd873006cb713fe411306c7d182c344905e101fb1167d301" -dependencies = [ - "dtor-proc-macro", -] - -[[package]] -name = "dtor-proc-macro" -version = "0.0.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f678cf4a922c215c63e0de95eb1ff08a958a81d47e485cf9da1e27bf6305cfa5" - [[package]] name = "dunce" version = "1.0.5" @@ -2437,6 +2312,65 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "foyer-common" +version = "0.22.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "643b47d510032a01e5af70dca288b0c5ec531646c1a8392e793fb5b0c13bef30" +dependencies = [ + "anyhow", + "bytes", + "cfg-if 1.0.4", + "foyer-tokio", + "mixtrics", + "parking_lot", + "pin-project", + "twox-hash", +] + +[[package]] +name = "foyer-intrusive-collections" +version = "0.10.0-dev" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e4fee46bea69e0596130e3210e65d3424e0ac1e6df3bde6636304bdf1ca4a3b" +dependencies = [ + "memoffset", +] + +[[package]] +name = "foyer-memory" +version = "0.22.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae51f5089f3d9025ae77f17ea66d2489ac0f82fbb1d91462807bea174d486d82" +dependencies = [ + "anyhow", + "asyncband", + "bitflags 2.11.0", + "datasketches", + "equivalent", + "foyer-common", + "foyer-intrusive-collections", + "foyer-tokio", + "futures-util", + "hashbrown 0.17.1", + "itertools 0.15.0", + "mixtrics", + "parking_lot", + "paste", + "pin-project", + "serde", + "tracing", +] + +[[package]] +name = "foyer-tokio" +version = "0.22.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6741d1133dfaab64d3f8a9290bbfed3685084add28f8b6ee7a6264aa4dc26918" +dependencies = [ + "tokio", +] + [[package]] name = "frostem" version = "1.20260821.5" @@ -2451,8 +2385,8 @@ checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" [[package]] name = "fsst" -version = "11.0.1-beta.0" -source = "git+https://github.com/lance-format/lance.git?rev=db211492fc5cd9da5642d7234d9682de9be77c19#db211492fc5cd9da5642d7234d9682de9be77c19" +version = "13.0.0-beta.18" +source = "git+https://github.com/lance-format/lance.git?rev=68c12dfd7efe02f90ad2d7f3a239a7eb64884e57#68c12dfd7efe02f90ad2d7f3a239a7eb64884e57" dependencies = [ "arrow-array", "rand 0.9.2", @@ -2801,27 +2735,25 @@ dependencies = [ [[package]] name = "goosefs-sdk" -version = "0.1.9" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e1ea4eee6dcbc31b25ab4fd577adc55b677d2bed3aa3016c44c58fbe1b2298a5" +checksum = "1e471f7bce40936cf2023855c7350e31e11b3f703eb7cb2ce73c9ffc451f87d2" dependencies = [ "arc-swap", "async-trait", "bytes", "dashmap", "fastrand", + "foyer-common", + "foyer-memory", "futures", "hostname", "io-uring", "itoa", "libc", "lru", - "memmap2", - "moka", "prost", - "prost-types", "rand 0.9.2", - "reqwest 0.12.28", "serde", "thiserror 2.0.18", "tokio", @@ -2955,18 +2887,21 @@ checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" [[package]] name = "hf-xet" -version = "1.5.2" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "430b33fa84f92796d4d263070b6c0d3ca219df7b9a0e1853ee431029b1612bcd" +checksum = "c237ef4fb0ce1962a5117f8bd8c74454b41629826a9df17d14a1840ca18f0754" dependencies = [ + "anyhow", "async-trait", "bytes", "http 1.4.0", "more-asserts", "serde", + "serde_json", "thiserror 2.0.18", "tokio", "tokio-util", + "tokio_with_wasm", "tracing", "uuid", "xet-client", @@ -3065,12 +3000,6 @@ version = "1.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" -[[package]] -name = "httpdate" -version = "1.0.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" - [[package]] name = "humantime" version = "2.3.0" @@ -3100,7 +3029,6 @@ dependencies = [ "http 1.4.0", "http-body 1.0.1", "httparse", - "httpdate", "itoa", "pin-project-lite", "pin-utils", @@ -3124,7 +3052,6 @@ dependencies = [ "tokio", "tokio-rustls", "tower-service", - "webpki-roots", ] [[package]] @@ -3461,12 +3388,6 @@ dependencies = [ "serde", ] -[[package]] -name = "is_terminal_polyfill" -version = "1.70.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" - [[package]] name = "itertools" version = "0.11.0" @@ -3485,6 +3406,15 @@ dependencies = [ "either", ] +[[package]] +name = "itertools" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b4baf93f58d4425749ca49a51c50ebab072c5df6994d08fed93541c331481dc" +dependencies = [ + "either", +] + [[package]] name = "itoa" version = "1.0.18" @@ -3654,8 +3584,8 @@ checksum = "e037a2e1d8d5fdbd49b16a4ea09d5d6401c1f29eca5ff29d03d3824dba16256a" [[package]] name = "lance" -version = "11.0.1-beta.0" -source = "git+https://github.com/lance-format/lance.git?rev=db211492fc5cd9da5642d7234d9682de9be77c19#db211492fc5cd9da5642d7234d9682de9be77c19" +version = "13.0.0-beta.18" +source = "git+https://github.com/lance-format/lance.git?rev=68c12dfd7efe02f90ad2d7f3a239a7eb64884e57#68c12dfd7efe02f90ad2d7f3a239a7eb64884e57" dependencies = [ "arc-swap", "arrow", @@ -3671,6 +3601,7 @@ dependencies = [ "async-recursion", "async-trait", "async_cell", + "blake3", "byteorder", "bytes", "chrono", @@ -3702,7 +3633,7 @@ dependencies = [ "lance-tokenizer", "log", "moka", - "object_store", + "object_store 0.14.2", "permutation", "pin-project", "prost", @@ -3726,8 +3657,8 @@ dependencies = [ [[package]] name = "lance-arrow" -version = "11.0.1-beta.0" -source = "git+https://github.com/lance-format/lance.git?rev=db211492fc5cd9da5642d7234d9682de9be77c19#db211492fc5cd9da5642d7234d9682de9be77c19" +version = "13.0.0-beta.18" +source = "git+https://github.com/lance-format/lance.git?rev=68c12dfd7efe02f90ad2d7f3a239a7eb64884e57#68c12dfd7efe02f90ad2d7f3a239a7eb64884e57" dependencies = [ "arrow-array", "arrow-buffer", @@ -3749,7 +3680,7 @@ dependencies = [ [[package]] name = "lance-arrow-scalar" version = "58.0.0" -source = "git+https://github.com/lance-format/lance.git?rev=db211492fc5cd9da5642d7234d9682de9be77c19#db211492fc5cd9da5642d7234d9682de9be77c19" +source = "git+https://github.com/lance-format/lance.git?rev=68c12dfd7efe02f90ad2d7f3a239a7eb64884e57#68c12dfd7efe02f90ad2d7f3a239a7eb64884e57" dependencies = [ "arrow-array", "arrow-buffer", @@ -3763,7 +3694,7 @@ dependencies = [ [[package]] name = "lance-arrow-stats" version = "58.0.0" -source = "git+https://github.com/lance-format/lance.git?rev=db211492fc5cd9da5642d7234d9682de9be77c19#db211492fc5cd9da5642d7234d9682de9be77c19" +source = "git+https://github.com/lance-format/lance.git?rev=68c12dfd7efe02f90ad2d7f3a239a7eb64884e57#68c12dfd7efe02f90ad2d7f3a239a7eb64884e57" dependencies = [ "arrow-array", "arrow-schema", @@ -3772,8 +3703,8 @@ dependencies = [ [[package]] name = "lance-bitpacking" -version = "11.0.1-beta.0" -source = "git+https://github.com/lance-format/lance.git?rev=db211492fc5cd9da5642d7234d9682de9be77c19#db211492fc5cd9da5642d7234d9682de9be77c19" +version = "13.0.0-beta.18" +source = "git+https://github.com/lance-format/lance.git?rev=68c12dfd7efe02f90ad2d7f3a239a7eb64884e57#68c12dfd7efe02f90ad2d7f3a239a7eb64884e57" dependencies = [ "arrayref", "crunchy", @@ -3815,8 +3746,8 @@ dependencies = [ [[package]] name = "lance-core" -version = "11.0.1-beta.0" -source = "git+https://github.com/lance-format/lance.git?rev=db211492fc5cd9da5642d7234d9682de9be77c19#db211492fc5cd9da5642d7234d9682de9be77c19" +version = "13.0.0-beta.18" +source = "git+https://github.com/lance-format/lance.git?rev=68c12dfd7efe02f90ad2d7f3a239a7eb64884e57#68c12dfd7efe02f90ad2d7f3a239a7eb64884e57" dependencies = [ "arrow-array", "arrow-buffer", @@ -3835,7 +3766,7 @@ dependencies = [ "log", "moka", "num_cpus", - "object_store", + "object_store 0.14.2", "pin-project", "prost", "quick_cache", @@ -3853,8 +3784,8 @@ dependencies = [ [[package]] name = "lance-datafusion" -version = "11.0.1-beta.0" -source = "git+https://github.com/lance-format/lance.git?rev=db211492fc5cd9da5642d7234d9682de9be77c19#db211492fc5cd9da5642d7234d9682de9be77c19" +version = "13.0.0-beta.18" +source = "git+https://github.com/lance-format/lance.git?rev=68c12dfd7efe02f90ad2d7f3a239a7eb64884e57#68c12dfd7efe02f90ad2d7f3a239a7eb64884e57" dependencies = [ "arrow", "arrow-array", @@ -3871,6 +3802,7 @@ dependencies = [ "datafusion-physical-expr", "datafusion-substrait", "futures", + "half", "jsonb", "lance-arrow", "lance-core", @@ -3885,8 +3817,8 @@ dependencies = [ [[package]] name = "lance-datagen" -version = "11.0.1-beta.0" -source = "git+https://github.com/lance-format/lance.git?rev=db211492fc5cd9da5642d7234d9682de9be77c19#db211492fc5cd9da5642d7234d9682de9be77c19" +version = "13.0.0-beta.18" +source = "git+https://github.com/lance-format/lance.git?rev=68c12dfd7efe02f90ad2d7f3a239a7eb64884e57#68c12dfd7efe02f90ad2d7f3a239a7eb64884e57" dependencies = [ "arrow", "arrow-array", @@ -3903,8 +3835,8 @@ dependencies = [ [[package]] name = "lance-derive" -version = "11.0.1-beta.0" -source = "git+https://github.com/lance-format/lance.git?rev=db211492fc5cd9da5642d7234d9682de9be77c19#db211492fc5cd9da5642d7234d9682de9be77c19" +version = "13.0.0-beta.18" +source = "git+https://github.com/lance-format/lance.git?rev=68c12dfd7efe02f90ad2d7f3a239a7eb64884e57#68c12dfd7efe02f90ad2d7f3a239a7eb64884e57" dependencies = [ "proc-macro2", "quote", @@ -3913,8 +3845,8 @@ dependencies = [ [[package]] name = "lance-encoding" -version = "11.0.1-beta.0" -source = "git+https://github.com/lance-format/lance.git?rev=db211492fc5cd9da5642d7234d9682de9be77c19#db211492fc5cd9da5642d7234d9682de9be77c19" +version = "13.0.0-beta.18" +source = "git+https://github.com/lance-format/lance.git?rev=68c12dfd7efe02f90ad2d7f3a239a7eb64884e57#68c12dfd7efe02f90ad2d7f3a239a7eb64884e57" dependencies = [ "arrow-arith", "arrow-array", @@ -3947,8 +3879,8 @@ dependencies = [ [[package]] name = "lance-file" -version = "11.0.1-beta.0" -source = "git+https://github.com/lance-format/lance.git?rev=db211492fc5cd9da5642d7234d9682de9be77c19#db211492fc5cd9da5642d7234d9682de9be77c19" +version = "13.0.0-beta.18" +source = "git+https://github.com/lance-format/lance.git?rev=68c12dfd7efe02f90ad2d7f3a239a7eb64884e57#68c12dfd7efe02f90ad2d7f3a239a7eb64884e57" dependencies = [ "arrow-arith", "arrow-array", @@ -3969,18 +3901,19 @@ dependencies = [ "lance-io", "log", "num-traits", - "object_store", + "object_store 0.14.2", "prost", "prost-build", "prost-types", + "serde", "tokio", "tracing", ] [[package]] name = "lance-geo" -version = "11.0.1-beta.0" -source = "git+https://github.com/lance-format/lance.git?rev=db211492fc5cd9da5642d7234d9682de9be77c19#db211492fc5cd9da5642d7234d9682de9be77c19" +version = "13.0.0-beta.18" +source = "git+https://github.com/lance-format/lance.git?rev=68c12dfd7efe02f90ad2d7f3a239a7eb64884e57#68c12dfd7efe02f90ad2d7f3a239a7eb64884e57" dependencies = [ "datafusion", "geo-traits", @@ -3994,8 +3927,8 @@ dependencies = [ [[package]] name = "lance-index" -version = "11.0.1-beta.0" -source = "git+https://github.com/lance-format/lance.git?rev=db211492fc5cd9da5642d7234d9682de9be77c19#db211492fc5cd9da5642d7234d9682de9be77c19" +version = "13.0.0-beta.18" +source = "git+https://github.com/lance-format/lance.git?rev=68c12dfd7efe02f90ad2d7f3a239a7eb64884e57#68c12dfd7efe02f90ad2d7f3a239a7eb64884e57" dependencies = [ "arc-swap", "arrow", @@ -4022,6 +3955,7 @@ dependencies = [ "geoarrow-array", "geoarrow-schema", "half", + "hex", "itertools 0.14.0", "jsonb", "lance-arrow", @@ -4042,7 +3976,7 @@ dependencies = [ "log", "ndarray", "num-traits", - "object_store", + "object_store 0.14.2", "prost", "prost-build", "prost-types", @@ -4058,12 +3992,13 @@ dependencies = [ "tempfile", "tokio", "tracing", + "twox-hash", ] [[package]] name = "lance-index-core" -version = "11.0.1-beta.0" -source = "git+https://github.com/lance-format/lance.git?rev=db211492fc5cd9da5642d7234d9682de9be77c19#db211492fc5cd9da5642d7234d9682de9be77c19" +version = "13.0.0-beta.18" +source = "git+https://github.com/lance-format/lance.git?rev=68c12dfd7efe02f90ad2d7f3a239a7eb64884e57#68c12dfd7efe02f90ad2d7f3a239a7eb64884e57" dependencies = [ "arrow-array", "arrow-schema", @@ -4081,12 +4016,13 @@ dependencies = [ "roaring", "serde", "serde_json", + "tokio", ] [[package]] name = "lance-io" -version = "11.0.1-beta.0" -source = "git+https://github.com/lance-format/lance.git?rev=db211492fc5cd9da5642d7234d9682de9be77c19#db211492fc5cd9da5642d7234d9682de9be77c19" +version = "13.0.0-beta.18" +source = "git+https://github.com/lance-format/lance.git?rev=68c12dfd7efe02f90ad2d7f3a239a7eb64884e57#68c12dfd7efe02f90ad2d7f3a239a7eb64884e57" dependencies = [ "arrow", "arrow-array", @@ -4098,15 +4034,17 @@ dependencies = [ "bytes", "chrono", "futures", + "goosefs-sdk", "http 1.4.0", "io-uring", "lance-core", "lance-namespace", "log", "moka", - "object_store", + "object_store 0.14.2", "object_store_opendal", "opendal", + "opendal-service-goosefs", "path_abs", "pin-project", "prost", @@ -4125,8 +4063,8 @@ dependencies = [ [[package]] name = "lance-linalg" -version = "11.0.1-beta.0" -source = "git+https://github.com/lance-format/lance.git?rev=db211492fc5cd9da5642d7234d9682de9be77c19#db211492fc5cd9da5642d7234d9682de9be77c19" +version = "13.0.0-beta.18" +source = "git+https://github.com/lance-format/lance.git?rev=68c12dfd7efe02f90ad2d7f3a239a7eb64884e57#68c12dfd7efe02f90ad2d7f3a239a7eb64884e57" dependencies = [ "arrow-array", "arrow-schema", @@ -4140,22 +4078,24 @@ dependencies = [ [[package]] name = "lance-namespace" -version = "11.0.1-beta.0" -source = "git+https://github.com/lance-format/lance.git?rev=db211492fc5cd9da5642d7234d9682de9be77c19#db211492fc5cd9da5642d7234d9682de9be77c19" +version = "13.0.0-beta.18" +source = "git+https://github.com/lance-format/lance.git?rev=68c12dfd7efe02f90ad2d7f3a239a7eb64884e57#68c12dfd7efe02f90ad2d7f3a239a7eb64884e57" dependencies = [ "arrow", "async-trait", "bytes", "lance-core", "lance-namespace-reqwest-client", + "serde", + "serde_json", "snafu", ] [[package]] name = "lance-namespace-reqwest-client" -version = "0.11.1" +version = "0.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d06b1fbb5d41f93bc652b61e2872af92e8a6c5f6b4ce8839a8ecfa05365d359" +checksum = "d8d23e54b1634d5bbb434f8dd33dc3c05f6e58d876a9a27b3b4aef58ddbe11af" dependencies = [ "reqwest 0.12.28", "serde", @@ -4167,8 +4107,8 @@ dependencies = [ [[package]] name = "lance-select" -version = "11.0.1-beta.0" -source = "git+https://github.com/lance-format/lance.git?rev=db211492fc5cd9da5642d7234d9682de9be77c19#db211492fc5cd9da5642d7234d9682de9be77c19" +version = "13.0.0-beta.18" +source = "git+https://github.com/lance-format/lance.git?rev=68c12dfd7efe02f90ad2d7f3a239a7eb64884e57#68c12dfd7efe02f90ad2d7f3a239a7eb64884e57" dependencies = [ "arrow-array", "arrow-buffer", @@ -4182,8 +4122,8 @@ dependencies = [ [[package]] name = "lance-table" -version = "11.0.1-beta.0" -source = "git+https://github.com/lance-format/lance.git?rev=db211492fc5cd9da5642d7234d9682de9be77c19#db211492fc5cd9da5642d7234d9682de9be77c19" +version = "13.0.0-beta.18" +source = "git+https://github.com/lance-format/lance.git?rev=68c12dfd7efe02f90ad2d7f3a239a7eb64884e57#68c12dfd7efe02f90ad2d7f3a239a7eb64884e57" dependencies = [ "arrow", "arrow-array", @@ -4202,7 +4142,7 @@ dependencies = [ "lance-io", "lance-select", "log", - "object_store", + "object_store 0.14.2", "prost", "prost-build", "prost-types", @@ -4221,8 +4161,8 @@ dependencies = [ [[package]] name = "lance-tokenizer" -version = "11.0.1-beta.0" -source = "git+https://github.com/lance-format/lance.git?rev=db211492fc5cd9da5642d7234d9682de9be77c19#db211492fc5cd9da5642d7234d9682de9be77c19" +version = "13.0.0-beta.18" +source = "git+https://github.com/lance-format/lance.git?rev=68c12dfd7efe02f90ad2d7f3a239a7eb64884e57#68c12dfd7efe02f90ad2d7f3a239a7eb64884e57" dependencies = [ "frostem", "icu_segmenter", @@ -4387,11 +4327,11 @@ dependencies = [ [[package]] name = "lru" -version = "0.18.4" +version = "0.16.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff9840bcc50b71349309900da0ce7279aa336ae71d73250b07998932c7d97c25" +checksum = "7f66e8d5d03f609abc3a39e6f08e4164ebf1447a732906d39eb9b99b7919ef39" dependencies = [ - "hashbrown 0.17.1", + "hashbrown 0.16.1", ] [[package]] @@ -4450,16 +4390,6 @@ dependencies = [ "thread-tree", ] -[[package]] -name = "md-5" -version = "0.10.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d89e7ee0cfbedfc4da3340218492196241d89eefb6dab27de5df917a6d2e78cf" -dependencies = [ - "cfg-if 1.0.4", - "digest 0.10.7", -] - [[package]] name = "md-5" version = "0.11.0" @@ -4470,15 +4400,6 @@ dependencies = [ "digest 0.11.3", ] -[[package]] -name = "mea" -version = "0.6.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6747f54621d156e1b47eb6b25f39a941b9fc347f98f67d25d8881ff99e8ed832" -dependencies = [ - "slab", -] - [[package]] name = "memchr" version = "2.8.0" @@ -4486,12 +4407,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79" [[package]] -name = "memmap2" -version = "0.9.11" +name = "memoffset" +version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d1219ed1b7f229ee7104d281dd01d6802fe28bb6e95d292942c4daacdeb798c0" +checksum = "488016bfae457b036d996092f6cb448677611ce4449e970ceaf42695203f218a" dependencies = [ - "libc", + "autocfg", ] [[package]] @@ -4531,6 +4452,16 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "mixtrics" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c46b5adfb7a3ae4996d327a5bdc90e78fec025806dd312bdbe6f07a755e0ec9" +dependencies = [ + "itertools 0.15.0", + "parking_lot", +] + [[package]] name = "moka" version = "0.12.15" @@ -4578,6 +4509,18 @@ dependencies = [ "rawpointer", ] +[[package]] +name = "nix" +version = "0.31.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf20d2fde8ff38632c426f1165ed7436270b44f199fc55284c38276f9db47c3d" +dependencies = [ + "bitflags 2.11.0", + "cfg-if 1.0.4", + "cfg_aliases", + "libc", +] + [[package]] name = "nom" version = "8.0.0" @@ -4743,9 +4686,37 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "622acbc9100d3c10e2ee15804b0caa40e55c933d5aa53814cd520805b7958a49" dependencies = [ "async-trait", - "base64 0.22.1", "bytes", "chrono", + "futures-channel", + "futures-core", + "futures-util", + "http 1.4.0", + "humantime", + "itertools 0.14.0", + "parking_lot", + "percent-encoding", + "thiserror 2.0.18", + "tokio", + "tracing", + "url", + "walkdir", + "wasm-bindgen-futures", + "web-time", +] + +[[package]] +name = "object_store" +version = "0.14.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1796bc93603f78c5760a69f2d58badc9618d22adade0a95385bb2adbae4eb94" +dependencies = [ + "async-trait", + "aws-lc-rs", + "base64 0.23.1", + "bytes", + "chrono", + "crc-fast", "form_urlencoded", "futures-channel", "futures-core", @@ -4755,14 +4726,14 @@ dependencies = [ "httparse", "humantime", "hyper", - "itertools 0.14.0", - "md-5 0.10.6", + "itertools 0.15.0", + "md-5", + "nix", "parking_lot", "percent-encoding", - "quick-xml 0.39.4", + "quick-xml 0.41.0", "rand 0.10.1", - "reqwest 0.12.28", - "ring", + "reqwest 0.13.4", "rustls-pki-types", "serde", "serde_json", @@ -4774,20 +4745,21 @@ dependencies = [ "walkdir", "wasm-bindgen-futures", "web-time", + "windows-sys 0.61.2", ] [[package]] name = "object_store_opendal" -version = "0.58.0" +version = "0.60.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "88f165780495c17aa3ce86846600504198c3fffd99073521552751c2430fa6ac" +checksum = "c479bcc317f0ed98972b7184e0f9cc1bea83a07f60cdade96d7207a48ebfb779" dependencies = [ "async-trait", + "asyncband", "bytes", "chrono", "futures", - "mea", - "object_store", + "object_store 0.14.2", "opendal", "pin-project", "tokio", @@ -4799,12 +4771,6 @@ version = "1.21.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" -[[package]] -name = "once_cell_polyfill" -version = "1.70.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" - [[package]] name = "oneshot" version = "0.1.13" @@ -4813,11 +4779,11 @@ checksum = "269bca4c2591a28585d6bf10d9ed0332b7d76900a1b02bec41bdc3a2cdcda107" [[package]] name = "opendal" -version = "0.58.2" +version = "0.59.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "33dbff14cc9bb085224256d6a81289d2f3202e85b06f408d42534b42162a4231" +checksum = "9fe43e16d96bed57937eb7c4a28559bf4c5fc4a3951656505d1fdd4f856349b4" dependencies = [ - "ctor 1.0.13", + "ctor", "opendal-core", "opendal-http-transport-reqwest", "opendal-layer-concurrent-limit", @@ -4837,9 +4803,9 @@ dependencies = [ [[package]] name = "opendal-core" -version = "0.58.2" +version = "0.59.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "48dbcef97d3eb7591db2c18d5cae95c836bcce07359b98d98dd6f4e861eb77b7" +checksum = "ddff5c648dcb3922c2bec71e0d26e881ddf9c367cb7c9a59ed7f2d39a4f38218" dependencies = [ "anyhow", "asyncband", @@ -4849,7 +4815,7 @@ dependencies = [ "http 1.4.0", "jiff", "log", - "md-5 0.11.0", + "md-5", "percent-encoding", "quick-xml 0.41.0", "reqsign-core", @@ -4863,9 +4829,9 @@ dependencies = [ [[package]] name = "opendal-http-transport-reqwest" -version = "0.58.2" +version = "0.59.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85663452ea32bbc17e8f79ab29788c846d116ec7de31451be9c787e462dcb36c" +checksum = "56fcee7ac4b4b815d38346ceb4f3a5d7ef18869c369e000cc7f622b51bcfa938" dependencies = [ "bytes", "futures", @@ -4877,9 +4843,9 @@ dependencies = [ [[package]] name = "opendal-layer-concurrent-limit" -version = "0.58.2" +version = "0.59.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "03f9e144b5228d741c3763ade8711d9b72e5fb6d998e779f2d7a09da0b5a3eba" +checksum = "b454e68e5f6b331bd079a2ee02ebe05078fc0ddaae6822767ef54ab3c2ac0aa8" dependencies = [ "asyncband", "futures", @@ -4889,9 +4855,9 @@ dependencies = [ [[package]] name = "opendal-layer-logging" -version = "0.58.2" +version = "0.59.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c2de17c61cd32e9d8d7d8efb91795e714dbccbafbc3c4e219e1542f4d6324161" +checksum = "e6fd30cc4be601d5adf4936f3bcc186be5e09046c85b4e0f4e6c47ac10966335" dependencies = [ "log", "opendal-core", @@ -4899,9 +4865,9 @@ dependencies = [ [[package]] name = "opendal-layer-retry" -version = "0.58.2" +version = "0.59.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e94db301964a25366090484d61e6da16d5979cc8faf02c3e12210dc74fafed38" +checksum = "1b2da6d89ef0392177f0410dead4b0f50ae7eed56033e00e6c5d78d618b49194" dependencies = [ "backon", "log", @@ -4910,9 +4876,9 @@ dependencies = [ [[package]] name = "opendal-layer-timeout" -version = "0.58.2" +version = "0.59.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08956ddda07465449bfd48825f4f0f25e0351278ac974eaa659895d9d74f2c80" +checksum = "337e46d134eb08beafea92fbe32fdb96d80750b711fea21b158ed3f5cb7ec40c" dependencies = [ "opendal-core", "tokio", @@ -4920,9 +4886,9 @@ dependencies = [ [[package]] name = "opendal-service-azblob" -version = "0.58.2" +version = "0.59.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d278d2fb57661947d1c9fb44047e432b2782c48dc085b7abc99fe6e18c26cf8" +checksum = "f79d8340796e728bf79248d183c59760996059c882c7b151845b8169103bf08d" dependencies = [ "base64 0.23.1", "bytes", @@ -4941,9 +4907,9 @@ dependencies = [ [[package]] name = "opendal-service-azdls" -version = "0.58.2" +version = "0.59.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d564484a8f7d091827e825cfc91ed45bd48e64d262451ee041fe843db81bd8a" +checksum = "be0a35156499ff5a95d8866d1baf4d77d22e4f14cf8d68806a7015ffe096cb3a" dependencies = [ "asyncband", "base64 0.23.1", @@ -4962,9 +4928,9 @@ dependencies = [ [[package]] name = "opendal-service-azure-common" -version = "0.58.2" +version = "0.59.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cfcc1bfdac4f54d9018c462dd32ad9e2f68fdf584f825c811550c8ffc87894cd" +checksum = "9dd98dfe60b62baa344f3fd4cee68dbb8538483569cded75c8b614c57880a9b9" dependencies = [ "http 1.4.0", "opendal-core", @@ -4972,9 +4938,9 @@ dependencies = [ [[package]] name = "opendal-service-cos" -version = "0.58.2" +version = "0.59.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb021c128ebde42e6f3e719d4cfed27e994017aa503a7a1e5818bdb61fd67dc9" +checksum = "fb37f62818e163131e9f588c45cf18998ba5136dc77d585996264fe6bde5c76f" dependencies = [ "bytes", "http 1.4.0", @@ -4989,9 +4955,9 @@ dependencies = [ [[package]] name = "opendal-service-gcs" -version = "0.58.2" +version = "0.59.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da1f2a8c975fd22fea01f0409bcb8774f1ac02ad79863a3490098203121555d3" +checksum = "658545a6d3ef1b650117abdb04ed6b8b9bafeae19ed8ace6a3c36212d5999fbe" dependencies = [ "async-trait", "bytes", @@ -5006,28 +4972,30 @@ dependencies = [ "serde", "serde_json", "tokio", + "uuid", ] [[package]] name = "opendal-service-goosefs" -version = "0.58.2" +version = "0.59.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89fd71b80078f2983bd363e322fbebfa6a46d5fc56f17e4f23d76d5edb31226b" +checksum = "18fb508375b4342d0cd29721ab5955f0ce26c2500fbb659d2f4a7c802005597f" dependencies = [ + "asyncband", "bytes", "goosefs-sdk", "log", "opendal-core", "serde", - "tokio", ] [[package]] name = "opendal-service-hf" -version = "0.58.2" +version = "0.59.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8a3c8ec0c2918fa23f258fa28822f222ec8c1e3a501ded665b78bc65a9d7734" +checksum = "3815e9695b4b73b4d038b55e335a6e2c17dc8da6997a8dc97705a3e86abe54aa" dependencies = [ + "asyncband", "bytes", "hf-xet", "http 1.4.0", @@ -5040,9 +5008,9 @@ dependencies = [ [[package]] name = "opendal-service-oss" -version = "0.58.2" +version = "0.59.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e3ce7a2ceb925e0f28f545b169eb57d04cec6116aae94b8584d2bdbe7d456c6" +checksum = "f09301e6909337878f95e5991447a875e93fc510295528685b931efdb28f3d7e" dependencies = [ "bytes", "http 1.4.0", @@ -5057,16 +5025,16 @@ dependencies = [ [[package]] name = "opendal-service-s3" -version = "0.58.2" +version = "0.59.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c64335f9f24ccb62ac36f1d976342b48611a75ba61979813a4f78a4ebd94de42" +checksum = "9de51cba3b83c8130e211045fd2b9d454c4b73f557ad88b71244c2ab943b2b05" dependencies = [ "base64 0.23.1", "bytes", "crc-fast", "http 1.4.0", "log", - "md-5 0.11.0", + "md-5", "opendal-core", "quick-xml 0.41.0", "reqsign-aws-v4", @@ -5078,9 +5046,9 @@ dependencies = [ [[package]] name = "opendal-service-tos" -version = "0.58.2" +version = "0.59.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70c3c507c3a565b2feb4c7b5f653436acd34ca59ffc842a7671b5498b13b76bf" +checksum = "25706a3288ae449555c9dbd5910474271d1bb99f86e7006c6d9f1753c2614fe0" dependencies = [ "bytes", "http 1.4.0", @@ -5492,9 +5460,9 @@ dependencies = [ [[package]] name = "quick-xml" -version = "0.39.4" +version = "0.41.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cdcc8dd4e2f670d309a5f0e83fe36dfdc05af317008fea29144da1a2ac858e5e" +checksum = "e660451e55124f798a69a5af3f49ccfbefbd41910eefd25caf2393e1f3473ec1" dependencies = [ "memchr", "serde", @@ -5502,9 +5470,9 @@ dependencies = [ [[package]] name = "quick-xml" -version = "0.41.0" +version = "0.42.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e660451e55124f798a69a5af3f49ccfbefbd41910eefd25caf2393e1f3473ec1" +checksum = "41b1177fdf999d2321d3fb46ff47159d9c1fb9ad66a4879f8c50a0b504615e9b" dependencies = [ "memchr", "serde", @@ -5844,9 +5812,9 @@ dependencies = [ [[package]] name = "reqsign-aws-core" -version = "3.1.0" +version = "3.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4d63b56638bb3cc7bd376a7cdce1ba3089777a08f47e4097888f2d784cc3f46c" +checksum = "d38c902d8ec745bd9eadd1abbcbe227f4c1fc2926b2ec8929f89234af3114500" dependencies = [ "bytes", "form_urlencoded", @@ -5854,7 +5822,7 @@ dependencies = [ "http 1.4.0", "log", "percent-encoding", - "quick-xml 0.41.0", + "quick-xml 0.42.0", "reqsign-core", "rust-ini", "serde", @@ -5865,14 +5833,14 @@ dependencies = [ [[package]] name = "reqsign-aws-v4" -version = "3.2.0" +version = "3.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4a0c499f4ed12d04c3d4c78fe4cb01aee22c9dae22848c14db2c6313d9df9f43" +checksum = "906bfd028f96bfe73fdb892ce26f3b3f99b4f30feb9bd024be58bd6657f68b2b" dependencies = [ "bytes", "http 1.4.0", "log", - "quick-xml 0.41.0", + "quick-xml 0.42.0", "reqsign-aws-core", "reqsign-core", "serde", @@ -5901,14 +5869,14 @@ dependencies = [ [[package]] name = "reqsign-core" -version = "3.3.0" +version = "3.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f4ac1510872d9481205975d264deb39c109797e5068cc882ed9064270eaae5fa" +checksum = "85fed85c436b6b920e4b36310f699a77771a91e38ae483df48447bd7d162671c" dependencies = [ "anyhow", + "asyncband", "base64 0.23.1", "bytes", - "futures", "hex", "hmac 0.13.0", "http 1.4.0", @@ -5925,9 +5893,9 @@ dependencies = [ [[package]] name = "reqsign-file-read-tokio" -version = "3.0.5" +version = "3.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95c3371bfc7e5c7f9627a04133af3583fd6c28715e7c83f79db38f3b384f535f" +checksum = "586a8e198057773f9616901fd723a4bc186812b9765bc0f20368661d6169d906" dependencies = [ "anyhow", "reqsign-core", @@ -5936,9 +5904,9 @@ dependencies = [ [[package]] name = "reqsign-google" -version = "3.1.0" +version = "3.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f81a9d38870892443489c0abb5332edfa81d5a14c437af9caef7c194897c92c1" +checksum = "719e55931cc81b8dcd69f95f0a2def55617cb76cb3aa2f8c439e10682e46fa10" dependencies = [ "bytes", "form_urlencoded", @@ -6024,7 +5992,6 @@ dependencies = [ "wasm-bindgen-futures", "wasm-streams 0.4.2", "web-sys", - "webpki-roots", ] [[package]] @@ -6037,6 +6004,7 @@ dependencies = [ "bytes", "futures-core", "futures-util", + "h2", "http 1.4.0", "http-body 1.0.1", "http-body-util", @@ -6567,7 +6535,6 @@ dependencies = [ "cfg-if 1.0.4", "cpufeatures 0.2.17", "digest 0.10.7", - "sha2-asm", ] [[package]] @@ -6581,15 +6548,6 @@ dependencies = [ "digest 0.11.3", ] -[[package]] -name = "sha2-asm" -version = "0.6.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b845214d6175804686b2bd482bcffe96651bb2d1200742b712003504a2dac1ab" -dependencies = [ - "cc", -] - [[package]] name = "sharded-slab" version = "0.1.7" @@ -7148,6 +7106,30 @@ dependencies = [ "tokio", ] +[[package]] +name = "tokio_with_wasm" +version = "0.8.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34e40fbbbd95441133fe9483f522db15dbfd26dc636164ebd8f2dd28759a6aa6" +dependencies = [ + "js-sys", + "tokio", + "tokio_with_wasm_proc", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", +] + +[[package]] +name = "tokio_with_wasm_proc" +version = "0.8.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d01145a2c788d6aae4cd653afec1e8332534d7d783d01897cefcafe4428de992" +dependencies = [ + "quote", + "syn 2.0.117", +] + [[package]] name = "toml_datetime" version = "1.1.0+spec-1.1.0" @@ -7187,7 +7169,6 @@ dependencies = [ "async-trait", "base64 0.22.1", "bytes", - "h2", "http 1.4.0", "http-body 1.0.1", "http-body-util", @@ -7196,7 +7177,6 @@ dependencies = [ "hyper-util", "percent-encoding", "pin-project", - "socket2", "sync_wrapper", "tokio", "tokio-stream", @@ -7513,12 +7493,6 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" -[[package]] -name = "utf8parse" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" - [[package]] name = "uuid" version = "1.24.0" @@ -7758,15 +7732,6 @@ dependencies = [ "rustls-pki-types", ] -[[package]] -name = "webpki-roots" -version = "1.0.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7dcd9d09a39985f5344844e66b0c530a33843579125f23e21e9f0f220850f22a" -dependencies = [ - "rustls-pki-types", -] - [[package]] name = "whoami" version = "2.1.1" @@ -8227,20 +8192,18 @@ dependencies = [ [[package]] name = "xet-client" -version = "1.5.2" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3e1e496dcbe6a09017acdfaf48e1a646735e7ff5b2a49e2c7e081cca77a59bc8" +checksum = "c3b8da8cc70aa2e3c500c0400e012df82c656ab9fca47f9f939fffc5afd89aca" dependencies = [ "anyhow", "async-trait", "base64 0.22.1", "bytes", - "clap", "crc32fast", "futures", "http 1.4.0", "hyper", - "lazy_static", "more-asserts", "rand 0.10.1", "redb", @@ -8254,8 +8217,8 @@ dependencies = [ "thiserror 2.0.18", "tokio", "tokio-retry", + "tokio_with_wasm", "tracing", - "tracing-subscriber", "url", "urlencoding", "web-time", @@ -8265,24 +8228,21 @@ dependencies = [ [[package]] name = "xet-core-structures" -version = "1.5.2" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb838aa8eb67d730af301584cf003caad407487606058292a6750711b603fbee" +checksum = "73503c223783dccc864abde22115e09d12f190448a0baf58ab2c54bc709e2f99" dependencies = [ "async-trait", "base64 0.22.1", "blake3", "bytemuck", "bytes", - "clap", "countio", - "csv", "futures", "futures-util", "getrandom 0.4.2", "heapify", "itertools 0.14.0", - "lazy_static", "lz4_flex", "more-asserts", "rand 0.10.1", @@ -8290,7 +8250,6 @@ dependencies = [ "safe-transmute", "serde", "static_assertions", - "tempfile", "thiserror 2.0.18", "tokio", "tokio-util", @@ -8302,32 +8261,31 @@ dependencies = [ [[package]] name = "xet-data" -version = "1.5.2" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67fd409bef621411a9d9013798540bb8036cb2678f03ab39af89a5e88034ed8c" +checksum = "c89052ec5dec2187cad30b86af92cc24fd61c4a57a795f1ff7ff5f38d49184eb" dependencies = [ "anyhow", "async-trait", "bytes", "chrono", - "clap", "gearhash", "http 1.4.0", "itertools 0.14.0", - "lazy_static", "more-asserts", "rand 0.10.1", "serde", "serde_json", - "sha2 0.10.9", + "sha2 0.11.0", "tempfile", "thiserror 2.0.18", "tokio", "tokio-util", + "tokio_with_wasm", "tracing", "url", "uuid", - "walkdir", + "web-time", "xet-client", "xet-core-structures", "xet-runtime", @@ -8335,9 +8293,9 @@ dependencies = [ [[package]] name = "xet-runtime" -version = "1.5.2" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "15d8f121c33866f7648b737abe70d0e2dd9c0af4ffdd7219207531d0283aa63d" +checksum = "af5c60d5eed38ab4c576f4421bae835e7bd07631fb381705605529d2015c106b" dependencies = [ "anyhow", "async-trait", @@ -8345,13 +8303,12 @@ dependencies = [ "chrono", "colored", "const-str", - "ctor 0.6.3", + "ctor", "dirs", "futures", "git-version", "humantime", "konst", - "lazy_static", "libc", "more-asserts", "oneshot", @@ -8365,9 +8322,11 @@ dependencies = [ "thiserror 2.0.18", "tokio", "tokio-util", + "tokio_with_wasm", "tracing", "tracing-appender", "tracing-subscriber", + "web-time", "whoami", "winapi", ] diff --git a/Cargo.toml b/Cargo.toml index 5b43998..18654b8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -22,14 +22,15 @@ crate-type = ["cdylib", "staticlib", "rlib"] # `lance_index::progress::IndexBuildProgress`; already in the graph # transitively via lance-index. async-trait = "0.1" -lance = { git = "https://github.com/lance-format/lance.git", rev = "db211492fc5cd9da5642d7234d9682de9be77c19", features = ["substrait"] } -lance-core = { git = "https://github.com/lance-format/lance.git", rev = "db211492fc5cd9da5642d7234d9682de9be77c19" } -lance-file = { git = "https://github.com/lance-format/lance.git", rev = "db211492fc5cd9da5642d7234d9682de9be77c19" } -lance-index = { git = "https://github.com/lance-format/lance.git", rev = "db211492fc5cd9da5642d7234d9682de9be77c19" } -lance-io = { git = "https://github.com/lance-format/lance.git", rev = "db211492fc5cd9da5642d7234d9682de9be77c19" } -lance-linalg = { git = "https://github.com/lance-format/lance.git", rev = "db211492fc5cd9da5642d7234d9682de9be77c19" } -lance-table = { git = "https://github.com/lance-format/lance.git", rev = "db211492fc5cd9da5642d7234d9682de9be77c19" } -lance-datafusion = { git = "https://github.com/lance-format/lance.git", rev = "db211492fc5cd9da5642d7234d9682de9be77c19", features = ["substrait"] } +# Keep the dependency on the segment-prefilter fix and its float-consistent PQ prerequisite. +lance = { git = "https://github.com/lance-format/lance.git", rev = "68c12dfd7efe02f90ad2d7f3a239a7eb64884e57", features = ["substrait"] } +lance-core = { git = "https://github.com/lance-format/lance.git", rev = "68c12dfd7efe02f90ad2d7f3a239a7eb64884e57" } +lance-file = { git = "https://github.com/lance-format/lance.git", rev = "68c12dfd7efe02f90ad2d7f3a239a7eb64884e57" } +lance-index = { git = "https://github.com/lance-format/lance.git", rev = "68c12dfd7efe02f90ad2d7f3a239a7eb64884e57" } +lance-io = { git = "https://github.com/lance-format/lance.git", rev = "68c12dfd7efe02f90ad2d7f3a239a7eb64884e57" } +lance-linalg = { git = "https://github.com/lance-format/lance.git", rev = "68c12dfd7efe02f90ad2d7f3a239a7eb64884e57" } +lance-table = { git = "https://github.com/lance-format/lance.git", rev = "68c12dfd7efe02f90ad2d7f3a239a7eb64884e57" } +lance-datafusion = { git = "https://github.com/lance-format/lance.git", rev = "68c12dfd7efe02f90ad2d7f3a239a7eb64884e57", features = ["substrait"] } datafusion = { version = "54.0.0", default-features = false } arrow = { version = "58.0.0", features = ["prettyprint", "ffi"] } arrow-array = "58.0.0" @@ -44,16 +45,16 @@ futures = "0.3" log = "0.4" libc = "0.2" # Explicitly install the HTTP transport when embedded in a static C/C++ executable. -opendal = { version = "=0.58.2", default-features = false, features = ["http-transport-reqwest"] } +opendal = { version = "=0.59.2", default-features = false, features = ["http-transport-reqwest"] } pin-project = "1.0" prost = "0.14" snafu = "0.9" uuid = { version = "1", features = ["v4"] } [dev-dependencies] -lance = { git = "https://github.com/lance-format/lance.git", rev = "db211492fc5cd9da5642d7234d9682de9be77c19", features = ["substrait"] } -lance-datagen = { git = "https://github.com/lance-format/lance.git", rev = "db211492fc5cd9da5642d7234d9682de9be77c19" } -lance-file = { git = "https://github.com/lance-format/lance.git", rev = "db211492fc5cd9da5642d7234d9682de9be77c19" } +lance = { git = "https://github.com/lance-format/lance.git", rev = "68c12dfd7efe02f90ad2d7f3a239a7eb64884e57", features = ["substrait"] } +lance-datagen = { git = "https://github.com/lance-format/lance.git", rev = "68c12dfd7efe02f90ad2d7f3a239a7eb64884e57" } +lance-file = { git = "https://github.com/lance-format/lance.git", rev = "68c12dfd7efe02f90ad2d7f3a239a7eb64884e57" } tokio = { version = "1", features = ["rt-multi-thread", "macros"] } arrow-array = "58.0.0" arrow-schema = "58.0.0" diff --git a/tests/c_api_test.rs b/tests/c_api_test.rs index c02cf45..1a25e34 100644 --- a/tests/c_api_test.rs +++ b/tests/c_api_test.rs @@ -7509,6 +7509,168 @@ fn test_scanner_nearest_with_ivf_pq_index() { unsafe { lance_dataset_close(ds) }; } +#[test] +fn test_scanner_nearest_4bit_pq_prefilter_consistency() { + for metric in [ + LanceMetricType::L2, + LanceMetricType::Cosine, + LanceMetricType::Dot, + ] { + // One 512-row partition exercises the bulk scoring path whose former + // quantized scores could select different candidates from an all-row mask. + let (_tmp, uri) = create_vector_dataset(512, 32); + let dataset = unsafe { lance_dataset_open(c_str(&uri).as_ptr(), ptr::null(), 0) }; + assert!(!dataset.is_null()); + let column = c_str("embedding"); + let name = c_str("pq_idx"); + let params = LanceVectorIndexParams { + index_type: LanceVectorIndexType::IvfPq, + metric, + num_partitions: 1, + num_sub_vectors: 8, + num_bits: 4, + max_iterations: 2, + hnsw_m: 0, + hnsw_ef_construction: 0, + sample_rate: 16, + }; + assert_eq!( + unsafe { + lance_dataset_create_vector_index( + dataset, + column.as_ptr(), + name.as_ptr(), + ¶ms, + false, + ) + }, + 0, + "{}", + take_last_error_message() + ); + let mut segment = [0_u8; 16]; + let mut count = 0; + assert_eq!( + unsafe { + lance_dataset_index_segments( + dataset, + name.as_ptr(), + segment.as_mut_ptr(), + 1, + &mut count, + ) + }, + 0 + ); + assert_eq!(count, 1); + let mut results = Vec::new(); + for has_filter in [false, true] { + let filter = c_str("id >= 0"); + let scanner = unsafe { + lance_scanner_new( + dataset, + ptr::null(), + if has_filter { + filter.as_ptr() + } else { + ptr::null() + }, + ) + }; + assert!(!scanner.is_null()); + let fragment = 0_u64; + assert_eq!( + unsafe { lance_scanner_set_fragment_ids(scanner, &fragment, 1) }, + 0 + ); + assert_eq!( + unsafe { lance_scanner_set_index_segments(scanner, segment.as_ptr(), 1) }, + 0 + ); + assert_eq!(unsafe { lance_scanner_set_prefilter(scanner, true) }, 0); + let query = [0.5_f32; 32]; + assert_eq!( + unsafe { + lance_scanner_nearest( + scanner, + column.as_ptr(), + query.as_ptr().cast(), + query.len(), + LanceDataType::Float32 as i32, + 10, + ) + }, + 0 + ); + assert_eq!(unsafe { lance_scanner_set_nprobes(scanner, 1) }, 0); + let mut captured = CapturedScanStatistics::default(); + assert_eq!( + unsafe { + lance_scanner_set_statistics_callback( + scanner, + Some(capture_scan_statistics), + (&mut captured as *mut CapturedScanStatistics).cast(), + ) + }, + 0 + ); + let batches = scan_all_rows_from_scanner(scanner); + let mut rows = Vec::new(); + for batch in &batches { + let ids = batch + .column_by_name("id") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap(); + let distances = batch + .column_by_name("_distance") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap(); + rows.extend( + ids.values() + .iter() + .copied() + .zip(distances.values().iter().copied()), + ); + } + assert_eq!(rows.len(), 10); + assert!(rows.iter().all(|(_, distance)| distance.is_finite())); + // Tie order is unspecified; compare candidate IDs and their PQ scores. + rows.sort_by_key(|(id, _)| *id); + results.push(rows); + assert_eq!(captured.calls, 1); + // Quantized fast-scan and filtered scoring must both report their fused work. + for name in ["index_query_prepare_time", "index_distance_topk_time"] { + assert!( + captured.metrics.iter().any(|(metric, kind, value)| { + metric == name + && *kind == LanceScanMetricKind::TimeNanoseconds + && *value > 0 + }), + "missing PQ timing: {name}: {:?}", + captured.metrics + ); + } + + let loads = captured + .metrics + .iter() + .filter(|(name, kind, _)| { + name == "prefilter_loads" && *kind == LanceScanMetricKind::Count + }) + .map(|(_, _, value)| *value) + .sum::(); + assert_eq!(loads, u64::from(has_filter)); + unsafe { lance_scanner_close(scanner) }; + } + assert_eq!(results[0], results[1]); + unsafe { lance_dataset_close(dataset) }; + } +} + #[test] fn test_scanner_adaptive_nprobes_and_approx_mode_validation_and_lifecycle() { let (_tmp, uri) = create_test_dataset(); @@ -7835,6 +7997,195 @@ fn test_scanner_nearest_full_snapshot_prefilter_statistics() { } } +#[test] +fn test_scanner_nearest_segment_prefilter_statistics() { + for stable_row_ids in [false, true] { + for segmented in [true, false] { + let (_tmp, uri) = create_multi_fragment_vector_dataset(3, 64, 8, stable_row_ids); + let dataset = unsafe { lance_dataset_open(c_str(&uri).as_ptr(), ptr::null(), 0) }; + assert!(!dataset.is_null()); + let covered = if segmented { vec![0] } else { vec![0, 1] }; + let first = build_vector_segment_bytes(dataset, LanceMetricType::L2, &covered); + let uuid = segment_uuid(first.as_ptr(), first.len()); + if segmented { + let second = build_vector_segment_bytes(dataset, LanceMetricType::L2, &[1]); + assert_eq!(commit_vector_segments(dataset, &[&first, &second]), 0); + } else { + assert_eq!(commit_vector_segments(dataset, &[&first]), 0); + } + let mut deleted = 0; + assert_eq!( + unsafe { lance_dataset_delete(dataset, c_str("id = 7").as_ptr(), &mut deleted) }, + 0 + ); + assert_eq!(deleted, 1); + for include_unindexed in [false, true] { + for filtered in [false, true] { + let filter = c_str("id >= 32"); + let scanner = unsafe { + lance_scanner_new( + dataset, + ptr::null(), + if filtered { + filter.as_ptr() + } else { + ptr::null() + }, + ) + }; + assert!(!scanner.is_null()); + // Keep the ANN scope narrower than the snapshot, optionally adding a flat-search tail. + let fragments = if include_unindexed { + vec![0_u64, 2] + } else { + vec![0_u64] + }; + assert_eq!( + unsafe { + lance_scanner_set_fragment_ids( + scanner, + fragments.as_ptr(), + fragments.len(), + ) + }, + 0 + ); + assert_eq!( + unsafe { lance_scanner_set_index_segments(scanner, uuid.as_ptr(), 1) }, + 0 + ); + assert_eq!(unsafe { lance_scanner_set_prefilter(scanner, true) }, 0); + let query = [0.0_f32; 8]; + assert_eq!( + unsafe { + lance_scanner_nearest( + scanner, + c_str("embedding").as_ptr(), + query.as_ptr().cast(), + query.len(), + LanceDataType::Float32 as i32, + 256, + ) + }, + 0 + ); + assert_eq!(unsafe { lance_scanner_set_nprobes(scanner, 2) }, 0); + let mut captured = CapturedScanStatistics::default(); + assert_eq!( + unsafe { + lance_scanner_set_statistics_callback( + scanner, + Some(capture_scan_statistics), + (&mut captured as *mut CapturedScanStatistics).cast(), + ) + }, + 0 + ); + let batches = scan_all_rows_from_scanner(scanner); + let mut ids = batches + .iter() + .flat_map(|batch| { + batch + .column_by_name("id") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap() + .values() + .to_vec() + }) + .collect::>(); + ids.sort_unstable(); + let expected = (0..64) + .chain(if include_unindexed { + 128..192 + } else { + 128..128 + }) + .filter(|id| *id != 7 && (!filtered || *id >= 32)) + .collect::>(); + assert_eq!( + ids, expected, + "stable={stable_row_ids}, segmented={segmented}, filtered={filtered}, tail={include_unindexed}" + ); + assert_eq!(captured.calls, 1); + // Zero is valid for short or uncontended stages; verify the callback + // contract independently of clock resolution and scheduling. + for name in [ + "ANNSubIndexExec_elapsed_compute", + "index_open_time", + "index_partition_load_time", + "index_partition_prepare_time", + "index_cpu_queue_wait_time", + "index_search_time", + "index_query_prepare_time", + "index_distance_topk_time", + "index_result_materialize_time", + ] { + assert!( + captured.metrics.iter().any(|(metric, kind, _)| { + metric == name && *kind == LanceScanMetricKind::TimeNanoseconds + }), + "missing ANN timing: {name}: {:?}", + captured.metrics + ); + } + + let loads = captured + .metrics + .iter() + .filter(|(name, kind, _)| { + name == "prefilter_loads" && *kind == LanceScanMetricKind::Count + }) + .map(|(_, _, value)| *value) + .sum::(); + // Correct rows alone do not detect rebuilding the complete segment allowlist. + assert_eq!( + loads, + u64::from(filtered || !segmented), + "stable={stable_row_ids}, segmented={segmented}, filtered={filtered}, tail={include_unindexed}" + ); + if segmented && !filtered { + // These counters expose the row-ID materialization cost independently + // of the index search and deletion-mask work that must still execute. + for name in [ + "prefilter_input_rows", + "prefilter_row_ids", + "prefilter_build_time", + "prefilter_load_time", + ] { + let value = captured + .metrics + .iter() + .filter(|(key, _, _)| key == name) + .map(|(_, _, value)| *value) + .sum::(); + assert_eq!( + value, 0, + "unexpected {name} for a complete unfiltered segment" + ); + } + } else { + // An absent metric also sums to zero above. Materializing cases + // must expose both timers, even when their durations are zero. + for name in ["prefilter_build_time", "prefilter_load_time"] { + assert!( + captured.metrics.iter().any(|(metric, kind, _)| { + metric == name && *kind == LanceScanMetricKind::TimeNanoseconds + }), + "missing prefilter timing: {name}: {:?}", + captured.metrics + ); + } + } + unsafe { lance_scanner_close(scanner) }; + } + } + unsafe { lance_dataset_close(dataset) }; + } + } +} + #[test] fn test_scanner_nearest_prefilter_with_fragment_ids_next() { let (_tmp, uri) = create_multi_fragment_vector_dataset(2, 32, 8, false);