diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 191457cf..1ff9b5bd 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -17,7 +17,7 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@v6 - name: Setup dependencies run: ./scripts/setup.sh @@ -40,7 +40,7 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@v6 - name: Setup dependencies run: ./scripts/setup.sh diff --git a/.github/workflows/claude-code-review.yml b/.github/workflows/claude-code-review.yml index f96cea7c..c8bc84ba 100644 --- a/.github/workflows/claude-code-review.yml +++ b/.github/workflows/claude-code-review.yml @@ -19,7 +19,7 @@ jobs: id-token: write steps: - name: Checkout repository - uses: actions/checkout@v4 + uses: actions/checkout@v6 with: fetch-depth: 0 - name: Run Claude Code Review diff --git a/native/Cargo.lock b/native/Cargo.lock index a39f1b10..1d3a2829 100644 --- a/native/Cargo.lock +++ b/native/Cargo.lock @@ -149,7 +149,7 @@ checksum = "36fa98bc79671c7981272d91a8753a928ff6a1cd8e4f20a44c45bd5d313840bf" dependencies = [ "bigdecimal", "bon", - "digest", + "digest 0.10.7", "log", "miniz_oxide", "num-bigint", @@ -741,7 +741,7 @@ dependencies = [ "once_cell", "percent-encoding", "regex-lite", - "sha2", + "sha2 0.10.9", "tracing", "url", ] @@ -840,7 +840,7 @@ dependencies = [ "p256", "percent-encoding", "ring", - "sha2", + "sha2 0.10.9", "subtle", "time", "tracing", @@ -875,7 +875,7 @@ dependencies = [ "md-5", "pin-project-lite", "sha1", - "sha2", + "sha2 0.10.9", "tracing", ] @@ -1156,7 +1156,7 @@ dependencies = [ "rustc_version", "serde", "serde_json", - "sha2", + "sha2 0.10.9", "time", "tracing", "url", @@ -1341,6 +1341,15 @@ dependencies = [ "generic-array", ] +[[package]] +name = "block-buffer" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2f6c7dbe95a6ed67ad9f18e57daf93a2f034c524b99fd2b76d18fdfeb6660aa" +dependencies = [ + "hybrid-array", +] + [[package]] name = "blocking" version = "1.6.2" @@ -1370,7 +1379,7 @@ version = "3.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "89ec27229c38ed0eb3c0feee3d2c1d6a4379ae44f418a29a658890e062d8f365" dependencies = [ - "darling 0.20.11", + "darling 0.21.3", "ident_case", "prettyplease", "proc-macro2", @@ -1704,6 +1713,12 @@ version = "0.9.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" +[[package]] +name = "const-oid" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6ef517f0926dd24a1582492c791b6a4818a4d94e789a334894aa15b0d12f55c" + [[package]] name = "const-random" version = "0.1.18" @@ -1765,6 +1780,15 @@ dependencies = [ "libc", ] +[[package]] +name = "cpufeatures" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b2a41393f66f16b0823bb79094d54ac5fbd34ab292ddafb9a0456ac9f87d201" +dependencies = [ + "libc", +] + [[package]] name = "crc" version = "3.4.0" @@ -1910,6 +1934,15 @@ dependencies = [ "typenum", ] +[[package]] +name = "crypto-common" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce6e4c961d6cd6c9a86db418387425e8bdeaf05b3c8bc1411e6dca4c252f1453" +dependencies = [ + "hybrid-array", +] + [[package]] name = "csv" version = "1.4.0" @@ -2082,7 +2115,7 @@ version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f1a467a65c5e759bce6e65eaf91cc29f466cdc57cb65777bd646872a8a1fd4de" dependencies = [ - "const-oid", + "const-oid 0.9.6", "zeroize", ] @@ -2133,12 +2166,23 @@ version = "0.10.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ - "block-buffer", - "const-oid", - "crypto-common", + "block-buffer 0.10.4", + "const-oid 0.9.6", + "crypto-common 0.1.7", "subtle", ] +[[package]] +name = "digest" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1dd6dbb5841937940781866fa1281a1ff7bd3bf827091440879f9994983d5c2" +dependencies = [ + "block-buffer 0.12.1", + "const-oid 0.10.2", + "crypto-common 0.2.2", +] + [[package]] name = "displaydoc" version = "0.2.5" @@ -2225,7 +2269,7 @@ dependencies = [ "base16ct", "crypto-bigint 0.4.9", "der", - "digest", + "digest 0.10.7", "ff", "generic-array", "group", @@ -2546,9 +2590,9 @@ checksum = "e6d5a32815ae3f33302d95fdcb2ce17862f8c65363dcfd29360480ba1001fc9c" [[package]] name = "futures" -version = "0.3.31" +version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "65bc07b1a8bc7c85c5f2e110c476c7389b4554ba72af57d8445ea63a576b0876" +checksum = "8b147ee9d1f6d097cef9ce628cd2ee62288d963e16fb287bd9286455b241382d" dependencies = [ "futures-channel", "futures-core", @@ -2561,9 +2605,9 @@ dependencies = [ [[package]] name = "futures-channel" -version = "0.3.31" +version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2dff15bf788c671c1934e366d07e30c1814a8ef514e1af724a602e8a2fbe1b10" +checksum = "07bbe89c50d7a535e539b8c17bc0b49bdb77747034daa8087407d655f3f7cc1d" dependencies = [ "futures-core", "futures-sink", @@ -2571,15 +2615,15 @@ dependencies = [ [[package]] name = "futures-core" -version = "0.3.31" +version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05f29059c0c2090612e8d742178b0580d2dc940c837851ad723096f87af6663e" +checksum = "7e3450815272ef58cec6d564423f6e755e25379b217b0bc688e295ba24df6b1d" [[package]] name = "futures-executor" -version = "0.3.31" +version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e28d1d997f585e54aebc3f97d39e72338912123a67330d723fdbb564d646c9f" +checksum = "baf29c38818342a3b26b5b923639e7b1f4a61fc5e76102d4b1981c6dc7a7579d" dependencies = [ "futures-core", "futures-task", @@ -2588,9 +2632,9 @@ dependencies = [ [[package]] name = "futures-io" -version = "0.3.31" +version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e5c1b78ca4aae1ac06c48a526a655760685149f0d465d21f37abfe57ce075c6" +checksum = "cecba35d7ad927e23624b22ad55235f2239cfa44fd10428eecbeba6d6a717718" [[package]] name = "futures-lite" @@ -2622,9 +2666,9 @@ dependencies = [ [[package]] name = "futures-macro" -version = "0.3.31" +version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" +checksum = "e835b70203e41293343137df5c0664546da5745f82ec9b84d40be8336958447b" dependencies = [ "proc-macro2", "quote", @@ -2633,15 +2677,15 @@ dependencies = [ [[package]] name = "futures-sink" -version = "0.3.31" +version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e575fab7d1e0dcb8d0c7bcf9a63ee213816ab51902e6d244a95819acacf1d4f7" +checksum = "c39754e157331b013978ec91992bde1ac089843443c49cbc7f46150b0fad0893" [[package]] name = "futures-task" -version = "0.3.31" +version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f90f7dce0722e95104fcb095585910c0977252f286e354b5e3bd38902cd99988" +checksum = "037711b3d59c33004d3856fbdc83b99d4ff37a24768fa1be9ce3538a1cde4393" [[package]] name = "futures-timer" @@ -2651,9 +2695,9 @@ checksum = "f288b0a4f20f9a56b5d1da57e2227c661b7b16168e2f72365f57b63326e29b24" [[package]] name = "futures-util" -version = "0.3.31" +version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9fa08315bb612088cc391249efdc3bc77536f16c91f6cf495e6fbe85b20a4a81" +checksum = "389ca41296e6190b48053de0321d02a77f32f8a5d2461dd38762c0593805c6d6" dependencies = [ "futures-channel", "futures-core", @@ -2663,7 +2707,6 @@ dependencies = [ "futures-task", "memchr", "pin-project-lite", - "pin-utils", "slab", ] @@ -2932,7 +2975,7 @@ version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" dependencies = [ - "digest", + "digest 0.10.7", ] [[package]] @@ -3064,6 +3107,15 @@ version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "135b12329e5e3ce057a9f972339ea52bc954fe1e9358ef27f95e89716fbc5424" +[[package]] +name = "hybrid-array" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9155a582abd142abc056962c29e3ce5ff2ad5469f4246b537ed42c5deba857da" +dependencies = [ + "typenum", +] + [[package]] name = "hyper" version = "0.14.32" @@ -3870,7 +3922,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d89e7ee0cfbedfc4da3340218492196241d89eefb6dab27de5df917a6d2e78cf" dependencies = [ "cfg-if", - "digest", + "digest 0.10.7", ] [[package]] @@ -4270,7 +4322,7 @@ dependencies = [ "serde", "serde_json", "serde_path_to_error", - "sha2", + "sha2 0.10.9", "thiserror 1.0.69", "url", ] @@ -4528,7 +4580,7 @@ checksum = "51f44edd08f51e2ade572f141051021c5af22677e42b7dd28a88155151c33594" dependencies = [ "ecdsa", "elliptic-curve", - "sha2", + "sha2 0.10.9", ] [[package]] @@ -5069,7 +5121,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "343d3bd7056eda839b03204e68deff7d1b13aba7af2b2fd16890697274262ee7" dependencies = [ "heck 0.5.0", - "itertools 0.13.0", + "itertools 0.14.0", "log", "multimap", "petgraph", @@ -5090,7 +5142,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "27c6023962132f4b30eb4c172c91ce92d933da334c59c23cddee82358ddafb0b" dependencies = [ "anyhow", - "itertools 0.13.0", + "itertools 0.14.0", "proc-macro2", "quote", "syn 2.0.114", @@ -6084,7 +6136,7 @@ dependencies = [ "serde", "serde_json", "sha1", - "sha2", + "sha2 0.10.9", "tokio", ] @@ -6688,8 +6740,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" dependencies = [ "cfg-if", - "cpufeatures", - "digest", + "cpufeatures 0.2.17", + "digest 0.10.7", ] [[package]] @@ -6699,8 +6751,19 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" dependencies = [ "cfg-if", - "cpufeatures", - "digest", + "cpufeatures 0.2.17", + "digest 0.10.7", +] + +[[package]] +name = "sha2" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "446ba717509524cb3f22f17ecc096f10f4822d76ab5c0b9822c5f9c284e825f4" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.0", + "digest 0.11.3", ] [[package]] @@ -6734,7 +6797,7 @@ version = "1.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "74233d3b3b2f6d4b006dc19dee745e73e2a6bfb6f93607cd3b02bd5b00797d7c" dependencies = [ - "digest", + "digest 0.10.7", "rand_core 0.6.4", ] @@ -7160,7 +7223,7 @@ dependencies = [ "jni", "lazy_static", "libc", - "lru 0.12.5", + "lru 0.16.3", "lz4_flex 0.11.5", "memmap2", "moka", @@ -7184,13 +7247,13 @@ dependencies = [ "serde", "serde_json", "serde_yaml", - "sha2", + "sha2 0.11.0", "stable_deref_trait", "tantivy", "tantivy-common", "tantivy-sstable", "tempfile", - "thiserror 1.0.69", + "thiserror 2.0.18", "time", "tokio", "url", @@ -7364,9 +7427,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.49.0" +version = "1.50.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72a2903cd7736441aac9df9d7688bd0ce48edccaadf181c3b90be801e81d3d86" +checksum = "27ad5e34374e03cfffefc301becb44e9dc3c17584f414349ebe29ed26661822d" dependencies = [ "bytes", "libc", @@ -7756,9 +7819,9 @@ checksum = "bc7d623258602320d5c55d1bc22793b57daff0ec7efc270ea7d55ce1d5f5471c" [[package]] name = "typenum" -version = "1.19.0" +version = "1.20.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "562d481066bde0658276a35467c4af00bdc6ee726305698a55b86e61d7ad82bb" +checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" [[package]] name = "typetag" diff --git a/native/Cargo.toml b/native/Cargo.toml index eb8c712d..937ba04b 100644 --- a/native/Cargo.toml +++ b/native/Cargo.toml @@ -27,7 +27,7 @@ serde_json = "1.0.142" crc32fast = "1.5" serde_yaml = "0.9" base64 = "0.22" -futures = "0.3.31" +futures = "0.3.32" once_cell = "1.19.0" bytesize = "2" @@ -48,7 +48,7 @@ quickwit-query = { git = "https://github.com/indextables/quickwit", rev = "3026b # Additional dependencies anyhow = "1.0" uuid = { version = "1.0", features = ["v4", "serde"] } -tokio = { version = "1.45", features = ["full"] } +tokio = { version = "1.50", features = ["full"] } lazy_static = "1.4" tempfile = "3" postcard = { version = "1.1", features = ["heapless"] } @@ -56,7 +56,7 @@ num_cpus = "1.0" memmap2 = "0.9.5" stable_deref_trait = "1.2" # For implementing StableDeref on Mmap wrapper libc = "0.2" -lru = "0.12" # LRU cache for searcher caching +lru = "0.16" # LRU cache for searcher caching lz4_flex = "0.11" # Pure Rust LZ4 for disk cache compression fs2 = "0.4" # Disk space detection for cache auto-sizing async-trait = "0.1" # For Storage trait implementation @@ -79,9 +79,9 @@ regex = "1" # Transaction log (Avro state format for indextables txlog) apache-avro = { version = "0.21", features = ["zstandard"] } -sha2 = "0.10" +sha2 = "0.11" flate2 = "1" -thiserror = "1" +thiserror = "2" parking_lot = "0.12" moka = { version = "0.12", features = ["sync"] } diff --git a/native/src/txlog/cache.rs b/native/src/txlog/cache.rs index 9daae967..77c64800 100644 --- a/native/src/txlog/cache.rs +++ b/native/src/txlog/cache.rs @@ -566,6 +566,17 @@ pub fn hash_snapshot_key(checkpoint_version: i64, post_cp_count: usize) -> i64 { mod tests { use super::*; + // Tests that touch the process-wide cache registry or the global manifest + // cache (especially the destructive `clear_all_caches()`) race with each + // other under cargo's parallel test runner. Serialize them on a shared + // mutex. `into_inner()` keeps the lock usable after a test panics (a failed + // assertion would otherwise poison it and cascade failures into later tests). + static GLOBAL_STATE_TEST_LOCK: std::sync::Mutex<()> = std::sync::Mutex::new(()); + + fn lock_global_state() -> std::sync::MutexGuard<'static, ()> { + GLOBAL_STATE_TEST_LOCK.lock().unwrap_or_else(|e| e.into_inner()) + } + #[test] fn test_cache_disabled() { let config = CacheConfig { enabled: false, ..Default::default() }; @@ -604,6 +615,7 @@ mod tests { #[test] fn test_global_manifest_cache() { + let _guard = lock_global_state(); let entries = Arc::new(vec![]); put_cached_manifest("test-manifest.avro", entries.clone()); let cached = get_cached_manifest("test-manifest.avro"); @@ -646,6 +658,7 @@ mod tests { #[test] fn test_get_or_create_cache_reuse() { + let _guard = lock_global_state(); let config = CacheConfig { version_ttl: Duration::from_secs(60), ..Default::default() }; let c1 = get_or_create_cache("test://table1-moka", config.clone()); let c2 = get_or_create_cache("test://table1-moka", config); @@ -655,6 +668,7 @@ mod tests { #[test] fn test_invalidate_table_cache() { + let _guard = lock_global_state(); let c = get_or_create_cache("test://table_inv_moka", CacheConfig::default()); c.put_version(1, vec![]); assert!(c.get_version(1).is_some()); @@ -664,6 +678,7 @@ mod tests { #[test] fn test_global_cache_stats() { + let _guard = lock_global_state(); let c = get_or_create_cache("test://table_stats_moka", CacheConfig::default()); c.get_version(99); // miss c.put_version(99, vec![]); @@ -675,6 +690,7 @@ mod tests { #[test] fn test_clear_all_caches_removes_all_entries() { + let _guard = lock_global_state(); // Populate caches for two distinct tables let c1 = get_or_create_cache("test://table_clear1_moka", CacheConfig::default()); let c2 = get_or_create_cache("test://table_clear2_moka", CacheConfig::default()); @@ -698,6 +714,7 @@ mod tests { #[test] fn test_clear_all_caches_then_repopulate() { + let _guard = lock_global_state(); // Verify the registry is usable after a global clear clear_all_caches(); diff --git a/pom.xml b/pom.xml index 016f6bdd..ce811925 100644 --- a/pom.xml +++ b/pom.xml @@ -47,7 +47,7 @@ 11 --> 11 UTF-8 - 5.10.0 + 5.12.2 ${project.build.directory}/rust @@ -75,14 +75,14 @@ com.fasterxml.jackson.core jackson-databind - 2.15.2 + 2.22.0 software.amazon.awssdk s3 - 2.21.29 + 2.43.0 test @@ -90,7 +90,7 @@ com.azure azure-storage-blob - 12.25.0 + 12.33.3 test @@ -98,7 +98,7 @@ com.azure azure-identity - 1.12.0 + 1.18.2 test @@ -106,7 +106,7 @@ com.azure azure-core - 1.48.0 + 1.57.1 test @@ -114,7 +114,7 @@ org.testcontainers testcontainers - 1.19.0 + 2.0.5 test @@ -122,7 +122,7 @@ com.google.cloud google-cloud-storage - 2.29.1 + 2.67.0 test @@ -130,7 +130,7 @@ com.google.cloud google-cloud-nio - 0.127.12 + 0.131.0 test @@ -138,14 +138,14 @@ org.mockito mockito-core - 5.7.0 + 5.23.0 test org.mockito mockito-junit-jupiter - 5.7.0 + 5.23.0 test @@ -153,13 +153,13 @@ org.apache.iceberg iceberg-core - 1.7.1 + 1.10.1 test org.apache.iceberg iceberg-api - 1.7.1 + 1.10.1 test @@ -185,7 +185,7 @@ org.apache.maven.plugins maven-compiler-plugin - 3.11.0 + 3.15.0 11 11 @@ -198,7 +198,7 @@ org.apache.maven.plugins maven-surefire-plugin - 3.1.2 + 3.5.6 none @@ -225,7 +225,7 @@ org.codehaus.mojo exec-maven-plugin - 3.1.0 + 3.6.3 @@ -256,7 +256,7 @@ org.apache.maven.plugins maven-resources-plugin - 3.3.1 + 3.5.0 copy-native-lib @@ -287,7 +287,7 @@ org.apache.maven.plugins maven-jar-plugin - 3.3.0 + 3.5.0 @@ -500,7 +500,7 @@ org.apache.maven.plugins maven-source-plugin - 3.3.0 + 3.4.0 attach-sources @@ -515,7 +515,7 @@ org.apache.maven.plugins maven-javadoc-plugin - 3.6.3 + 3.12.0 attach-javadocs @@ -535,7 +535,7 @@ org.apache.maven.plugins maven-gpg-plugin - 3.1.0 + 3.2.8 sign-artifacts @@ -558,7 +558,7 @@ org.sonatype.central central-publishing-maven-plugin - 0.4.0 + 0.10.0 true central