From e37a894076702ab69000141e0b5a8d36b0579fb9 Mon Sep 17 00:00:00 2001 From: Brenton Strickler Date: Fri, 3 Apr 2026 09:11:10 -0400 Subject: [PATCH 1/3] Fix disk_cache tests: replace sleep with flush_blocking MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The 5 failing Linux CI tests used sleep(100ms) to wait for the async background writer to complete, which was insufficient on the Linux runner. Replace all sleep-based synchronization with flush_blocking(), which properly waits for all pending writes — matching the pattern already used by the newer tests in this file. --- native/src/disk_cache/tests.rs | 33 ++++++++++++++------------------- 1 file changed, 14 insertions(+), 19 deletions(-) diff --git a/native/src/disk_cache/tests.rs b/native/src/disk_cache/tests.rs index 6b96c7a1..88813726 100644 --- a/native/src/disk_cache/tests.rs +++ b/native/src/disk_cache/tests.rs @@ -3,8 +3,6 @@ #[cfg(test)] mod tests { - use std::time::Duration; - use tempfile::TempDir; use super::super::*; @@ -79,7 +77,7 @@ mod tests { let data: Vec = (0..10000).map(|i| (i % 256) as u8).collect(); cache.put("s3://bucket/path", "split-001", "term", None, &data); - std::thread::sleep(Duration::from_millis(100)); + cache.flush_blocking(); let retrieved = cache.get("s3://bucket/path", "split-001", "term", None); assert!(retrieved.is_some()); @@ -96,7 +94,7 @@ mod tests { let range = Some(0u64..5000u64); cache.put("s3://bucket", "split-002", "idx", range.clone(), &data); - std::thread::sleep(Duration::from_millis(100)); + cache.flush_blocking(); let retrieved = cache.get("s3://bucket", "split-002", "idx", range); assert!(retrieved.is_some()); @@ -115,20 +113,20 @@ mod tests { let data: Vec = (0..10000).map(|i| (i % 256) as u8).collect(); cache.put("s3://bucket", "split-001", "term", None, &data); - std::thread::sleep(Duration::from_millis(100)); + cache.flush_blocking(); assert!(cache.get_total_bytes() > 0); assert_eq!(cache.get_split_count(), 1); assert_eq!(cache.get_component_count(), 1); cache.put("s3://bucket", "split-001", "idx", None, &data); - std::thread::sleep(Duration::from_millis(100)); + cache.flush_blocking(); assert_eq!(cache.get_split_count(), 1); assert_eq!(cache.get_component_count(), 2); cache.put("s3://bucket", "split-002", "term", None, &data); - std::thread::sleep(Duration::from_millis(100)); + cache.flush_blocking(); assert_eq!(cache.get_split_count(), 2); assert_eq!(cache.get_component_count(), 3); @@ -145,14 +143,14 @@ mod tests { cache.put("s3://bucket", "split-001", "term", None, &data); cache.put("s3://bucket", "split-001", "idx", None, &data); cache.put("s3://bucket", "split-002", "term", None, &data); - std::thread::sleep(Duration::from_millis(150)); + cache.flush_blocking(); assert_eq!(cache.get_split_count(), 2); assert_eq!(cache.get_component_count(), 3); let bytes_before = cache.get_total_bytes(); cache.evict_split("s3://bucket", "split-001"); - std::thread::sleep(Duration::from_millis(100)); + cache.flush_blocking(); assert_eq!(cache.get_split_count(), 1); assert_eq!(cache.get_component_count(), 1); @@ -177,7 +175,7 @@ mod tests { let data: Vec = (0..10000).map(|i| (i % 256) as u8).collect(); cache.put("s3://bucket", "split-001", "term", None, &data); - std::thread::sleep(Duration::from_millis(100)); + cache.flush_blocking(); let retrieved = cache.get("s3://bucket", "split-001", "term", None); assert!(retrieved.is_some()); @@ -216,10 +214,9 @@ mod tests { let data: Vec = (0..10000).map(|i| (i % 256) as u8).collect(); cache.put("s3://bucket", "split-001", "term", None, &data); cache.put("s3://bucket", "split-002", "idx", None, &data); - std::thread::sleep(Duration::from_millis(100)); + cache.flush_blocking(); cache.sync_manifest().unwrap(); - std::thread::sleep(Duration::from_millis(100)); } let manifest_path = cache_path.join(CACHE_SUBDIR).join("manifest.json"); @@ -249,7 +246,7 @@ mod tests { let data: Vec = (0..10000).map(|i| (i % 256) as u8).collect(); cache.put("s3://bucket", "split-001", "term", None, &data); - std::thread::sleep(Duration::from_millis(100)); + cache.flush_blocking(); cache.sync_manifest().unwrap(); } @@ -290,11 +287,9 @@ mod tests { for i in 0..6 { cache.put("s3://bucket", &format!("split-{:03}", i), "term", None, &data); - std::thread::sleep(Duration::from_millis(50)); + cache.flush_blocking(); } - std::thread::sleep(Duration::from_millis(200)); - let total_bytes = cache.get_total_bytes(); assert!(total_bytes <= 50000); @@ -333,7 +328,7 @@ mod tests { handle.join().unwrap(); } - std::thread::sleep(Duration::from_millis(200)); + cache.flush_blocking(); let mut read_handles = vec![]; for i in 0..4 { @@ -372,7 +367,7 @@ mod tests { cache.put("s3://bucket", "split-001", "idx", Some(0..5000), &data1); cache.put("s3://bucket", "split-001", "idx", Some(5000..10000), &data2); - std::thread::sleep(Duration::from_millis(100)); + cache.flush_blocking(); let r1 = cache.get("s3://bucket", "split-001", "idx", Some(0..5000)); let r2 = cache.get("s3://bucket", "split-001", "idx", Some(5000..10000)); @@ -407,7 +402,7 @@ mod tests { cache.put("s3://bucket-a", "split-001", "term", None, &data); cache.put("s3://bucket-b", "split-001", "term", None, &data); cache.put("azure://container", "split-001", "term", None, &data); - std::thread::sleep(Duration::from_millis(150)); + cache.flush_blocking(); assert!(cache.get("s3://bucket-a", "split-001", "term", None).is_some()); assert!(cache.get("s3://bucket-b", "split-001", "term", None).is_some()); From 63878a9373908ef3a557a5298702a13e29832e89 Mon Sep 17 00:00:00 2001 From: Brenton Strickler Date: Fri, 3 Apr 2026 11:59:28 -0400 Subject: [PATCH 2/3] Add diagnostics for intermittent multi-segment docid mapping test failure MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The test_multi_segment_docid_mapping_correctness test fails intermittently on Linux with a file slice bounds error (19139 > 18320) when opening the split bundle. This suggests the bundle footer records offsets that exceed the actual file size — possibly a non-deterministic flush ordering issue in BundleDirectory under the 15MB minimum heap. Changes: - Add split file size logging and bundle footer hex dump on failure to capture the exact state when the error occurs - Enable RUST_BACKTRACE=1 in CI Rust test step for full stack traces - Remove --lib --tests flags from cargo test (run all tests) --- .github/workflows/ci.yml | 8 ++++++-- native/src/parquet_companion/indexing.rs | 17 ++++++++++++++++- 2 files changed, 22 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 191457cf..9baaf66e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -29,7 +29,9 @@ jobs: run: make test - name: Rust tests - run: cd native && cargo test --lib --tests + run: cd native && cargo test + env: + RUST_BACKTRACE: 1 build-and-test-linux: name: Build & Test (Linux X64) @@ -52,4 +54,6 @@ jobs: run: make test - name: Rust tests - run: cd native && cargo test --lib --tests + run: cd native && cargo test + env: + RUST_BACKTRACE: 1 diff --git a/native/src/parquet_companion/indexing.rs b/native/src/parquet_companion/indexing.rs index da1de61e..31d7d412 100644 --- a/native/src/parquet_companion/indexing.rs +++ b/native/src/parquet_companion/indexing.rs @@ -2495,10 +2495,25 @@ mod tests { // Read the manifest from the split bundle let split_bytes = std::fs::read(&split_path).unwrap(); + eprintln!( + "🔍 TEST: Split file size = {} bytes, path = {}", + split_bytes.len(), + split_path.display() + ); let file_slice = tantivy::directory::FileSlice::new(Arc::new( tantivy::directory::OwnedBytes::new(split_bytes), )); - let bundle = quickwit_directories::BundleDirectory::open_split(file_slice).unwrap(); + let bundle = quickwit_directories::BundleDirectory::open_split(file_slice) + .unwrap_or_else(|e| { + // On failure, dump the bundle footer for diagnosis + let split_bytes2 = std::fs::read(&split_path).unwrap(); + let len = split_bytes2.len(); + let footer_preview = if len > 256 { &split_bytes2[len-256..] } else { &split_bytes2[..] }; + eprintln!("❌ BundleDirectory::open_split FAILED: {}", e); + eprintln!(" Split file size: {} bytes", len); + eprintln!(" Last 256 bytes (hex): {:02x?}", footer_preview); + panic!("BundleDirectory::open_split failed: {}", e); + }); use tantivy::directory::Directory; use crate::parquet_companion::manifest_io::{MANIFEST_FILENAME, deserialize_manifest}; From 5dbf75f463cce267099aa3aec862f13f226e94d7 Mon Sep 17 00:00:00 2001 From: Brenton Strickler Date: Fri, 3 Apr 2026 12:23:35 -0400 Subject: [PATCH 3/3] Fix CI: skip doc tests (rustdoc not available for cdylib crate) cargo test without --lib --tests tries to run doc tests via rustdoc, which fails on the self-hosted macOS runner with "No such file or directory". Restore --lib --tests to skip doc tests while keeping RUST_BACKTRACE=1 for the intermittent failure diagnostics. --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9baaf66e..5fdceee8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -29,7 +29,7 @@ jobs: run: make test - name: Rust tests - run: cd native && cargo test + run: cd native && cargo test --lib --tests env: RUST_BACKTRACE: 1 @@ -54,6 +54,6 @@ jobs: run: make test - name: Rust tests - run: cd native && cargo test + run: cd native && cargo test --lib --tests env: RUST_BACKTRACE: 1