feat: On disk cache for split footers

config/quickwit.yaml

@@ -138,6 +138,7 @@ indexer:
 # searcher:
 #   fast_field_cache_capacity: 1G
 #   split_footer_cache_capacity: 500M
+#   split_footer_disk_cache_capacity: 1G
 #   partial_request_cache_capacity: 64M
 #   max_num_concurrent_split_streams: 100
 #   max_num_concurrent_split_searches: 100

docs/configuration/node-config.md

@@ -200,6 +200,7 @@ This section contains the configuration options for a Searcher.
 | `aggregation_bucket_limit` | Determines the maximum number of buckets returned to the client. | `65000` |
 | `fast_field_cache_capacity` | Fast field in memory cache capacity on a Searcher. If your filter by dates, run aggregations, range queries, or even for tracing, it might worth increasing this parameter. The [metrics](../reference/metrics.md) starting by `quickwit_cache_fastfields_cache` can help you make an informed choice when setting this value. | `1G` |
 | `split_footer_cache_capacity` | Split footer in memory cache (it is essentially the hotcache) capacity on a Searcher.| `500M` |
+| `split_footer_disk_cache_capacity` | Opt-in, persistent on-disk cache for split footers, stored in the data directory under `searcher-split-footer-cache/`. When set, split footers survive a searcher restart, avoiding refetching the hotcache from the object store on startup. Disabled if unspecified. | |
 | `partial_request_cache_capacity` | Partial request in memory cache capacity on a Searcher. Cache intermediate state for a request, possibly making subsequent requests faster. It can be disabled by setting the size to `0`. | `64M` |
 | `max_num_concurrent_split_searches` | Maximum number of concurrent split search requests running on a Searcher. | `100` |
 | `split_cache` | Searcher split cache configuration options defined in the section below. Cache disabled if unspecified. | |

docs/overview/concepts/querying.md

@@ -98,6 +98,7 @@ In memory:
 On disk:
 
 - The split cache stores entire splits on disk. It can be enabled by setting the `split_cache` configuration fields. This cache can help reduce object store costs and load. Searchers populate this cache when splits are created or queried and evict them with a simple LRU strategy.
+- The split footer disk cache persists split footers on disk. It can be enabled by setting the `split_footer_disk_cache_capacity` configuration value. It backs the in-memory footer cache with a persistent LRU tier so that, after a searcher restart, footers do not have to be refetched from the object store. Cached footers are promoted back into memory on the first access.
 
 Learn more about cache parameters in the [searcher configuration docs](../../configuration/node-config.md#searcher-configuration).
 

quickwit/quickwit-config/src/node_config/mod.rs

@@ -274,6 +274,8 @@ pub struct SearcherConfig {
     pub aggregation_bucket_limit: u32,
     pub fast_field_cache_capacity: ByteSize,
     pub split_footer_cache_capacity: ByteSize,
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub split_footer_disk_cache_capacity: Option<ByteSize>,
     pub partial_request_cache_capacity: ByteSize,
     pub predicate_cache_capacity: ByteSize,
     pub max_num_concurrent_split_searches: usize,
@@ -333,6 +335,7 @@ impl Default for SearcherConfig {
         SearcherConfig {
             fast_field_cache_capacity: ByteSize::gb(1),
             split_footer_cache_capacity: ByteSize::mb(500),
+            split_footer_disk_cache_capacity: None,
             partial_request_cache_capacity: ByteSize::mb(64),
             predicate_cache_capacity: ByteSize::mb(256),
             max_num_concurrent_split_searches: 100,

quickwit/quickwit-config/src/node_config/serialize.rs

@@ -663,6 +663,7 @@ mod tests {
                 aggregation_bucket_limit: 500_000,
                 fast_field_cache_capacity: ByteSize::gb(10),
                 split_footer_cache_capacity: ByteSize::gb(1),
+                split_footer_disk_cache_capacity: None,
                 partial_request_cache_capacity: ByteSize::mb(64),
                 predicate_cache_capacity: ByteSize::mb(256),
                 max_num_concurrent_split_searches: 150,

quickwit/quickwit-search/src/leaf.rs

@@ -35,8 +35,8 @@ use quickwit_query::query_ast::{
 };
 use quickwit_query::tokenizers::TokenizerManager;
 use quickwit_storage::{
-    BundleStorage, ByteRangeCache, MemorySizedCache, OwnedBytes, SplitCache, Storage,
-    StorageResolver, TimeoutAndRetryStorage, wrap_storage_with_cache,
+    BundleStorage, ByteRangeCache, OwnedBytes, SplitCache, Storage, StorageResolver,
+    TieredSizedCache, TimeoutAndRetryStorage, wrap_storage_with_cache,
 };
 use tantivy::aggregation::agg_req::{AggregationVariants, Aggregations};
 use tantivy::aggregation::{AggContextParams, AggregationLimitsGuard};
@@ -60,10 +60,10 @@ use crate::{QuickwitAggregations, SearchError};
 async fn get_split_footer_from_cache_or_fetch(
     index_storage: Arc<dyn Storage>,
     split_and_footer_offsets: &SplitIdAndFooterOffsets,
-    footer_cache: &MemorySizedCache<String>,
+    footer_cache: &TieredSizedCache<String>,
 ) -> anyhow::Result<OwnedBytes> {
     {
-        let possible_val = footer_cache.get(&split_and_footer_offsets.split_id);
+        let possible_val = footer_cache.get(&split_and_footer_offsets.split_id).await;
         if let Some(footer_data) = possible_val {
             return Ok(footer_data);
         }
@@ -84,10 +84,12 @@ async fn get_split_footer_from_cache_or_fetch(
             )
         })?;
 
-    footer_cache.put(
-        split_and_footer_offsets.split_id.to_owned(),
-        footer_data_opt.clone(),
-    );
+    footer_cache
+        .put(
+            split_and_footer_offsets.split_id.to_owned(),
+            footer_data_opt.clone(),
+        )
+        .await;
 
     Ok(footer_data_opt)
 }

quickwit/quickwit-search/src/lib.rs

@@ -345,7 +345,7 @@ pub async fn single_node_search(
     let search_job_placer = SearchJobPlacer::new(searcher_pool.clone());
     let cluster_client = ClusterClient::new(search_job_placer);
     let searcher_config = SearcherConfig::default();
-    let searcher_context = Arc::new(SearcherContext::new(searcher_config, None));
+    let searcher_context = Arc::new(SearcherContext::new(searcher_config, None, None));
     let search_service = Arc::new(SearchServiceImpl::new(
         metastore.clone(),
         storage_resolver,

quickwit/quickwit-search/src/service.rs

@@ -29,7 +29,8 @@ use quickwit_proto::search::{
     SearchResponse, SnippetRequest,
 };
 use quickwit_storage::{
-    MemorySizedCache, QuickwitCache, SplitCache, StorageCache, StorageResolver,
+    DiskSizedCache, MemorySizedCache, QuickwitCache, SplitCache, StorageCache, StorageResolver,
+    TieredSizedCache,
 };
 use tantivy::aggregation::AggregationLimitsGuard;
 
@@ -405,8 +406,8 @@ pub struct SearcherContext {
     pub fast_fields_cache: Arc<dyn StorageCache>,
     /// Limit the concurrency for small snappy interactive searches.
     pub search_permit_provider: SearchPermitProvider,
-    /// Split footer cache.
-    pub split_footer_cache: MemorySizedCache<String>,
+    /// Split footer cache. In-memory, optionally backed by a persistent on-disk tier.
+    pub split_footer_cache: TieredSizedCache<String>,
     /// Per-split and per-query cache.
     pub leaf_search_cache: LeafSearchCache,
     /// Per-split and per-predicate cache.
@@ -432,16 +433,23 @@ impl SearcherContext {
     #[cfg(test)]
     pub fn for_test() -> SearcherContext {
         let searcher_config = SearcherConfig::default();
-        SearcherContext::new(searcher_config, None)
+        SearcherContext::new(searcher_config, None, None)
     }
 
-    /// Creates a new searcher context, given a searcher config, and an optional `SplitCache`.
-    pub fn new(searcher_config: SearcherConfig, split_cache_opt: Option<Arc<SplitCache>>) -> Self {
+    /// Creates a new searcher context, given a searcher config, an optional `SplitCache` and an
+    /// optional persistent (on-disk) split footer cache.
+    pub fn new(
+        searcher_config: SearcherConfig,
+        split_cache_opt: Option<Arc<SplitCache>>,
+        split_footer_disk_cache_opt: Option<DiskSizedCache<String>>,
+    ) -> Self {
         let capacity_in_bytes = searcher_config.split_footer_cache_capacity.as_u64() as usize;
         let global_split_footer_cache = MemorySizedCache::with_capacity_in_bytes(
             capacity_in_bytes,
             &quickwit_storage::STORAGE_METRICS.split_footer_cache,
         );
+        let split_footer_cache =
+            TieredSizedCache::new(global_split_footer_cache, split_footer_disk_cache_opt);
         let leaf_search_split_semaphore = SearchPermitProvider::new(
             searcher_config.max_num_concurrent_split_searches,
             searcher_config.warmup_memory_budget,
@@ -465,7 +473,7 @@ impl SearcherContext {
             fast_fields_cache: storage_long_term_cache,
             predicate_cache: predicate_cache.into(),
             search_permit_provider: leaf_search_split_semaphore,
-            split_footer_cache: global_split_footer_cache,
+            split_footer_cache,
             leaf_search_cache,
             list_fields_cache,
             split_cache_opt,

quickwit/quickwit-search/src/tests.rs

@@ -1125,7 +1125,7 @@ async fn test_search_util(test_sandbox: &TestSandbox, query: &str) -> Vec<u32> {
         ..Default::default()
     });
     let searcher_context: Arc<SearcherContext> =
-        Arc::new(SearcherContext::new(SearcherConfig::default(), None));
+        Arc::new(SearcherContext::new(SearcherConfig::default(), None, None));
 
     let agg_limits = searcher_context.get_aggregation_limits();
 
@@ -1765,7 +1765,7 @@ async fn test_single_node_list_terms() -> anyhow::Result<()> {
         .into_iter()
         .map(|split| extract_split_and_footer_offsets(&split.split_metadata))
         .collect();
-    let searcher_context = Arc::new(SearcherContext::new(SearcherConfig::default(), None));
+    let searcher_context = Arc::new(SearcherContext::new(SearcherConfig::default(), None, None));
 
     {
         let request = ListTermsRequest {

quickwit/quickwit-serve/src/lib.rs

@@ -113,7 +113,7 @@ use quickwit_search::{
     SearchJobPlacer, SearchService, SearchServiceClient, SearcherContext, SearcherPool,
     create_search_client_from_channel, start_searcher_service,
 };
-use quickwit_storage::{SplitCache, StorageResolver};
+use quickwit_storage::{DiskSizedCache, SplitCache, StorageResolver};
 use tcp_listener::TcpListenerResolver;
 use tokio::sync::oneshot;
 use tonic_health::ServingStatus;
@@ -658,9 +658,32 @@ pub async fn serve_quickwit(
             None
         };
 
+    let split_footer_disk_cache_opt: Option<DiskSizedCache<String>> = if let Some(capacity) =
+        node_config.searcher_config.split_footer_disk_cache_capacity
+    {
+        match DiskSizedCache::open(
+            node_config
+                .data_dir_path
+                .join("searcher-split-footer-cache"),
+            capacity.as_u64(),
+            &quickwit_storage::STORAGE_METRICS.split_footer_disk_cache,
+        )
+        .await
+        {
+            Ok(disk_cache) => Some(disk_cache),
+            Err(error) => {
+                error!(%error, "failed to open the persistent split footer cache, disabling it");
+                None
+            }
+        }
+    } else {
+        None
+    };
+
     let searcher_context = Arc::new(SearcherContext::new(
         node_config.searcher_config.clone(),
         split_cache_opt,
+        split_footer_disk_cache_opt,
     ));
 
     let (search_job_placer, search_service) = setup_searcher(
@@ -1581,7 +1604,8 @@ mod tests {
     #[tokio::test]
     async fn test_setup_searcher() {
         let node_config = NodeConfig::for_test();
-        let searcher_context = Arc::new(SearcherContext::new(SearcherConfig::default(), None));
+        let searcher_context =
+            Arc::new(SearcherContext::new(SearcherConfig::default(), None, None));
         let metastore = metastore_for_test();
         let (change_stream, change_stream_tx) = ClusterChangeStream::new_unbounded();
         let storage_resolver = StorageResolver::unconfigured();

quickwit/quickwit-serve/src/soft_delete_api/handler.rs

@@ -207,8 +207,11 @@ mod tests {
         let search_job_placer = SearchJobPlacer::new(searcher_pool.clone());
         let cluster_client = ClusterClient::new(search_job_placer);
         let searcher_config = SearcherConfig::default();
-        let searcher_context =
-            Arc::new(quickwit_search::SearcherContext::new(searcher_config, None));
+        let searcher_context = Arc::new(quickwit_search::SearcherContext::new(
+            searcher_config,
+            None,
+            None,
+        ));
         let search_service: Arc<dyn SearchService> = Arc::new(SearchServiceImpl::new(
             sandbox.metastore(),
             sandbox.storage_resolver(),