diff --git a/be/src/cloud/cloud_meta_mgr.cpp b/be/src/cloud/cloud_meta_mgr.cpp index a582f83991e7bf..98f896ea61cef8 100644 --- a/be/src/cloud/cloud_meta_mgr.cpp +++ b/be/src/cloud/cloud_meta_mgr.cpp @@ -668,7 +668,14 @@ Status CloudMetaMgr::sync_tablet_rowsets_unlocked(CloudTablet* tablet, SyncRowsetStats* sync_stats) { using namespace std::chrono; - TEST_SYNC_POINT_RETURN_WITH_VALUE("CloudMetaMgr::sync_tablet_rowsets", Status::OK(), tablet); + // Forward `options` so a test callback can react to sync_delete_bitmap the + // way the real body does below (the delete-bitmap sync is gated on it), i.e. + // a mock can drop the bitmap when the flag is false. The pointer is appended + // before the macro's own return-value slot, which callbacks read via + // args.back(), so existing callbacks that only read args[0]/the ret pair are + // unaffected. + TEST_SYNC_POINT_RETURN_WITH_VALUE("CloudMetaMgr::sync_tablet_rowsets", Status::OK(), tablet, + &options); DBUG_EXECUTE_IF("CloudMetaMgr::sync_tablet_rowsets.before.inject_error", { auto target_tablet_id = dp->param("tablet_id", -1); auto target_table_id = dp->param("table_id", -1); diff --git a/be/src/cloud/cloud_storage_engine.cpp b/be/src/cloud/cloud_storage_engine.cpp index 0dc8321c2c4e3a..5b917528fa60ae 100644 --- a/be/src/cloud/cloud_storage_engine.cpp +++ b/be/src/cloud/cloud_storage_engine.cpp @@ -103,6 +103,54 @@ CloudStorageEngine::CloudStorageEngine(const EngineOptions& options) _cumulative_compaction_policies[CUMULATIVE_TIME_SERIES_POLICY] = std::make_shared(); _startup_timepoint = std::chrono::system_clock::now(); + // Build the dedicated, bounded query-cache delta pre-sync pool up front (see + // the member declaration): a decision fan-out must never find it null, and the + // destructor's stop() drains it before _meta_mgr/_tablet_mgr die. A build + // failure here means the process cannot create threads at startup, already + // fatal, so CHECK rather than limp on with a null pool. + // + // Report a misconfigured sizing loudly instead of silently coercing it: an + // operator who set an invalid value must see why the BE refused to start rather + // than run with a capacity that differs from the config (invalid state fails, it + // never silently continues). A max of >= 1 is required (a zero-width pool can run + // nothing) and the queue bound must be non-negative. + int32_t qc_sync_threads = config::query_cache_delta_sync_thread; + int32_t qc_sync_queue = config::query_cache_delta_sync_max_pending_tasks; + CHECK_GE(qc_sync_threads, 1) << "query_cache_delta_sync_thread must be >= 1, got " + << qc_sync_threads; + CHECK_GE(qc_sync_queue, 0) << "query_cache_delta_sync_max_pending_tasks must be >= 0, got " + << qc_sync_queue; + // Pre-start all workers here (min == max), at engine construction, OFF the query + // admission path. The decision fan-out submits from operator init, which runs on + // the bounded light_work_pool, and ThreadPool creates workers synchronously + // inside submit_func when the pool is short of the demand (creation can take + // hundreds of ms). A lazily grown pool (min = 0) would move that creation latency + // onto the query's critical path and, worse, could hand the fan-out a submit that + // enqueued a task yet returned an error at zero live threads. A fixed pool sized + // up front keeps submit_func pure enqueue (it never creates a thread, so it never + // blocks), so the fast-fail budget is spent on the sync RPCs rather than on + // spawning threads. The cost is a fixed set of mostly-idle workers on every cloud + // BE even when this opt-in feature is unused; that steady footprint is the + // deliberate trade for predictable, non-blocking admission. + Status qc_pool_st = ThreadPoolBuilder("QueryCacheDeltaSyncThreadPool") + .set_min_threads(qc_sync_threads) + .set_max_threads(qc_sync_threads) + .set_max_queue_size(qc_sync_queue) + .build(&_query_cache_delta_sync_pool); + CHECK(qc_pool_st.ok()) << "failed to build QueryCacheDeltaSyncThreadPool: " << qc_pool_st; + // A successful build does NOT prove the workers exist: ThreadPool::init() + // deliberately ignores per-thread creation failures (a worker can be created + // later on submit). Verify the fixed worker set actually started and fail + // construction if not, so the fan-out's "submit never creates a thread" property + // is a guarantee, not a hope -- without this a startup thread shortage would + // silently push synchronous thread creation back onto the query admission path, + // exactly the latency this fixed pool exists to avoid. num_threads() counts + // started plus pending-start workers, so it equals the request precisely when + // every worker was spawned. + CHECK_EQ(_query_cache_delta_sync_pool->num_threads(), qc_sync_threads) + << "QueryCacheDeltaSyncThreadPool started " + << _query_cache_delta_sync_pool->num_threads() << " of " << qc_sync_threads + << " workers"; } CloudStorageEngine::~CloudStorageEngine() { @@ -274,6 +322,20 @@ void CloudStorageEngine::stop() { } } + // Drain the query-cache delta pre-sync fan-out FIRST, before the pools its + // sync_rowsets callbacks depend on: a merge-on-write delete-bitmap sync + // submits file-read work to _sync_delete_bitmap_thread_pool / + // _calc_tablet_delete_bitmap_task_thread_pool and blocks on it. Shutting a + // dependency pool down first would discard its queued callbacks and leave a + // query-cache worker blocked forever, hanging this drain and BE shutdown. + // Draining this pool first lets its in-flight workers finish against the + // still-live dependency pools. stop() also runs before any member is + // destroyed (~CloudStorageEngine calls it first), so those tasks see live + // _meta_mgr/_tablet_mgr as well. + if (_query_cache_delta_sync_pool) { + _query_cache_delta_sync_pool->shutdown(); + } + if (_base_compaction_thread_pool) { _base_compaction_thread_pool->shutdown(); } diff --git a/be/src/cloud/cloud_storage_engine.h b/be/src/cloud/cloud_storage_engine.h index b939bbf094a1d5..9ae1edabb5aac8 100644 --- a/be/src/cloud/cloud_storage_engine.h +++ b/be/src/cloud/cloud_storage_engine.h @@ -176,6 +176,11 @@ class CloudStorageEngine final : public BaseStorageEngine { return *_sync_load_for_tablets_thread_pool; } + // Dedicated bounded pool for the query-cache incremental decision's per-tablet + // rowset pre-sync. Always non-null (built in the constructor), so callers need + // no null check. See the member declaration and cloud/config.h. + ThreadPool& query_cache_delta_sync_pool() const { return *_query_cache_delta_sync_pool; } + ThreadPool& warmup_cache_async_thread_pool() const { return *_warmup_cache_async_thread_pool; } Status register_compaction_stop_token(CloudTabletSPtr tablet, int64_t initiator); @@ -234,6 +239,16 @@ class CloudStorageEngine final : public BaseStorageEngine { std::shared_ptr _cloud_warm_up_manager; std::unique_ptr _tablet_hotspot; std::unique_ptr _sync_load_for_tablets_thread_pool; + // Dedicated bounded pool for the query-cache incremental decision's per-tablet + // rowset pre-sync (QueryCacheRuntime::_presync_cloud_delta_tablets). Built in + // the constructor -- not open() -- so a decision fan-out never dereferences a + // null pool and unit fixtures that construct the engine without open() have + // it; drained in stop() (which the destructor calls first) so every in-flight + // task joins before _meta_mgr/_tablet_mgr are torn down. Declared after those + // managers so reverse-order member destruction is a second guarantee of the + // same ordering. Kept separate from the FE-warmup SyncLoadForTabletsThreadPool + // so a meta-service brownout cannot couple the two paths. + std::unique_ptr _query_cache_delta_sync_pool; std::unique_ptr _warmup_cache_async_thread_pool; std::unique_ptr _cloud_snapshot_mgr; diff --git a/be/src/cloud/config.cpp b/be/src/cloud/config.cpp index f03f28c0c85a37..5c5376c49a703b 100644 --- a/be/src/cloud/config.cpp +++ b/be/src/cloud/config.cpp @@ -38,6 +38,10 @@ DEFINE_Int64(tablet_cache_shards, "16"); DEFINE_mInt32(tablet_sync_interval_s, "1800"); DEFINE_mInt32(init_scanner_sync_rowsets_parallelism, "10"); DEFINE_mInt32(sync_rowsets_slow_threshold_ms, "1000"); +DEFINE_mInt32(query_cache_decision_sync_timeout_ms, "2000"); +DEFINE_Int32(query_cache_delta_sync_thread, "16"); +DEFINE_Int32(query_cache_delta_sync_max_pending_tasks, "2048"); +DEFINE_mInt32(query_cache_max_concurrent_decision_sync, "32"); DEFINE_mInt64(min_compaction_failure_interval_ms, "5000"); DEFINE_mInt64(base_compaction_freeze_interval_s, "1800"); diff --git a/be/src/cloud/config.h b/be/src/cloud/config.h index 63355451c8c218..887bb60b5d8dfe 100644 --- a/be/src/cloud/config.h +++ b/be/src/cloud/config.h @@ -70,6 +70,54 @@ DECLARE_mInt32(tablet_sync_interval_s); // parallelism for scanner init where may issue RPCs to sync rowset meta from MS DECLARE_mInt32(init_scanner_sync_rowsets_parallelism); DECLARE_mInt32(sync_rowsets_slow_threshold_ms); +// Fast-fail budget (ms) for the query-cache incremental-merge decision's +// pre-sync rowset fan-out. That decision runs in operator init on a bounded +// query-admission thread pool (the BE light_work_pool, contractually "must be +// light, not locked"), so a meta-service brownout that stalls the sync must not +// hold that thread for the full RPC retry budget (tens of seconds): sustained, +// it would exhaust the pool and reject query admission cluster-wide. When the +// fan-out does not finish within this budget the decision abandons the wait and +// falls back to one full recompute (the scan node's own async sync still brings +// the view up for the actual scan). A healthy sync is milliseconds, far under +// this, so it only trips under real meta-service degradation and leaves the +// steady-state incremental path unchanged. A value <= 0 disables cloud +// incremental merge outright: the decision skips the pre-sync fan-out entirely +// (launching nothing) and falls every scanned tablet back to a full recompute, +// a fail-safe rather than a useful setting. Cloud only. +DECLARE_mInt32(query_cache_decision_sync_timeout_ms); + +// The dedicated, bounded thread pool that runs the query-cache incremental +// decision's per-tablet rowset pre-sync (CloudStorageEngine owns it, created at +// construction and drained in stop()). It is deliberately NOT the shared +// SyncLoadForTabletsThreadPool: that pool is on the FE-driven warmup path, and +// giving the decision fan-out its own pool keeps a meta-service brownout from +// coupling the two. `query_cache_delta_sync_thread` bounds the concurrent +// syncs; `query_cache_delta_sync_max_pending_tasks` caps the queue so a brownout +// (every sync stalls in retry_rpc while new stale queries keep enqueuing) cannot +// grow the backlog without bound -- a submit that would exceed the cap fails +// fast and that tablet falls back to a full recompute. Cloud only. +DECLARE_Int32(query_cache_delta_sync_thread); +DECLARE_Int32(query_cache_delta_sync_max_pending_tasks); + +// Upper bound on how many query-cache incremental decisions may block in the +// decision-sync wait at the same time. That wait runs on brpc's light work pool +// (query admission; "must be light, not locked"), and single-flight coalesces only +// IDENTICAL cache keys, so under a meta-service brownout a wave of DISTINCT stale +// keys could otherwise park every light-pool worker for the full decision-sync +// timeout and starve unrelated fragment admission (the paired scan and cache-source +// operators can each park one on a first-call race, so the pressure is per operator, +// not just per query). The effective bound is the smaller of this value and half the +// ACTUAL light-pool width (the configured brpc_light_work_pool_threads, or +// max(128, 4*cores) when left at -1), so even a shrunk pool keeps spare workers for +// that admission -- this value only tightens below that floor. A query arriving over +// the bound skips incremental merge this round and recomputes in full (always correct, +// the right posture for an optimization under load). A non-positive value does not +// disable the guard; it falls back to the width-derived half. The fully non-blocking +// alternative is to drive the decision +// off a pipeline dependency, mirroring OlapScanLocalState::_cloud_tablet_dependency; +// that is a larger cross-operator change left as a follow-up. Mutable so it can be +// retuned online. Cloud only. +DECLARE_mInt32(query_cache_max_concurrent_decision_sync); // Cloud compaction config DECLARE_mInt64(min_compaction_failure_interval_ms); diff --git a/be/src/common/metrics/doris_metrics.cpp b/be/src/common/metrics/doris_metrics.cpp index de596407e512b3..2cec7cce201de5 100644 --- a/be/src/common/metrics/doris_metrics.cpp +++ b/be/src/common/metrics/doris_metrics.cpp @@ -57,6 +57,10 @@ DEFINE_COUNTER_METRIC_PROTOTYPE_3ARG(query_cache_incremental_fallback_total, Met DEFINE_COUNTER_METRIC_PROTOTYPE_3ARG(query_cache_write_back_total, MetricUnit::REQUESTS, "Query cache entries handed to the cache to be written back " "(full or merged); admission may still turn one down."); +DEFINE_COUNTER_METRIC_PROTOTYPE_3ARG(query_cache_decision_sync_time_ms, MetricUnit::MILLISECONDS, + "Cumulative wall time the cloud incremental-merge decision " + "spent blocking on its pre-sync rowset fan-out."); +DEFINE_GAUGE_CORE_METRIC_PROTOTYPE_2ARG(query_cache_presync_inflight, MetricUnit::NOUNIT); DEFINE_COUNTER_METRIC_PROTOTYPE_5ARG(push_requests_success_total, MetricUnit::REQUESTS, "", push_requests_total, Labels({{"status", "SUCCESS"}})); DEFINE_COUNTER_METRIC_PROTOTYPE_5ARG(push_requests_fail_total, MetricUnit::REQUESTS, "", @@ -303,6 +307,8 @@ DorisMetrics::DorisMetrics() : _metric_registry(_s_registry_name) { INT_COUNTER_METRIC_REGISTER(_server_metric_entity, query_cache_stale_hit_total); INT_COUNTER_METRIC_REGISTER(_server_metric_entity, query_cache_incremental_fallback_total); INT_COUNTER_METRIC_REGISTER(_server_metric_entity, query_cache_write_back_total); + INT_COUNTER_METRIC_REGISTER(_server_metric_entity, query_cache_decision_sync_time_ms); + INT_GAUGE_METRIC_REGISTER(_server_metric_entity, query_cache_presync_inflight); INT_COUNTER_METRIC_REGISTER(_server_metric_entity, push_requests_success_total); INT_COUNTER_METRIC_REGISTER(_server_metric_entity, push_requests_fail_total); diff --git a/be/src/common/metrics/doris_metrics.h b/be/src/common/metrics/doris_metrics.h index 1679f28d775a8d..be1e6fda6f4ad3 100644 --- a/be/src/common/metrics/doris_metrics.h +++ b/be/src/common/metrics/doris_metrics.h @@ -64,6 +64,16 @@ class DorisMetrics { IntCounter* query_cache_stale_hit_total = nullptr; IntCounter* query_cache_incremental_fallback_total = nullptr; IntCounter* query_cache_write_back_total = nullptr; + // Cumulative wall time the cloud incremental-merge decision spent blocking + // on its pre-sync rowset fan-out. That sync runs during operator init, not + // inside a profiled scan node, so without this counter its (potentially + // meta-service-bound) latency is invisible. + IntCounter* query_cache_decision_sync_time_ms = nullptr; + // Live count of cloud pre-sync single-flight entries currently registered + // (owned or draining). Steady state is near zero; a persistently rising value + // would flag a registry leak or a stuck fan-out. Also makes the coalescing + // mechanism observable (how many concurrent identical fan-outs are in flight). + IntGauge* query_cache_presync_inflight = nullptr; IntCounter* push_requests_success_total = nullptr; IntCounter* push_requests_fail_total = nullptr; diff --git a/be/src/exec/operator/cache_source_operator.cpp b/be/src/exec/operator/cache_source_operator.cpp index d83c83788c0eb7..a56ae61150ab75 100644 --- a/be/src/exec/operator/cache_source_operator.cpp +++ b/be/src/exec/operator/cache_source_operator.cpp @@ -20,13 +20,14 @@ #include #include +#include "cloud/config.h" #include "common/status.h" #include "core/block/block.h" #include "exec/operator/operator.h" #include "exec/pipeline/dependency.h" +#include "runtime/runtime_state.h" namespace doris { -class RuntimeState; Status CacheSourceLocalState::init(RuntimeState* state, LocalStateInfo& info) { RETURN_IF_ERROR(Base::init(state, info)); @@ -95,8 +96,38 @@ Status CacheSourceLocalState::init(RuntimeState* state, LocalStateInfo& info) { // MISS and INCREMENTAL rebuild the entry (from scratch / by merge), unless // the decision already knows the merged entry could never fit the entry // limits (write_back_feasible == false). - _need_insert_cache = - _cache_decision->key_valid && !hit_cache && _cache_decision->write_back_feasible; + // A query reading through a warmed-up-layout knob (both cloud only) must + // not write back either, because its scan is not guaranteed to represent + // exactly the queried version the entry would be stamped with. Both + // knobs walk the version graph by warmed-up preference and, unlike the + // plain capture, never clip an edge whose end lies past the requested + // window: the read may OVERSHOOT the queried version (a warmed + // compaction rowset spanning it drags the path beyond), and under + // freshness tolerance it may also STOP BELOW it (the walk halts at the + // warmed boundary). Neither knob carries an affectQueryResult + // annotation, so such an entry shares its cache key with plain runs of + // the same statement, which would then serve (exact hit) or extend + // (incremental merge) a base whose content does not match its version + // stamp: missing rows where the read stopped short, double-counted rows + // where it overshot. Suppressing the write-back keeps these reads pure + // consumers: they may still serve an exact HIT filled by a plain run + // (precise data, no worse than asked). + // + // Carve-out for the prefer knob on cloud merge-on-write UNIQUE tables: + // CloudTablet::capture_consistent_versions_unlocked honors + // enable_prefer_cached_rowset only for non-MOW tables (it guards on + // !enable_unique_key_merge_on_write()), so a MOW query that set prefer but + // not freshness still reads the exact queried version. That fill is + // version-exact and cacheable; suppressing it would keep a cloud MOW table + // with prefer set from ever populating the cache. Freshness tolerance has + // no such MOW guard (it is honored for MOW too), so it still forces + // suppression regardless of table type. + const bool inexact_version_fill = + config::is_cloud_mode() && + (state->enable_query_freshness_tolerance() || + (state->enable_prefer_cached_rowset() && !cache_param.is_merge_on_write)); + _need_insert_cache = _cache_decision->key_valid && !hit_cache && + _cache_decision->write_back_feasible && !inexact_version_fill; _insert_delta_count = _is_incremental ? _cache_decision->cached_delta_count + 1 : 0; if (hit_cache || _is_incremental) { diff --git a/be/src/exec/pipeline/pipeline_fragment_context.cpp b/be/src/exec/pipeline/pipeline_fragment_context.cpp index def2cf473aea77..ef069881471a1c 100644 --- a/be/src/exec/pipeline/pipeline_fragment_context.cpp +++ b/be/src/exec/pipeline/pipeline_fragment_context.cpp @@ -1605,8 +1605,32 @@ Status PipelineFragmentContext::_create_operator(ObjectPool* pool, const TPlanNo auto cache_node_id = _params.local_params[0].per_node_scan_ranges.begin()->first; auto cache_source_id = next_operator_id(); if (_query_cache_runtime == nullptr) { + TQueryCacheParam runtime_param = _params.fragment.query_cache_param; + // BE-side mirror of the FE incremental knob gates: a same-version + // FE already cleared allow_incremental under an active freshness/ + // prefer-cached knob, making this a no-op; it closes the rolling- + // upgrade window where an older FE without those gates still + // requests incremental while a knob is active, which would defeat + // the knob's warmed-read intent (never correctness: the delta + // capture is version-exact either way). Applied here, the single + // point where the fragment-wide runtime is born, so the cache + // source and the scan can never disagree on it; cloud-only since + // both knobs are inert on local storage. The whole decision is a + // pure function (unit-tested) so this stays trivial wiring; + // deliberately NOT part of the cache key (key identity must not + // depend on who asks). + runtime_param.allow_incremental = + QueryCacheRuntime::gate_allow_incremental_for_cloud_knobs( + runtime_param.allow_incremental, runtime_param.is_merge_on_write, + config::is_cloud_mode(), + _runtime_state->enable_query_freshness_tolerance(), + _runtime_state->enable_prefer_cached_rowset()); + // `runtime_param` is a throwaway built solely to carry the gated + // flag into the runtime, so move it in: the by-value constructor + // then move-constructs `_param` and no second O(tablets) deep copy + // of TQueryCacheParam happens on this admission path. _query_cache_runtime = - std::make_shared(_params.fragment.query_cache_param); + std::make_shared(std::move(runtime_param)); } op = std::make_shared(pool, cache_node_id, cache_source_id, _params.fragment.query_cache_param, diff --git a/be/src/runtime/query_cache/query_cache.cpp b/be/src/runtime/query_cache/query_cache.cpp index d7e042e0301a24..f6ce1e4963b754 100644 --- a/be/src/runtime/query_cache/query_cache.cpp +++ b/be/src/runtime/query_cache/query_cache.cpp @@ -17,9 +17,19 @@ #include "runtime/query_cache/query_cache.h" +#include +#include +#include #include +#include +#include +#include +#include "cloud/cloud_meta_mgr.h" +#include "cloud/cloud_storage_engine.h" +#include "cloud/cloud_tablet.h" #include "cloud/config.h" +#include "common/exception.h" #include "common/logging.h" #include "common/metrics/doris_metrics.h" #include "cpp/sync_point.h" @@ -28,6 +38,10 @@ #include "storage/rowset/rowset_reader.h" #include "storage/tablet/base_tablet.h" #include "storage/tablet/tablet_meta.h" +#include "util/countdown_latch.h" +#include "util/cpu_info.h" +#include "util/defer_op.h" +#include "util/threadpool.h" namespace doris { @@ -155,11 +169,12 @@ std::shared_ptr QueryCacheRuntime::get_or_make_decis } // Build the candidate outside the lock: a stale-entry decision walks - // tablet metadata under the header lock and, on merge-on-write tables, - // scans the delete bitmap, so its cost grows with the tablets per - // instance and the bitmap size. This runtime is fragment-wide, so a - // single lock-scoped build would serialize even unrelated instances' - // keys behind one slow decision. Racing callers of the same instance + // tablet metadata under the header lock, on merge-on-write tables scans + // the delete bitmap, and on cloud tablets may first sync the tablet view + // from the meta service (an RPC), so its cost grows with the tablets per + // instance, the bitmap size and the sync latency. This runtime is + // fragment-wide, so a single lock-scoped build would serialize even + // unrelated instances' keys behind one slow decision. Racing callers of the same instance // build duplicate candidates; the loser adopts the winner's decision // below and its own candidate releases with the shared_ptr (the entry // pin and any captured delta read sources go with it), which only costs @@ -235,13 +250,6 @@ bool QueryCacheRuntime::_try_prepare_incremental(const std::vectorincremental_fallback_reason = "cloud mode"; - return false; - } - int64_t cached_version = decision->handle.get_cache_version(); if (cached_version >= decision->current_version) { // The entry is newer than what this replica is asked to read (e.g. the @@ -259,9 +267,19 @@ bool QueryCacheRuntime::_try_prepare_incremental(const std::vector presync_reasons; + if (config::is_cloud_mode()) { + presync_reasons = _presync_cloud_delta_tablets(_cache, decision->cache_key, scan_ranges, + decision->current_version); + } + for (const auto& scan_range : scan_ranges) { int64_t tablet_id = scan_range.scan_range.palo_scan_range.tablet_id; - if (!_capture_tablet_delta(tablet_id, cached_version, decision)) { + if (!_capture_tablet_delta(tablet_id, cached_version, presync_reasons, decision)) { return false; } } @@ -282,9 +300,831 @@ bool QueryCacheRuntime::_try_prepare_incremental(const std::vector QueryCacheRuntime::_presync_active_waiters {0}; + +// Refund the flight's one-lump memory charge on its final teardown. release() is a +// plain atomic subtract, so this destructor never allocates and never throws (safe on +// any thread that drops the last ref -- a pool worker, a waiter, or a losing candidate +// being discarded). Skipped when the flight was never armed (mem_tracker null). +QueryCache::PresyncFlight::~PresyncFlight() { + if (mem_tracker != nullptr) { + mem_tracker->release(tracked_bytes); + } +} + +std::unordered_map QueryCacheRuntime::_presync_cloud_delta_tablets( + QueryCache* cache, const std::string& cache_key, + const std::vector& scan_ranges, int64_t current_version) { + std::unordered_map fallback_reasons; + + // (c) A non-positive budget disables cloud incremental merge: return WITHOUT + // launching any work. The previous code launched the fan-out and abandoned + // it on an instantly expired wait, spawning tasks whose only effect was to + // be orphaned. Fall every scanned tablet back so the whole instance + // recomputes in full. Checked before touching the engine so the semantics + // (and its unit test) do not depend on engine state. + if (config::query_cache_decision_sync_timeout_ms <= 0) { + for (const auto& scan_range : scan_ranges) { + fallback_reasons[scan_range.scan_range.palo_scan_range.tablet_id] = + "cloud incremental sync disabled"; + } + return fallback_reasons; + } + + // Reached only under config::is_cloud_mode() (the caller gates on it), but + // cloud_unique_id is a mutable config, so is_cloud_mode() can flip true on a + // live LOCAL deployment whose engine stays a local StorageEngine. Degrade + // gracefully rather than aborting in the hard CHECK inside to_cloud(): fall + // every tablet back, mirroring the per-tablet cloud-cast fallback below. + auto* engine = dynamic_cast(&ExecEnv::GetInstance()->storage_engine()); + if (engine == nullptr) { + for (const auto& scan_range : scan_ranges) { + fallback_reasons[scan_range.scan_range.palo_scan_range.tablet_id] = + "storage engine is not cloud"; + } + return fallback_reasons; + } + + // (a) The BE is tearing down: do not add new sync work. Fall every scanned + // tablet back rather than race teardown. + if (engine->stopped()) { + for (const auto& scan_range : scan_ranges) { + fallback_reasons[scan_range.scan_range.palo_scan_range.tablet_id] = + "be is stopping, sync skipped"; + } + return fallback_reasons; + } + + // Bound how many decision-sync waits may block the query-admission light pool at + // once. The fan-out wait below runs on that pool ("must be light, not locked"), and + // single-flight coalesces only IDENTICAL keys, so under a meta-service brownout a + // wave of DISTINCT stale keys (and, on a first-call race, the paired scan and cache- + // source operators each waiting) could otherwise park every admission worker for the + // full timeout and starve unrelated fragments. A soft counting cap keeps spare + // workers free: a query over the cap skips incremental this round and recomputes in + // full (always correct). + // + // The ceiling is the SMALLER of the tunable (query_cache_max_concurrent_decision_sync, + // default 32) and half the ACTUAL light-pool width. The width must be figured in, not + // just the config: brpc_light_work_pool_threads is operator-configurable, and on a + // pool shrunk to e.g. 16 a fixed 32 would never bind -- every worker could still park. + // The width is recomputed with the same formula internal_service.cpp sizes the pool by + // (the configured thread count, or max(128, 4*cores) when left at -1), so any pool + // keeps a worker reserve. The config only tightens below that floor; a non-positive + // config does NOT disable the guard (which would reopen the exact starvation this + // prevents) but falls back to the width-derived half. The fetch_add-then-back-out on a + // miss allows a brief boundary overshoot (harmless for a pool guard) while reading the + // mutable configs each call. Placed before the registry and fan-out so a rejected + // query does no work at all, and the Defer releases the slot on every admitted exit + // path (including the timeout and unwind paths below). + const int light_pool_width = config::brpc_light_work_pool_threads != -1 + ? config::brpc_light_work_pool_threads + : std::max(128, CpuInfo::num_cores() * 4); + // Integer half: a degenerate 1-thread light pool floors to 0, so the guard then + // admits nobody and every stale query falls back rather than parking the sole worker + // for the decision timeout (the worker-reserve guarantee would otherwise be violated + // exactly where it matters most). + const int width_cap = light_pool_width / 2; + int wait_cap = config::query_cache_max_concurrent_decision_sync; + if (wait_cap <= 0 || wait_cap > width_cap) { + wait_cap = width_cap; + } + const bool wait_admitted = + _presync_active_waiters.fetch_add(1, std::memory_order_acq_rel) < wait_cap; + if (!wait_admitted) { + _presync_active_waiters.fetch_sub(1, std::memory_order_acq_rel); + for (const auto& scan_range : scan_ranges) { + fallback_reasons[scan_range.scan_range.palo_scan_range.tablet_id] = + "cloud decision sync at capacity"; + } + return fallback_reasons; + } + Defer release_wait_slot( + [&] { _presync_active_waiters.fetch_sub(1, std::memory_order_acq_rel); }); + + // Coalesce identical concurrent fan-outs through the cache's single-flight + // registry. Runtimes are private per fragment context, so without this, N + // concurrent identical queries hitting the same stale key would each submit + // the full per-tablet fan-out: the duplicates waste the bounded queue's + // capacity (rejecting tablets that would have fit as one fan-out) and, + // during a meta-service brownout, park pool workers on the same per-tablet + // _sync_meta_lock, denying admission to unrelated keys. The first arrival + // (the flight owner) submits the fan-out; later identical arrivals join as + // waiters on the SAME completion latch, each under its own deadline. The + // flight's shared pieces are exactly what a private fan-out uses; a null + // cache (test-only) skips the registry and keeps the fan-out private. + bool flight_owner = true; + std::shared_ptr flight; + std::string flight_key; + auto make_flight = [&] { + // Attribute the flight's retained memory to the STABLE query-cache MemTracker + // rather than to the transient per-query tracker of whichever admission thread + // happened to create the flight (the flight outlives that query: coalesced + // waiters and still-running fan-out tasks keep its shared pieces alive past the + // query's teardown). + // + // The whole flight's estimated retained bytes are charged as ONE lump via the + // manual consume()/release() side-channel on the query-cache MemTrackerLimiter + // -- the SAME idiom the LRU layer uses for its tracking_bytes + // (QueryCache::insert above). consume()/release() are plain atomic counter ops: + // they NEVER allocate and NEVER throw, so the matching release runs safely in + // ~PresyncFlight (a noexcept destructor) on whatever pool/waiter thread drops + // the last ref. An earlier version charged the two hot buffers through a + // DorisVector whose deleter had to SWITCH the thread's MemTrackerLimiter, but + // that switch pushes onto a std::vector (attach_limiter_tracker) and can throw + // bad_alloc under memory pressure, and a throw from a shared_ptr deleter runs + // inside a noexcept destructor and std::terminates the BE. The single manual + // lump keeps stable attribution without that terminate risk, lets every buffer + // be a plain make_shared, and (unlike the earlier two-buffer-only charge) covers + // ALL of the flight's significant allocations -- the object, the + // per_range_reason / tablet_ids / slot_counted O(tablets) buffers, the control + // blocks, and the registry key -- so the query-cache tracker does not + // underreport retained RSS while flights and tombstones accumulate during a + // meta-service brownout. (BE has no global new/malloc hook -- + // be/src/runtime/memory/jemalloc_hook.cpp aliases malloc/free straight to + // jemalloc with no tracker interaction, and there is no global ::operator new + // override -- so absent this manual charge a plain new/std::vector/std::string + // is invisible to every MemTrackerLimiter and shows up only in process RSS.) + // The lifetimes coincide: publish_slot (captured by every task) holds the + // flight shared_ptr, so the flight object outlives every buffer, and + // ~PresyncFlight releases exactly when its last ref -- and thus all the buffers + // -- is gone. + // + // This is a tracking charge, not a process reservation. The O(tablets) pieces + // are small metadata (a pointer / an int64 / an atomic per tablet), doubly + // bounded by the concurrency cap (query_cache_max_concurrent_decision_sync) and + // the fan-out pool's fixed queue, and are dwarfed by the delta scan data this + // decision gates -- a query near the process limit is tipped over by that scan, + // not by these vectors. If an allocation here does fail under pressure it + // surfaces as an ordinary per-query bad_alloc (the admission caller fails that + // one query; a fan-out task's catch below turns its own into a fallback), never + // data corruption or a BE crash, so a try_reserve for metadata this small and + // this well-bounded is deliberately not taken. + auto qc_limiter = ExecEnv::GetInstance()->query_cache_mem_tracker(); + const int64_t slot_n = static_cast(scan_ranges.size()); + + auto f = std::make_shared(); + // Index-aligned so the fan-out tasks write disjoint slots without a lock; an + // empty (nullptr) slot means synced (or skipped as non-append-only), a + // non-empty slot is the fallback reason (a static literal). Each piece sits + // behind its own shared_ptr because the fan-out is waited on with a fast-fail + // deadline: on a timeout a waiter returns while still-running tasks keep writing + // their slots, so the storage must outlive every frame -- the shared_ptr each + // task captured keeps it alive until the last one finishes. Only read on the + // completed path. + f->per_range_reason = std::make_shared>(scan_ranges.size()); + f->fanout_done = std::make_shared(static_cast(scan_ranges.size())); + // Per-slot single-owner claim flags; see the claim comments at the submit loop + // below. (Value-initialized to false.) + f->slot_counted = std::make_shared>>(scan_ranges.size()); + // Set true when the LAST interested waiter abandons the flight (its deadline + // passed): a not-yet-started task then skips its sync RPC instead of spending + // the full retry budget, so a meta-service brownout drains the bounded queue + // fast rather than running abandoned work to completion and starving the pool. + f->abandoned = std::make_shared>(false); + // Snapshot the owner's tablet ids in slot order. Every waiter (this owner and + // any that coalesce onto the flight) builds its fallback map from THESE ids, not + // from its own scan-range order: build_cache_key sorts tablet ids into the key, + // so two runtimes whose scan ranges hold the same tablets in a different order + // share this flight, and keying the shared index-aligned slots off a joiner's + // own order would pin the owner's per-slot reason to the wrong tablet. + auto ids = std::make_shared>(); + ids->reserve(scan_ranges.size()); + for (const auto& scan_range : scan_ranges) { + ids->push_back(scan_range.scan_range.palo_scan_range.tablet_id); + } + f->tablet_ids = std::move(ids); + // The flight's own registry key (a plain std::string); on the null-cache test + // path flight_key is empty, so this is a harmless no-op there. + f->key = flight_key; + f->slots = scan_ranges.size(); + f->waiters = 1; + // Estimate the flight's retained bytes and charge them as one lump. Covered: + // the object; the three O(tablets) buffers (per_range_reason 8B, tablet_ids 8B, + // slot_counted ~1B per tablet); BOTH copies of the registry key (PresyncFlight:: + // key and the registry map node's own key are separate std::string buffers); the + // N queued fan-out task closures (submit_func heap-wraps each lambda, which + // captures the shared pieces -- a handful of shared_ptrs plus small scalars -- so + // a per-task slack covers each); and a fixed slack for the ~6 shared_ptr control + // blocks and the latch/atomic bodies. Arm the release (mem_tracker + + // tracked_bytes) BEFORE the consume so any later unwind still balances; + // ~PresyncFlight releases the same tracked_bytes off the same tracker. + constexpr int64_t kPerTaskClosureBytes = 128; // ~a handful of shared_ptrs + scalars + const int64_t flight_tracked_bytes = + static_cast(sizeof(QueryCache::PresyncFlight)) + + slot_n * static_cast(sizeof(const char*) + sizeof(int64_t) + + sizeof(std::atomic) + kPerTaskClosureBytes) + + 2 * static_cast(f->key.size()) + 6 * 64; + f->tracked_bytes = flight_tracked_bytes; + f->mem_tracker = qc_limiter; + qc_limiter->consume(flight_tracked_bytes); + return f; + }; + if (cache != nullptr) { + // flight_key embeds the digest and the SORTED tablet set + version, and + // is injective: build_cache_key appends the version as decimal digits, so + // this '@' is the LAST one in the string (no digit is '@') and splitting + // at the final '@' recovers exactly (cache_key, version) even though the + // binary cache_key may itself contain '@' bytes. Equal flight_key + // therefore means the identical tablet set at the identical version -- + // correct to coalesce, never a cross-key alias. + flight_key = cache_key; + flight_key += '@'; + flight_key += std::to_string(current_version); + // Double-checked registration. Check under the lock FIRST (the common + // coalescing case joins without allocating anything); only on a miss do we + // build the O(tablets) candidate -- OUTSIDE the lock, because holding the + // BE-global registry lock across that allocation would serialize every + // cloud incremental query, and a stampeding follower must not build a + // candidate it immediately discards -- then re-lock and re-check, since a + // concurrent owner may have inserted in the gap (we then join theirs and + // free the candidate). A found flight always covers the identical tablet + // set and slot count (the key invariant above), so assert rather than + // branch; joining a flight whose original waiters all already timed out is + // intentional -- it rides the draining fan-out instead of submitting a + // DUPLICATE one, which is what keeps a sustained brownout from recreating + // wave after wave of identical syncs. The entry is reaped only once the + // latch drains -- by the last waiter (leave_flight) or, if none is left, + // by the final task (publish_slot) -- so it never lingers. + { + std::lock_guard reg_lock(cache->_presync_flights_lock); + auto it = cache->_presync_flights.find(flight_key); + if (it != cache->_presync_flights.end()) { + DCHECK_EQ(it->second->slots, scan_ranges.size()); + flight = it->second; + ++flight->waiters; + flight_owner = false; + } + } + if (flight_owner) { + auto candidate = make_flight(); + std::lock_guard reg_lock(cache->_presync_flights_lock); + // try_emplace does the re-check and the insert in ONE hash of this long + // key (digest + sorted tablet set + version): it inserts `candidate` + // only when no concurrent owner beat us to the key in the unlocked gap, + // otherwise it leaves the map untouched and hands back the existing + // flight to join (and `candidate` is dropped). This replaces a find() + // plus a second operator[] that would hash the key twice under the + // BE-global registry lock. + auto [it, inserted] = cache->_presync_flights.try_emplace(flight_key, candidate); + if (inserted) { + flight = candidate; + DorisMetrics::instance()->query_cache_presync_inflight->increment(1); + } else { + DCHECK_EQ(it->second->slots, scan_ranges.size()); + flight = it->second; + ++flight->waiters; + flight_owner = false; + } + } + } else { + flight = make_flight(); + } + auto per_range_reason = flight->per_range_reason; + auto fanout_done = flight->fanout_done; + auto slot_counted = flight->slot_counted; + auto abandoned = flight->abandoned; + // The owner's tablet ids in slot order; every waiter (owner and coalesced + // joiners) keys the merged reasons off THESE, never off its own scan-range + // order (build_cache_key sorts, so a joiner's order may differ). Shared for + // read only, so a plain copy of the shared_ptr is enough. + auto flight_tablet_ids = flight->tablet_ids; + // Leave the flight on EVERY exit path -- normal return AND an exception + // unwinding out of the submit loop or the merge below -- via RAII, so a + // decrement can never be skipped and strand the registry entry. Two rules: + // (1) Reap the entry only once the fan-out has actually DRAINED (latch at + // 0), NOT merely when the last waiter's deadline expired. Erasing while + // tasks are still mid-RPC would let the next identical query find no + // entry, become a fresh owner, and submit a DUPLICATE fan-out; under a + // sustained meta-service brownout that recreates wave after wave of + // duplicate syncs -- the exact bounded-pool pressure single-flight + // exists to prevent. So while tasks remain, KEEP the entry: post-timeout + // arrivals then join this draining flight (see the join above) instead + // of piling on. If a waiter is present when the latch drains it reaps + // here; if every waiter already left (a timed-out tombstone), the FINAL + // task reaps in publish_slot when it drives the latch to 0 and sees zero + // waiters -- so a drained tombstone is never stranded, even with no + // future identical query. Every slot is published exactly once (run, + // skip, inline-reject, or submit-throw fallback), so the latch always + // reaches 0 in bounded time (except at pool shutdown, which discards + // queued-but-unstarted tasks; that strand is a process-teardown residual + // reclaimed when the cache is destroyed, not runtime growth), and a + // drained flight makes wait_for return immediately for a joiner. + // (2) A last leaver that leaves while tasks still run -- its deadline + // expired, OR an exception (bad_alloc) unwound the owner mid-submit -- + // sets `abandoned` UNDER the registry lock (so a joiner cannot observe + // the entry pre-abandon), telling not-yet-started tasks to skip so the + // pool drains fast instead of running work no one will read. NOT + // per-waiter: a timed-out waiter must not cancel work a still-waiting + // sibling needs. A private (null-cache) flight has exactly one waiter, + // so this degenerates to plain abandon-if-tasks-remain. + bool timed_out = false; + Defer leave_flight([&] { + if (cache != nullptr) { + std::lock_guard reg_lock(cache->_presync_flights_lock); + bool last = (--flight->waiters == 0); + bool drained = (fanout_done->count() == 0); + if (last && drained) { + // Guard `it->second == flight`: a prior tombstone for this key may + // already have been reaped and a fresh flight inserted; erase only + // our own. + auto it = cache->_presync_flights.find(flight_key); + if (it != cache->_presync_flights.end() && it->second == flight) { + cache->_presync_flights.erase(it); + DorisMetrics::instance()->query_cache_presync_inflight->increment(-1); + } + } else if (last) { + // Last waiter leaving with the fan-out still running (timeout or an + // owner exception): keep the tombstone and abandon so queued tasks + // skip; the final task reaps it in publish_slot once it drains. + abandoned->store(true, std::memory_order_release); + } + } else if (fanout_done->count() != 0) { + // Private (null-cache) flight, tasks still running: abandon so they + // skip. There is no registry entry to reap. + abandoned->store(true, std::memory_order_release); + } + }); + // Called ONLY by whoever already won the exclusive slot claim (the slot_counted + // exchange below), so it is the sole writer of the slot: publish its reason (a + // null/empty reason means synced OK / skipped, leaving the default-empty slot) + // and release the latch exactly once. The claim winner counting down is what + // lets the caller move the slot out the instant the latch settles. + // + // `reason` is a `const char*` (a static literal), NOT a std::string, and the + // slot is a `const char*` too, so storing it is a plain pointer write that + // CANNOT allocate or throw. This matters because this also runs on the + // bad_alloc unwind (publish_unsubmitted below): a throwing store inside that + // Defer would std::terminate. A null/empty reason means synced OK / skipped and + // leaves the default-null slot. The latch count_down and the reap always run, + // so a slot never strands the flight; the tablet-id map materializes the + // reason strings once, after the latch settles. + auto publish_slot = [fanout_done, per_range_reason, cache, flight](size_t idx, + const char* reason) { + if (reason != nullptr && reason[0] != '\0') { + (*per_range_reason)[idx] = reason; + } + fanout_done->count_down(); + // Reap-on-drain by the FINAL publisher when no waiter is left. If this + // count_down drained the latch and every interested waiter has already + // left (a timed-out tombstone, or an owner that unwound on bad_alloc), no + // leave_flight will run to reap it -- so reap here. The count()==0 + // pre-check takes only the latch's own lock, NOT the BE-global registry + // lock (the latch saturates at 0, so once true it stays true, and every + // slot publishes exactly once); only the settling publisher takes the + // registry lock, where it re-reads waiters and erases only its own entry. + // A joiner that raced in (++waiters under the same lock) makes waiters != 0, + // so this skips and that joiner reaps on its own leave. `cache` is null on + // the test-only private path (no registry). + if (cache != nullptr && fanout_done->count() == 0) { + std::lock_guard reg_lock(cache->_presync_flights_lock); + if (flight->waiters == 0) { + auto it = cache->_presync_flights.find(flight->key); + if (it != cache->_presync_flights.end() && it->second == flight) { + cache->_presync_flights.erase(it); + DorisMetrics::instance()->query_cache_presync_inflight->increment(-1); + } + } + } + }; + + // Run the per-tablet syncs on the engine-owned, bounded query-cache delta + // pre-sync pool -- a DEDICATED pool, not the shared SyncLoadForTablets warmup + // pool, so a meta-service brownout cannot couple the two paths. Properties + // this relies on: (1) fixed width + a bounded queue (cloud/config.h + // query_cache_delta_sync_*), so a brownout can neither spawn unbounded + // concurrent syncs nor grow the backlog without bound (a submit past the queue + // cap fails fast and that tablet falls back); (2) CloudStorageEngine::stop() + // drains this pool (joins running tasks, discards queued ones) and the + // destructor calls stop() before _meta_mgr/_tablet_mgr are destroyed, so a + // task that outlived its timed-out query still runs against a live engine and + // none can survive it. The raw detached bthreads this replaced were joined by + // nothing, so one sleeping in retry_rpc could wake after the engine was freed. + // + // Fast-fail is preserved: this runs in operator init on the bounded query + // admission pool (light_work_pool, "must be light, not locked"). submit_func + // never creates a worker -- the pool's fixed worker set (min == max) is + // pre-started AND verified present in the engine ctor (a CHECK_EQ on num_threads, + // because ThreadPool::init swallows creation failures), so submit is pure enqueue: + // it returns immediately at capacity/shutdown and never blocks on thread creation, + // keeping thread-creation latency off this critical path. The whole fan-out plus + // wait is then bounded by ONE absolute deadline (query_cache_decision_sync_ + // timeout_ms from the start below), which the pure-enqueue submit loop cannot + // exhaust before the wait even begins. A healthy sync is milliseconds, far under + // the budget, so the steady-state path is unchanged; this only trips under real + // meta-service degradation. Every task records its own sync error into its slot so + // no failure aborts the others. + // + // The tasks do no explicit MemTracker/ResourceContext attach: this is a detached + // engine-layer sync with no live query context to charge to (the query may have + // already timed out and returned, so attaching to it would be both unavailable + // here -- this is a static engine path -- and WRONG: sync_rowsets' allocations + // land in the tablet cache's own structures (rowset metas, delete bitmap) that + // outlive this decision, so charging them to the query-cache or a per-query + // tracker would mis-attribute tablet-cache memory to the wrong owner). Their + // allocations fall to the worker thread's default accounting, matching the + // same-shaped sync on the same kind of pool by CloudStorageEngine's other + // sync_rowsets caller, CloudBackendService::sync_load_for_tablets (the FE warmup + // path, be/src/cloud/cloud_backend_service.cpp), which likewise submits the sync + // to its pool with no attach. + auto sync_fanout_start = std::chrono::steady_clock::now(); + if (!flight_owner) { + // Joined an in-flight fan-out for the same (cache_key, version): the + // owner's tasks fill the shared slots; just wait on the shared latch + // below under this waiter's own deadline. + TEST_SYNC_POINT("QueryCacheRuntime::_presync_cloud_delta_tablets.follower_joined"); + } else { + auto& pool = engine->query_cache_delta_sync_pool(); + // Safe to capture the engine by raw pointer: stop() drains this pool + // before the engine (and _meta_mgr/_tablet_mgr) is destroyed, so a + // running task always sees a live engine. + CloudStorageEngine* engine_ptr = engine; + // Guarantee the latch always settles even if the submit loop unwinds + // partway (submit_func can throw bad_alloc heap-allocating a task closure + // under memory pressure). `submitted` counts the slots the loop has fully + // handled (a task was enqueued, or the inline-reject path published it); + // on an unwind this Defer publishes exactly the slots NOT yet reached, so + // the CountDownLatch still reaches 0. It must publish only [submitted, N): + // slots already submitted are owned by their (possibly still-queued) tasks + // and stealing their claim would falsely report a scheduled sync as + // unscheduled -- so on the normal path (submitted == N) this publishes + // nothing. A settling latch is what lets the leave Defer's reap-on-drain + // rule eventually remove the registry entry instead of stranding it. + size_t submitted = 0; + Defer publish_unsubmitted([&] { + for (size_t j = submitted; j < scan_ranges.size(); ++j) { + if (!(*slot_counted)[j].exchange(true, std::memory_order_acq_rel)) { + publish_slot(j, "cloud rowset sync not scheduled"); + } + } + }); + for (size_t i = 0; i < scan_ranges.size(); ++i) { + int64_t tablet_id = scan_ranges[i].scan_range.palo_scan_range.tablet_id; + Status submit_st = pool.submit_func([tablet_id, current_version, abandoned, i, + engine_ptr, publish_slot, slot_counted]() { + // Claim this slot atomically BEFORE any external work. If the inline + // submit-failure path already claimed it -- the narrow enqueue-then-fail + // case where do_submit queued this task but then returned an error at zero + // live workers, and the caller finished the slot inline -- bail before the + // sync RPC: the caller owns the slot and the count-down, and this task's + // result would be discarded. Claiming before the RPC (an exchange, not a + // plain load that a concurrent inline claim could race between the read + // and the RPC) is what keeps an enqueue-then-fail task that a later worker + // picks up from firing a meta-service call for an already-settled query. + if ((*slot_counted)[i].exchange(true, std::memory_order_acq_rel)) { + return; + } + // This task now exclusively owns the slot and MUST publish it exactly once + // on every exit path so the caller's bounded wait can settle. Empty reason + // means synced OK (or skipped as non-append-only); a non-empty reason + // falls this tablet back. + // A static literal (const char*), never an owning std::string, so + // publishing it in `done` cannot allocate on the bad_alloc unwind + // path (see publish_slot's contract above). + const char* reason = nullptr; + Defer done([&] { publish_slot(i, reason); }); + // Guard the whole per-tablet body: get_tablet fans out through + // CloudTabletMgr's single-flight loader, which THROWS + // std::system_error when a duplicate concurrent load's + // CountdownEvent wait returns non-zero (be/src/cloud/ + // cloud_tablet_mgr.cpp), and sync_rowsets can surface a bad_alloc + // (or a callee throw) under memory pressure. FunctionRunnable::run + // invokes this closure with no catch, so an escaped exception on + // this engine-pool worker would std::terminate the whole BE -- + // defeating the per-tablet fallback contract (any sync failure + // must cost only the incremental merge, never the process). The + // catch below converts it to a fallback reason. + try { + // (a) Shutdown began, or the interested waiters all abandoned the + // wait (their deadlines passed, or an owner unwound on bad_alloc): + // bail before a fresh sync RPC so the pool drains quickly instead of + // spending the full RPC retry budget on work the timed-out waiters + // will not read. A coalesced joiner MAY later reuse this slot (it + // rides the draining tombstone), so the reason must be accurate: + // keep "be is stopping" for a genuine teardown, but give the + // abandonment case a distinct reason (covering BOTH the timeout and + // the owner-unwind causes) so a joiner on a HEALTHY BE never sees a + // false shutdown diagnosis in its profile. + if (engine_ptr->stopped()) { + reason = "be is stopping, sync skipped"; + return; + } + if (abandoned->load(std::memory_order_acquire)) { + reason = "cloud rowset sync abandoned (all waiters left)"; + return; + } + auto tablet_res = ExecEnv::get_tablet(tablet_id); + if (!tablet_res) { + // Publish the failure instead of leaving the slot empty: an + // empty reason tells _capture_tablet_delta the view is synced, + // so its own get_tablet would RETRY this very cache-miss + // meta-service load synchronously on the admission thread, + // outside the fast-fail deadline -- during a brownout (the one + // situation where this load fails slowly) that repeats the + // slow RPC on the pool this budget exists to protect. Falling + // back is also never wrong here: this worker-side load is + // strictly conservative (a transient failure only costs the + // incremental merge, never correctness). + reason = "cloud tablet load failed"; + return; + } + BaseTabletSPtr tablet = std::move(tablet_res.value()); + const bool append_only = tablet->keys_type() == KeysType::DUP_KEYS || + (tablet->keys_type() == KeysType::UNIQUE_KEYS && + tablet->enable_unique_key_merge_on_write()); + if (!append_only) { + // Skip the RPC entirely: _capture_tablet_delta rejects this + // tablet at its keys-type check before it would consume a sync. + return; + } + auto cloud_tablet = std::dynamic_pointer_cast(std::move(tablet)); + if (cloud_tablet == nullptr) { + // is_cloud_mode() flipped on a live local deployment + // (cloud_unique_id is a mutable config): degrade to a full + // recompute rather than dereference the failed cast downstream. + reason = "tablet is not a cloud tablet"; + return; + } + SyncOptions options; + options.query_version = current_version; + options.merge_schema = true; + // The history-rewrite check reads the delete bitmap this sync merges, + // so request it explicitly rather than inherit the struct default (which + // a refactor could flip with no compile error): a merge-on-write delta + // cannot be classified on a bitmap the sync did not bring up to the + // queried version. This is the IDENTICAL sync shape the normal cloud MOW + // scan issues (olap_scan_operator.cpp sets query_version + merge_schema + // and leaves sync_delete_bitmap at its true default, cloud_tablet.h), and + // the classification then reads the very same tablet_meta()->delete_ + // bitmap() with no lazy per-rowset re-fetch, so it is no more exposed than + // the scan whose result it caches. + // + // It is NOT fully immune, but the gap is pre-existing and shared with + // that scan, not introduced here. sync_rowsets' fast path (_max_version + // >= query_version, cloud_tablet.cpp) returns without consulting + // sync_delete_bitmap, so if the tablet is already resident with rowsets + // synced to the queried version while its bitmap was left behind, this + // sync no-ops and the check reads the short bitmap. A full-compaction + // SUBMISSION creates exactly that state: it cache-miss-loads the tablet + // with sync_delete_bitmap=false (task_worker_pool.cpp and cloud_ + // compaction_action.cpp both pass false for FULL_COMPACTION), advancing + // _max_version ahead of the bitmap for the span of an admin-triggered + // full compaction that lands on a query-serving BE (with + // enable_compaction_rw_separation compaction and queries never share a + // BE's tablet cache, so it cannot happen). In that window a plain MOW + // scan already returns un-deduplicated rows; the only thing this feature + // adds is that a mis-classified delta is written back, so the wrong + // result persists in the entry instead of being a one-shot -- bounded by + // the same window, since committing the compaction swaps the rowsets and + // rebases the entry. The clean repair is in the storage layer (have the + // fast path require bitmap coverage too, or keep a sync_delete_bitmap= + // false load out of the query-visible cache), which fixes the scan and + // this path together; it is left as a follow-up rather than widened into + // this change. A delta rowset a compaction folded away in the meantime is + // a separate, benign case -- the capture below fails to find it and falls + // the tablet back to a full recompute. + options.sync_delete_bitmap = true; + // Re-check abandonment/shutdown right before the sync: a task that + // passed the earlier checks may have sat here while its waiters all + // timed out. sync_rowsets serializes same-tablet syncs on + // _sync_meta_lock and early-returns once one succeeds, but on a + // persistent brownout each DISTINCT-key task that reaches it re-runs + // the full retry budget (distinct keys do not share this flight's + // `abandoned` flag); skipping here when nobody will read the result + // keeps abandoned work from occupying the bounded pool and prolonging + // teardown. This narrows the window to the common case (abandonment + // during the wait, before the sync starts); it cannot be fully closed + // at this layer, because a task already inside sync_rowsets blocked on + // _sync_meta_lock does not re-check -- threading cancellation into + // sync_rowsets is a storage-layer change left as a follow-up. + if (engine_ptr->stopped()) { + reason = "be is stopping, sync skipped"; + return; + } + if (abandoned->load(std::memory_order_acquire)) { + reason = "cloud rowset sync abandoned (all waiters left)"; + return; + } + Status st = cloud_tablet->sync_rowsets(options); + if (!st.ok()) { + // Unlike the other fallback reasons (expected data shapes), a + // failed sync is an infrastructure error: log the status so the + // profile reason correlates to a cause. Throttled with + // LOG_EVERY_N because a meta-service brownout fails this for + // every stale tablet of every query at once; the underlying RPC + // errors are already logged per retry inside retry_rpc, so a + // sampled line here is enough to correlate without a storm, and + // the exact reason still reaches the user via the query profile. + LOG_EVERY_N(WARNING, 100) + << "query cache incremental merge falls back, cloud rowset sync " + "failed" + << ", tablet_id=" << tablet_id << ", status=" << st.to_string(); + reason = "cloud rowset sync failed"; + } + } catch (const doris::Exception& e) { + // A Doris-specific throw (e.g. surfaced by a nested THROW_IF_ERROR): + // catch it BEFORE the generic std::exception so the Doris error code + // survives into the log, per the standard catch order + // doris::Exception -> std::exception -> ... (be/src/common/AGENTS.md). + // Same fallback as below: the tablet recomputes in full rather than + // the escaped exception terminating the BE. + reason = "cloud tablet sync raised"; + LOG_EVERY_N(WARNING, 100) + << "query cache incremental merge falls back, cloud tablet sync raised" + << ", tablet_id=" << tablet_id << ", error=" << e.code() << ": " + << e.to_string(); + } catch (const std::exception& e) { + // Any other escaped exception (single-flight std::system_error, a + // bad_alloc, a callee throw) falls this tablet back instead of + // terminating the BE. `done` then publishes this reason, so the + // empty-slot "synced OK" path (which would make + // _capture_tablet_delta reissue the load synchronously) is never + // taken on a throw. Throttled like the sync-failed log because a + // brownout raises it for many tablets of many queries at once. + reason = "cloud tablet sync raised"; + LOG_EVERY_N(WARNING, 100) + << "query cache incremental merge falls back, cloud tablet sync raised" + << ", tablet_id=" << tablet_id << ", what=" << e.what(); + } catch (...) { + reason = "cloud tablet sync raised"; + LOG_EVERY_N(WARNING, 100) << "query cache incremental merge falls back, cloud " + "tablet sync raised (non-std exception)" + << ", tablet_id=" << tablet_id; + } + }); + if (!submit_st.ok()) { + // Pool shutting down or queue at capacity: do not block. Claim the + // slot for the inline fallback. do_submit rejects shutdown and + // capacity before enqueue, so the closure will not run; on the + // narrow path where it did enqueue before failing (thread creation + // at zero live workers), the exchange makes exactly one of this + // path and the queued task own the slot -- if the task claimed it + // first it will publish, so do nothing here. + if (!(*slot_counted)[i].exchange(true, std::memory_order_acq_rel)) { + publish_slot(i, "cloud rowset sync not scheduled"); + } + } + // Slot i is now fully handled (a task was enqueued above, or the + // inline-reject path published it); advance the safety-net floor so + // the unwind Defer never re-touches it. Reached only when submit_func + // did NOT throw -- a throw skips this and leaves slot i to the Defer. + submitted = i + 1; + } + } + + // ONE absolute deadline across the whole operation, anchored at the fan-out + // start above rather than restarted here. With the pre-started pool the submit + // loop is pure enqueue and returns in microseconds, so remaining is essentially + // the full budget; pinning the wait to the same start keeps the TOTAL budget at + // query_cache_decision_sync_timeout_ms even if a degraded pool ever made + // submission slow, instead of granting the full timeout again after submission + // already consumed part of it. + int64_t decision_sync_timeout_ms = config::query_cache_decision_sync_timeout_ms; + // Test seam: lets a test pin a deterministic deadline per coalescing role. When + // two runtimes share one flight, a test that wants to observe the intermediate + // "first waiter left, second still waiting" state needs the leave order fixed by + // construction (give the owner a shorter timeout than the joining follower) + // rather than by wall-clock start staggering, which a descheduled CI runner could + // invert. `flight_owner` is the role discriminator (true for the runtime that + // created the flight, false for one that joined it). A no-op in production (no + // callback registered); this function is static, so the role, not `this`, is what + // a test can key on. + TEST_SYNC_POINT_CALLBACK("QueryCacheRuntime::_presync_cloud_delta_tablets.deadline_ms", + flight_owner, &decision_sync_timeout_ms); + auto sync_deadline = sync_fanout_start + std::chrono::milliseconds(decision_sync_timeout_ms); + auto sync_now = std::chrono::steady_clock::now(); + // Keep full clock precision here -- do NOT truncate to whole milliseconds. + // Truncating would shave the sub-millisecond submit time off the budget and let + // the total blocked time dip just under query_cache_decision_sync_timeout_ms. + // wait_for never returns before its duration elapses, so with the raw remaining + // the total time from sync_fanout_start stays >= the budget. + auto remaining = sync_deadline > sync_now ? (sync_deadline - sync_now) + : std::chrono::steady_clock::duration::zero(); + // The leave-the-flight bookkeeping (decrement waiters, reap on drain, mark + // abandoned) runs in the `leave_flight` Defer established above, so it fires + // on every exit including an unwind. That Defer keys its reap/abandon decision + // off the latch count (`drained`), NOT off `timed_out` -- a last leaver can + // exit with `timed_out==false` (an owner unwinding on bad_alloc mid-submit) + // yet with tasks still running, and only the latch count captures that. Record + // the timeout result here; `timed_out` is read only by the fallback below. + timed_out = !fanout_done->wait_for(remaining); + DorisMetrics::instance()->query_cache_decision_sync_time_ms->increment( + std::chrono::duration_cast(std::chrono::steady_clock::now() - + sync_fanout_start) + .count()); + + if (timed_out) { + // The fan-out outran its fast-fail budget. Do NOT read per_range_reason + // (still-running tasks are writing it); fall every scanned tablet back so + // the whole instance recomputes in full. A distinct reason from a hard + // sync failure so the profile tells a slow meta service apart from a + // failing one. Throttled like the sync-failure log above: a brownout + // trips this for every stale tablet of every query at once, and this + // timeout is the primary operator-visible symptom of that brownout, so a + // sampled line correlates it without a storm (the per-query reason still + // reaches the user via the profile). + LOG_EVERY_N(WARNING, 100) + << "query cache incremental merge falls back, cloud rowset sync did not finish" + << " within query_cache_decision_sync_timeout_ms=" + << config::query_cache_decision_sync_timeout_ms << "ms"; + for (const auto& scan_range : scan_ranges) { + fallback_reasons[scan_range.scan_range.palo_scan_range.tablet_id] = + "cloud rowset sync timed out"; + } + return fallback_reasons; + } + // Merge the index-aligned slots into a tablet-id map for the capture loop. + // Safe to read now: the fan-out completed, so no task is still writing. COPY + // rather than move: the slots are shared with every other runtime coalesced + // onto this flight, and each builds its own map from them. Key each slot off + // the OWNER's tablet id (flight_tablet_ids), NOT this waiter's scan_ranges: + // build_cache_key sorts tablet ids, so a joiner whose scan ranges differ + // from the owner's only in order still shares this flight, and reading a + // slot through the joiner's own position would misassign the owner's + // per-slot reason to the wrong tablet (a failed tablet's reason lands on a + // sibling, and the truly failed tablet, left with no reason, is treated as + // synced -- so _capture_tablet_delta reissues a synchronous get_tablet on + // the admission thread outside the fast-fail budget, exactly the brownout + // stall the presync exists to avoid). Scan ranges carry distinct tablet ids + // in practice (each tablet is scanned once per query), but were one to + // appear twice the fan-out would only issue an idempotent redundant sync + // (serialized inside sync_rowsets by _sync_meta_lock) and this map, keyed by + // tablet id, would still resolve to one consistent reason for it -- so a + // duplicate is harmless, not something to guard against with an unreachable + // de-dup branch. + for (size_t i = 0; i < per_range_reason->size(); ++i) { + // A null slot is a synced-OK / skipped tablet; a non-null slot is a static + // reason literal (publish_slot only stores non-empty ones). Materialize the + // literal into the tablet-id map here, once, now that the fan-out has drained. + if ((*per_range_reason)[i] != nullptr) { + fallback_reasons[(*flight_tablet_ids)[i]] = (*per_range_reason)[i]; + } + } + return fallback_reasons; +} + +bool QueryCacheRuntime::_capture_tablet_delta( + int64_t tablet_id, int64_t cached_version, + const std::unordered_map& presync_reasons, + QueryCacheInstanceDecision* decision) { + if (config::is_cloud_mode()) { + // A CloudTablet keeps its rowset list (and for merge-on-write tables + // its delete bitmap) as a lazily synced copy of the meta service, and + // this decision runs before the scan node's own sync round, so the + // local view had to be brought up to the queried version first. + // _try_prepare_incremental already did that for every append-only + // tablet in one parallel fork-join (_presync_cloud_delta_tablets), + // which is the single sync per tablet; this loop only consumes its + // outcome. A recorded reason means the sync could not vouch for the + // view (a failed cast on a misconfigured deployment, an infrastructure + // sync failure, the fast-fail budget expiring on a slow meta service, + // or the pool refusing the sync outright), so fall back to a full + // recompute. Consumed BEFORE get_tablet below for two reasons: a refused + // submit loads nothing, so this recorded reason is more precise than the + // generic cache-only miss get_tablet would report; and a sync that + // FAILED may still leave a stale copy cached from an earlier query, which + // the cache-only get_tablet would return -- classifying on that unsynced + // view would be wrong, so the reason must veto it first. No reason means + // the view + // is synced up to query_version, and the history-rewrite check below + // reads the tablet's delete bitmap as the sync brought it to that + // version (sync_delete_bitmap=true) -- exactly as complete as the + // merge-on-write read path itself sees it, and no more: the + // classification inherits that read path's completeness, including any + // of its known limitations (e.g. a bitmap left short by a prior + // sync_delete_bitmap=false materialization of the tablet), rather than + // adding a stronger guarantee of its own. The scan node's later sync + // hits the same no-op shortcut, so the RPC count per query does not + // grow. + auto reason_it = presync_reasons.find(tablet_id); + if (reason_it != presync_reasons.end()) { + decision->incremental_fallback_reason = reason_it->second; + return false; + } + } + // Cache-only lookup (force_use_only_cached=true): the presync fan-out just + // synced this tablet, so in the common case it is still cached here and this + // is a hit with no RPC. If capacity pressure evicted it in the narrow window + // between that sync and this consume, a plain get_tablet would take the + // synchronous cache-miss meta-service load on THIS light admission thread -- + // the very blocking window the presync fast-fail budget caps, reopened after + // the fact. Forcing cache-only turns that miss into an immediate error that + // falls the tablet back to a full recompute instead; the fallback is never + // wrong (a missed incremental costs only recompute, never correctness), and + // the scan node's own later get_tablet still performs any genuine reload off + // this thread. The "tablet not found" reason is deliberately shared with a + // genuinely absent tablet id: both are simply "not resident locally", and + // this path cannot (nor needs to) tell an eviction from a never-loaded id. + // In non-cloud mode the flag is ignored (StorageEngine::get_tablet reads the + // resident tablet map, which never evicts), so this is a no-op there. + auto tablet_res = ExecEnv::get_tablet(tablet_id, /*sync_stats=*/nullptr, + /*force_use_only_cached=*/true); if (!tablet_res) { decision->incremental_fallback_reason = "tablet not found"; return false; @@ -357,13 +1197,20 @@ bool QueryCacheRuntime::_capture_tablet_delta(int64_t tablet_id, int64_t cached_ // input, which the stale sweep then hands to the unused-rowset GC. // Classification would see no rewrite and merge the cached rows with // their replacements: a wrong entry, stored at the current version. - // Holding the rowsets stops that, in start_delete_unused_rowset(): it - // collects a retired rowset only once nothing else references it, and - // only what it collects gets remove_rowset_delete_bitmap(), which drops - // every version of that rowset's bitmap. The other remover, the key - // ranges that the stale sweep queues onto the pre-window rowsets, is - // held off by the delta capture instead: that queue is released only - // once the swept path's own rowsets are unreferenced. + // Holding the rowsets stops that: both storage engines drop a retired + // rowset's bitmap only once nothing else references it, so raising the + // use count with this pin blocks the drop in either deployment mode. + // Local storage does it in start_delete_unused_rowset(): it collects a + // retired rowset only once nothing else references it, and only what it + // collects gets remove_rowset_delete_bitmap(), which drops every version + // of that rowset's bitmap. Cloud does it in + // CloudTablet::remove_unused_rowsets(), which skips any rowset whose + // use_count() still exceeds one before the same remove_rowset_delete_bitmap(). + // The other remover, the key ranges the stale sweep queues onto the + // pre-window rowsets (the engine's unused-delete-bitmap queue locally, + // CloudTablet::_unused_delete_bitmap in cloud), is held off by the delta + // capture instead: that queue is released only once the swept path's own + // rowsets are unreferenced. std::vector cached_side_pin; if (merge_on_write) { { @@ -429,7 +1276,10 @@ bool QueryCacheRuntime::_delta_rewrites_history(BaseTablet& tablet, // gone is rejected before we get here. Capture and scan are not atomic, // so a sweep landing exactly in between (with the aggregated keys also // already recycled) can still hide a rewrite: a known narrow race the - // fallback does not cover, shared with the baseline exact-hit path. + // fallback does not cover, the same one any merge-on-write snapshot read + // faces when a concurrent sweep re-stamps and recycles a marker (the + // exact-hit cache path is not itself exposed, since it serves cached + // blocks without re-reading the bitmap). // Pending entries use DeleteBitmap::TEMP_VERSION_COMMON (= 0) and stay // below the window as well. // diff --git a/be/src/runtime/query_cache/query_cache.h b/be/src/runtime/query_cache/query_cache.h index f727159eac2915..a2a66f2d6f4016 100644 --- a/be/src/runtime/query_cache/query_cache.h +++ b/be/src/runtime/query_cache/query_cache.h @@ -39,6 +39,7 @@ #include "runtime/exec_env.h" #include "runtime/memory/lru_cache_policy.h" #include "runtime/memory/mem_tracker.h" +#include "util/countdown_latch.h" #include "util/lru_cache.h" #include "util/slice.h" #include "util/time.h" @@ -257,6 +258,80 @@ class QueryCache : public LRUCachePolicy { void insert(const CacheKey& key, int64_t version, CacheResult& result, const std::vector& solt_orders, int64_t cache_size, int64_t delta_count = 0); + + // ---- cloud presync single-flight registry (internal to QueryCacheRuntime) ---- + // Fragment runtimes are private per PipelineFragmentContext, so without this + // registry N concurrent identical queries hitting the same stale entry would + // each submit the full per-tablet sync fan-out to the (bounded) engine pool: + // duplicates waste queue capacity and, during a meta-service brownout, park + // workers on the same per-tablet sync lock, denying admission to unrelated + // keys. The registry, keyed by (cache_key, version), lets the first arriving + // runtime own the fan-out while later identical runtimes wait on the SAME + // completion latch under their own deadlines. It lives here because this + // cache object is the one BE-global thing every runtime already shares. + struct PresyncFlight { + // The same shared pieces a private fan-out uses (see + // _presync_cloud_delta_tablets); sharing the struct just widens who may + // wait on them. Tasks capture the individual shared_ptrs, so the pieces + // outlive both the registry entry and every waiter. + // Per-slot fallback reason, or nullptr/"" for a synced-OK (or skipped) + // tablet. A `const char*` to a static string literal, NEVER an owning + // std::string: publishing a slot is then a plain pointer store that cannot + // allocate, so it is safe on the bad_alloc unwind path (publish_unsubmitted) + // where constructing a std::string could throw again inside a Defer and + // std::terminate. The merged tablet-id map materializes the strings once, + // after the fan-out latch settles. A plain std::vector; its O(tablets) buffer + // is covered by the flight's single lump charge to the query-cache MemTracker + // (see make_flight and ~PresyncFlight), not an allocator-aware container, so + // nothing in its teardown switches trackers or can throw. + std::shared_ptr> per_range_reason; + std::shared_ptr fanout_done; + std::shared_ptr>> slot_counted; + std::shared_ptr> abandoned; + // The owner's tablet ids in slot order, so the index-aligned slots map + // back to tablets by the OWNER's positions, which every waiter shares. + // A joining runtime must NOT key the merged reasons off its own + // scan-range order: build_cache_key sorts tablet ids, so two runtimes + // whose scan ranges differ only in order share a flight, and reading a + // slot through the joiner's own positions would misassign the owner's + // per-slot reason to the wrong tablet. Written once at creation, read + // (never mutated) by every waiter, so no lock is needed. A plain + // std::vector whose O(tablets) buffer is covered by the flight's single lump + // charge to the query-cache MemTracker (see make_flight and ~PresyncFlight). + std::shared_ptr> tablet_ids; + // This flight's own registry key (cache_key + '@' + version). Stored so + // the fan-out's final publisher can reap a drained-but-waiterless flight + // by key without capturing the string into every task closure -- see the + // reap-on-drain rule below. Written once at creation, then read-only. + std::string key; + // Slot count, to sanity-check that a joining runtime's scan-range set + // matches. cache-key equality implies it (the key embeds the sorted + // tablet set), so it is an invariant, DCHECK_EQ'd at the join site. + size_t slots = 0; + // Interested waiters (owner included), guarded by _presync_flights_lock. + // The registry entry is reaped only once the fan-out has DRAINED (latch + // at 0): either by the last waiter to leave (its leave_flight Defer) or, + // if every waiter left before the fan-out drained (a timed-out tombstone), + // by the final task that drives the latch to 0 when it observes zero + // waiters. A last leaver that leaves while tasks still run keeps the entry + // as a tombstone and sets `abandoned` so not-yet-started tasks skip. + int waiters = 0; + + // The query-cache MemTracker and the estimated retained bytes make_flight + // charged as one lump for THIS flight (object + all O(tablets) buffers + + // control blocks + key). The destructor releases exactly this many bytes off + // this tracker, so a flight is charged on creation and refunded on its final + // teardown (whether it wins the registry, is dropped as a losing candidate, or + // is reaped as a tombstone). release() is a plain atomic op, so the destructor + // never allocates and never throws. Null tracker means unaccounted (never armed + // -- e.g. an exception before make_flight finished), in which case the refund is + // skipped and nothing was consumed. + std::shared_ptr mem_tracker; + int64_t tracked_bytes = 0; + ~PresyncFlight(); + }; + std::mutex _presync_flights_lock; + std::unordered_map> _presync_flights; }; // The per-fragment-instance decision of how the query cache participates in the @@ -337,8 +412,13 @@ class QueryCacheRuntime { public: // `cache` is injectable for tests; production callers pass nullptr and the // global instance is used. - explicit QueryCacheRuntime(const TQueryCacheParam& param, QueryCache* cache = nullptr) - : _param(param), _cache(cache != nullptr ? cache : QueryCache::instance()) {} + // Takes the param BY VALUE and moves it into `_param`, so a caller that no + // longer needs its copy (the fragment-context gate builds a mutated + // `runtime_param` solely to pass here) can `std::move` it in and avoid a second + // O(tablets) deep copy of TQueryCacheParam; an lvalue caller (the tests) still + // copies exactly once, as before. + explicit QueryCacheRuntime(TQueryCacheParam param, QueryCache* cache = nullptr) + : _param(std::move(param)), _cache(cache != nullptr ? cache : QueryCache::instance()) {} QueryCache* cache() const { return _cache; } @@ -347,6 +427,46 @@ class QueryCacheRuntime { // threaded, before any local state init), so no locking is needed. void disable_for_binlog_scan() { _binlog_scan = true; } + // BE-side mirror of the FE incremental knob gates (QueryCacheNormalizer. + // computeAllowIncremental): freshness tolerance defeats an incremental merge + // for every table (the delta capture is version-exact by design, forcing the + // un-warmed reads the query chose to skip), and prefer-cached-rowset does so + // for every non-merge-on-write index (a MOW read is version-exact regardless + // of the knob, see CloudTablet::capture_consistent_versions_unlocked). A + // same-version FE already clears allow_incremental in these cases, making + // this re-derivation a no-op; it exists for the rolling-upgrade window where + // an older FE without the knob gates still sets allow_incremental while a + // knob is active. Intent, not correctness: the merged entry such a query + // would produce is version-exact either way; the gate preserves the knob's + // "skip un-warmed reads" promise. With the optional is_merge_on_write field + // absent (that same older FE), prefer suppresses MOW too, the conservative + // direction. Pure so the truth table is unit-testable; the caller (the + // fragment context, which has the session variables) applies it cloud-only, + // since both knobs are inert on local storage. + static bool cloud_knobs_suppress_incremental(bool freshness_tolerance_active, + bool prefer_cached_rowset_active, + bool is_merge_on_write) { + return freshness_tolerance_active || (prefer_cached_rowset_active && !is_merge_on_write); + } + + // The whole gate as a pure function, so the wiring (cloud-only, gated by the + // FE's request and the table type) is unit-testable without standing up a + // fragment and constructing operators: returns the effective + // allow_incremental. The fragment context passes config::is_cloud_mode() and + // its session variables. The gate is a no-op off cloud (the knobs are inert + // on local storage) and only ever clears a request, never grants one. + static bool gate_allow_incremental_for_cloud_knobs(bool requested_allow_incremental, + bool is_merge_on_write, bool cloud_mode, + bool freshness_tolerance_active, + bool prefer_cached_rowset_active) { + if (cloud_mode && requested_allow_incremental && + cloud_knobs_suppress_incremental(freshness_tolerance_active, + prefer_cached_rowset_active, is_merge_on_write)) { + return false; + } + return requested_allow_incremental; + } + // Idempotent: the first call for a given instance (identified by the cache // key derived from its scan ranges) makes the decision, later calls return // the same decision object. Never returns nullptr. @@ -361,6 +481,33 @@ class QueryCacheRuntime { std::lock_guard lock(_lock); _decisions[cache_key] = std::move(decision); } + + // Exercise the cloud presync fan-out (a private static) directly, so unit + // tests can assert the <= 0 skip-launch and the engine-stopped bail without + // standing up a full fragment. The default null cache skips the single- + // flight registry (a private, uncoalesced fan-out), which is exactly what + // these early-path tests need. + static std::unordered_map presync_cloud_delta_tablets_for_test( + const std::vector& scan_ranges, int64_t current_version, + QueryCache* cache = nullptr, const std::string& cache_key = "") { + return _presync_cloud_delta_tablets(cache, cache_key, scan_ranges, current_version); + } + + // Drives the concurrency-cap counter so a test can simulate the cap being + // reached without standing up real concurrent parked waiters. + static std::atomic& presync_active_waiters_for_test() { return _presync_active_waiters; } + + // Drives _capture_tablet_delta (a private member) directly so a test can + // prove the capture-site get_tablet is cache-only: with no recorded presync + // reason but the tablet absent from the cloud tablet cache (evicted after a + // successful presync), the classification must fall back WITHOUT reissuing a + // synchronous meta-service load on this admission thread. + bool capture_tablet_delta_for_test( + int64_t tablet_id, int64_t cached_version, + const std::unordered_map& presync_reasons, + QueryCacheInstanceDecision* decision) { + return _capture_tablet_delta(tablet_id, cached_version, presync_reasons, decision); + } #endif private: @@ -373,10 +520,38 @@ class QueryCacheRuntime { bool _try_prepare_incremental(const std::vector& scan_ranges, QueryCacheInstanceDecision* decision); + // Cloud only: bring every append-only scanned tablet's local view up to the + // queried version in parallel (a fork-join of sync_rowsets, matching the + // scan node's own tablet-sync fan-out) before the serial per-tablet capture + // below runs, instead of issuing those RPCs one tablet at a time on the + // shared prepare thread. This is the single sync per tablet: _capture_ + // tablet_delta consumes the result here and does not sync again. Concurrent + // identical runtimes are coalesced through the cache's single-flight + // registry keyed by (cache_key, current_version): the first arrival owns + // the fan-out, later ones wait on the same completion latch under their own + // deadlines (a null `cache` skips coalescing, test-only). The number of + // decisions that may block in the fan-out wait at once is soft-capped + // (config::query_cache_max_concurrent_decision_sync) so a meta-service brownout + // cannot park the whole light admission pool; a decision arriving over the cap + // returns every tablet as "cloud decision sync at capacity" and recomputes in + // full. Returns the per-tablet fallback reasons the sync produced (keyed by + // tablet id, only failures present): the over-capacity reason above, a cast + // failure ("tablet is not a cloud tablet"), a worker-side load failure ("cloud + // tablet load failed"), an infrastructure sync failure ("cloud rowset sync + // failed"), or a raised exception ("cloud tablet sync raised"). Tablets that are + // not append-only are skipped (no wasted RPC); _capture_tablet_delta rejects them + // at its own keys-type check. + static std::unordered_map _presync_cloud_delta_tablets( + QueryCache* cache, const std::string& cache_key, + const std::vector& scan_ranges, int64_t current_version); + // Validate one tablet for incremental merge and capture its delta read // source of (cached_version, current_version]. On any failure records the - // fallback reason in the decision and returns false. + // fallback reason in the decision and returns false. In cloud mode the view + // sync already ran in _presync_cloud_delta_tablets; its per-tablet failure + // reasons arrive through presync_reasons. bool _capture_tablet_delta(int64_t tablet_id, int64_t cached_version, + const std::unordered_map& presync_reasons, QueryCacheInstanceDecision* decision); // Merge-on-write only: true if any delete-bitmap entry stamped with a @@ -398,6 +573,11 @@ class QueryCacheRuntime { // Shared by every instance whose cache key cannot be built (see // get_or_make_decision): one immutable MISS decision, one log line. std::shared_ptr _invalid_decision; + + // BE-global soft cap counter for concurrent decision-sync waiters (see + // _presync_cloud_delta_tablets). Static because the bound is process-wide across + // every fragment runtime, not per instance. + static std::atomic _presync_active_waiters; }; } // namespace doris diff --git a/be/test/exec/operator/query_cache_operator_test.cpp b/be/test/exec/operator/query_cache_operator_test.cpp index 177e1aef7f96ee..96f23024a27ae4 100644 --- a/be/test/exec/operator/query_cache_operator_test.cpp +++ b/be/test/exec/operator/query_cache_operator_test.cpp @@ -20,6 +20,7 @@ #include #include +#include "cloud/config.h" #include "core/block/block.h" #include "exec/operator/cache_sink_operator.h" #include "exec/operator/cache_source_operator.h" @@ -50,7 +51,11 @@ class QueryCacheMockChildOperator : public OperatorXBase { }; struct QueryCacheOperatorTest : public ::testing::Test { + std::string _saved_deploy_mode; + std::string _saved_cloud_unique_id; void SetUp() override { + _saved_deploy_mode = config::deploy_mode; + _saved_cloud_unique_id = config::cloud_unique_id; state = std::make_shared(); state->_batch_size = 10; child_op = std::make_unique(); @@ -77,6 +82,15 @@ struct QueryCacheOperatorTest : public ::testing::Test { state.reset(); source.reset(); sink.reset(); + // Restore deploy_mode exception-safely: the cloud write-back tests set it + // to "cloud" mid-body; an inline restore would leak "cloud" into a later + // test (e.g. test_write_back_kept_on_local_with_freshness_set) if anything + // between the set and the restore threw. Restore cloud_unique_id too: + // is_cloud_mode() is deploy_mode=="cloud" || !cloud_unique_id.empty(), so + // a sibling test that leaves cloud_unique_id set would otherwise flip this + // suite's local-mode assumption and fail the local write-back test. + config::deploy_mode = _saved_deploy_mode; + config::cloud_unique_id = _saved_cloud_unique_id; } void create_local_state() { shared_state = sink->create_shared_state(); @@ -1016,4 +1030,212 @@ TEST_F(QueryCacheOperatorTest, test_failed_final_delta_merge_publishes_nothing) EXPECT_EQ(stale_handle.get_cache_version(), 100); } +TEST_F(QueryCacheOperatorTest, test_freshness_fill_does_not_write_back) { + // A cloud query under freshness tolerance may legally read below the + // queried version (its version-graph walk halts at the warmed boundary), + // yet the entry would be stamped with the queried version; and since the + // freshness knobs carry no affectQueryResult annotation, that entry would + // share its cache key with freshness-free runs of the same statement, + // poisoning their exact hits and incremental merges with the missing tail + // rows. So such a query must be a pure cache consumer: the write-back is + // suppressed at init, and nothing may land in the cache even after blocks + // flow through. + config::deploy_mode = "cloud"; + state->set_query_freshness_tolerance_ms(5000); + + sink = std::make_unique(); + TQueryCacheParam cache_param; + cache_param.node_id = 0; + cache_param.digest = "test_digest"; + cache_param.output_slot_mapping[0] = 0; + cache_param.tablet_to_range.insert({42, "test"}); + cache_param.force_refresh_query_cache = false; + cache_param.entry_max_bytes = 1024 * 1024; + cache_param.entry_max_rows = 1000; + source = std::make_unique( + &pool, /*plan_node_id=*/0, /*operator_id=*/0, cache_param, + std::make_shared(cache_param, query_cache)); + EXPECT_TRUE(source->set_child(child_op)); + child_op->_mock_row_desc.reset( + new MockRowDescriptor {{std::make_shared()}, &pool}); + create_local_state(); + + EXPECT_FALSE(source_local_state->_need_insert_cache); + { + auto block = ColumnHelper::create_block({1, 2, 3, 4, 5}); + auto st = sink->sink(state.get(), &block, true); + EXPECT_TRUE(st.ok()) << st.msg(); + } + { + Block block; + bool eos = false; + auto st = source->get_block(state.get(), &block, &eos); + EXPECT_TRUE(st.ok()) << st.msg(); + EXPECT_TRUE(eos); + EXPECT_EQ(block.rows(), 5); + EXPECT_EQ(query_cache->get_element_count(), 0); + } +} + +TEST_F(QueryCacheOperatorTest, test_prefer_cached_fill_does_not_write_back) { + // Prefer-cached-rowset (cloud) may OVERSHOOT the queried version: unlike + // the plain capture, its version-graph walk never clips an edge whose end + // lies past the requested window, so a warmed compaction rowset spanning + // the queried version drags the read beyond it while the entry would + // still be stamped with the queried version. Sharing the cache key with + // plain runs (no affectQueryResult annotation), such an entry would serve + // ahead-of-version rows on an exact hit and double-count the overshot + // range on an incremental merge. So a prefer-cached query must not write + // back either. + config::deploy_mode = "cloud"; + state->set_enable_prefer_cached_rowset(true); + + sink = std::make_unique(); + TQueryCacheParam cache_param; + cache_param.node_id = 0; + cache_param.digest = "test_digest"; + cache_param.output_slot_mapping[0] = 0; + cache_param.tablet_to_range.insert({42, "test"}); + cache_param.force_refresh_query_cache = false; + cache_param.entry_max_bytes = 1024 * 1024; + cache_param.entry_max_rows = 1000; + source = std::make_unique( + &pool, /*plan_node_id=*/0, /*operator_id=*/0, cache_param, + std::make_shared(cache_param, query_cache)); + EXPECT_TRUE(source->set_child(child_op)); + child_op->_mock_row_desc.reset( + new MockRowDescriptor {{std::make_shared()}, &pool}); + create_local_state(); + + EXPECT_FALSE(source_local_state->_need_insert_cache); + { + auto block = ColumnHelper::create_block({1, 2, 3, 4, 5}); + auto st = sink->sink(state.get(), &block, true); + EXPECT_TRUE(st.ok()) << st.msg(); + } + { + Block block; + bool eos = false; + auto st = source->get_block(state.get(), &block, &eos); + EXPECT_TRUE(st.ok()) << st.msg(); + EXPECT_TRUE(eos); + EXPECT_EQ(block.rows(), 5); + EXPECT_EQ(query_cache->get_element_count(), 0); + } +} + +TEST_F(QueryCacheOperatorTest, test_write_back_kept_on_cloud_without_freshness) { + // The freshness suppression must not clip an ordinary cloud query: with + // the tolerance unset, a cloud MISS still writes its result back. + config::deploy_mode = "cloud"; + + sink = std::make_unique(); + TQueryCacheParam cache_param; + cache_param.node_id = 0; + cache_param.digest = "test_digest"; + cache_param.output_slot_mapping[0] = 0; + cache_param.tablet_to_range.insert({42, "test"}); + cache_param.force_refresh_query_cache = false; + cache_param.entry_max_bytes = 1024 * 1024; + cache_param.entry_max_rows = 1000; + source = std::make_unique( + &pool, /*plan_node_id=*/0, /*operator_id=*/0, cache_param, + std::make_shared(cache_param, query_cache)); + EXPECT_TRUE(source->set_child(child_op)); + child_op->_mock_row_desc.reset( + new MockRowDescriptor {{std::make_shared()}, &pool}); + create_local_state(); + + EXPECT_TRUE(source_local_state->_need_insert_cache); +} + +TEST_F(QueryCacheOperatorTest, test_write_back_kept_on_local_with_freshness_set) { + // Freshness tolerance is inert on local storage (the scan clamps it to + // disabled), so a local deployment must keep writing back even when the + // session carries a tolerance value: the suppression is cloud-only. + state->set_query_freshness_tolerance_ms(5000); + + sink = std::make_unique(); + TQueryCacheParam cache_param; + cache_param.node_id = 0; + cache_param.digest = "test_digest"; + cache_param.output_slot_mapping[0] = 0; + cache_param.tablet_to_range.insert({42, "test"}); + cache_param.force_refresh_query_cache = false; + cache_param.entry_max_bytes = 1024 * 1024; + cache_param.entry_max_rows = 1000; + source = std::make_unique( + &pool, /*plan_node_id=*/0, /*operator_id=*/0, cache_param, + std::make_shared(cache_param, query_cache)); + EXPECT_TRUE(source->set_child(child_op)); + child_op->_mock_row_desc.reset( + new MockRowDescriptor {{std::make_shared()}, &pool}); + create_local_state(); + + EXPECT_TRUE(source_local_state->_need_insert_cache); +} + +TEST_F(QueryCacheOperatorTest, test_prefer_cached_fill_on_cloud_mow_writes_back) { + // Prefer-cached-rowset is ignored by cloud storage for a merge-on-write + // UNIQUE table (CloudTablet::capture_consistent_versions_unlocked guards the + // prefer branch on !enable_unique_key_merge_on_write()), so such a read + // still lands on the exact queried version. Its fill is therefore + // version-exact and safe to cache; FE reports the table type through + // TQueryCacheParam.is_merge_on_write so BE keeps the write-back rather than + // suppressing it. Mirrors test_prefer_cached_fill_does_not_write_back with + // the MOW flag set: the flag is what flips the outcome. + config::deploy_mode = "cloud"; + state->set_enable_prefer_cached_rowset(true); + + sink = std::make_unique(); + TQueryCacheParam cache_param; + cache_param.node_id = 0; + cache_param.digest = "test_digest"; + cache_param.output_slot_mapping[0] = 0; + cache_param.tablet_to_range.insert({42, "test"}); + cache_param.force_refresh_query_cache = false; + cache_param.entry_max_bytes = 1024 * 1024; + cache_param.entry_max_rows = 1000; + cache_param.__set_is_merge_on_write(true); + source = std::make_unique( + &pool, /*plan_node_id=*/0, /*operator_id=*/0, cache_param, + std::make_shared(cache_param, query_cache)); + EXPECT_TRUE(source->set_child(child_op)); + child_op->_mock_row_desc.reset( + new MockRowDescriptor {{std::make_shared()}, &pool}); + create_local_state(); + + EXPECT_TRUE(source_local_state->_need_insert_cache); +} + +TEST_F(QueryCacheOperatorTest, test_freshness_fill_on_cloud_mow_still_suppressed) { + // The MOW carve-out is prefer-only: freshness tolerance has no MOW guard on + // the storage side (CloudTablet honors it for every table type), so a + // freshness read can still stop below or overshoot the queried version even + // on a merge-on-write table. Its fill must stay suppressed regardless of the + // is_merge_on_write flag -- proving the carve-out did not widen to freshness. + config::deploy_mode = "cloud"; + state->set_query_freshness_tolerance_ms(5000); + + sink = std::make_unique(); + TQueryCacheParam cache_param; + cache_param.node_id = 0; + cache_param.digest = "test_digest"; + cache_param.output_slot_mapping[0] = 0; + cache_param.tablet_to_range.insert({42, "test"}); + cache_param.force_refresh_query_cache = false; + cache_param.entry_max_bytes = 1024 * 1024; + cache_param.entry_max_rows = 1000; + cache_param.__set_is_merge_on_write(true); + source = std::make_unique( + &pool, /*plan_node_id=*/0, /*operator_id=*/0, cache_param, + std::make_shared(cache_param, query_cache)); + EXPECT_TRUE(source->set_child(child_op)); + child_op->_mock_row_desc.reset( + new MockRowDescriptor {{std::make_shared()}, &pool}); + create_local_state(); + + EXPECT_FALSE(source_local_state->_need_insert_cache); +} + } // namespace doris diff --git a/be/test/exec/pipeline/query_cache_test.cpp b/be/test/exec/pipeline/query_cache_test.cpp index 4f79f4e8943146..958f6c199ff3a4 100644 --- a/be/test/exec/pipeline/query_cache_test.cpp +++ b/be/test/exec/pipeline/query_cache_test.cpp @@ -28,11 +28,15 @@ #include #include #include +#include #include #include +#include "cloud/cloud_storage_engine.h" +#include "cloud/cloud_tablet.h" #include "cloud/config.h" #include "common/config.h" +#include "common/exception.h" #include "common/metrics/doris_metrics.h" #include "core/data_type/data_type_number.h" #include "cpp/sync_point.h" @@ -40,6 +44,7 @@ #include "json2pb/json_to_pb.h" #include "storage/data_dir.h" #include "storage/options.h" +#include "storage/rowset/rowset_factory.h" #include "storage/rowset/rowset_meta.h" #include "storage/storage_engine.h" #include "storage/tablet/base_tablet.h" @@ -49,6 +54,7 @@ #include "testutil/column_helper.h" #include "util/debug_points.h" #include "util/defer_op.h" +#include "util/threadpool.h" #include "util/uid_util.h" namespace doris { @@ -413,6 +419,375 @@ TEST_F(QueryCacheTest, runtime_decision_hit) { fallback_before); } +// (c) query_cache_decision_sync_timeout_ms <= 0 disables cloud incremental +// merge: the presync must spawn NO sync work and fall every scanned tablet back. +// No storage engine is installed, so the <= 0 guard returning cleanly (before +// to_cloud()) is itself proof nothing launched -- any launch would have +// dereferenced the absent engine. The sync-point counter double-confirms zero +// sync_rowsets calls. +TEST_F(QueryCacheTest, presync_skips_launch_when_timeout_non_positive) { + auto* sp = SyncPoint::get_instance(); + sp->enable_processing(); + std::atomic sync_calls {0}; + sp->set_call_back("CloudMetaMgr::sync_tablet_rowsets", + [&](auto&&) { sync_calls.fetch_add(1); }); + Defer sp_defer {[sp] { + sp->disable_processing(); + sp->clear_all_call_backs(); + }}; + + auto saved = config::query_cache_decision_sync_timeout_ms; + config::query_cache_decision_sync_timeout_ms = 0; + Defer cfg_defer {[&] { config::query_cache_decision_sync_timeout_ms = saved; }}; + + auto scan_ranges = make_scan_ranges(101, "100"); + auto reasons = QueryCacheRuntime::presync_cloud_delta_tablets_for_test(scan_ranges, 100); + + ASSERT_EQ(reasons.count(101), 1); + EXPECT_EQ(reasons[101], "cloud incremental sync disabled"); + EXPECT_EQ(sync_calls.load(), 0); +} + +// (a) A stopped engine must bail before launching: positive budget, but +// stopped() is true, so no sync_rowsets runs and every tablet falls back. The +// stopped() check precedes the pool access, so an un-opened engine's null pool +// is never touched. +TEST_F(QueryCacheTest, presync_bails_when_engine_stopped) { + auto engine = std::make_unique(EngineOptions {}); + engine->stop(); // sets _stopped = true; safe on an un-opened engine. + ExecEnv::GetInstance()->set_storage_engine(std::move(engine)); + Defer eng_defer {[] { ExecEnv::GetInstance()->set_storage_engine(nullptr); }}; + + auto* sp = SyncPoint::get_instance(); + sp->enable_processing(); + std::atomic sync_calls {0}; + sp->set_call_back("CloudMetaMgr::sync_tablet_rowsets", + [&](auto&&) { sync_calls.fetch_add(1); }); + Defer sp_defer {[sp] { + sp->disable_processing(); + sp->clear_all_call_backs(); + }}; + + auto saved = config::query_cache_decision_sync_timeout_ms; + config::query_cache_decision_sync_timeout_ms = 2000; // positive: would launch if not stopped. + Defer cfg_defer {[&] { config::query_cache_decision_sync_timeout_ms = saved; }}; + + auto scan_ranges = make_scan_ranges(42, "100"); + auto reasons = QueryCacheRuntime::presync_cloud_delta_tablets_for_test(scan_ranges, 100); + + ASSERT_EQ(reasons.count(42), 1); + EXPECT_EQ(reasons[42], "be is stopping, sync skipped"); + EXPECT_EQ(sync_calls.load(), 0); +} + +// (finding 1) is_cloud_mode() can flip true on a live LOCAL deployment (the +// mutable cloud_unique_id), leaving a local StorageEngine installed. Reaching +// the fan-out then must degrade gracefully, not abort in the hard CHECK inside +// to_cloud(): the engine-type dynamic_cast fails, so every tablet falls back +// with "storage engine is not cloud" and no sync is issued. +TEST_F(QueryCacheTest, presync_falls_back_on_non_cloud_engine) { + auto saved_mode = config::deploy_mode; + config::deploy_mode = "cloud"; // is_cloud_mode() true while the engine stays local + ExecEnv::GetInstance()->set_storage_engine(std::make_unique(EngineOptions {})); + Defer eng_defer {[&] { + ExecEnv::GetInstance()->set_storage_engine(nullptr); + config::deploy_mode = saved_mode; + }}; + + auto* sp = SyncPoint::get_instance(); + sp->enable_processing(); + std::atomic sync_calls {0}; + sp->set_call_back("CloudMetaMgr::sync_tablet_rowsets", + [&](auto&&) { sync_calls.fetch_add(1); }); + Defer sp_defer {[sp] { + sp->disable_processing(); + sp->clear_all_call_backs(); + }}; + + auto saved = config::query_cache_decision_sync_timeout_ms; + config::query_cache_decision_sync_timeout_ms = + 2000; // positive: would launch on a cloud engine. + Defer cfg_defer {[&] { config::query_cache_decision_sync_timeout_ms = saved; }}; + + auto scan_ranges = make_scan_ranges(42, "100"); + auto reasons = QueryCacheRuntime::presync_cloud_delta_tablets_for_test(scan_ranges, 100); + + ASSERT_EQ(reasons.count(42), 1); + EXPECT_EQ(reasons[42], "storage engine is not cloud"); + EXPECT_EQ(sync_calls.load(), 0); +} + +// The dedicated pre-sync pool refuses work (here: shut down, the same rejection a +// full bounded queue gives) while the engine itself is still live, so the fan-out +// is reached rather than short-cut by the stopped() guard. Every submit_func then +// fails, and the inline submit-failure path must fall each tablet back AND release +// its own latch slot via publish_slot so the bounded wait still settles instead of +// hanging on a slot no task will ever count down. Two tablets prove the latch +// reaches N purely through the inline claim. No sync RPC is issued (rejected before +// enqueue, so the closure never runs). +TEST_F(QueryCacheTest, presync_falls_back_when_pool_rejects_submit) { + auto saved_mode = config::deploy_mode; + config::deploy_mode = "cloud"; + auto engine = std::make_unique(EngineOptions {}); + auto* engine_ptr = engine.get(); + ExecEnv::GetInstance()->set_storage_engine(std::move(engine)); + Defer eng_defer {[&] { + ExecEnv::GetInstance()->set_storage_engine(nullptr); + config::deploy_mode = saved_mode; + }}; + // Shut the pool so do_submit rejects before enqueue; the engine stays live + // (not stopped()), so the fan-out is entered and hits the inline fallback. + engine_ptr->query_cache_delta_sync_pool().shutdown(); + + auto* sp = SyncPoint::get_instance(); + sp->enable_processing(); + std::atomic sync_calls {0}; + sp->set_call_back("CloudMetaMgr::sync_tablet_rowsets", + [&](auto&&) { sync_calls.fetch_add(1); }); + Defer sp_defer {[sp] { + sp->disable_processing(); + sp->clear_all_call_backs(); + }}; + + auto saved = config::query_cache_decision_sync_timeout_ms; + config::query_cache_decision_sync_timeout_ms = 2000; // positive: a healthy pool would launch. + Defer cfg_defer {[&] { config::query_cache_decision_sync_timeout_ms = saved; }}; + + auto scan_ranges = make_scan_ranges(42, "100"); + scan_ranges.push_back(make_scan_ranges(43, "100").front()); + auto reasons = QueryCacheRuntime::presync_cloud_delta_tablets_for_test(scan_ranges, 100); + + ASSERT_EQ(reasons.count(42), 1); + ASSERT_EQ(reasons.count(43), 1); + EXPECT_EQ(reasons[42], "cloud rowset sync not scheduled"); + EXPECT_EQ(reasons[43], "cloud rowset sync not scheduled"); + EXPECT_EQ(sync_calls.load(), 0); +} + +TEST_F(QueryCacheTest, presync_over_concurrency_cap_falls_back) { + // The decision-sync wait runs on the query-admission light pool. When the + // concurrent-waiter soft cap (config::query_cache_max_concurrent_decision_sync) is + // already reached, a new decision must NOT enter that wait: doing so would park + // yet another admission worker, and under a brownout of DISTINCT (non-coalescing) + // keys that is exactly what would starve the whole pool. It falls every scanned + // tablet back to a full recompute instead. Simulate the cap being reached by + // presetting the global counter, then assert the fan-out is never launched and + // every tablet carries the over-capacity reason. Pre-fix (no cap) this same call + // proceeds into the fan-out and returns scheduling reasons, not this reason. + auto saved_mode = config::deploy_mode; + config::deploy_mode = "cloud"; + auto engine = std::make_unique(EngineOptions {}); + ExecEnv::GetInstance()->set_storage_engine(std::move(engine)); + Defer eng_defer {[&] { + ExecEnv::GetInstance()->set_storage_engine(nullptr); + config::deploy_mode = saved_mode; + }}; + + auto* sp = SyncPoint::get_instance(); + sp->enable_processing(); + std::atomic sync_calls {0}; + sp->set_call_back("CloudMetaMgr::sync_tablet_rowsets", + [&](auto&&) { sync_calls.fetch_add(1); }); + Defer sp_defer {[sp] { + sp->disable_processing(); + sp->clear_all_call_backs(); + }}; + + auto saved_timeout = config::query_cache_decision_sync_timeout_ms; + config::query_cache_decision_sync_timeout_ms = 2000; // positive: a healthy pool would launch. + auto saved_cap = config::query_cache_max_concurrent_decision_sync; + config::query_cache_max_concurrent_decision_sync = 1; + Defer cfg_defer {[&] { + config::query_cache_decision_sync_timeout_ms = saved_timeout; + config::query_cache_max_concurrent_decision_sync = saved_cap; + }}; + + // Simulate the single slot already held by one parked waiter. + QueryCacheRuntime::presync_active_waiters_for_test().store(1, std::memory_order_release); + Defer waiter_defer {[] { + QueryCacheRuntime::presync_active_waiters_for_test().store(0, std::memory_order_release); + }}; + + auto scan_ranges = make_scan_ranges(42, "100"); + scan_ranges.push_back(make_scan_ranges(43, "100").front()); + auto reasons = QueryCacheRuntime::presync_cloud_delta_tablets_for_test(scan_ranges, 100); + + ASSERT_EQ(reasons.count(42), 1); + ASSERT_EQ(reasons.count(43), 1); + EXPECT_EQ(reasons[42], "cloud decision sync at capacity"); + EXPECT_EQ(reasons[43], "cloud decision sync at capacity"); + // The cap gate returns BEFORE the registry and fan-out, so no sync RPC is issued, + // and the rejected path takes no slot (fetch_add then immediate fetch_sub), so the + // counter is left exactly as preset. + EXPECT_EQ(sync_calls.load(), 0); + EXPECT_EQ(QueryCacheRuntime::presync_active_waiters_for_test().load(), 1); +} + +TEST_F(QueryCacheTest, presync_cap_clamps_to_configured_light_pool_width) { + // The waiter cap must derive from the ACTUAL light-pool width, not just the config. + // brpc_light_work_pool_threads is operator-configurable; shrunk to 16, the effective + // cap is width/2 = 8, so the config default of 32 does NOT bind and the 9th concurrent + // waiter still falls back. Pre-fix (a fixed 32) the counter at 8 is under the cap, so + // this same call would ADMIT and enter the fan-out instead of rejecting -- exactly the + // starvation a small configured pool would suffer. + auto saved_mode = config::deploy_mode; + config::deploy_mode = "cloud"; + auto engine = std::make_unique(EngineOptions {}); + ExecEnv::GetInstance()->set_storage_engine(std::move(engine)); + Defer eng_defer {[&] { + ExecEnv::GetInstance()->set_storage_engine(nullptr); + config::deploy_mode = saved_mode; + }}; + + auto* sp = SyncPoint::get_instance(); + sp->enable_processing(); + std::atomic sync_calls {0}; + sp->set_call_back("CloudMetaMgr::sync_tablet_rowsets", + [&](auto&&) { sync_calls.fetch_add(1); }); + Defer sp_defer {[sp] { + sp->disable_processing(); + sp->clear_all_call_backs(); + }}; + + auto saved_timeout = config::query_cache_decision_sync_timeout_ms; + config::query_cache_decision_sync_timeout_ms = 2000; // positive: a healthy pool would launch. + auto saved_cap = config::query_cache_max_concurrent_decision_sync; + auto saved_pool = config::brpc_light_work_pool_threads; + // A 16-thread pool caps effective waiters at 8, below the untouched config of 32. + config::brpc_light_work_pool_threads = 16; + config::query_cache_max_concurrent_decision_sync = 32; + Defer cfg_defer {[&] { + config::query_cache_decision_sync_timeout_ms = saved_timeout; + config::query_cache_max_concurrent_decision_sync = saved_cap; + config::brpc_light_work_pool_threads = saved_pool; + }}; + + // Exactly the width-derived cap (16/2) already held: the next waiter is over it even + // though the config cap (32) is not. + QueryCacheRuntime::presync_active_waiters_for_test().store(8, std::memory_order_release); + Defer waiter_defer {[] { + QueryCacheRuntime::presync_active_waiters_for_test().store(0, std::memory_order_release); + }}; + + auto scan_ranges = make_scan_ranges(42, "100"); + auto reasons = QueryCacheRuntime::presync_cloud_delta_tablets_for_test(scan_ranges, 100); + + ASSERT_EQ(reasons.count(42), 1); + EXPECT_EQ(reasons[42], "cloud decision sync at capacity"); + EXPECT_EQ(sync_calls.load(), 0); + EXPECT_EQ(QueryCacheRuntime::presync_active_waiters_for_test().load(), 8); +} + +TEST_F(QueryCacheTest, presync_single_thread_light_pool_never_parks_sole_worker) { + // A 1-thread light pool floors the width-derived cap (1/2) to 0, so NO decision-sync + // waiter may be admitted: parking the only worker for the timeout would starve all + // unrelated fragment admission. Every stale query falls back immediately, even with + // the counter at 0. Pre-fix (max(1, width/2)) the cap was 1, so the first waiter was + // admitted and parked the sole worker. + auto saved_mode = config::deploy_mode; + config::deploy_mode = "cloud"; + auto engine = std::make_unique(EngineOptions {}); + ExecEnv::GetInstance()->set_storage_engine(std::move(engine)); + Defer eng_defer {[&] { + ExecEnv::GetInstance()->set_storage_engine(nullptr); + config::deploy_mode = saved_mode; + }}; + + auto* sp = SyncPoint::get_instance(); + sp->enable_processing(); + std::atomic sync_calls {0}; + sp->set_call_back("CloudMetaMgr::sync_tablet_rowsets", + [&](auto&&) { sync_calls.fetch_add(1); }); + Defer sp_defer {[sp] { + sp->disable_processing(); + sp->clear_all_call_backs(); + }}; + + auto saved_timeout = config::query_cache_decision_sync_timeout_ms; + config::query_cache_decision_sync_timeout_ms = 2000; + auto saved_cap = config::query_cache_max_concurrent_decision_sync; + auto saved_pool = config::brpc_light_work_pool_threads; + config::brpc_light_work_pool_threads = 1; // width/2 floors to 0 -> admit nobody + config::query_cache_max_concurrent_decision_sync = 32; + Defer cfg_defer {[&] { + config::query_cache_decision_sync_timeout_ms = saved_timeout; + config::query_cache_max_concurrent_decision_sync = saved_cap; + config::brpc_light_work_pool_threads = saved_pool; + }}; + + // Even the FIRST waiter (counter 0) is rejected on a 1-thread pool. + QueryCacheRuntime::presync_active_waiters_for_test().store(0, std::memory_order_release); + Defer waiter_defer {[] { + QueryCacheRuntime::presync_active_waiters_for_test().store(0, std::memory_order_release); + }}; + + auto scan_ranges = make_scan_ranges(42, "100"); + auto reasons = QueryCacheRuntime::presync_cloud_delta_tablets_for_test(scan_ranges, 100); + + ASSERT_EQ(reasons.count(42), 1); + EXPECT_EQ(reasons[42], "cloud decision sync at capacity"); + EXPECT_EQ(sync_calls.load(), 0); + // The rejected fetch_add was immediately backed out, so the counter is left at 0. + EXPECT_EQ(QueryCacheRuntime::presync_active_waiters_for_test().load(), 0); +} + +// The BE-side mirror of the FE incremental knob gates, applied where the +// fragment context creates the shared runtime. A same-version FE already +// cleared allow_incremental in these cases; the mirror exists for the +// rolling-upgrade window where an older FE without the knob gates still +// requests incremental while a knob is active. The truth table is the FE +// gate's, verbatim: freshness suppresses for every table type, prefer-cached +// only for non-merge-on-write (a MOW read is version-exact regardless of the +// knob), no active knob suppresses nothing. +TEST_F(QueryCacheTest, cloud_knobs_suppress_incremental_truth_table) { + // Freshness tolerance active: suppresses regardless of the other inputs. + EXPECT_TRUE(QueryCacheRuntime::cloud_knobs_suppress_incremental(true, false, false)); + EXPECT_TRUE(QueryCacheRuntime::cloud_knobs_suppress_incremental(true, false, true)); + EXPECT_TRUE(QueryCacheRuntime::cloud_knobs_suppress_incremental(true, true, false)); + EXPECT_TRUE(QueryCacheRuntime::cloud_knobs_suppress_incremental(true, true, true)); + // Prefer-cached-rowset active: suppresses only non-MOW. + EXPECT_TRUE(QueryCacheRuntime::cloud_knobs_suppress_incremental(false, true, false)); + EXPECT_FALSE(QueryCacheRuntime::cloud_knobs_suppress_incremental(false, true, true)); + // No knob active: never suppresses. + EXPECT_FALSE(QueryCacheRuntime::cloud_knobs_suppress_incremental(false, false, false)); + EXPECT_FALSE(QueryCacheRuntime::cloud_knobs_suppress_incremental(false, false, true)); +} + +// The whole gate the fragment context applies at runtime creation, as a pure +// function. Covers the two wrappers the truth table above does not: the +// cloud-only guard, and that the gate only ever clears an FE request (never +// grants one). The fragment context's own call site is trivial wiring on top of +// this (it passes config::is_cloud_mode() and the session getters). +TEST_F(QueryCacheTest, gate_allow_incremental_for_cloud_knobs) { + // Cloud + a suppressing knob (freshness) + FE requested it: cleared. This is + // the rolling-upgrade case an older FE without the gates produces. + EXPECT_FALSE(QueryCacheRuntime::gate_allow_incremental_for_cloud_knobs( + /*requested=*/true, /*is_mow=*/false, /*cloud=*/true, /*freshness=*/true, + /*prefer=*/false)); + // Same knob, but LOCAL storage: the knobs are inert, so the gate must not + // fire (local incremental keeps working under an inert freshness setting). + EXPECT_TRUE(QueryCacheRuntime::gate_allow_incremental_for_cloud_knobs( + /*requested=*/true, /*is_mow=*/false, /*cloud=*/false, /*freshness=*/true, + /*prefer=*/false)); + // Cloud + prefer-cached, but the index is MOW: prefer does not apply to MOW, + // so the request survives (the MOW read is version-exact regardless). + EXPECT_TRUE(QueryCacheRuntime::gate_allow_incremental_for_cloud_knobs( + /*requested=*/true, /*is_mow=*/true, /*cloud=*/true, /*freshness=*/false, + /*prefer=*/true)); + // The gate never grants: FE did not request incremental, so a knob (or its + // absence) leaves it off. + EXPECT_FALSE(QueryCacheRuntime::gate_allow_incremental_for_cloud_knobs( + /*requested=*/false, /*is_mow=*/false, /*cloud=*/true, /*freshness=*/true, + /*prefer=*/false)); + EXPECT_FALSE(QueryCacheRuntime::gate_allow_incremental_for_cloud_knobs( + /*requested=*/false, /*is_mow=*/false, /*cloud=*/true, /*freshness=*/false, + /*prefer=*/false)); + // Cloud, request set, but no knob active: passes through untouched. + EXPECT_TRUE(QueryCacheRuntime::gate_allow_incremental_for_cloud_knobs( + /*requested=*/true, /*is_mow=*/false, /*cloud=*/true, /*freshness=*/false, + /*prefer=*/false)); +} + TEST_F(QueryCacheTest, runtime_decision_force_refresh) { std::unique_ptr cache(QueryCache::create_global_cache(1024 * 1024)); auto scan_ranges = make_scan_ranges(42, "100"); @@ -585,29 +960,6 @@ TEST_F(QueryCacheTest, take_delta_read_source) { EXPECT_EQ(decision->take_delta_read_source(42), nullptr); } -TEST_F(QueryCacheTest, runtime_decision_stale_incremental_cloud_mode) { - std::unique_ptr cache(QueryCache::create_global_cache(1024 * 1024)); - auto scan_ranges = make_scan_ranges(42, "100"); - auto cache_param = make_cache_param(42); - cache_param.__set_allow_incremental(true); - - std::string cache_key; - int64_t version = 0; - EXPECT_TRUE(QueryCache::build_cache_key(scan_ranges, cache_param, &cache_key, &version).ok()); - insert_entry(cache.get(), cache_key, 50, 0); - - // Incremental merge only supports local storage for now. - std::string saved_deploy_mode = config::deploy_mode; - config::deploy_mode = "cloud"; - QueryCacheRuntime runtime(cache_param, cache.get()); - auto decision = runtime.get_or_make_decision(scan_ranges); - config::deploy_mode = saved_deploy_mode; - - EXPECT_EQ(decision->mode, QueryCacheInstanceDecision::Mode::MISS); - EXPECT_TRUE(decision->key_valid); - EXPECT_EQ(decision->incremental_fallback_reason, "cloud mode"); -} - // Exercises the per-tablet part of the incremental decision against a real // (metadata-only) tablet registered in a real storage engine: capturing the // delta read source never touches segment files, so no data is needed. @@ -1058,13 +1410,30 @@ TEST_F(QueryCacheIncrementalTest, fallback_on_partial_capture) { EXPECT_EQ(decision->incremental_fallback_reason, "delta versions not capturable"); } +TEST_F(QueryCacheIncrementalTest, mismatched_engine_falls_back) { + // is_cloud_mode() can flip on a live local deployment (cloud_unique_id is a + // mutable config) while the installed engine stays a local StorageEngine. The + // decision must degrade the misconfiguration to a full recompute instead of + // aborting in the hard CHECK inside to_cloud(): the pre-sync's engine-type + // dynamic_cast fails first, so every scanned tablet falls back with "storage + // engine is not cloud" (earlier and more precise than the per-tablet cast). + create_tablet(TKeysType::DUP_KEYS, false, {{0, 50}, {51, 100}}); + std::string saved_deploy_mode = config::deploy_mode; + config::deploy_mode = "cloud"; + auto decision = make_stale_decision(); + config::deploy_mode = saved_deploy_mode; + EXPECT_EQ(decision->mode, QueryCacheInstanceDecision::Mode::MISS); + EXPECT_TRUE(decision->key_valid); + EXPECT_EQ(decision->incremental_fallback_reason, "storage engine is not cloud"); +} + TEST_F(QueryCacheIncrementalTest, concurrent_racers_share_one_decision) { // The candidate decision is built outside the runtime lock (a stale - // merge-on-write decision may scan a large delete bitmap there), so two - // operators of the same instance can race: the sync point parks both - // threads right before publication, forcing two live candidates. Exactly - // one must win, both callers must observe the winner, and the metrics - // must be settled once. + // decision may scan a large delete bitmap there, and on cloud tablets + // first sync the view from the meta service), so two operators of the + // same instance can race: the sync point parks both threads right before + // publication, forcing two live candidates. Exactly one must win, both + // callers must observe the winner, and the metrics must be settled once. create_tablet(TKeysType::DUP_KEYS, false, {{0, 50}, {51, 100}}); auto scan_ranges = make_scan_ranges(kTabletId, "100"); auto cache_param = make_cache_param(kTabletId); @@ -1140,4 +1509,1662 @@ TEST_F(QueryCacheIncrementalTest, concurrent_racers_settle_fallback_once) { fallback_before + 1); } +// Exercises the incremental decision against a real CloudTablet materialized +// through the regular cache-miss load of ExecEnv::get_tablet, with the two +// meta-service RPCs intercepted through sync points (the same pattern as +// cloud_schema_change_job_test): get_tablet_meta serves a local meta and the +// loader's first sync_tablet_rowsets installs the requested rowsets. The +// decision's own sync is then either short-cut by the local max version +// covering query_version, or (as the second sync call) answered with an +// injected result without changing the view. Capturing the delta read source +// never touches rowset data, so none exists. +class QueryCacheCloudIncrementalTest : public testing::Test { +protected: + static constexpr int64_t kTabletId = 15673; + + void SetUp() override { + _saved_deploy_mode = config::deploy_mode; + config::deploy_mode = "cloud"; + auto engine = std::make_unique(EngineOptions {}); + _engine = engine.get(); + ExecEnv::GetInstance()->set_storage_engine(std::move(engine)); + _cache.reset(QueryCache::create_global_cache(1024 * 1024)); + _sp = SyncPoint::get_instance(); + _sp->enable_processing(); + } + + void TearDown() override { + // Drain the dedicated delta-sync pool BEFORE tearing anything down: a + // task's publish_slot reap now touches _cache->_presync_flights_lock (the + // round-2 tombstone reap), so a worker that resumes after _cache is freed + // would use-after-free the registry lock (the 200ms cushions the tests add + // do not guarantee a descheduled worker finished). wait() blocks while the + // cache, engine, and SyncPoint callbacks are all still alive; a task's + // remaining publish_slot work triggers no SyncPoint callback, so waiting + // with the callbacks still installed is safe. Only then clear callbacks and + // destroy the cache/engine -- mirroring production, where + // CloudStorageEngine::stop() drains this pool before the cache is torn down. + if (_engine != nullptr) { + _engine->query_cache_delta_sync_pool().wait(); + } + _sp->disable_processing(); + _sp->clear_all_call_backs(); + _cache.reset(); + ExecEnv::GetInstance()->set_storage_engine(nullptr); + _engine = nullptr; + config::deploy_mode = _saved_deploy_mode; + } + + // Builds metadata-only rowsets for `versions` and installs them into the + // tablet's view, exactly like a production sync would. Also records them + // in _installed_rowsets for tests that stamp delete-bitmap entries onto + // specific rowsets. + void install_rowsets(CloudTablet* tablet, + const std::vector>& versions) { + std::vector rowsets; + rowsets.reserve(versions.size()); + for (auto [start, end] : versions) { + auto rs_meta = std::make_shared(); + rs_meta->set_rowset_type(BETA_ROWSET); + rs_meta->set_version({start, end}); + rs_meta->set_rowset_id(_engine->next_rowset_id()); + rs_meta->set_num_segments(1); + rs_meta->set_tablet_schema(tablet->tablet_meta()->tablet_schema()); + RowsetSharedPtr rowset; + Status st = RowsetFactory::create_rowset(nullptr, "", rs_meta, &rowset); + EXPECT_TRUE(st.ok()) << st; + if (st.ok()) { + rowsets.push_back(std::move(rowset)); + } + } + _installed_rowsets.insert(_installed_rowsets.end(), rowsets.begin(), rowsets.end()); + std::unique_lock lock(tablet->get_header_lock()); + tablet->add_rowsets(std::move(rowsets), /*version_overlap=*/false, lock); + } + + void intercept_tablet_load( + std::vector> versions, Status decision_sync_result, + TKeysType::type keys_type = TKeysType::DUP_KEYS, bool enable_merge_on_write = false, + std::vector> decision_sync_installs = {}, + // Delete-bitmap marks {index into _installed_rowsets, version} the + // decision's sync stamps AFTER installing its rowsets, but ONLY when + // that sync carried sync_delete_bitmap (mirroring the real body). + // Lets a test both exercise cloud sync-completeness and GUARD the + // flag: a rewrite entry the loader's sync never carried, that only a + // sync_delete_bitmap=true decision sync brings, then feeds the + // history check. + std::vector> decision_sync_bitmap_marks = {}) { + _sp->set_call_back("CloudMetaMgr::get_tablet_meta", + [keys_type, enable_merge_on_write](auto&& args) { + TTabletSchema schema; + schema.keys_type = keys_type; + auto meta = std::make_shared( + 1, 2, kTabletId, 15674, 4, 5, schema, 6, + std::unordered_map {{7, 8}}, + UniqueId(9, 10), TTabletType::TABLET_TYPE_DISK, + TCompressionType::LZ4F, 0, enable_merge_on_write); + // Pin the RUNNING state (also the ctor default) explicitly: a + // tablet that is not RUNNING takes sync_if_not_running(), which + // clears the just-installed view and re-syncs through the same + // interception, breaking both the version view and _sync_calls. + meta->set_tablet_state(TABLET_RUNNING); + *try_any_cast(args[1]) = std::move(meta); + try_any_cast_ret(args)->second = true; + }); + _sp->set_call_back( + "CloudMetaMgr::sync_tablet_rowsets", + [this, versions = std::move(versions), decision_sync_result, + decision_sync_installs = std::move(decision_sync_installs), + decision_sync_bitmap_marks = std::move(decision_sync_bitmap_marks)](auto&& args) { + auto* tablet = try_any_cast(args[0]); + const auto* options = try_any_cast(args[1]); + auto* ret = try_any_cast_ret(args); + if (_sync_calls.fetch_add(1) == 0) { + // The loader's sync: install the requested rowsets. + install_rowsets(tablet, versions); + } else { + // The incremental decision's sync: install the extra + // rowsets, if any, stamp any delete-bitmap marks it is + // responsible for bringing, and inject the result. + install_rowsets(tablet, decision_sync_installs); + // Gate the marks on sync_delete_bitmap exactly as the real + // sync body does (cloud_meta_mgr.cpp): a sync that did not + // carry the flag brings no delete bitmap. This makes the + // mock flag-faithful, so a test can prove the pre-sync set + // the flag -- flip it to false and the mark is dropped. + if (options->sync_delete_bitmap) { + for (auto [rs_idx, ver] : decision_sync_bitmap_marks) { + // .at(): a mistyped index in a future test should + // fail loudly, not be silent out-of-bounds UB. + tablet->tablet_meta()->delete_bitmap().add( + {_installed_rowsets.at(rs_idx)->rowset_id(), 0, ver}, 7); + } + } + ret->first = decision_sync_result; + } + ret->second = true; + }); + } + + // A stale entry of version 50 while the query reads version 100 with + // allow_incremental set, mirroring the local-storage fixture. + std::shared_ptr make_stale_decision() { + auto scan_ranges = make_scan_ranges(kTabletId, "100"); + auto cache_param = make_cache_param(kTabletId); + cache_param.__set_allow_incremental(true); + std::string cache_key; + int64_t version = 0; + EXPECT_TRUE( + QueryCache::build_cache_key(scan_ranges, cache_param, &cache_key, &version).ok()); + insert_entry(_cache.get(), cache_key, 50, 0); + QueryCacheRuntime runtime(cache_param, _cache.get()); + return runtime.get_or_make_decision(scan_ranges); + } + + std::string _saved_deploy_mode; + CloudStorageEngine* _engine = nullptr; + std::unique_ptr _cache; + SyncPoint* _sp = nullptr; + std::atomic _sync_calls {0}; + // The rowsets the loader's sync installed, for tests that stamp + // delete-bitmap entries onto specific rowsets. + std::vector _installed_rowsets; +}; + +TEST_F(QueryCacheCloudIncrementalTest, incremental_success_on_synced_view) { + // The loader installs a view that already covers query_version 100, so + // the decision's sync_rowsets() takes its no-op shortcut (the sync point + // sees exactly one call) and the decision merges incrementally, exactly + // like on local storage. + intercept_tablet_load({{0, 50}, {51, 100}}, Status::OK()); + int64_t stale_hits_before = DorisMetrics::instance()->query_cache_stale_hit_total->value(); + auto decision = make_stale_decision(); + EXPECT_EQ(DorisMetrics::instance()->query_cache_stale_hit_total->value(), + stale_hits_before + 1); + EXPECT_EQ(decision->mode, QueryCacheInstanceDecision::Mode::INCREMENTAL); + EXPECT_EQ(decision->cached_version, 50); + EXPECT_TRUE(decision->incremental_fallback_reason.empty()); + EXPECT_EQ(_sync_calls.load(), 1); + auto source = decision->take_delta_read_source(kTabletId); + ASSERT_NE(source, nullptr); + EXPECT_EQ(source->rs_splits.size(), 1); + EXPECT_EQ(source->rs_splits.front().rs_reader->rowset()->start_version(), 51); + EXPECT_EQ(source->rs_splits.front().rs_reader->rowset()->end_version(), 100); +} + +TEST_F(QueryCacheCloudIncrementalTest, fallback_on_sync_failure) { + // The local view (max 50) does not cover query_version 100, so the + // decision must sync from the meta service first; the injected failure + // falls the query back to a full recompute. + intercept_tablet_load({{0, 50}}, Status::InternalError("injected sync failure")); + int64_t fallbacks_before = + DorisMetrics::instance()->query_cache_incremental_fallback_total->value(); + auto decision = make_stale_decision(); + EXPECT_EQ(DorisMetrics::instance()->query_cache_incremental_fallback_total->value(), + fallbacks_before + 1); + EXPECT_EQ(decision->mode, QueryCacheInstanceDecision::Mode::MISS); + EXPECT_EQ(decision->incremental_fallback_reason, "cloud rowset sync failed"); + EXPECT_EQ(_sync_calls.load(), 2); +} + +TEST_F(QueryCacheCloudIncrementalTest, incremental_after_decision_sync_installs_delta) { + // The flagship cloud composition: the local view (max 50) does not cover + // query_version 100, so the decision's sync really goes to the meta + // service; the injected response installs the missing delta rowset, and + // the capture on the freshly synced view then merges incrementally. + intercept_tablet_load({{0, 50}}, Status::OK(), TKeysType::DUP_KEYS, + /*enable_merge_on_write=*/false, + /*decision_sync_installs=*/ {{51, 100}}); + int64_t stale_hits_before = DorisMetrics::instance()->query_cache_stale_hit_total->value(); + auto decision = make_stale_decision(); + EXPECT_EQ(DorisMetrics::instance()->query_cache_stale_hit_total->value(), + stale_hits_before + 1); + EXPECT_EQ(decision->mode, QueryCacheInstanceDecision::Mode::INCREMENTAL); + EXPECT_TRUE(decision->incremental_fallback_reason.empty()); + EXPECT_EQ(_sync_calls.load(), 2); + auto source = decision->take_delta_read_source(kTabletId); + ASSERT_NE(source, nullptr); + EXPECT_EQ(source->rs_splits.size(), 1); + EXPECT_EQ(source->rs_splits.front().rs_reader->rowset()->start_version(), 51); + EXPECT_EQ(source->rs_splits.front().rs_reader->rowset()->end_version(), 100); +} + +TEST_F(QueryCacheCloudIncrementalTest, fallback_when_synced_view_misses_delta) { + // The decision's sync succeeds (injected OK) but leaves the view at max + // 50, so the delta versions (51, 100] are not on the local version + // graph: the quiet capture returns an empty read source and the decision + // falls back instead of silently dropping the delta rows. + intercept_tablet_load({{0, 50}}, Status::OK()); + auto decision = make_stale_decision(); + EXPECT_EQ(decision->mode, QueryCacheInstanceDecision::Mode::MISS); + EXPECT_EQ(decision->incremental_fallback_reason, "delta versions not capturable"); + EXPECT_EQ(_sync_calls.load(), 2); +} + +TEST_F(QueryCacheCloudIncrementalTest, mow_pure_append_incremental) { + // A merge-on-write UNIQUE cloud tablet whose delta window carries no + // delete-bitmap entry (the hourly-append pattern) merges incrementally, + // exactly like on local storage: the history-rewrite check runs on the + // freshly synced bitmap and finds nothing. + intercept_tablet_load({{0, 50}, {51, 100}}, Status::OK(), TKeysType::UNIQUE_KEYS, + /*enable_merge_on_write=*/true); + auto decision = make_stale_decision(); + EXPECT_EQ(decision->mode, QueryCacheInstanceDecision::Mode::INCREMENTAL); + EXPECT_TRUE(decision->incremental_fallback_reason.empty()); + EXPECT_EQ(_sync_calls.load(), 1); + auto source = decision->take_delta_read_source(kTabletId); + ASSERT_NE(source, nullptr); + EXPECT_EQ(source->rs_splits.size(), 1); +} + +TEST_F(QueryCacheCloudIncrementalTest, mow_history_rewrite_falls_back) { + // A delete-bitmap entry inside the delta window that targets the + // baseline rowset (a backfill rewrote a key predating the cached + // version): the same history-rewrite fallback as on local storage, on a + // bitmap the cloud sync is responsible for keeping complete. + intercept_tablet_load({{0, 50}, {51, 100}}, Status::OK(), TKeysType::UNIQUE_KEYS, + /*enable_merge_on_write=*/true); + auto tablet_res = ExecEnv::get_tablet(kTabletId); + ASSERT_TRUE(tablet_res.has_value()) << tablet_res.error(); + ASSERT_EQ(_installed_rowsets.size(), 2); + tablet_res.value()->tablet_meta()->delete_bitmap().add( + {_installed_rowsets[0]->rowset_id(), 0, 60}, 7); + + int64_t fallbacks_before = + DorisMetrics::instance()->query_cache_incremental_fallback_total->value(); + auto decision = make_stale_decision(); + EXPECT_EQ(DorisMetrics::instance()->query_cache_incremental_fallback_total->value(), + fallbacks_before + 1); + EXPECT_EQ(decision->mode, QueryCacheInstanceDecision::Mode::MISS); + EXPECT_EQ(decision->incremental_fallback_reason, "delta rewrites history rows"); + // The loader already brought the view up to 100, so the decision's sync + // took the no-op shortcut: still exactly one meta-service round trip. + EXPECT_EQ(_sync_calls.load(), 1); +} + +TEST_F(QueryCacheCloudIncrementalTest, not_incremental_skips_decision_sync) { + // allow_incremental unset: the stale entry is a plain miss and the + // decision never reaches the per-tablet path, so it must not load the + // tablet or talk to the meta service at all (no hidden RPC cost when the + // feature is off). + intercept_tablet_load({{0, 50}}, Status::OK()); + auto scan_ranges = make_scan_ranges(kTabletId, "100"); + auto cache_param = make_cache_param(kTabletId); + std::string cache_key; + int64_t version = 0; + EXPECT_TRUE(QueryCache::build_cache_key(scan_ranges, cache_param, &cache_key, &version).ok()); + insert_entry(_cache.get(), cache_key, 50, 0); + QueryCacheRuntime runtime(cache_param, _cache.get()); + auto decision = runtime.get_or_make_decision(scan_ranges); + EXPECT_EQ(decision->mode, QueryCacheInstanceDecision::Mode::MISS); + EXPECT_TRUE(decision->incremental_fallback_reason.empty()); + EXPECT_EQ(_sync_calls.load(), 0); +} + +TEST_F(QueryCacheCloudIncrementalTest, agg_keys_rejected_before_decision_sync) { + // The keys-type check precedes the cloud sync, so an AGG tablet is + // rejected after the loader's round trip but without a second one: no + // meta-service RPC is spent on a tablet that can never merge. + intercept_tablet_load({{0, 50}}, Status::OK(), TKeysType::AGG_KEYS); + int64_t fallbacks_before = + DorisMetrics::instance()->query_cache_incremental_fallback_total->value(); + auto decision = make_stale_decision(); + EXPECT_EQ(DorisMetrics::instance()->query_cache_incremental_fallback_total->value(), + fallbacks_before + 1); + EXPECT_EQ(decision->mode, QueryCacheInstanceDecision::Mode::MISS); + EXPECT_EQ(decision->incremental_fallback_reason, "keys type not append-only"); + EXPECT_EQ(_sync_calls.load(), 1); +} + +TEST_F(QueryCacheCloudIncrementalTest, newer_entry_rejected_before_tablet_access) { + // The cached entry (version 200) is newer than the queried version 100: + // rejected before the per-tablet loop, so the tablet is never loaded and + // the meta service is never contacted. + intercept_tablet_load({{0, 50}}, Status::OK()); + auto scan_ranges = make_scan_ranges(kTabletId, "100"); + auto cache_param = make_cache_param(kTabletId); + cache_param.__set_allow_incremental(true); + std::string cache_key; + int64_t version = 0; + EXPECT_TRUE(QueryCache::build_cache_key(scan_ranges, cache_param, &cache_key, &version).ok()); + insert_entry(_cache.get(), cache_key, 200, 0); + QueryCacheRuntime runtime(cache_param, _cache.get()); + auto decision = runtime.get_or_make_decision(scan_ranges); + EXPECT_EQ(decision->mode, QueryCacheInstanceDecision::Mode::MISS); + EXPECT_EQ(decision->incremental_fallback_reason, "cached entry is newer"); + EXPECT_EQ(_sync_calls.load(), 0); +} + +TEST_F(QueryCacheCloudIncrementalTest, presync_tablet_load_failure_falls_back) { + // The pre-sync fan-out issues the first ExecEnv::get_tablet; a meta-service + // failure injected there fails that load on the WORKER thread, and the task + // publishes "cloud tablet load failed" into its slot. The capture loop must + // consume that reason and fall back WITHOUT retrying the cache-miss load on + // the admission thread: during a brownout (the one situation where this + // load fails slowly) a capture-side retry would repeat the slow RPC on the + // bounded admission pool, outside the fast-fail deadline. This exercises + // the one pre-sync branch the other cloud tests never reach: they all load + // the tablet successfully before syncing. + std::atomic meta_loads {0}; + _sp->set_call_back("CloudMetaMgr::get_tablet_meta", [&meta_loads](auto&& args) { + meta_loads.fetch_add(1); + auto* ret = try_any_cast_ret(args); + ret->first = Status::InternalError("injected: tablet meta unavailable"); + ret->second = true; + }); + int64_t fallbacks_before = + DorisMetrics::instance()->query_cache_incremental_fallback_total->value(); + auto decision = make_stale_decision(); + EXPECT_EQ(DorisMetrics::instance()->query_cache_incremental_fallback_total->value(), + fallbacks_before + 1); + EXPECT_EQ(decision->mode, QueryCacheInstanceDecision::Mode::MISS); + EXPECT_EQ(decision->incremental_fallback_reason, "cloud tablet load failed"); + // The pre-sync get_tablet failed before any sync, so no meta-service rowset + // sync RPC was ever issued. + EXPECT_EQ(_sync_calls.load(), 0); + // Exactly ONE load: the worker's. Two would mean the capture loop retried + // the failed load on the admission thread (the pre-fix behavior); the + // published reason above is the second witness of the same ordering (a + // capture-side retry would report its own "tablet not found" instead). + EXPECT_EQ(meta_loads.load(), 1); +} + +TEST_F(QueryCacheCloudIncrementalTest, presync_decision_sync_raise_falls_back) { + // The worker-side task body calls get_tablet and sync_rowsets. get_tablet fans out + // through CloudTabletMgr's single-flight loader, which THROWS (std::system_error) + // when a duplicate concurrent load's CountdownEvent wait fails, and sync_rowsets + // can surface a bad_alloc under memory pressure. FunctionRunnable::run invokes the + // task with no catch, so pre-fix an escaped exception std::terminates the BE and + // defeats the fallback contract (a sync failure must cost only the incremental + // merge, never the process). The task-body guard must convert ANY throw into a + // per-tablet fallback. Inject the throw at the decision's sync: it propagates + // uncaught through CloudTablet::sync_rowsets (no try/catch there, its RAII locks + // release on unwind) into the guard, exactly as the single-flight throw would. + _sp->set_call_back("CloudMetaMgr::get_tablet_meta", [](auto&& args) { + TTabletSchema schema; + schema.keys_type = TKeysType::DUP_KEYS; + auto meta = std::make_shared(1, 2, kTabletId, 15674, 4, 5, schema, 6, + std::unordered_map {{7, 8}}, + UniqueId(9, 10), TTabletType::TABLET_TYPE_DISK, + TCompressionType::LZ4F, 0, false); + meta->set_tablet_state(TABLET_RUNNING); + *try_any_cast(args[1]) = std::move(meta); + try_any_cast_ret(args)->second = true; + }); + _sp->set_call_back("CloudMetaMgr::sync_tablet_rowsets", [this](auto&& args) { + auto* tablet = try_any_cast(args[0]); + if (_sync_calls.fetch_add(1) == 0) { + // The loader's sync brings a stale view (< the queried version 100), so the + // decision's own sync below actually runs and can raise. + install_rowsets(tablet, {{0, 50}}); + try_any_cast_ret(args)->second = true; + return; + } + // The decision's sync, inside the guarded task body: raise. + throw std::runtime_error("injected: decision sync raised"); + }); + int64_t fallbacks_before = + DorisMetrics::instance()->query_cache_incremental_fallback_total->value(); + auto decision = make_stale_decision(); + EXPECT_EQ(DorisMetrics::instance()->query_cache_incremental_fallback_total->value(), + fallbacks_before + 1); + EXPECT_EQ(decision->mode, QueryCacheInstanceDecision::Mode::MISS); + EXPECT_EQ(decision->incremental_fallback_reason, "cloud tablet sync raised"); + // Two syncs entered: the loader's (installed the stale view) and the decision's + // (raised, then converted to a fallback by the guard rather than escaping to + // terminate the BE). + EXPECT_EQ(_sync_calls.load(), 2); +} + +TEST_F(QueryCacheCloudIncrementalTest, presync_decision_sync_raises_doris_exception_falls_back) { + // Same guard, but the decision sync raises a doris::Exception (what a nested + // THROW_IF_ERROR surfaces). The task body catches doris::Exception BEFORE the + // generic std::exception (the standard catch order per be/src/common/AGENTS.md), so + // the Doris error code reaches the log, and converts it to the same per-tablet + // fallback rather than letting it escape and terminate the BE. + _sp->set_call_back("CloudMetaMgr::get_tablet_meta", [](auto&& args) { + TTabletSchema schema; + schema.keys_type = TKeysType::DUP_KEYS; + auto meta = std::make_shared(1, 2, kTabletId, 15674, 4, 5, schema, 6, + std::unordered_map {{7, 8}}, + UniqueId(9, 10), TTabletType::TABLET_TYPE_DISK, + TCompressionType::LZ4F, 0, false); + meta->set_tablet_state(TABLET_RUNNING); + *try_any_cast(args[1]) = std::move(meta); + try_any_cast_ret(args)->second = true; + }); + _sp->set_call_back("CloudMetaMgr::sync_tablet_rowsets", [this](auto&& args) { + auto* tablet = try_any_cast(args[0]); + if (_sync_calls.fetch_add(1) == 0) { + install_rowsets(tablet, {{0, 50}}); + try_any_cast_ret(args)->second = true; + return; + } + throw doris::Exception(ErrorCode::INTERNAL_ERROR, + "injected: decision sync doris exception"); + }); + int64_t fallbacks_before = + DorisMetrics::instance()->query_cache_incremental_fallback_total->value(); + auto decision = make_stale_decision(); + EXPECT_EQ(DorisMetrics::instance()->query_cache_incremental_fallback_total->value(), + fallbacks_before + 1); + EXPECT_EQ(decision->mode, QueryCacheInstanceDecision::Mode::MISS); + EXPECT_EQ(decision->incremental_fallback_reason, "cloud tablet sync raised"); + EXPECT_EQ(_sync_calls.load(), 2); +} + +TEST_F(QueryCacheCloudIncrementalTest, capture_site_get_tablet_is_cache_only_on_eviction) { + // The presync fan-out synced this tablet, but capacity pressure evicted it + // from the cloud tablet cache before the serial capture consumed the result. + // With no recorded presync reason, _capture_tablet_delta's capture-site + // get_tablet must be cache-only (force_use_only_cached=true): a miss becomes + // an immediate fallback instead of a synchronous meta-service reload on this + // light admission thread (the very blocking window the presync fast-fail + // budget caps, which a plain reload would reopen after the fact). Both + // callbacks below serve a valid tablet, so a regression to a plain get_tablet + // would SUCCEED the reload and fail this test on the load count rather than + // crash, keeping the revert a clean red. + std::atomic meta_loads {0}; + _sp->set_call_back("CloudMetaMgr::get_tablet_meta", [&meta_loads](auto&& args) { + ++meta_loads; + TTabletSchema schema; + schema.keys_type = TKeysType::DUP_KEYS; + auto meta = std::make_shared(1, 2, kTabletId, 15674, 4, 5, schema, 6, + std::unordered_map {{7, 8}}, + UniqueId(9, 10), TTabletType::TABLET_TYPE_DISK, + TCompressionType::LZ4F, 0, false); + meta->set_tablet_state(TABLET_RUNNING); + *try_any_cast(args[1]) = std::move(meta); + try_any_cast_ret(args)->second = true; + }); + _sp->set_call_back("CloudMetaMgr::sync_tablet_rowsets", [this](auto&& args) { + auto* tablet = try_any_cast(args[0]); + install_rowsets(tablet, {{0, 50}, {51, 100}}); + try_any_cast_ret(args)->second = true; + }); + + QueryCacheRuntime runtime(make_cache_param(kTabletId), _cache.get()); + QueryCacheInstanceDecision decision; + decision.current_version = 100; + decision.cached_version = 50; + // Empty presync reasons + a tablet absent from the cloud tablet cache (never + // loaded here) reproduce "synced OK by presync, then evicted". + bool incremental = runtime.capture_tablet_delta_for_test(kTabletId, 50, {}, &decision); + EXPECT_FALSE(incremental); + // Shared with a genuinely absent tablet: both are "not resident locally". + EXPECT_EQ(decision.incremental_fallback_reason, "tablet not found"); + // The load-bearing assertion: the cache-only lookup issued NO meta-service + // RPC. A plain get_tablet would reload the evicted tablet here (meta_loads + // == 1), which is exactly the admission-thread blocking this fix removes. + EXPECT_EQ(meta_loads.load(), 0); +} + +TEST_F(QueryCacheCloudIncrementalTest, unique_non_mow_rejected_before_decision_sync) { + // A UNIQUE_KEYS tablet WITHOUT merge-on-write (the legacy merge-on-read + // unique table) is not append-only either, so the keys-type check rejects + // it before the cloud sync, exactly like an AGG tablet. Exercises the + // enable_unique_key_merge_on_write()==false leg of the append-only test: + // the merge-on-write cases take its true leg, AGG never evaluates it (short + // -circuited by the keys-type compare), so this is the one input that drives + // that leg false. + intercept_tablet_load({{0, 50}}, Status::OK(), TKeysType::UNIQUE_KEYS, + /*enable_merge_on_write=*/false); + int64_t fallbacks_before = + DorisMetrics::instance()->query_cache_incremental_fallback_total->value(); + auto decision = make_stale_decision(); + EXPECT_EQ(DorisMetrics::instance()->query_cache_incremental_fallback_total->value(), + fallbacks_before + 1); + EXPECT_EQ(decision->mode, QueryCacheInstanceDecision::Mode::MISS); + EXPECT_EQ(decision->incremental_fallback_reason, "keys type not append-only"); + // Rejected at the keys-type check, after the loader's single round trip and + // without a second (decision) sync. + EXPECT_EQ(_sync_calls.load(), 1); +} + +TEST_F(QueryCacheCloudIncrementalTest, mow_decision_sync_brings_bitmap_detects_rewrite) { + // Cloud sync-completeness, GUARDING options.sync_delete_bitmap=true: the + // loader view stops at 50 with no delta and no delete-bitmap entry, so the + // decision's sync really runs (_sync_calls==2). The fixture mock is + // flag-faithful -- it stamps the injected rewrite mark only when that sync + // carried sync_delete_bitmap=true (the CloudMetaMgr::sync_tablet_rowsets sync + // point forwards the SyncOptions, mirroring the real body's gate). Because the + // pre-sync sets the flag, the mark on the baseline rowset (version 60, inside + // the delta window) lands, the history-rewrite check reads that freshly-synced + // bitmap, and the query falls back. If a refactor flipped the flag to false, + // the mock would drop the mark just as a real meta-service would drop the + // bitmap sync, the rewrite would go undetected, and this test would fail + // (INCREMENTAL instead of MISS) -- so the load-bearing MoW invariant is + // UT-guarded, not just documented. mow_history_rewrite_falls_back cannot show + // this: there the loader carried the view to 100, so the decision sync was a + // no-op and the mark was hand-placed. + intercept_tablet_load({{0, 50}}, Status::OK(), TKeysType::UNIQUE_KEYS, + /*enable_merge_on_write=*/true, + /*decision_sync_installs=*/ {{51, 100}}, + /*decision_sync_bitmap_marks=*/ {{0, 60}}); + int64_t fallbacks_before = + DorisMetrics::instance()->query_cache_incremental_fallback_total->value(); + auto decision = make_stale_decision(); + EXPECT_EQ(DorisMetrics::instance()->query_cache_incremental_fallback_total->value(), + fallbacks_before + 1); + EXPECT_EQ(decision->mode, QueryCacheInstanceDecision::Mode::MISS); + EXPECT_EQ(decision->incremental_fallback_reason, "delta rewrites history rows"); + // The loader's view (max 50) did not cover the query, so the decision's sync + // really ran -- two round trips -- and it is that second sync that brought + // the rewrite evidence the classification tripped on. + EXPECT_EQ(_sync_calls.load(), 2); +} + +TEST_F(QueryCacheCloudIncrementalTest, mow_cloud_cached_side_not_pinnable_falls_back) { + // The cloud analogue of mow_cached_side_not_pinnable_falls_back, exercising a + // DEFENSIVE branch that is unreachable after a real cloud sync: the meta + // service serves a gapless committed set, so a synced [0,max] view never has + // a hole below the cached version and [0,cached] is always pinnable. The test + // FORGES the hole ([41,50] missing) with install_rowsets; since the loader + // already carries the view to 100 the decision sync short-cuts (_sync_calls + // ==1), so the pin runs against that loader-installed forged view. It + // confirms the pin-and-endpoint guard is wired on the cloud path (the block + // is now shared, not cloud-early-returned) and would fire if the gapless + // invariant were ever broken: the cached-side pin walks only [0,40] and the + // endpoint check -- short of cached version 50 -- rejects it (not emptiness). + intercept_tablet_load({{0, 40}, {51, 80}, {81, 100}}, Status::OK(), TKeysType::UNIQUE_KEYS, + /*enable_merge_on_write=*/true); + int64_t fallbacks_before = + DorisMetrics::instance()->query_cache_incremental_fallback_total->value(); + auto decision = make_stale_decision(); + EXPECT_EQ(DorisMetrics::instance()->query_cache_incremental_fallback_total->value(), + fallbacks_before + 1); + EXPECT_EQ(decision->mode, QueryCacheInstanceDecision::Mode::MISS); + EXPECT_EQ(decision->incremental_fallback_reason, "cached versions not pinnable"); + // The loader carried the view to 100, so the decision's sync took its no-op + // shortcut: one meta-service round trip. + EXPECT_EQ(_sync_calls.load(), 1); +} + +TEST_F(QueryCacheCloudIncrementalTest, mow_decision_sync_clean_bitmap_stays_incremental) { + // Positive complement (and non-vacuousness proof) of + // mow_decision_sync_brings_bitmap_detects_rewrite: identical shape -- loader + // view 50, the decision's real sync (_sync_calls==2) installs the delta + // (50,100] on a merge-on-write tablet -- but with NO rewrite mark. The + // history-rewrite check runs on the freshly synced bitmap, finds nothing, and + // the merge stays INCREMENTAL. This shows the sibling's fallback is driven by + // its injected mark and not by the decision-sync-on-MoW path itself: same + // path, no mark, no fallback. + intercept_tablet_load({{0, 50}}, Status::OK(), TKeysType::UNIQUE_KEYS, + /*enable_merge_on_write=*/true, + /*decision_sync_installs=*/ {{51, 100}}); + auto decision = make_stale_decision(); + EXPECT_EQ(decision->mode, QueryCacheInstanceDecision::Mode::INCREMENTAL); + EXPECT_TRUE(decision->incremental_fallback_reason.empty()); + EXPECT_EQ(_sync_calls.load(), 2); + auto source = decision->take_delta_read_source(kTabletId); + ASSERT_NE(source, nullptr); + EXPECT_EQ(source->rs_splits.size(), 1); + EXPECT_EQ(source->rs_splits.front().rs_reader->rowset()->start_version(), 51); + EXPECT_EQ(source->rs_splits.front().rs_reader->rowset()->end_version(), 100); +} + +TEST_F(QueryCacheCloudIncrementalTest, presync_fans_out_across_tablets_and_merges_reason) { + // Two append-only tablets in one instance drive the parallel pre-sync + // fan-out (_presync_cloud_delta_tablets) at N>1 -- every other cloud test + // here exercises it only at N=1. T1's loader view already covers the queried + // version, so its decision sync short-cuts; T2's stops short, so its decision + // sync runs and is injected to fail. The fan-out must sync both tablets, the + // post-join merge must surface T2's failure keyed by its tablet id, and the + // capture loop must fall the whole decision back on it -- exercising the + // multi-tablet fan-out, the per-index reason merge, and the capture-loop + // consumption together, none of which the single-tablet cases reach. + constexpr int64_t kTabletId2 = 15680; + + // get_tablet_meta serves a DUP_KEYS meta stamped with the REQUESTED tablet + // id (args[0]), so both scan ranges materialize as distinct tablets. It is + // stateless, so the concurrent fan-out loads need no synchronization here. + _sp->set_call_back("CloudMetaMgr::get_tablet_meta", [](auto&& args) { + auto tid = try_any_cast(args[0]); + TTabletSchema schema; + schema.keys_type = TKeysType::DUP_KEYS; + auto meta = std::make_shared( + 1, 2, tid, 15674, 4, 5, schema, 6, std::unordered_map {{7, 8}}, + UniqueId(9, 10), TTabletType::TABLET_TYPE_DISK, TCompressionType::LZ4F, 0, false); + meta->set_tablet_state(TABLET_RUNNING); + *try_any_cast(args[1]) = std::move(meta); + try_any_cast_ret(args)->second = true; + }); + + // sync_tablet_rowsets fires concurrently across the fan-out bthreads; a mutex + // guards the per-tablet call count and the metadata-only install (which draws + // from the shared engine's rowset-id generator). Per tablet the first call is + // the loader: T1 gets a view covering 100 (its later decision sync short-cuts, + // so it never calls back a second time), T2 stops at 50 so its decision sync + // runs as the second call and is failed there. add_rowsets takes each + // tablet's own header lock, so the two installs never contend. + std::mutex mu; + std::unordered_map calls; + _sp->set_call_back("CloudMetaMgr::sync_tablet_rowsets", [&](auto&& args) { + auto* tablet = try_any_cast(args[0]); + auto* ret = try_any_cast_ret(args); + std::lock_guard lk(mu); + if (calls[tablet->tablet_id()]++ == 0) { + std::vector> versions = + tablet->tablet_id() == kTabletId + ? std::vector> {{0, 50}, {51, 100}} + : std::vector> {{0, 50}}; + std::vector rowsets; + for (auto [start, end] : versions) { + auto rs_meta = std::make_shared(); + rs_meta->set_rowset_type(BETA_ROWSET); + rs_meta->set_version({start, end}); + rs_meta->set_rowset_id(_engine->next_rowset_id()); + rs_meta->set_num_segments(1); + rs_meta->set_tablet_schema(tablet->tablet_meta()->tablet_schema()); + RowsetSharedPtr rowset; + if (RowsetFactory::create_rowset(nullptr, "", rs_meta, &rowset).ok()) { + rowsets.push_back(std::move(rowset)); + } + } + std::unique_lock lock(tablet->get_header_lock()); + tablet->add_rowsets(std::move(rowsets), /*version_overlap=*/false, lock); + } else { + // T2's decision sync (T1's short-cuts and never reaches here). + ret->first = Status::InternalError("injected sync failure"); + } + ret->second = true; + }); + + // A stale entry (version 50) keyed over both tablets, queried at 100. + std::vector scan_ranges = make_scan_ranges(kTabletId, "100"); + scan_ranges.push_back(make_scan_ranges(kTabletId2, "100").front()); + auto cache_param = make_cache_param(kTabletId); + cache_param.tablet_to_range.insert({kTabletId2, "range"}); + cache_param.__set_allow_incremental(true); + std::string cache_key; + int64_t version = 0; + EXPECT_TRUE(QueryCache::build_cache_key(scan_ranges, cache_param, &cache_key, &version).ok()); + insert_entry(_cache.get(), cache_key, 50, 0); + QueryCacheRuntime runtime(cache_param, _cache.get()); + + int64_t fallbacks_before = + DorisMetrics::instance()->query_cache_incremental_fallback_total->value(); + auto decision = runtime.get_or_make_decision(scan_ranges); + EXPECT_EQ(DorisMetrics::instance()->query_cache_incremental_fallback_total->value(), + fallbacks_before + 1); + EXPECT_EQ(decision->mode, QueryCacheInstanceDecision::Mode::MISS); + EXPECT_EQ(decision->incremental_fallback_reason, "cloud rowset sync failed"); + // Both tablets fanned out and were synced: T1 once (loader; decision + // short-cut) and T2 twice (loader + the failed decision sync). + std::lock_guard lk(mu); + EXPECT_EQ(calls[kTabletId], 1); + EXPECT_EQ(calls[kTabletId2], 2); +} + +TEST_F(QueryCacheCloudIncrementalTest, pool_rejection_skips_admission_tablet_load) { + // Composition of two degradations: the dedicated pre-sync pool refuses the + // sync (here: shut down, the same inline rejection a full bounded queue + // gives) AND the tablet is absent from the local tablet cache (nothing -- + // loader or pre-sync, whose closure never ran -- ever loaded it). The + // decision must consume the recorded "not scheduled" reason BEFORE touching + // the tablet: the capture loop's get_tablet would otherwise take the + // synchronous cache-miss meta-service load on the admission thread -- the + // blocking window the fast-fail budget exists to cap -- to fetch a tablet + // whose decision is already a known fallback. The meta-load counter proves + // the admission path stayed load-free; a regression in the consumption + // order shows up twice (a nonzero count, and the weaker "tablet not found" + // reason from the failed load displacing the pool's). + std::atomic meta_loads {0}; + _sp->set_call_back("CloudMetaMgr::get_tablet_meta", [&](auto&& args) { + meta_loads.fetch_add(1); + auto* ret = try_any_cast_ret(args); + ret->first = Status::InternalError("meta service unreachable"); + ret->second = true; + }); + _engine->query_cache_delta_sync_pool().shutdown(); + + int64_t fallbacks_before = + DorisMetrics::instance()->query_cache_incremental_fallback_total->value(); + auto decision = make_stale_decision(); + EXPECT_EQ(DorisMetrics::instance()->query_cache_incremental_fallback_total->value(), + fallbacks_before + 1); + EXPECT_EQ(decision->mode, QueryCacheInstanceDecision::Mode::MISS); + EXPECT_EQ(decision->incremental_fallback_reason, "cloud rowset sync not scheduled"); + EXPECT_EQ(meta_loads.load(), 0); + EXPECT_EQ(_sync_calls.load(), 0); +} + +TEST_F(QueryCacheCloudIncrementalTest, presync_single_flight_coalesces_and_unrelated_key_proceeds) { + // Two fragment runtimes over the SAME stale key must coalesce onto ONE + // presync fan-out through the cache's single-flight registry (runtimes are + // private per fragment context, so without the registry each would submit + // the full fan-out), while a runtime over an UNRELATED key builds its own + // flight and completes even though the first flight is parked on a slow + // meta service. Deterministic: the shared flight's decision sync parks on a + // gate, the second runtime's join is witnessed by the follower_joined sync + // point, and only then does the unrelated key run and the gate open. + constexpr int64_t kOtherTablet = 15690; + _sp->set_call_back("CloudMetaMgr::get_tablet_meta", [](auto&& args) { + auto tid = try_any_cast(args[0]); + TTabletSchema schema; + schema.keys_type = TKeysType::DUP_KEYS; + auto meta = std::make_shared( + 1, 2, tid, 15674, 4, 5, schema, 6, std::unordered_map {{7, 8}}, + UniqueId(9, 10), TTabletType::TABLET_TYPE_DISK, TCompressionType::LZ4F, 0, false); + meta->set_tablet_state(TABLET_RUNNING); + *try_any_cast(args[1]) = std::move(meta); + try_any_cast_ret(args)->second = true; + }); + std::mutex mu; + std::unordered_map calls; + std::atomic gate_open {false}; + std::atomic gated_sync_parked {false}; + _sp->set_call_back("CloudMetaMgr::sync_tablet_rowsets", [&](auto&& args) { + auto* tablet = try_any_cast(args[0]); + auto* ret = try_any_cast_ret(args); + ret->second = true; + int n; + { + std::lock_guard lk(mu); + n = calls[tablet->tablet_id()]++; + } + if (n == 0) { + // Loader sync: a view that stops short of query version 100, so + // the decision sync really runs. Serialized under the test mutex + // (install draws from the shared engine rowset-id generator). + std::lock_guard lk(mu); + install_rowsets(tablet, {{0, 50}}); + return; + } + if (tablet->tablet_id() == kTabletId) { + gated_sync_parked = true; + while (!gate_open.load()) { + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + } + } + std::lock_guard lk(mu); + install_rowsets(tablet, {{51, 100}}); + }); + + auto ranges_a = make_scan_ranges(kTabletId, "100"); + auto param_a = make_cache_param(kTabletId); + param_a.__set_allow_incremental(true); + std::string key_a; + int64_t version = 0; + EXPECT_TRUE(QueryCache::build_cache_key(ranges_a, param_a, &key_a, &version).ok()); + insert_entry(_cache.get(), key_a, 50, 0); + auto ranges_c = make_scan_ranges(kOtherTablet, "100"); + auto param_c = make_cache_param(kOtherTablet); + param_c.__set_allow_incremental(true); + std::string key_c; + EXPECT_TRUE(QueryCache::build_cache_key(ranges_c, param_c, &key_c, &version).ok()); + insert_entry(_cache.get(), key_c, 50, 0); + + QueryCacheRuntime runtime_a1(param_a, _cache.get()); + QueryCacheRuntime runtime_a2(param_a, _cache.get()); + QueryCacheRuntime runtime_c(param_c, _cache.get()); + + std::atomic follower_joined {false}; + _sp->set_call_back("QueryCacheRuntime::_presync_cloud_delta_tablets.follower_joined", + [&](auto&&) { follower_joined = true; }); + + std::shared_ptr d1; + std::shared_ptr d2; + std::thread t1; + std::thread t2; + // A fatal ASSERT below returns while a spawned thread is still joinable; its + // std::thread destructor would std::terminate the whole UT binary and mask + // the real failure. Open the gate (so a parked worker can finish) and join + // both on every exit, mirroring race_two_callers. + Defer thread_guard {[&] { + gate_open = true; + if (t1.joinable()) { + t1.join(); + } + if (t2.joinable()) { + t2.join(); + } + }}; + + t1 = std::thread([&] { d1 = runtime_a1.get_or_make_decision(ranges_a); }); + for (int i = 0; i < 10000 && !gated_sync_parked.load(); ++i) { + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + } + ASSERT_TRUE(gated_sync_parked.load()); + + t2 = std::thread([&] { d2 = runtime_a2.get_or_make_decision(ranges_a); }); + for (int i = 0; i < 10000 && !follower_joined.load(); ++i) { + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + } + ASSERT_TRUE(follower_joined.load()); + + // With the shared flight parked, an unrelated key runs end to end on its + // own flight: registry entries do not interfere across keys. + auto d3 = runtime_c.get_or_make_decision(ranges_c); + EXPECT_EQ(d3->mode, QueryCacheInstanceDecision::Mode::INCREMENTAL); + + gate_open = true; + t1.join(); + t2.join(); + EXPECT_EQ(d1->mode, QueryCacheInstanceDecision::Mode::INCREMENTAL); + EXPECT_EQ(d2->mode, QueryCacheInstanceDecision::Mode::INCREMENTAL); + { + std::lock_guard lk(mu); + // ONE shared fan-out for the coalesced key: the loader plus the single + // gated decision sync. The joined runtime added no sync of its own; the + // unrelated tablet ran its own loader + decision pair. + EXPECT_EQ(calls[kTabletId], 2); + EXPECT_EQ(calls[kOtherTablet], 2); + } + // All waiters left, so the registry drained (BE UT compiles with + // -fno-access-control, so peek at the internals directly). + { + std::lock_guard lk(_cache->_presync_flights_lock); + EXPECT_TRUE(_cache->_presync_flights.empty()); + } +} + +TEST_F(QueryCacheCloudIncrementalTest, + presync_single_flight_last_leaver_keeps_tombstone_reaped_on_drain) { + // Two coalesced waiters expire their OWN deadlines while the shared decision + // sync is parked on a slow meta service. Three properties: + // (1) The FIRST leaver must NOT abandon/erase -- a timed-out waiter cannot + // cancel work the still-waiting sibling is entitled to. + // (2) The LAST leaver, seeing the fan-out still un-drained (task parked), + // marks the flight abandoned but KEEPS the registry entry as a + // tombstone. Erasing on timeout would let the next identical query + // become a fresh owner and submit a DUPLICATE fan-out during the very + // brownout single-flight exists to smooth. + // (3) The tombstone is reaped only once the fan-out DRAINS: after the parked + // task returns (latch hits 0), a later identical arrival joins the + // drained flight, reuses its settled slots, and erases it on leave. + // To observe the intermediate "first left, second still waiting" state the two + // waiters are given DETERMINISTIC per-runtime deadlines via the + // ..._presync_cloud_delta_tablets.deadline_ms test seam (below): the owner gets + // 2000ms, the follower 4000ms. Because the follower cannot even start until the + // owner's task is already parked (its sync_fanout_start is strictly later), the + // owner's deadline (t_owner + 2000) is ALWAYS earlier than the follower's + // (t_follower + 4000 >= t_owner + 4000), so the owner times out first by + // construction -- no wall-clock staggering, and no CI deschedule can invert the + // order. The 2000ms owner deadline also gives the follower the suite's standard + // 2000ms of scheduling headroom to REGISTER while the owner is still waiting + // (the two-waiter overlap the intermediate assertions observe) -- the same + // headroom every other parked-coalescing test gets from the 2000ms default + // timeout, so this test is not the flakiest link. (The prior versions staggered + // starts by a fixed 500ms sleep, then used 500/1000ms deadlines whose 500ms + // registration window was tighter than the suite convention.) + int32_t saved_timeout = config::query_cache_decision_sync_timeout_ms; + // Kept > 0 so the early enable check passes; the per-runtime seam overrides the + // actual deadline for both waiters, so this nominal value is otherwise unused here. + config::query_cache_decision_sync_timeout_ms = 2000; + Defer restore_timeout {[&] { config::query_cache_decision_sync_timeout_ms = saved_timeout; }}; + + _sp->set_call_back("CloudMetaMgr::get_tablet_meta", [](auto&& args) { + TTabletSchema schema; + schema.keys_type = TKeysType::DUP_KEYS; + auto meta = std::make_shared(1, 2, kTabletId, 15674, 4, 5, schema, 6, + std::unordered_map {{7, 8}}, + UniqueId(9, 10), TTabletType::TABLET_TYPE_DISK, + TCompressionType::LZ4F, 0, false); + meta->set_tablet_state(TABLET_RUNNING); + *try_any_cast(args[1]) = std::move(meta); + try_any_cast_ret(args)->second = true; + }); + std::mutex mu; + std::atomic syncs {0}; + std::atomic gate_open {false}; + std::atomic gated_sync_parked {false}; + std::atomic gated_sync_returned {false}; + _sp->set_call_back("CloudMetaMgr::sync_tablet_rowsets", [&](auto&& args) { + auto* tablet = try_any_cast(args[0]); + auto* ret = try_any_cast_ret(args); + ret->second = true; + if (syncs.fetch_add(1) == 0) { + std::lock_guard lk(mu); + install_rowsets(tablet, {{0, 50}}); + return; + } + gated_sync_parked = true; + while (!gate_open.load()) { + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + } + // On release the parked decision sync completes the view to the queried + // version, so the later reaping arrival can go INCREMENTAL off the + // settled slot -- proving the reuse path, not just the erase. + { + std::lock_guard lk(mu); + install_rowsets(tablet, {{51, 100}}); + } + gated_sync_returned = true; + }); + + auto ranges = make_scan_ranges(kTabletId, "100"); + auto param = make_cache_param(kTabletId); + param.__set_allow_incremental(true); + std::string key; + int64_t version = 0; + EXPECT_TRUE(QueryCache::build_cache_key(ranges, param, &key, &version).ok()); + insert_entry(_cache.get(), key, 50, 0); + QueryCacheRuntime runtime_a(param, _cache.get()); + QueryCacheRuntime runtime_b(param, _cache.get()); + + // Deterministic deadlines by coalescing role: the flight owner gets 2000ms and a + // joining follower 4000ms, so the owner times out first by construction (see the + // header comment), independent of scheduling jitter. runtime_a starts first and + // owns the flight; runtime_b joins it. + _sp->set_call_back("QueryCacheRuntime::_presync_cloud_delta_tablets.deadline_ms", + [](auto&& args) { + bool is_owner = try_any_cast(args[0]); + *try_any_cast(args[1]) = is_owner ? 2000 : 4000; + }); + + std::atomic follower_joined {false}; + _sp->set_call_back("QueryCacheRuntime::_presync_cloud_delta_tablets.follower_joined", + [&](auto&&) { follower_joined = true; }); + + std::shared_ptr d1; + std::shared_ptr d2; + std::thread t1; + std::thread t2; + // A fatal ASSERT below returns while a spawned thread is still joinable; its + // std::thread destructor would std::terminate the whole UT binary and mask + // the real failure. Open the gate (so the parked worker can finish) and join + // both on every exit, mirroring race_two_callers. + Defer thread_guard {[&] { + gate_open = true; + if (t1.joinable()) { + t1.join(); + } + if (t2.joinable()) { + t2.join(); + } + }}; + + t1 = std::thread([&] { d1 = runtime_a.get_or_make_decision(ranges); }); + for (int i = 0; i < 10000 && !gated_sync_parked.load(); ++i) { + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + } + ASSERT_TRUE(gated_sync_parked.load()); + + // The follower can start as soon as the owner's task has parked: its longer + // per-runtime deadline (4000ms vs the owner's 2000ms) guarantees the owner times + // out first regardless of when exactly the follower joins, so no start stagger. + t2 = std::thread([&] { d2 = runtime_b.get_or_make_decision(ranges); }); + for (int i = 0; i < 10000 && !follower_joined.load(); ++i) { + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + } + ASSERT_TRUE(follower_joined.load()); + + // Capture the shared flight while both wait: present, two waiters, not abandoned. + std::shared_ptr flight; + { + std::lock_guard lk(_cache->_presync_flights_lock); + ASSERT_EQ(_cache->_presync_flights.size(), 1); + flight = _cache->_presync_flights.begin()->second; + EXPECT_EQ(flight->waiters, 2); + } + EXPECT_FALSE(flight->abandoned->load()); + + // The owner times out first (it started ~500ms earlier). As the FIRST leaver + // it must NOT abandon or erase: exactly one waiter left, entry still + // registered, flag still clear. + t1.join(); + EXPECT_EQ(d1->mode, QueryCacheInstanceDecision::Mode::MISS); + EXPECT_EQ(d1->incremental_fallback_reason, "cloud rowset sync timed out"); + { + std::lock_guard lk(_cache->_presync_flights_lock); + EXPECT_EQ(_cache->_presync_flights.size(), 1); + EXPECT_EQ(flight->waiters, 1); + } + EXPECT_FALSE(flight->abandoned->load()); + + // The follower times out next. As the LAST leaver, with the fan-out still + // parked (un-drained), it marks the flight abandoned but KEEPS the tombstone + // -- it is NOT erased on timeout (that is the codex-flagged premature-erase + // regression this design fixes). + t2.join(); + EXPECT_EQ(d2->mode, QueryCacheInstanceDecision::Mode::MISS); + EXPECT_EQ(d2->incremental_fallback_reason, "cloud rowset sync timed out"); + EXPECT_TRUE(flight->abandoned->load()); + { + std::lock_guard lk(_cache->_presync_flights_lock); + ASSERT_EQ(_cache->_presync_flights.size(), 1); + EXPECT_EQ(_cache->_presync_flights.begin()->second, flight); + EXPECT_EQ(flight->waiters, 0); + } + + // Release the parked task and let it fully return. As it drives the latch to 0 + // with ZERO waiters remaining (both timed out into the tombstone above), the + // FINAL task reaps the tombstone itself in publish_slot -- no later arrival is + // needed (the round-2 reap-by-final-task fix; the round-1 code left this + // waiterless tombstone stranded). NOTE: an identical arrival here would NOT + // join and reuse the slot -- it would find the registry already reaped by the + // final task and re-own, so the two-waiter tombstone's final-task reap is what + // this phase proves (join-not-reown is proved by + // presync_single_flight_post_timeout_arrival_joins_not_reowns). + gate_open = true; + for (int i = 0; i < 10000 && !gated_sync_returned.load(); ++i) { + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + } + ASSERT_TRUE(gated_sync_returned.load()); + bool reaped = false; + for (int i = 0; i < 10000; ++i) { + { + std::lock_guard lk(_cache->_presync_flights_lock); + if (_cache->_presync_flights.empty()) { + reaped = true; + break; + } + } + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + } + EXPECT_TRUE(reaped) << "the final task must reap the waiterless tombstone on drain"; + EXPECT_EQ(flight->fanout_done->count(), 0u); + // The parked task published exactly once (loader + the single decision sync); + // no leaver re-submitted a duplicate fan-out. + EXPECT_EQ(syncs.load(), 2); + // Deterministically drain the callback shell's unwind after its last store, mirroring + // decision_sync_timeout_falls_back, before TearDown clears the sync points: pool.wait() + // hard-synchronizes the task's exit instead of betting a fixed sleep outlasts it. + _engine->query_cache_delta_sync_pool().wait(); +} + +TEST_F(QueryCacheCloudIncrementalTest, + presync_single_flight_post_timeout_arrival_joins_not_reowns) { + // The codex-flagged brownout regression, directly: after every original + // waiter times out on a parked (still-running) fan-out, a NEW identical query + // must JOIN that draining flight, NOT become a fresh owner and submit a + // DUPLICATE fan-out (which would double the bounded-pool pressure wave after + // wave during a sustained meta-service brownout). The owner times out while + // its single decision sync is parked, leaving an abandoned-but-kept tombstone; + // a second runtime then arrives, joins, and adds NO sync of its own -- proven + // by the sync-call count staying at loader + the one parked decision sync. + int32_t saved_timeout = config::query_cache_decision_sync_timeout_ms; + config::query_cache_decision_sync_timeout_ms = 500; + Defer restore_timeout {[&] { config::query_cache_decision_sync_timeout_ms = saved_timeout; }}; + + _sp->set_call_back("CloudMetaMgr::get_tablet_meta", [](auto&& args) { + TTabletSchema schema; + schema.keys_type = TKeysType::DUP_KEYS; + auto meta = std::make_shared(1, 2, kTabletId, 15674, 4, 5, schema, 6, + std::unordered_map {{7, 8}}, + UniqueId(9, 10), TTabletType::TABLET_TYPE_DISK, + TCompressionType::LZ4F, 0, false); + meta->set_tablet_state(TABLET_RUNNING); + *try_any_cast(args[1]) = std::move(meta); + try_any_cast_ret(args)->second = true; + }); + std::mutex mu; + std::atomic syncs {0}; + std::atomic gate_open {false}; + std::atomic gated_sync_parked {false}; + std::atomic gated_sync_returned {false}; + _sp->set_call_back("CloudMetaMgr::sync_tablet_rowsets", [&](auto&& args) { + auto* tablet = try_any_cast(args[0]); + auto* ret = try_any_cast_ret(args); + ret->second = true; + if (syncs.fetch_add(1) == 0) { + std::lock_guard lk(mu); + install_rowsets(tablet, {{0, 50}}); + return; + } + gated_sync_parked = true; + while (!gate_open.load()) { + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + } + { + std::lock_guard lk(mu); + install_rowsets(tablet, {{51, 100}}); + } + gated_sync_returned = true; + }); + + auto ranges = make_scan_ranges(kTabletId, "100"); + auto param = make_cache_param(kTabletId); + param.__set_allow_incremental(true); + std::string key; + int64_t version = 0; + EXPECT_TRUE(QueryCache::build_cache_key(ranges, param, &key, &version).ok()); + insert_entry(_cache.get(), key, 50, 0); + QueryCacheRuntime runtime_owner(param, _cache.get()); + QueryCacheRuntime runtime_late(param, _cache.get()); + + std::atomic late_joined {false}; + _sp->set_call_back("QueryCacheRuntime::_presync_cloud_delta_tablets.follower_joined", + [&](auto&&) { late_joined = true; }); + + std::shared_ptr d_owner; + std::shared_ptr d_late; + std::thread t_owner; + std::thread t_late; + Defer thread_guard {[&] { + gate_open = true; + if (t_owner.joinable()) { + t_owner.join(); + } + if (t_late.joinable()) { + t_late.join(); + } + }}; + + t_owner = std::thread([&] { d_owner = runtime_owner.get_or_make_decision(ranges); }); + for (int i = 0; i < 10000 && !gated_sync_parked.load(); ++i) { + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + } + ASSERT_TRUE(gated_sync_parked.load()); + + // The owner times out while the decision sync is still parked, leaving an + // abandoned tombstone (the fan-out is un-drained, so it is NOT erased). + t_owner.join(); + EXPECT_EQ(d_owner->mode, QueryCacheInstanceDecision::Mode::MISS); + EXPECT_EQ(d_owner->incremental_fallback_reason, "cloud rowset sync timed out"); + { + std::lock_guard lk(_cache->_presync_flights_lock); + ASSERT_EQ(_cache->_presync_flights.size(), 1); + EXPECT_TRUE(_cache->_presync_flights.begin()->second->abandoned->load()); + } + + // A NEW identical query arrives while the fan-out is still parked. It must + // JOIN the tombstone (witnessed by the follower_joined sync point), not + // re-own and submit a second fan-out. + t_late = std::thread([&] { d_late = runtime_late.get_or_make_decision(ranges); }); + for (int i = 0; i < 10000 && !late_joined.load(); ++i) { + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + } + ASSERT_TRUE(late_joined.load()); + + // Release the parked task; the late arrival (waiting on the shared latch) + // wakes, reuses the completed view, goes INCREMENTAL, and reaps the flight. + gate_open = true; + t_late.join(); + EXPECT_EQ(d_late->mode, QueryCacheInstanceDecision::Mode::INCREMENTAL); + // The crux: exactly ONE fan-out ran (loader + the single parked decision + // sync). The late arrival added no sync -- it joined instead of re-owning. + EXPECT_EQ(syncs.load(), 2); + { + std::lock_guard lk(_cache->_presync_flights_lock); + EXPECT_TRUE(_cache->_presync_flights.empty()); + } + for (int i = 0; i < 10000 && !gated_sync_returned.load(); ++i) { + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + } + ASSERT_TRUE(gated_sync_returned.load()); + // Drain the task's tail after its last store deterministically (pool.wait() hard- + // synchronizes the callback shell's exit) before TearDown clears the sync points. + _engine->query_cache_delta_sync_pool().wait(); +} + +TEST_F(QueryCacheCloudIncrementalTest, + presync_single_flight_tombstone_reaped_on_drain_without_arrival) { + // The round-2 tombstone-leak fix: a flight whose sole/last waiter times out + // while a task is still parked is KEPT as a tombstone, but once the fan-out + // DRAINS it must be reaped by the FINAL task itself, even if NO later + // identical query ever arrives to join and reap it. Without this the entry + // leaks for the BE lifetime on exactly the brownout path the feature targets. + // A single owner times out; the parked task is then released and drains the + // latch with ZERO waiters; the registry must end empty with no reaping + // arrival. (Pre-fix -- reap only in leave_flight -- this asserts a permanent + // leak: the final _presync_flights.empty() never holds.) + int32_t saved_timeout = config::query_cache_decision_sync_timeout_ms; + config::query_cache_decision_sync_timeout_ms = 500; + Defer restore_timeout {[&] { config::query_cache_decision_sync_timeout_ms = saved_timeout; }}; + + _sp->set_call_back("CloudMetaMgr::get_tablet_meta", [](auto&& args) { + TTabletSchema schema; + schema.keys_type = TKeysType::DUP_KEYS; + auto meta = std::make_shared(1, 2, kTabletId, 15674, 4, 5, schema, 6, + std::unordered_map {{7, 8}}, + UniqueId(9, 10), TTabletType::TABLET_TYPE_DISK, + TCompressionType::LZ4F, 0, false); + meta->set_tablet_state(TABLET_RUNNING); + *try_any_cast(args[1]) = std::move(meta); + try_any_cast_ret(args)->second = true; + }); + std::mutex mu; + std::atomic syncs {0}; + std::atomic gate_open {false}; + std::atomic gated_sync_parked {false}; + std::atomic gated_sync_returned {false}; + _sp->set_call_back("CloudMetaMgr::sync_tablet_rowsets", [&](auto&& args) { + auto* tablet = try_any_cast(args[0]); + auto* ret = try_any_cast_ret(args); + ret->second = true; + if (syncs.fetch_add(1) == 0) { + std::lock_guard lk(mu); + install_rowsets(tablet, {{0, 50}}); + return; + } + gated_sync_parked = true; + while (!gate_open.load()) { + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + } + gated_sync_returned = true; + }); + + auto ranges = make_scan_ranges(kTabletId, "100"); + auto param = make_cache_param(kTabletId); + param.__set_allow_incremental(true); + std::string key; + int64_t version = 0; + EXPECT_TRUE(QueryCache::build_cache_key(ranges, param, &key, &version).ok()); + insert_entry(_cache.get(), key, 50, 0); + // Baseline the query-cache MemTracker AFTER the cache entry is in (that charge is + // a constant offset for the rest of the test). make_flight charges the whole + // flight's estimated retained bytes (object + the per_range_reason / tablet_ids / + // slot_counted O(tablets) buffers + control blocks + key) to this SAME stable + // limiter as ONE lump, and ~PresyncFlight releases exactly that lump, so + // consumption must rise while the flight is live and return to EXACTLY this baseline + // once the flight is reaped and destroyed -- the balance that proves the charge is + // stable and fully refunded. The byte-exact EQ below also + // depends on this fixture keeping the query-cache tracker otherwise quiescent + // between baseline and assertion (one tiny cache entry, no eviction, no second + // flight); do not add cache traffic in between without switching to a tolerance + // band. + const int64_t qc_mem_baseline = + ExecEnv::GetInstance()->query_cache_mem_tracker()->consumption(); + QueryCacheRuntime runtime_owner(param, _cache.get()); + + std::shared_ptr d; + std::thread t; + Defer thread_guard {[&] { + gate_open = true; + if (t.joinable()) { + t.join(); + } + }}; + + t = std::thread([&] { d = runtime_owner.get_or_make_decision(ranges); }); + for (int i = 0; i < 10000 && !gated_sync_parked.load(); ++i) { + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + } + ASSERT_TRUE(gated_sync_parked.load()); + + // The owner times out while its task is parked, leaving an abandoned tombstone + // with zero waiters and the fan-out not yet drained. + t.join(); + EXPECT_EQ(d->mode, QueryCacheInstanceDecision::Mode::MISS); + EXPECT_EQ(d->incremental_fallback_reason, "cloud rowset sync timed out"); + std::shared_ptr flight; + { + std::lock_guard lk(_cache->_presync_flights_lock); + ASSERT_EQ(_cache->_presync_flights.size(), 1); + flight = _cache->_presync_flights.begin()->second; + EXPECT_EQ(flight->waiters, 0); + } + EXPECT_TRUE(flight->abandoned->load()); + // The live flight's buffers are charged to the stable query-cache MemTracker. + EXPECT_GT(ExecEnv::GetInstance()->query_cache_mem_tracker()->consumption(), qc_mem_baseline) + << "the live flight's buffers must be charged to the query-cache MemTracker"; + + // Release the parked task; with zero waiters, the FINAL task reaps the + // tombstone itself -- no later identical query is ever issued. + gate_open = true; + for (int i = 0; i < 10000 && !gated_sync_returned.load(); ++i) { + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + } + ASSERT_TRUE(gated_sync_returned.load()); + bool reaped = false; + for (int i = 0; i < 10000; ++i) { + { + std::lock_guard lk(_cache->_presync_flights_lock); + if (_cache->_presync_flights.empty()) { + reaped = true; + break; + } + } + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + } + EXPECT_TRUE(reaped) << "the final task must reap the drained tombstone with no arrival"; + // Drain the pool so the parked task's closure (a holder of the flight's shared + // pieces) is destroyed, then drop the test's own captured reference. With the + // registry entry reaped and no waiters, the flight and all its buffers are now + // freed, so the query-cache MemTracker must return to EXACTLY the pre-flight + // baseline -- proving the O(tablets) buffers were charged to the stable limiter + // and fully released on reap (no leak, no over-release). This also removes the + // former unwind cushion: pool.wait() hard-synchronizes the callback shell's exit. + _engine->query_cache_delta_sync_pool().wait(); + flight.reset(); + EXPECT_EQ(ExecEnv::GetInstance()->query_cache_mem_tracker()->consumption(), qc_mem_baseline) + << "reaping the flight must release its charge back to the query-cache MemTracker"; +} + +TEST_F(QueryCacheCloudIncrementalTest, + presync_single_flight_reversed_order_maps_reasons_by_tablet) { + // The single-flight coalescing hazard the shared slots must survive: + // build_cache_key sorts tablet ids, so two runtimes whose scan ranges hold + // the same two tablets in a DIFFERENT order share one flight. The owner's + // fan-out writes the index-aligned slots in the OWNER's order; a follower + // that keyed the merged reasons off its OWN (reversed) order would misassign + // the failed tablet's reason to its sibling, leave the truly failed tablet + // unreasoned, and so drive a synchronous admission-thread get_tablet for it + // outside the fast-fail budget -- the exact brownout stall the presync + // exists to avoid. With the reasons keyed off the owner's tablet ids, the + // follower must reach the SAME decision as the owner: MISS, keyed to the + // failed tablet. (Pre-fix this test fails: the follower's fallback reason is + // a capture-side one for the sibling, not the owner's sync-failure reason.) + constexpr int64_t kTabletId2 = 15680; + + _sp->set_call_back("CloudMetaMgr::get_tablet_meta", [](auto&& args) { + auto tid = try_any_cast(args[0]); + TTabletSchema schema; + schema.keys_type = TKeysType::DUP_KEYS; + auto meta = std::make_shared( + 1, 2, tid, 15674, 4, 5, schema, 6, std::unordered_map {{7, 8}}, + UniqueId(9, 10), TTabletType::TABLET_TYPE_DISK, TCompressionType::LZ4F, 0, false); + meta->set_tablet_state(TABLET_RUNNING); + *try_any_cast(args[1]) = std::move(meta); + try_any_cast_ret(args)->second = true; + }); + std::mutex mu; + std::unordered_map calls; + std::atomic gate_open {false}; + std::atomic gated_sync_parked {false}; + _sp->set_call_back("CloudMetaMgr::sync_tablet_rowsets", [&](auto&& args) { + auto* tablet = try_any_cast(args[0]); + auto* ret = try_any_cast_ret(args); + ret->second = true; + int n; + { + std::lock_guard lk(mu); + n = calls[tablet->tablet_id()]++; + } + if (n == 0) { + // Loader: a view stopping short of 100 so the decision sync runs. + std::lock_guard lk(mu); + install_rowsets(tablet, {{0, 50}}); + return; + } + // Decision sync. The failing tablet parks first, so the follower joins + // the still-live flight before the fan-out settles; then it fails. The + // other tablet's decision sync succeeds. + if (tablet->tablet_id() == kTabletId2) { + gated_sync_parked = true; + while (!gate_open.load()) { + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + } + ret->first = Status::InternalError("injected sync failure"); + return; + } + std::lock_guard lk(mu); + install_rowsets(tablet, {{51, 100}}); + }); + + // One stale entry keyed over both tablets, queried at 100. The owner scans + // [kTabletId, kTabletId2]; the follower scans the SAME tablets REVERSED. + auto ranges_owner = make_scan_ranges(kTabletId, "100"); + ranges_owner.push_back(make_scan_ranges(kTabletId2, "100").front()); + auto ranges_follower = make_scan_ranges(kTabletId2, "100"); + ranges_follower.push_back(make_scan_ranges(kTabletId, "100").front()); + auto param = make_cache_param(kTabletId); + param.tablet_to_range.insert({kTabletId2, "range"}); + param.__set_allow_incremental(true); + std::string key; + int64_t version = 0; + EXPECT_TRUE(QueryCache::build_cache_key(ranges_owner, param, &key, &version).ok()); + // Sanity: the reversed order yields the identical key (so they coalesce). + std::string key_reversed; + EXPECT_TRUE(QueryCache::build_cache_key(ranges_follower, param, &key_reversed, &version).ok()); + ASSERT_EQ(key, key_reversed); + insert_entry(_cache.get(), key, 50, 0); + + QueryCacheRuntime runtime_owner(param, _cache.get()); + QueryCacheRuntime runtime_follower(param, _cache.get()); + + std::atomic follower_joined {false}; + _sp->set_call_back("QueryCacheRuntime::_presync_cloud_delta_tablets.follower_joined", + [&](auto&&) { follower_joined = true; }); + + std::shared_ptr d_owner; + std::shared_ptr d_follower; + std::thread t_owner; + std::thread t_follower; + Defer thread_guard {[&] { + gate_open = true; + if (t_owner.joinable()) { + t_owner.join(); + } + if (t_follower.joinable()) { + t_follower.join(); + } + }}; + + t_owner = std::thread([&] { d_owner = runtime_owner.get_or_make_decision(ranges_owner); }); + for (int i = 0; i < 10000 && !gated_sync_parked.load(); ++i) { + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + } + ASSERT_TRUE(gated_sync_parked.load()); + + t_follower = std::thread( + [&] { d_follower = runtime_follower.get_or_make_decision(ranges_follower); }); + for (int i = 0; i < 10000 && !follower_joined.load(); ++i) { + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + } + ASSERT_TRUE(follower_joined.load()); + + gate_open = true; + t_owner.join(); + t_follower.join(); + + // Both decisions fall the whole instance back, keyed to the SAME failed + // tablet: the owner surfaces kTabletId2's sync failure, and the reversed + // follower -- reading the owner-ordered slots by the owner's tablet ids -- + // surfaces the identical reason rather than misattributing it. + EXPECT_EQ(d_owner->mode, QueryCacheInstanceDecision::Mode::MISS); + EXPECT_EQ(d_owner->incremental_fallback_reason, "cloud rowset sync failed"); + EXPECT_EQ(d_follower->mode, QueryCacheInstanceDecision::Mode::MISS); + EXPECT_EQ(d_follower->incremental_fallback_reason, "cloud rowset sync failed"); + { + std::lock_guard lk(mu); + // ONE shared fan-out: each tablet synced its loader plus its single + // decision sync. The coalesced follower added no sync of its own. + EXPECT_EQ(calls[kTabletId], 2); + EXPECT_EQ(calls[kTabletId2], 2); + } + { + std::lock_guard lk(_cache->_presync_flights_lock); + EXPECT_TRUE(_cache->_presync_flights.empty()); + } +} + +TEST_F(QueryCacheCloudIncrementalTest, presync_single_flight_coalesces_mow_tablet) { + // Coalescing over a MERGE-ON-WRITE tablet: the owner's presync carries + // options.sync_delete_bitmap=true (a MoW read is delete-bitmap-sensitive), + // and a coalesced follower REUSES that one sync rather than issuing its own. + // Both must reach the SAME decision off the shared, bitmap-complete view. + // Covers the MoW x single-flight combination the DUP_KEYS coalesce tests do + // not: the DUP tests prove the registry mechanics, the single-runtime MoW + // tests prove the delete-bitmap capture, and this proves a follower riding + // the owner's bitmap sync classifies identically (clean bitmap + pure-append + // delta -> both INCREMENTAL). + _sp->set_call_back("CloudMetaMgr::get_tablet_meta", [](auto&& args) { + auto tid = try_any_cast(args[0]); + TTabletSchema schema; + schema.keys_type = TKeysType::UNIQUE_KEYS; + auto meta = std::make_shared( + 1, 2, tid, 15674, 4, 5, schema, 6, std::unordered_map {{7, 8}}, + UniqueId(9, 10), TTabletType::TABLET_TYPE_DISK, TCompressionType::LZ4F, 0, + /*enable_merge_on_write=*/true); + meta->set_tablet_state(TABLET_RUNNING); + *try_any_cast(args[1]) = std::move(meta); + try_any_cast_ret(args)->second = true; + }); + std::mutex mu; + std::unordered_map calls; + std::atomic gate_open {false}; + std::atomic gated_sync_parked {false}; + std::atomic saw_sync_delete_bitmap {false}; + _sp->set_call_back("CloudMetaMgr::sync_tablet_rowsets", [&](auto&& args) { + auto* tablet = try_any_cast(args[0]); + const auto* options = try_any_cast(args[1]); + auto* ret = try_any_cast_ret(args); + ret->second = true; + int n; + { + std::lock_guard lk(mu); + n = calls[tablet->tablet_id()]++; + } + if (n == 0) { + std::lock_guard lk(mu); + install_rowsets(tablet, {{0, 50}}); + return; + } + // Decision sync on a MoW tablet must carry the delete-bitmap sync. + if (options != nullptr && options->sync_delete_bitmap) { + saw_sync_delete_bitmap = true; + } + gated_sync_parked = true; + while (!gate_open.load()) { + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + } + std::lock_guard lk(mu); + install_rowsets(tablet, {{51, 100}}); + }); + + auto ranges = make_scan_ranges(kTabletId, "100"); + auto param = make_cache_param(kTabletId); + param.__set_allow_incremental(true); + std::string key; + int64_t version = 0; + EXPECT_TRUE(QueryCache::build_cache_key(ranges, param, &key, &version).ok()); + insert_entry(_cache.get(), key, 50, 0); + QueryCacheRuntime runtime_a(param, _cache.get()); + QueryCacheRuntime runtime_b(param, _cache.get()); + + std::atomic follower_joined {false}; + _sp->set_call_back("QueryCacheRuntime::_presync_cloud_delta_tablets.follower_joined", + [&](auto&&) { follower_joined = true; }); + + std::shared_ptr d1; + std::shared_ptr d2; + std::thread t1; + std::thread t2; + Defer thread_guard {[&] { + gate_open = true; + if (t1.joinable()) { + t1.join(); + } + if (t2.joinable()) { + t2.join(); + } + }}; + + t1 = std::thread([&] { d1 = runtime_a.get_or_make_decision(ranges); }); + for (int i = 0; i < 10000 && !gated_sync_parked.load(); ++i) { + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + } + ASSERT_TRUE(gated_sync_parked.load()); + t2 = std::thread([&] { d2 = runtime_b.get_or_make_decision(ranges); }); + for (int i = 0; i < 10000 && !follower_joined.load(); ++i) { + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + } + ASSERT_TRUE(follower_joined.load()); + + gate_open = true; + t1.join(); + t2.join(); + // Clean bitmap + pure-append delta: both coalesced MoW waiters go INCREMENTAL + // and, riding the same sync, reach the identical decision. + EXPECT_EQ(d1->mode, QueryCacheInstanceDecision::Mode::INCREMENTAL); + EXPECT_EQ(d2->mode, QueryCacheInstanceDecision::Mode::INCREMENTAL); + EXPECT_EQ(d1->mode, d2->mode); + EXPECT_EQ(d1->incremental_fallback_reason, d2->incremental_fallback_reason); + // The owner's shared decision sync carried the MoW delete-bitmap sync. + EXPECT_TRUE(saw_sync_delete_bitmap.load()); + { + std::lock_guard lk(mu); + // ONE shared fan-out: loader + the single gated decision sync. The + // follower rode the owner's MoW sync, adding none of its own. + EXPECT_EQ(calls[kTabletId], 2); + } + { + std::lock_guard lk(_cache->_presync_flights_lock); + EXPECT_TRUE(_cache->_presync_flights.empty()); + } +} + +TEST_F(QueryCacheCloudIncrementalTest, decision_sync_timeout_falls_back) { + // A meta-service brownout: the decision's rowset sync stalls past the + // fast-fail budget (query_cache_decision_sync_timeout_ms). The decision must + // abandon the wait and fall the query back to a full recompute rather than + // hold its (bounded) admission thread for the RPC retry budget. Made + // deterministic with a gate that keeps the decision sync blocked until AFTER + // the assertion, so the decision's wait_for is guaranteed to expire first; + // the test then releases it and waits for the detached sync to fully exit + // its callback before teardown, so the callback never runs after the fixture + // (and the std::function it runs in) is torn down. + int32_t saved_timeout = config::query_cache_decision_sync_timeout_ms; + config::query_cache_decision_sync_timeout_ms = 50; + Defer restore_timeout {[&] { config::query_cache_decision_sync_timeout_ms = saved_timeout; }}; + + std::atomic release_gate {false}; + std::atomic decision_sync_returned {false}; + _sp->set_call_back("CloudMetaMgr::get_tablet_meta", [](auto&& args) { + TTabletSchema schema; + schema.keys_type = TKeysType::DUP_KEYS; + auto meta = std::make_shared(1, 2, kTabletId, 15674, 4, 5, schema, 6, + std::unordered_map {{7, 8}}, + UniqueId(9, 10), TTabletType::TABLET_TYPE_DISK, + TCompressionType::LZ4F, 0, false); + meta->set_tablet_state(TABLET_RUNNING); + *try_any_cast(args[1]) = std::move(meta); + try_any_cast_ret(args)->second = true; + }); + _sp->set_call_back("CloudMetaMgr::sync_tablet_rowsets", + [this, &release_gate, &decision_sync_returned](auto&& args) { + auto* tablet = try_any_cast(args[0]); + auto* ret = try_any_cast_ret(args); + ret->second = true; + if (_sync_calls.fetch_add(1) == 0) { + // Loader sync: install a view that does NOT cover + // query 100, so the decision sync really runs. + install_rowsets(tablet, {{0, 50}}); + return; + } + // The decision's sync stalls until the test releases + // the gate (after it has asserted the fast-fail). + while (!release_gate.load()) { + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + } + // Last statement: signals the callback body is done, + // so the test can wait for it before teardown. + decision_sync_returned = true; + }); + + int64_t fallbacks_before = + DorisMetrics::instance()->query_cache_incremental_fallback_total->value(); + int64_t decision_sync_ms_before = + DorisMetrics::instance()->query_cache_decision_sync_time_ms->value(); + auto decision = make_stale_decision(); + EXPECT_EQ(decision->mode, QueryCacheInstanceDecision::Mode::MISS); + EXPECT_EQ(decision->incremental_fallback_reason, "cloud rowset sync timed out"); + EXPECT_EQ(DorisMetrics::instance()->query_cache_incremental_fallback_total->value(), + fallbacks_before + 1); + // The timeout path still records the wall time it blocked before giving up + // (steady_clock taken before the fan-out, read again after wait_for expires), + // so the sync-time counter advances by at least the budget. This is the one + // test that proves the metric is wired on the fast-fail path; wait_for is + // guaranteed not to return before its duration elapses, so >= budget is not + // a timing bet. + EXPECT_GE(DorisMetrics::instance()->query_cache_decision_sync_time_ms->value() - + decision_sync_ms_before, + config::query_cache_decision_sync_timeout_ms); + + // Release the stalled sync and wait for its callback to fully exit before + // teardown clears the sync points (destroying the callback while it runs is + // UB). The detached fan-out task reaches its decision sync eventually even + // on a slow host, so this converges. + release_gate = true; + for (int i = 0; i < 2000 && !decision_sync_returned.load(); ++i) { + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + } + ASSERT_TRUE(decision_sync_returned.load()); + // Both syncs ran once the fan-out joined: the loader and the (stalled) + // decision sync. The decision just did not wait for the second to finish. + EXPECT_EQ(_sync_calls.load(), 2); + // The decision deliberately abandons its fan-out future, so there is no + // handle to join the detached sync on. This margin covers the microsecond + // stack unwind of the callback and its SyncPoint shell after the last store + // above, plus the task's tail (releasing a non-last CloudTablet ref -- the + // tablet cache still holds one, so nothing is destroyed here -- and the + // fork-join bookkeeping). Drain it deterministically with pool.wait(), which hard- + // synchronizes the callback shell's exit, rather than betting a fixed sleep outlasts + // that tail before TearDown clears the sync points. + _engine->query_cache_delta_sync_pool().wait(); +} + } // namespace doris diff --git a/be/test/testutil/mock/mock_runtime_state.h b/be/test/testutil/mock/mock_runtime_state.h index 82c7d788943e4f..961c00f5daef14 100644 --- a/be/test/testutil/mock/mock_runtime_state.h +++ b/be/test/testutil/mock/mock_runtime_state.h @@ -69,6 +69,14 @@ class MockRuntimeState : public RuntimeState { void set_enable_strict_cast(bool enable) { _query_options.__set_enable_strict_cast(enable); } + void set_query_freshness_tolerance_ms(int64_t ms) { + _query_options.__set_query_freshness_tolerance_ms(ms); + } + + void set_enable_prefer_cached_rowset(bool enable) { + _query_options.__set_enable_prefer_cached_rowset(enable); + } + bool enable_local_exchange() const override { return true; } WorkloadGroupPtr workload_group() override { return _workload_group; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/normalize/QueryCacheNormalizer.java b/fe/fe-core/src/main/java/org/apache/doris/planner/normalize/QueryCacheNormalizer.java index 5212c5a5fd6533..a522dcb991a685 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/normalize/QueryCacheNormalizer.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/normalize/QueryCacheNormalizer.java @@ -24,6 +24,7 @@ import org.apache.doris.catalog.KeysType; import org.apache.doris.catalog.OlapTable; import org.apache.doris.catalog.Tablet; +import org.apache.doris.common.Config; import org.apache.doris.common.Pair; import org.apache.doris.planner.AggregationNode; import org.apache.doris.planner.ExchangeNode; @@ -100,6 +101,7 @@ private Optional setQueryCacheParam( queryCacheParam.setEntryMaxBytes(sessionVariable.getQueryCacheEntryMaxBytes()); queryCacheParam.setEntryMaxRows(sessionVariable.getQueryCacheEntryMaxRows()); queryCacheParam.setAllowIncremental(computeAllowIncremental(cachePoint, sessionVariable)); + queryCacheParam.setIsMergeOnWrite(computeIsMergeOnWrite(cachePoint)); queryCacheParam.setOutputSlotMapping( cachePoint.cacheRoot.getOutputTupleIds() @@ -198,20 +200,107 @@ private boolean computeAllowIncremental(CachePoint cachePoint, SessionVariable s long selectIndexId = scanNode.getSelectedIndexId() == -1 ? olapTable.getBaseIndexId() : scanNode.getSelectedIndexId(); - // Note: judged on the selected index, not the base table: a DUP table - // may serve the query from an aggregated materialized view, whose data - // is no longer append-only. + // The scanned index must be append-only, guaranteeing "cached snapshot + + // delta rowsets == new snapshot". Note: judged on the selected index, not + // the base table -- a DUP table may serve the query from an aggregated + // materialized view, whose data is no longer append-only. + // + // DUP_KEYS is always append-only. A merge-on-write UNIQUE index is + // append-only as long as a load does not touch pre-existing keys, which + // covers the common "hourly append plus occasional backfill" pattern: BE + // verifies per tablet through the delete bitmap of the delta window and + // falls back to a full recompute for the rare load that rewrites history. + // A merge-on-read UNIQUE index resolves duplicates by merging across + // rowsets at read time, so a delta-only scan cannot stand alone there; + // AGG tables merge rows inside the storage layer likewise. KeysType keysType = olapTable.getKeysTypeByIndexId(selectIndexId); - if (keysType == KeysType.DUP_KEYS) { + boolean mergeOnWrite = + keysType == KeysType.UNIQUE_KEYS && olapTable.getEnableUniqueKeyMergeOnWrite(); + if (keysType != KeysType.DUP_KEYS && !mergeOnWrite) { + return false; + } + // Freshness tolerance and prefer-cached-rowset (both cloud-only in effect) + // trade exactness for speed and locality: the scan may read a warmed-up layout + // that stops below the queried version (freshness halts at the warmed boundary) + // or reaches beyond it (neither walk clips an edge spanning it). An incremental + // merge would defeat such a knob, because the delta capture always targets the + // exact queried version (it must, to keep the merged entry correct), forcing the + // un-warmed reads the query chose to skip. The suppression is mode-gated + // (Config.isCloudMode()) through a pure helper: these knobs are INERT on local + // storage, so suppressing incremental there would only forgo it for no + // correctness benefit (the local read is version-exact regardless). Correctness + // against version-inexact reads does NOT rest on this per-query gate anyway (an + // entry filled by such a read could still be reused by a knob-free query sharing + // the cache key): on cloud the BE suppresses their write-back, so no entry whose + // content mismatches its version stamp exists. Extracting the pure, + // mode-parameterized helper is what lets the unit test exercise BOTH modes + // directly, without planning under a flipped cloud flag (which casts + // SystemInfoService to CloudSystemInfoService and fails). + if (cloudKnobsSuppressIncremental(Config.isCloudMode(), + sessionVariable.getQueryFreshnessToleranceMs(), + sessionVariable.getEnablePreferCachedRowset(), mergeOnWrite)) { + return false; + } + return true; + } + + // Whether the cloud warmed-read knobs suppress incremental merge for this query. + // Pure and mode-parameterized (see the call site for why): only cloud mode honors + // freshness-tolerance / prefer-cached-rowset, so in shared-nothing mode they are + // inert and never suppress. Mirrors the BE gate + // QueryCacheRuntime::cloud_knobs_suppress_incremental, plus the cloud-mode guard. + @VisibleForTesting + public static boolean cloudKnobsSuppressIncremental(boolean cloudMode, + long queryFreshnessToleranceMs, boolean enablePreferCachedRowset, + boolean mergeOnWrite) { + if (!cloudMode) { + return false; + } + // Freshness tolerance may read a layout below the queried version, so an + // incremental merge (which targets the exact version) would defeat it: suppress + // for every table type. + if (queryFreshnessToleranceMs > 0) { return true; } - // A merge-on-write UNIQUE index is append-only as long as a load does - // not touch pre-existing keys, which covers the common "hourly append - // plus occasional backfill" pattern: BE verifies per tablet through - // the delete bitmap of the delta window and falls back to a full - // recompute for the rare load that rewrites history. A merge-on-read - // UNIQUE index resolves duplicates by merging across rowsets at read - // time, so a delta-only scan cannot stand alone there. + // Prefer-cached-rowset is honored by the storage layer only for non-MOW tables + // (CloudTablet::capture_consistent_versions_unlocked guards on + // !enable_unique_key_merge_on_write()), so a MOW query reads the exact queried + // version regardless of the knob and stays incrementally mergeable; only non-MOW + // is suppressed. + return enablePreferCachedRowset && !mergeOnWrite; + } + + // Whether the selected index of the cache point's underlying scan is a merge-on- + // write UNIQUE table. BE consumes this in the cloud cache write-back gate: cloud + // ignores enable_prefer_cached_rowset for a MOW table, so a prefer-only MOW read is + // version-exact and its fill must not be suppressed. Reported independently of + // allow_incremental because the write-back gate also applies to a MISS (a MOW MISS + // under prefer must still be cached). Determined on the selected index, mirroring + // computeAllowIncremental. + // + // Resolves the scan through the (possibly multi-level) unary chain, not just the + // direct child: incremental merge requires a DIRECT-child scan (a nested agg is not + // mergeable, see computeAllowIncremental), but this write-back gate only needs the + // scan's TABLE TYPE, which is well-defined for a nested-agg cache point too. Reading + // it through the chain lets a nested-agg-over-MOW query populate the cache under + // prefer (its read is still version-exact) instead of being over-suppressed just + // because the scan is not the direct child. Returns false if the chain branches or + // bottoms out at a non-scan (no single underlying olap scan), leaving BE at its safe + // suppress default. + private boolean computeIsMergeOnWrite(CachePoint cachePoint) { + PlanNode node = cachePoint.cacheRoot.getChild(0); + while (node != null && !(node instanceof OlapScanNode) && node.getChildren().size() == 1) { + node = node.getChild(0); + } + if (!(node instanceof OlapScanNode)) { + return false; + } + OlapScanNode scanNode = (OlapScanNode) node; + OlapTable olapTable = scanNode.getOlapTable(); + long selectIndexId = scanNode.getSelectedIndexId() == -1 + ? olapTable.getBaseIndexId() + : scanNode.getSelectedIndexId(); + KeysType keysType = olapTable.getKeysTypeByIndexId(selectIndexId); return keysType == KeysType.UNIQUE_KEYS && olapTable.getEnableUniqueKeyMergeOnWrite(); } diff --git a/fe/fe-core/src/test/java/org/apache/doris/planner/QueryCacheNormalizerTest.java b/fe/fe-core/src/test/java/org/apache/doris/planner/QueryCacheNormalizerTest.java index d7cb0e7d94c434..2ce51c9e6041cc 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/planner/QueryCacheNormalizerTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/planner/QueryCacheNormalizerTest.java @@ -124,6 +124,19 @@ protected void runBeforeAll() throws Exception { + "distributed by hash(k1) buckets 3\n" + "properties('replication_num' = '1', 'enable_unique_key_merge_on_write' = 'true')"; + // Merge-on-write with a COMPOSITE key so the distribution column k1 is not + // unique: grouping the inner agg by k1 (the distribution column) stays a + // real, colocate (single-phase, no shuffle) aggregation, so the outer agg + // nests above it inside the scan fragment. That produces a genuinely nested + // cache point over a MOW scan, which the nested-MOW test below needs. + String uniqueMowMultiTable = "create table db1.uniq_mow_multi(" + + " k1 int,\n" + + " k2 int,\n" + + " v1 int)\n" + + "UNIQUE KEY(k1, k2)\n" + + "distributed by hash(k1) buckets 3\n" + + "properties('replication_num' = '1', 'enable_unique_key_merge_on_write' = 'true')"; + String uniqueMorTable = "create table db1.uniq_mor(" + " k1 int,\n" + " v1 int)\n" @@ -139,7 +152,7 @@ protected void runBeforeAll() throws Exception { + "properties('replication_num' = '1')"; createTables(nonPart, part1, part2, multiLeveParts, variantTable, uniqueMowTable, - uniqueMorTable, aggTable); + uniqueMowMultiTable, uniqueMorTable, aggTable); connectContext.getSessionVariable().setDisableNereidsRules("PRUNE_EMPTY_PARTITION"); connectContext.getSessionVariable().setEnableQueryCache(true); @@ -420,6 +433,9 @@ public void testAllowIncremental() throws Exception { TQueryCacheParam dupTwoPhase = getQueryCacheParam( "select k2, sum(v1) as v from db1.part1 group by k2"); Assertions.assertTrue(dupTwoPhase.allow_incremental); + // A DUP table is not merge-on-write: the flag BE reads for its cloud + // write-back gate stays false. + Assertions.assertFalse(dupTwoPhase.is_merge_on_write); // One-phase aggregation finalizes inside the cached fragment: its // output has no downstream merge, so emitting cached and delta @@ -436,12 +452,17 @@ public void testAllowIncremental() throws Exception { TQueryCacheParam uniqueMow = getQueryCacheParam( "select v1, count(*) as v from db1.uniq_mow group by v1"); Assertions.assertTrue(uniqueMow.allow_incremental); + // FE reports the merge-on-write table type so BE can keep the cloud + // write-back for a prefer-only MOW read (which stays version-exact). + Assertions.assertTrue(uniqueMow.is_merge_on_write); // UNIQUE merge-on-read resolves duplicates by merging across // rowsets at read time: a delta-only scan cannot stand alone. TQueryCacheParam uniqueMor = getQueryCacheParam( "select v1, count(*) as v from db1.uniq_mor group by v1"); Assertions.assertFalse(uniqueMor.allow_incremental); + // Merge-on-read UNIQUE is not merge-on-write: the flag is false. + Assertions.assertFalse(uniqueMor.is_merge_on_write); // AGG_KEYS merges rows inside the storage layer. TQueryCacheParam aggKeys = getQueryCacheParam( @@ -465,11 +486,96 @@ public void testAllowIncremental() throws Exception { + " (select k1, count(*) cnt from db1.non_part group by k1) x" + " group by cnt"); Assertions.assertFalse(nestedAgg.allow_incremental); + + // A genuinely nested cache point over a MOW scan. The inner agg groups by + // the distribution column k1 (non-unique here, since the key is (k1, k2)), + // so it is a local single-phase aggregation with no shuffle, and the outer + // agg's partial nests above it inside the scan fragment. The cache point is + // therefore the OUTER agg, whose direct child is the inner agg rather than + // the scan, so incremental is off (nested). But the write-back gate must + // still resolve the scan's table type through the unary chain, so + // is_merge_on_write is true -- letting a nested-agg-over-MOW query populate + // the cache under prefer instead of being over-suppressed just because the + // scan is not the direct child. (Pre-fix, the direct-child-only check + // reported false here.) The direct (non-nested) MOW case, where the cache + // point sits right on the scan and incremental IS allowed, is covered by + // uniqueMow above. + TQueryCacheParam nestedMow = getQueryCacheParam( + "select cnt, count(*) from" + + " (select k1, count(*) cnt from db1.uniq_mow_multi group by k1) x" + + " group by cnt"); + Assertions.assertFalse(nestedMow.allow_incremental); + Assertions.assertTrue(nestedMow.is_merge_on_write); + + // Query freshness tolerance and prefer-cached-rowset are cloud-only knobs; + // in the local (shared-nothing) harness they are inert, so the mode-gated + // suppression no longer forgoes incremental for an otherwise-eligible query + // (the local read is version-exact regardless). The cloud-mode suppression + // itself is verified directly on the pure helper in + // testCloudKnobsSuppressIncremental below, since this planning harness cannot + // flip to cloud mode (the cast to CloudSystemInfoService fails). + connectContext.getSessionVariable().queryFreshnessToleranceMs = 5000; + try { + TQueryCacheParam withFreshness = getQueryCacheParam( + "select k2, sum(v1) as v from db1.part1 group by k2"); + Assertions.assertTrue(withFreshness.allow_incremental); + } finally { + connectContext.getSessionVariable().queryFreshnessToleranceMs = -1; + } + + connectContext.getSessionVariable().enablePreferCachedRowset = true; + try { + TQueryCacheParam withPreferCached = getQueryCacheParam( + "select k2, sum(v1) as v from db1.part1 group by k2"); + Assertions.assertTrue(withPreferCached.allow_incremental); + // The MOW carve-out is mode-independent (a MOW read is version-exact + // whether the knob is inert locally or explicitly ignored on cloud), and + // its is_merge_on_write flag -- BE's write-back gate -- is reported + // regardless of mode. + TQueryCacheParam mowWithPreferCached = getQueryCacheParam( + "select v1, count(*) as v from db1.uniq_mow group by v1"); + Assertions.assertTrue(mowWithPreferCached.allow_incremental); + Assertions.assertTrue(mowWithPreferCached.is_merge_on_write); + } finally { + connectContext.getSessionVariable().enablePreferCachedRowset = false; + } } finally { connectContext.getSessionVariable().setEnableQueryCacheIncremental(false); } } + @Test + public void testCloudKnobsSuppressIncremental() { + // Local (shared-nothing) mode: the cloud warmed-read knobs are inert and never + // suppress incremental, whatever their values -- so a local query that happens + // to set them keeps incremental. This is the mode the planning harness above + // runs in; here the mode-gate itself is pinned directly. + Assertions.assertFalse( + QueryCacheNormalizer.cloudKnobsSuppressIncremental(false, 5000, false, false)); + Assertions.assertFalse( + QueryCacheNormalizer.cloudKnobsSuppressIncremental(false, 0, true, false)); + Assertions.assertFalse( + QueryCacheNormalizer.cloudKnobsSuppressIncremental(false, 5000, true, true)); + // Cloud mode, freshness tolerance active: suppresses for every table type. + Assertions.assertTrue( + QueryCacheNormalizer.cloudKnobsSuppressIncremental(true, 5000, false, false)); + Assertions.assertTrue( + QueryCacheNormalizer.cloudKnobsSuppressIncremental(true, 5000, false, true)); + Assertions.assertTrue( + QueryCacheNormalizer.cloudKnobsSuppressIncremental(true, 5000, true, true)); + // Cloud mode, prefer-cached-rowset active: suppresses only non-MOW (a MOW read + // stays version-exact, cloud ignores the knob for it). + Assertions.assertTrue( + QueryCacheNormalizer.cloudKnobsSuppressIncremental(true, 0, true, false)); + Assertions.assertFalse( + QueryCacheNormalizer.cloudKnobsSuppressIncremental(true, 0, true, true)); + // Cloud mode, no knob active: never suppresses. + Assertions.assertFalse( + QueryCacheNormalizer.cloudKnobsSuppressIncremental(true, 0, false, false)); + Assertions.assertFalse( + QueryCacheNormalizer.cloudKnobsSuppressIncremental(true, 0, false, true)); + } + private String getDigest(String sql) throws Exception { return Hex.encodeHexString(getQueryCacheParam(sql).digest); } diff --git a/gensrc/thrift/QueryCache.thrift b/gensrc/thrift/QueryCache.thrift index 67c31cd4bc6120..eaab618ae10e7b 100644 --- a/gensrc/thrift/QueryCache.thrift +++ b/gensrc/thrift/QueryCache.thrift @@ -77,4 +77,15 @@ struct TQueryCacheParam { // cannot be captured (e.g. merged away by compaction), when the delta // contains delete predicates, or when the delta rewrites history rows. 8: optional bool allow_incremental + + // Whether the scanned index is a merge-on-write UNIQUE table. BE needs this + // for the cloud write-back gate: enable_prefer_cached_rowset lets the storage + // layer read a warmed-up layout that may not match the queried version, so BE + // suppresses the cache write-back of such a read. But on cloud that knob is + // ignored for a merge-on-write table (CloudTablet::capture_consistent_ + // versions_unlocked guards it on !enable_unique_key_merge_on_write()), so a + // MOW query still reads the exact version and its fill is safe to cache. FE + // reports the table type here (on the selected index) so BE can keep the + // write-back for a prefer-only MOW query instead of forgoing its cache. + 9: optional bool is_merge_on_write } diff --git a/regression-test/suites/query_p0/cache/query_cache_incremental.groovy b/regression-test/suites/query_p0/cache/query_cache_incremental.groovy index 9ee2d876550e3c..831e303650ddc5 100644 --- a/regression-test/suites/query_p0/cache/query_cache_incremental.groovy +++ b/regression-test/suites/query_p0/cache/query_cache_incremental.groovy @@ -20,14 +20,17 @@ // delta rowsets since the cached version and merging them with the cached // partial aggregation blocks. Every query below is checked against the same // query with the cache disabled, so results stay verified even where -// incremental merge falls back to a full recompute (e.g. cloud mode). On -// local storage the suite additionally proves through the BE metrics that -// the incremental path really fires: the early stale queries must increase -// query_cache_stale_hit_total, and the two designed fallback phases (a -// delete predicate in the delta, a merge-on-write backfill that rewrites -// history) must increase query_cache_incremental_fallback_total. The final -// phase covers a long-dormant entry whose first reuse faces a large, -// many-rowset delta that goes through the parallel scanner builder. +// incremental merge falls back to a full recompute (e.g. when the delta was +// merged away by a compaction). The suite additionally proves through the BE +// metrics that the incremental path really fires: the early stale queries +// must increase query_cache_stale_hit_total, and the two designed fallback +// phases (a delete predicate in the delta, a merge-on-write backfill that +// rewrites history) must increase query_cache_incremental_fallback_total. +// The final phase covers a long-dormant entry whose first reuse faces a +// large, many-rowset delta that goes through the parallel scanner builder. +// This holds in cloud mode too: the decision first brings the tablet view up +// to the queried version (usually a no-op, the scan would sync anyway) and +// then takes the same incremental path as local storage. suite("query_cache_incremental") { def querySql = """ SELECT @@ -72,16 +75,38 @@ suite("query_cache_incremental") { return total } - // Besides result consistency, prove on local storage that the stale entry - // was really merged incrementally (Mode::INCREMENTAL), not recomputed: - // only the incremental path increases query_cache_stale_hit_total, and no + // Besides result consistency, prove that the stale entry was really + // merged incrementally (Mode::INCREMENTAL), not recomputed: only the + // incremental path increases query_cache_stale_hit_total, and no // concurrent suite touches it because the session switch defaults to off. - // Cloud mode always falls back by design, so it only checks consistency. - // Residual caveat (here and in checkIncrementalFallback): a - // memory-pressure prune evicting the entry inside the tiny fill-to-assert - // window surfaces as a plain MISS and would fail the assertion; accepted - // as rare, and a rerun re-establishes the counters from fresh deltas. + // Compaction is kept out of the way in every mode: every table here sets + // disable_auto_compaction, so no background compaction can merge the delta + // away mid-window and turn the expected incremental hit into a fallback + // (this matters most on cloud, where compaction is driven externally). + // + // The stale_hit assertion runs on non-cloud only. Summed across BEs it is a + // deterministic INCREMENTAL signal solely when the fill and the reuse land + // on the same BE, which holds on the single-BE local cluster but not on + // multi-BE cloud. Two cloud channels flip it with no product-correctness + // meaning: a transient meta-service hiccup during the decision's view sync + // turns the expected stale hit into a full-recompute fallback (stale_hit + // stays flat, the fallback counter moves instead), and because the entry is + // BE-local a query routed to a BE that does not hold it recomputes in full + // there. A retry repairs neither: a fallback re-bases nothing, and a + // no-entry BE fills at the current version so its later runs are exact hits, + // never the stale hit. Cloud still exercises the whole incremental flow + // through checkConsistency (end-to-end result correctness on a real cloud + // cluster); that the cloud decision truly reaches Mode::INCREMENTAL is + // proven deterministically by the BE unit tests (QueryCacheCloudIncremental + // Test), which drive the cloud sync-then-capture path without cluster + // routing in play. A local memory-pressure prune evicting the entry inside + // the tiny fill-to-assert window is the one accepted residual (rare; a rerun + // re-establishes the counters). def checkStaleIncremental = { String sqlText -> + // On cloud the metric endpoints are not even sampled: a slow or briefly + // unreachable BE metrics port would hang or throw in sumBeMetric before + // a merely gated assertion, re-introducing the very topology-dependent + // flakiness this gate exists to remove (and the samples would go unused). if (isCloudMode()) { checkConsistency(sqlText) return @@ -93,8 +118,17 @@ suite("query_cache_incremental") { "expected a stale incremental cache hit, but query_cache_stale_hit_total stayed at ${before}") } - // Prove that a designed fallback phase really took the fallback path. + // Prove that a designed fallback phase really took the fallback path. Like the + // stale_hit assertion, the metric check is restricted to non-cloud: the entry + // is BE-local, so on a multi-BE cloud cluster the query can route to a BE that + // does not hold the stale entry, take a plain MISS (which is not a fallback, + // so the counter does not move), and spuriously fail. Cloud still verifies the + // fallback phase's result correctness through checkConsistency; that the cloud + // fallback classification itself fires is proven deterministically by the BE + // unit tests (QueryCacheCloudIncrementalTest.mow_history_rewrite_falls_back, + // fallback_on_sync_failure, and the delete-predicate/version-gap cases). def checkIncrementalFallback = { String sqlText -> + // Cloud skips the sampling too, for the same reason as above. if (isCloudMode()) { checkConsistency(sqlText) return @@ -131,7 +165,8 @@ suite("query_cache_incremental") { DISTRIBUTED BY HASH(user_id) BUCKETS 3 PROPERTIES ( - "replication_num" = "1" + "replication_num" = "1", + "disable_auto_compaction" = "true" ) """ @@ -168,13 +203,15 @@ suite("query_cache_incremental") { ('2026-01-13',${100 + i},'/inc',${10 * i}) """ if (i == 1) { - // The hot partition holds just two data rowsets here, far from - // both the merge-count threshold and any compaction, so on local - // storage this stale query must take the incremental path. + // The hot partition holds just two data rowsets here, well below + // the merge-count threshold, and auto-compaction is disabled on the + // table, so this stale query must take the incremental path on both + // deployment modes. checkStaleIncremental(querySql) } else { - // Later rounds may legitimately recompute in full (merge-count - // threshold, background compaction), so assert correctness only. + // Later rounds may legitimately recompute in full (the merge-count + // threshold forces one every query_cache_max_incremental_merge_count + // merges), so assert correctness only. checkConsistency(querySql) } } @@ -215,7 +252,8 @@ suite("query_cache_incremental") { PROPERTIES ( "replication_num" = "1", - "enable_unique_key_merge_on_write" = "true" + "enable_unique_key_merge_on_write" = "true", + "disable_auto_compaction" = "true" ) """ def uniqueQuerySql = """ @@ -249,9 +287,11 @@ suite("query_cache_incremental") { // entry. sql "INSERT INTO test_query_cache_incremental_mow VALUES ('2026-01-01',1,100)" checkIncrementalFallback(uniqueQuerySql) - // After the re-base, pure appends take the incremental path again (not - // asserted through the metric: by now enough rowsets piled up that a - // background compaction could legitimately force a full recompute). + // After the re-base, pure appends take the incremental path again. Left as + // a correctness-only check (not metric-asserted): the earlier rounds already + // pin the incremental path through the metric on non-cloud, so repeating a + // stale_hit assertion here would add nothing while carrying the same cloud + // caveats that already make checkStaleIncremental result-only there. sql "INSERT INTO test_query_cache_incremental_mow VALUES ('2026-01-06',200,7)" checkConsistency(uniqueQuerySql) order_qt_mow_final "${uniqueQuerySql}" @@ -285,7 +325,8 @@ suite("query_cache_incremental") { DISTRIBUTED BY HASH(user_id) BUCKETS 1 PROPERTIES ( - "replication_num" = "1" + "replication_num" = "1", + "disable_auto_compaction" = "true" ) """ def dormantQuerySql = """ @@ -312,10 +353,11 @@ suite("query_cache_incremental") { sql "INSERT INTO test_query_cache_incremental_dormant VALUES ${values}" } // The first query after the idle stretch must still take the incremental - // path on local storage and agree with the uncached result. The loads - // landed moments ago, so a boundary-crossing base compaction inside this - // window is as unlikely as in the first-round assertions above (an - // in-window cumulative compaction keeps the capture, and therefore the - // assertion, intact). + // path on both deployment modes and agree with the uncached result. + // Auto-compaction is disabled on the table, so the 30-rowset delta stays + // intact and no boundary-crossing compaction can turn this into a fallback. + // checkStaleIncremental asserts the stale_hit metric on non-cloud (where it + // is deterministic) and verifies result consistency on cloud, so no + // cluster-routing or transient-sync flake reaches this assertion. checkStaleIncremental(dormantQuerySql) }