From 860115c67698d3a81716cea34980303ad8a726cc Mon Sep 17 00:00:00 2001 From: Benedict Jin Date: Sun, 19 Jul 2026 14:45:20 +0800 Subject: [PATCH 1/4] [feature](query-cache) Extend stale-entry incremental merge to cloud mode The base incremental merge (#65482) only ran in the shared-nothing (local storage) deployment. This extends it to the compute-storage decoupled (cloud) mode. In cloud mode a tablet's rowset list, and for merge-on-write tables its delete bitmap, are a lazily synced copy of the meta service. The incremental decision runs in operator init, before the scan node's own sync round, so it must first bring each stale tablet's local view up to the queried version and only then capture the delta. Decision path: - _presync_cloud_delta_tablets fans the per-tablet meta-service syncs out in parallel (reusing init_scanner_sync_rowsets_parallelism) and waits on them with a fast-fail budget instead of blocking. The decision runs on the bounded query-admission pool (light_work_pool), so a meta-service brownout that stalled these RPCs for the full retry budget could exhaust the pool and reject query admission cluster-wide. If the fan-out does not finish within query_cache_decision_sync_timeout_ms (new cloud BE config, default 2000, changeable at runtime) the decision abandons the wait and falls the whole instance back to a full recompute. A value <= 0 disables cloud incremental merge as a fail-safe. - The sync brings the delete bitmap up to the queried version (sync_delete_bitmap=true), so the history-rewrite classification reads exactly what the merge-on-write read path sees. The later scan-node sync takes the same no-op shortcut (its local view is already at the queried version), so it issues no extra meta-service RPC and the RPC count per query does not grow. - While the classification reads the cached-side delete bitmap, the cached rowsets are pinned so a concurrent unused-rowset GC cannot retire the evidence. Both engines drop a retired rowset's bitmap only once nothing else references it, so raising the use count blocks the drop in either deployment mode. Correctness guard: a query that reads a version-inexact view (query_freshness_tolerance_ms or enable_prefer_cached_rowset) does not use incremental merge, since the delta capture always targets the exact queried version. The exclusion is a deliberately mode-agnostic gate. On cloud such a query also skips the query cache write-back, so no entry whose content mismatches its version stamp is ever created. Observability: new counter query_cache_decision_sync_time_ms records the wall time the decision spent syncing tablet views before capturing the delta. It only advances in cloud mode. Tests: BE unit tests cover the cloud decision paths (sync-then-capture, the timeout fallback, the merge-on-write bitmap rewrite check, the non-cloud-tablet and load-failure fallbacks, the no-op shortcut, and the write-back suppression). The FE QueryCacheNormalizerTest covers the freshness and prefer-cached gate. The query_cache_incremental regression suite now runs in cloud mode too, with auto-compaction disabled on its tables so the stale-hit metric assertions stay deterministic. --- be/src/cloud/cloud_meta_mgr.cpp | 9 +- be/src/cloud/config.cpp | 1 + be/src/cloud/config.h | 17 + be/src/common/metrics/doris_metrics.cpp | 4 + be/src/common/metrics/doris_metrics.h | 5 + .../exec/operator/cache_source_operator.cpp | 26 +- be/src/runtime/query_cache/query_cache.cpp | 277 ++++++- be/src/runtime/query_cache/query_cache.h | 19 +- .../operator/query_cache_operator_test.cpp | 159 ++++ be/test/exec/pipeline/query_cache_test.cpp | 687 +++++++++++++++++- be/test/testutil/mock/mock_runtime_state.h | 8 + .../normalize/QueryCacheNormalizer.java | 27 + .../planner/QueryCacheNormalizerTest.java | 31 + .../cache/query_cache_incremental.groovy | 86 ++- 14 files changed, 1263 insertions(+), 93 deletions(-) diff --git a/be/src/cloud/cloud_meta_mgr.cpp b/be/src/cloud/cloud_meta_mgr.cpp index a582f83991e7bf..98f896ea61cef8 100644 --- a/be/src/cloud/cloud_meta_mgr.cpp +++ b/be/src/cloud/cloud_meta_mgr.cpp @@ -668,7 +668,14 @@ Status CloudMetaMgr::sync_tablet_rowsets_unlocked(CloudTablet* tablet, SyncRowsetStats* sync_stats) { using namespace std::chrono; - TEST_SYNC_POINT_RETURN_WITH_VALUE("CloudMetaMgr::sync_tablet_rowsets", Status::OK(), tablet); + // Forward `options` so a test callback can react to sync_delete_bitmap the + // way the real body does below (the delete-bitmap sync is gated on it), i.e. + // a mock can drop the bitmap when the flag is false. The pointer is appended + // before the macro's own return-value slot, which callbacks read via + // args.back(), so existing callbacks that only read args[0]/the ret pair are + // unaffected. + TEST_SYNC_POINT_RETURN_WITH_VALUE("CloudMetaMgr::sync_tablet_rowsets", Status::OK(), tablet, + &options); DBUG_EXECUTE_IF("CloudMetaMgr::sync_tablet_rowsets.before.inject_error", { auto target_tablet_id = dp->param("tablet_id", -1); auto target_table_id = dp->param("table_id", -1); diff --git a/be/src/cloud/config.cpp b/be/src/cloud/config.cpp index f03f28c0c85a37..54008ddc2fb655 100644 --- a/be/src/cloud/config.cpp +++ b/be/src/cloud/config.cpp @@ -38,6 +38,7 @@ DEFINE_Int64(tablet_cache_shards, "16"); DEFINE_mInt32(tablet_sync_interval_s, "1800"); DEFINE_mInt32(init_scanner_sync_rowsets_parallelism, "10"); DEFINE_mInt32(sync_rowsets_slow_threshold_ms, "1000"); +DEFINE_mInt32(query_cache_decision_sync_timeout_ms, "2000"); DEFINE_mInt64(min_compaction_failure_interval_ms, "5000"); DEFINE_mInt64(base_compaction_freeze_interval_s, "1800"); diff --git a/be/src/cloud/config.h b/be/src/cloud/config.h index 63355451c8c218..02c0c1a6de38de 100644 --- a/be/src/cloud/config.h +++ b/be/src/cloud/config.h @@ -70,6 +70,23 @@ DECLARE_mInt32(tablet_sync_interval_s); // parallelism for scanner init where may issue RPCs to sync rowset meta from MS DECLARE_mInt32(init_scanner_sync_rowsets_parallelism); DECLARE_mInt32(sync_rowsets_slow_threshold_ms); +// Fast-fail budget (ms) for the query-cache incremental-merge decision's +// pre-sync rowset fan-out. That decision runs in operator init on a bounded +// query-admission thread pool (the BE light_work_pool, contractually "must be +// light, not locked"), so a meta-service brownout that stalls the sync must not +// hold that thread for the full RPC retry budget (tens of seconds): sustained, +// it would exhaust the pool and reject query admission cluster-wide. When the +// fan-out does not finish within this budget the decision abandons the wait and +// falls back to one full recompute (the scan node's own async sync still brings +// the view up for the actual scan). A healthy sync is milliseconds, far under +// this, so it only trips under real meta-service degradation and leaves the +// steady-state incremental path unchanged. A value <= 0 makes the wait expire +// immediately, so the decision falls back to a full recompute whenever a sync +// is still in flight (a fan-out that already finished -- every tablet a no-op +// -- still proceeds correctly): that disables cloud incremental merge under any +// real sync latency rather than blocking, a fail-safe, not a useful setting. +// Cloud only. +DECLARE_mInt32(query_cache_decision_sync_timeout_ms); // Cloud compaction config DECLARE_mInt64(min_compaction_failure_interval_ms); diff --git a/be/src/common/metrics/doris_metrics.cpp b/be/src/common/metrics/doris_metrics.cpp index de596407e512b3..6b789749c5c75d 100644 --- a/be/src/common/metrics/doris_metrics.cpp +++ b/be/src/common/metrics/doris_metrics.cpp @@ -57,6 +57,9 @@ DEFINE_COUNTER_METRIC_PROTOTYPE_3ARG(query_cache_incremental_fallback_total, Met DEFINE_COUNTER_METRIC_PROTOTYPE_3ARG(query_cache_write_back_total, MetricUnit::REQUESTS, "Query cache entries handed to the cache to be written back " "(full or merged); admission may still turn one down."); +DEFINE_COUNTER_METRIC_PROTOTYPE_3ARG(query_cache_decision_sync_time_ms, MetricUnit::MILLISECONDS, + "Cumulative wall time the cloud incremental-merge decision " + "spent blocking on its pre-sync rowset fan-out."); DEFINE_COUNTER_METRIC_PROTOTYPE_5ARG(push_requests_success_total, MetricUnit::REQUESTS, "", push_requests_total, Labels({{"status", "SUCCESS"}})); DEFINE_COUNTER_METRIC_PROTOTYPE_5ARG(push_requests_fail_total, MetricUnit::REQUESTS, "", @@ -303,6 +306,7 @@ DorisMetrics::DorisMetrics() : _metric_registry(_s_registry_name) { INT_COUNTER_METRIC_REGISTER(_server_metric_entity, query_cache_stale_hit_total); INT_COUNTER_METRIC_REGISTER(_server_metric_entity, query_cache_incremental_fallback_total); INT_COUNTER_METRIC_REGISTER(_server_metric_entity, query_cache_write_back_total); + INT_COUNTER_METRIC_REGISTER(_server_metric_entity, query_cache_decision_sync_time_ms); INT_COUNTER_METRIC_REGISTER(_server_metric_entity, push_requests_success_total); INT_COUNTER_METRIC_REGISTER(_server_metric_entity, push_requests_fail_total); diff --git a/be/src/common/metrics/doris_metrics.h b/be/src/common/metrics/doris_metrics.h index 1679f28d775a8d..47c9806dc3cf1b 100644 --- a/be/src/common/metrics/doris_metrics.h +++ b/be/src/common/metrics/doris_metrics.h @@ -64,6 +64,11 @@ class DorisMetrics { IntCounter* query_cache_stale_hit_total = nullptr; IntCounter* query_cache_incremental_fallback_total = nullptr; IntCounter* query_cache_write_back_total = nullptr; + // Cumulative wall time the cloud incremental-merge decision spent blocking + // on its pre-sync rowset fan-out. That sync runs during operator init, not + // inside a profiled scan node, so without this counter its (potentially + // meta-service-bound) latency is invisible. + IntCounter* query_cache_decision_sync_time_ms = nullptr; IntCounter* push_requests_success_total = nullptr; IntCounter* push_requests_fail_total = nullptr; diff --git a/be/src/exec/operator/cache_source_operator.cpp b/be/src/exec/operator/cache_source_operator.cpp index d83c83788c0eb7..83936946cc394d 100644 --- a/be/src/exec/operator/cache_source_operator.cpp +++ b/be/src/exec/operator/cache_source_operator.cpp @@ -20,13 +20,14 @@ #include #include +#include "cloud/config.h" #include "common/status.h" #include "core/block/block.h" #include "exec/operator/operator.h" #include "exec/pipeline/dependency.h" +#include "runtime/runtime_state.h" namespace doris { -class RuntimeState; Status CacheSourceLocalState::init(RuntimeState* state, LocalStateInfo& info) { RETURN_IF_ERROR(Base::init(state, info)); @@ -95,8 +96,27 @@ Status CacheSourceLocalState::init(RuntimeState* state, LocalStateInfo& info) { // MISS and INCREMENTAL rebuild the entry (from scratch / by merge), unless // the decision already knows the merged entry could never fit the entry // limits (write_back_feasible == false). - _need_insert_cache = - _cache_decision->key_valid && !hit_cache && _cache_decision->write_back_feasible; + // A query reading through a warmed-up-layout knob (both cloud only) must + // not write back either, because its scan is not guaranteed to represent + // exactly the queried version the entry would be stamped with. Both + // knobs walk the version graph by warmed-up preference and, unlike the + // plain capture, never clip an edge whose end lies past the requested + // window: the read may OVERSHOOT the queried version (a warmed + // compaction rowset spanning it drags the path beyond), and under + // freshness tolerance it may also STOP BELOW it (the walk halts at the + // warmed boundary). Neither knob carries an affectQueryResult + // annotation, so such an entry shares its cache key with plain runs of + // the same statement, which would then serve (exact hit) or extend + // (incremental merge) a base whose content does not match its version + // stamp: missing rows where the read stopped short, double-counted rows + // where it overshot. Suppressing the write-back keeps these reads pure + // consumers: they may still serve an exact HIT filled by a plain run + // (precise data, no worse than asked). + const bool inexact_version_fill = + config::is_cloud_mode() && (state->enable_query_freshness_tolerance() || + state->enable_prefer_cached_rowset()); + _need_insert_cache = _cache_decision->key_valid && !hit_cache && + _cache_decision->write_back_feasible && !inexact_version_fill; _insert_delta_count = _is_incremental ? _cache_decision->cached_delta_count + 1 : 0; if (hit_cache || _is_incremental) { diff --git a/be/src/runtime/query_cache/query_cache.cpp b/be/src/runtime/query_cache/query_cache.cpp index d7e042e0301a24..ccbb6ca66c79d3 100644 --- a/be/src/runtime/query_cache/query_cache.cpp +++ b/be/src/runtime/query_cache/query_cache.cpp @@ -17,8 +17,17 @@ #include "runtime/query_cache/query_cache.h" +#include +#include +#include +#include #include +#include +#include +#include +#include "cloud/cloud_meta_mgr.h" +#include "cloud/cloud_tablet.h" #include "cloud/config.h" #include "common/logging.h" #include "common/metrics/doris_metrics.h" @@ -155,11 +164,12 @@ std::shared_ptr QueryCacheRuntime::get_or_make_decis } // Build the candidate outside the lock: a stale-entry decision walks - // tablet metadata under the header lock and, on merge-on-write tables, - // scans the delete bitmap, so its cost grows with the tablets per - // instance and the bitmap size. This runtime is fragment-wide, so a - // single lock-scoped build would serialize even unrelated instances' - // keys behind one slow decision. Racing callers of the same instance + // tablet metadata under the header lock, on merge-on-write tables scans + // the delete bitmap, and on cloud tablets may first sync the tablet view + // from the meta service (an RPC), so its cost grows with the tablets per + // instance, the bitmap size and the sync latency. This runtime is + // fragment-wide, so a single lock-scoped build would serialize even + // unrelated instances' keys behind one slow decision. Racing callers of the same instance // build duplicate candidates; the loser adopts the winner's decision // below and its own candidate releases with the shared_ptr (the entry // pin and any captured delta read sources go with it), which only costs @@ -235,13 +245,6 @@ bool QueryCacheRuntime::_try_prepare_incremental(const std::vectorincremental_fallback_reason = "cloud mode"; - return false; - } - int64_t cached_version = decision->handle.get_cache_version(); if (cached_version >= decision->current_version) { // The entry is newer than what this replica is asked to read (e.g. the @@ -259,9 +262,19 @@ bool QueryCacheRuntime::_try_prepare_incremental(const std::vector presync_reasons; + if (config::is_cloud_mode()) { + presync_reasons = + _presync_cloud_delta_tablets(scan_ranges, decision->current_version); + } + for (const auto& scan_range : scan_ranges) { int64_t tablet_id = scan_range.scan_range.palo_scan_range.tablet_id; - if (!_capture_tablet_delta(tablet_id, cached_version, decision)) { + if (!_capture_tablet_delta(tablet_id, cached_version, presync_reasons, decision)) { return false; } } @@ -282,8 +295,187 @@ bool QueryCacheRuntime::_try_prepare_incremental(const std::vector QueryCacheRuntime::_presync_cloud_delta_tablets( + const std::vector& scan_ranges, int64_t current_version) { + // Index-aligned so the fork-joined tasks write disjoint slots without a + // lock; an empty slot means synced (or skipped as non-append-only), a + // non-empty slot is the fallback reason. Held behind a shared_ptr because + // the fan-out below is waited on with a fast-fail deadline: on a timeout + // this frame returns while the still-running tasks keep writing their + // slots, so the storage must outlive the frame -- the shared_ptr each task + // captured keeps it alive until the last one finishes. Only read on the + // completed (non-timeout) path, where every task is done. + auto per_range_reason = std::make_shared>(scan_ranges.size()); + std::vector> tasks; + tasks.reserve(scan_ranges.size()); + for (size_t i = 0; i < scan_ranges.size(); ++i) { + int64_t tablet_id = scan_ranges[i].scan_range.palo_scan_range.tablet_id; + tasks.emplace_back([tablet_id, current_version, per_range_reason, i]() -> Status { + auto tablet_res = ExecEnv::get_tablet(tablet_id); + if (!tablet_res) { + // _capture_tablet_delta reports "tablet not found" from its own + // get_tablet; nothing to sync. Should that later get_tablet + // succeed instead (a transient failure here), it takes the + // cache-miss load, which itself syncs rowsets AND the delete + // bitmap up to the visible version -- that load-bearing + // invariant (plus the endpoint check downstream) is what keeps + // a tablet that skipped this pre-sync safe to capture. + return Status::OK(); + } + BaseTabletSPtr tablet = std::move(tablet_res.value()); + const bool append_only = + tablet->keys_type() == KeysType::DUP_KEYS || + (tablet->keys_type() == KeysType::UNIQUE_KEYS && + tablet->enable_unique_key_merge_on_write()); + if (!append_only) { + // Skip the RPC entirely: _capture_tablet_delta rejects this + // tablet at its keys-type check before it would consume a sync. + return Status::OK(); + } + auto cloud_tablet = std::dynamic_pointer_cast(std::move(tablet)); + if (cloud_tablet == nullptr) { + // is_cloud_mode() flipped on a live local deployment + // (cloud_unique_id is a mutable config): degrade to a full + // recompute rather than dereference the failed cast downstream. + (*per_range_reason)[i] = "tablet is not a cloud tablet"; + return Status::OK(); + } + SyncOptions options; + options.query_version = current_version; + options.merge_schema = true; + // The history-rewrite check reads the delete bitmap this sync merges, + // so pin the dependency explicitly rather than inherit the struct + // default (which a refactor could flip with no compile error): a + // merge-on-write delta cannot be classified on a bitmap the sync did + // not bring up to the queried version. + options.sync_delete_bitmap = true; + Status st = cloud_tablet->sync_rowsets(options); + if (!st.ok()) { + // Unlike the other fallback reasons (expected data shapes), a + // failed sync is an infrastructure error: log the status so the + // profile reason correlates to a cause. Throttled with + // LOG_EVERY_N because a meta-service brownout fails this for + // every stale tablet of every query at once; the underlying RPC + // errors are already logged per retry inside retry_rpc, so a + // sampled line here is enough to correlate without a storm, and + // the exact reason still reaches the user via the query profile. + LOG_EVERY_N(WARNING, 100) + << "query cache incremental merge falls back, cloud rowset sync failed" + << ", tablet_id=" << tablet_id << ", status=" << st.to_string(); + (*per_range_reason)[i] = "cloud rowset sync failed"; + } + return Status::OK(); + }); + } + // Fan the syncs out asynchronously and wait on the result with a fast-fail + // budget instead of blocking unconditionally. This decision runs in + // operator init on a bounded query-admission pool (the BE light_work_pool, + // contractually "must be light, not locked"), so a meta-service brownout + // that stalls these RPCs must not hold that thread for the full RPC retry + // budget (tens of seconds): sustained, it would exhaust the pool and reject + // query admission cluster-wide. If the fan-out does not finish within + // query_cache_decision_sync_timeout_ms the decision abandons the wait and + // falls the whole instance back to a full recompute; the still-running + // tasks are correctness-harmless and bounded: on a slow-but-healthy sync + // each merely advances the tablet view early (work the scan node's own + // async sync would otherwise do), and on a failing sync each is at worst one + // extra bounded sync attempt that changes no result; either way they stay + // cheap while pending (small per-task captures; the brpc I/O yields the + // pthread rather than pinning it) and self-limiting once the meta service + // recovers, and keep their slot storage alive through the shared_ptr they + // captured. A healthy sync is + // milliseconds, far under the budget, so this only trips under real + // meta-service degradation and leaves the steady-state incremental path + // unchanged. + // + // Parallelism reuses init_scanner_sync_rowsets_parallelism on purpose: this + // is the same per-tablet rowset-sync fan-out the scan node runs, and one + // shared knob keeps the two fan-outs' budgets aligned rather than letting a + // cache-only config silently drift apart from it. Every task swallows its + // own error into its slot and returns OK, which is load-bearing: + // bthread_fork_join stops DISPATCHING the tasks queued after the first one + // that returns non-OK (see cloud_meta_mgr.cpp), so a task that propagated + // its sync error would leave the tablets queued behind it un-synced; + // recording the failure into the slot keeps every tablet's sync attempted. + // The blocking time (bounded by the budget) is folded into + // query_cache_decision_sync_time_ms. + auto sync_fanout_start = std::chrono::steady_clock::now(); + std::future fanout_done; + // Clamp the shared parallelism knob to >= 1: bthread_fork_join waits for a + // free slot before dispatching its first task (count 0 >= concurrency 0), + // so a misconfigured 0/negative would park the driver bthread forever with + // nothing to notify it. On the scan node that surfaces as a hung query; on + // this fast-fail path the caller's wait_for still expires and falls back, so + // the deadlock would instead leak the driver bthread and its task closures + // silently, once per incremental candidate. A local floor avoids that. + Status launch_st = cloud::bthread_fork_join( + std::move(tasks), std::max(1, config::init_scanner_sync_rowsets_parallelism), + &fanout_done); + bool timed_out = false; + if (!launch_st.ok()) { + // Could not even spawn the fan-out driver bthread: fall back rather than + // block, the same as a timeout (its future would never be fulfilled). + timed_out = true; + } else if (fanout_done.wait_for(std::chrono::milliseconds( + config::query_cache_decision_sync_timeout_ms)) != + std::future_status::ready) { + timed_out = true; + } else { + // Completed within budget. Every task returns OK by construction (each + // swallows its own error into its slot), so the join status is always + // OK; assert it rather than discard, so a future refactor that lets a + // task propagate an error trips here instead of silently leaving the + // tablets queued behind it un-synced. + Status join_st = fanout_done.get(); + DCHECK(join_st.ok()) << join_st; + } + DorisMetrics::instance()->query_cache_decision_sync_time_ms->increment( + std::chrono::duration_cast( + std::chrono::steady_clock::now() - sync_fanout_start) + .count()); + + std::unordered_map fallback_reasons; + if (timed_out) { + // The fan-out outran its fast-fail budget. Do NOT read per_range_reason + // (the detached tasks are still writing it); fall every scanned tablet + // back so the whole instance recomputes in full. A distinct reason from + // a hard sync failure so the profile tells a slow meta service apart + // from a failing one. Throttled like the sync-failure log above: a + // brownout trips this for every stale tablet of every query at once, and + // this timeout is the primary operator-visible symptom of that brownout, + // so a sampled line correlates it without a storm (the per-query reason + // still reaches the user via the profile). + LOG_EVERY_N(WARNING, 100) + << "query cache incremental merge falls back, cloud rowset sync did not finish" + << " within query_cache_decision_sync_timeout_ms=" + << config::query_cache_decision_sync_timeout_ms << "ms"; + for (const auto& scan_range : scan_ranges) { + fallback_reasons[scan_range.scan_range.palo_scan_range.tablet_id] = + "cloud rowset sync timed out"; + } + return fallback_reasons; + } + // Merge the index-aligned slots into a tablet-id map for the capture loop. + // Safe to read now: the fan-out completed, so no task is still writing. + // Scan ranges carry distinct tablet ids in practice (each tablet is scanned + // once per query), but were one to appear twice the fan-out would only + // issue an idempotent redundant sync (serialized inside sync_rowsets by + // _sync_meta_lock) and this map, keyed by tablet id, would still resolve to + // one consistent reason for it -- so a duplicate is harmless, not something + // to guard against with an unreachable de-dup branch. + for (size_t i = 0; i < per_range_reason->size(); ++i) { + if (!(*per_range_reason)[i].empty()) { + fallback_reasons[scan_ranges[i].scan_range.palo_scan_range.tablet_id] = + std::move((*per_range_reason)[i]); + } + } + return fallback_reasons; +} + +bool QueryCacheRuntime::_capture_tablet_delta( + int64_t tablet_id, int64_t cached_version, + const std::unordered_map& presync_reasons, + QueryCacheInstanceDecision* decision) { auto tablet_res = ExecEnv::get_tablet(tablet_id); if (!tablet_res) { decision->incremental_fallback_reason = "tablet not found"; @@ -305,6 +497,35 @@ bool QueryCacheRuntime::_capture_tablet_delta(int64_t tablet_id, int64_t cached_ return false; } + if (config::is_cloud_mode()) { + // A CloudTablet keeps its rowset list (and for merge-on-write tables + // its delete bitmap) as a lazily synced copy of the meta service, and + // this decision runs before the scan node's own sync round, so the + // local view had to be brought up to the queried version first. + // _try_prepare_incremental already did that for every append-only + // tablet in one parallel fork-join (_presync_cloud_delta_tablets), + // which is the single sync per tablet; this loop only consumes its + // outcome. A recorded reason means the sync could not vouch for the + // view (a failed cast on a misconfigured deployment, an infrastructure + // sync failure, or the fast-fail budget expiring on a slow meta + // service), so fall back to a full recompute. No reason means the view + // is synced up to query_version, and the history-rewrite check below + // reads the tablet's delete bitmap as the sync brought it to that + // version (sync_delete_bitmap=true) -- exactly as complete as the + // merge-on-write read path itself sees it, and no more: the + // classification inherits that read path's completeness, including any + // of its known limitations (e.g. a bitmap left short by a prior + // sync_delete_bitmap=false materialization of the tablet), rather than + // adding a stronger guarantee of its own. The scan node's later sync + // hits the same no-op shortcut, so the RPC count per query does not + // grow. + auto reason_it = presync_reasons.find(tablet_id); + if (reason_it != presync_reasons.end()) { + decision->incremental_fallback_reason = reason_it->second; + return false; + } + } + // quiet: on a version-graph miss (the delta merged away by compaction, an // expected per-query situation) this option makes the capture API neither // log nor error; it returns whatever PREFIX of the path it walked before @@ -357,13 +578,20 @@ bool QueryCacheRuntime::_capture_tablet_delta(int64_t tablet_id, int64_t cached_ // input, which the stale sweep then hands to the unused-rowset GC. // Classification would see no rewrite and merge the cached rows with // their replacements: a wrong entry, stored at the current version. - // Holding the rowsets stops that, in start_delete_unused_rowset(): it - // collects a retired rowset only once nothing else references it, and - // only what it collects gets remove_rowset_delete_bitmap(), which drops - // every version of that rowset's bitmap. The other remover, the key - // ranges that the stale sweep queues onto the pre-window rowsets, is - // held off by the delta capture instead: that queue is released only - // once the swept path's own rowsets are unreferenced. + // Holding the rowsets stops that: both storage engines drop a retired + // rowset's bitmap only once nothing else references it, so raising the + // use count with this pin blocks the drop in either deployment mode. + // Local storage does it in start_delete_unused_rowset(): it collects a + // retired rowset only once nothing else references it, and only what it + // collects gets remove_rowset_delete_bitmap(), which drops every version + // of that rowset's bitmap. Cloud does it in + // CloudTablet::remove_unused_rowsets(), which skips any rowset whose + // use_count() still exceeds one before the same remove_rowset_delete_bitmap(). + // The other remover, the key ranges the stale sweep queues onto the + // pre-window rowsets (the engine's unused-delete-bitmap queue locally, + // CloudTablet::_unused_delete_bitmap in cloud), is held off by the delta + // capture instead: that queue is released only once the swept path's own + // rowsets are unreferenced. std::vector cached_side_pin; if (merge_on_write) { { @@ -429,7 +657,10 @@ bool QueryCacheRuntime::_delta_rewrites_history(BaseTablet& tablet, // gone is rejected before we get here. Capture and scan are not atomic, // so a sweep landing exactly in between (with the aggregated keys also // already recycled) can still hide a rewrite: a known narrow race the - // fallback does not cover, shared with the baseline exact-hit path. + // fallback does not cover, the same one any merge-on-write snapshot read + // faces when a concurrent sweep re-stamps and recycles a marker (the + // exact-hit cache path is not itself exposed, since it serves cached + // blocks without re-reading the bitmap). // Pending entries use DeleteBitmap::TEMP_VERSION_COMMON (= 0) and stay // below the window as well. // diff --git a/be/src/runtime/query_cache/query_cache.h b/be/src/runtime/query_cache/query_cache.h index f727159eac2915..e726a79937e0a0 100644 --- a/be/src/runtime/query_cache/query_cache.h +++ b/be/src/runtime/query_cache/query_cache.h @@ -373,10 +373,27 @@ class QueryCacheRuntime { bool _try_prepare_incremental(const std::vector& scan_ranges, QueryCacheInstanceDecision* decision); + // Cloud only: bring every append-only scanned tablet's local view up to the + // queried version in parallel (a fork-join of sync_rowsets, matching the + // scan node's own tablet-sync fan-out) before the serial per-tablet capture + // below runs, instead of issuing those RPCs one tablet at a time on the + // shared prepare thread. This is the single sync per tablet: _capture_ + // tablet_delta consumes the result here and does not sync again. Returns + // the per-tablet fallback reasons the sync produced (keyed by tablet id, + // only failures present): a cast failure ("tablet is not a cloud tablet") + // or an infrastructure sync failure ("cloud rowset sync failed"). Tablets + // that are not append-only are skipped (no wasted RPC); _capture_tablet_ + // delta rejects them at its own keys-type check. + static std::unordered_map _presync_cloud_delta_tablets( + const std::vector& scan_ranges, int64_t current_version); + // Validate one tablet for incremental merge and capture its delta read // source of (cached_version, current_version]. On any failure records the - // fallback reason in the decision and returns false. + // fallback reason in the decision and returns false. In cloud mode the view + // sync already ran in _presync_cloud_delta_tablets; its per-tablet failure + // reasons arrive through presync_reasons. bool _capture_tablet_delta(int64_t tablet_id, int64_t cached_version, + const std::unordered_map& presync_reasons, QueryCacheInstanceDecision* decision); // Merge-on-write only: true if any delete-bitmap entry stamped with a diff --git a/be/test/exec/operator/query_cache_operator_test.cpp b/be/test/exec/operator/query_cache_operator_test.cpp index 177e1aef7f96ee..943d38e4d2a2f0 100644 --- a/be/test/exec/operator/query_cache_operator_test.cpp +++ b/be/test/exec/operator/query_cache_operator_test.cpp @@ -20,6 +20,7 @@ #include #include +#include "cloud/config.h" #include "core/block/block.h" #include "exec/operator/cache_sink_operator.h" #include "exec/operator/cache_source_operator.h" @@ -50,7 +51,11 @@ class QueryCacheMockChildOperator : public OperatorXBase { }; struct QueryCacheOperatorTest : public ::testing::Test { + std::string _saved_deploy_mode; + std::string _saved_cloud_unique_id; void SetUp() override { + _saved_deploy_mode = config::deploy_mode; + _saved_cloud_unique_id = config::cloud_unique_id; state = std::make_shared(); state->_batch_size = 10; child_op = std::make_unique(); @@ -77,6 +82,15 @@ struct QueryCacheOperatorTest : public ::testing::Test { state.reset(); source.reset(); sink.reset(); + // Restore deploy_mode exception-safely: the cloud write-back tests set it + // to "cloud" mid-body; an inline restore would leak "cloud" into a later + // test (e.g. test_write_back_kept_on_local_with_freshness_set) if anything + // between the set and the restore threw. Restore cloud_unique_id too: + // is_cloud_mode() is deploy_mode=="cloud" || !cloud_unique_id.empty(), so + // a sibling test that leaves cloud_unique_id set would otherwise flip this + // suite's local-mode assumption and fail the local write-back test. + config::deploy_mode = _saved_deploy_mode; + config::cloud_unique_id = _saved_cloud_unique_id; } void create_local_state() { shared_state = sink->create_shared_state(); @@ -1016,4 +1030,149 @@ TEST_F(QueryCacheOperatorTest, test_failed_final_delta_merge_publishes_nothing) EXPECT_EQ(stale_handle.get_cache_version(), 100); } +TEST_F(QueryCacheOperatorTest, test_freshness_fill_does_not_write_back) { + // A cloud query under freshness tolerance may legally read below the + // queried version (its version-graph walk halts at the warmed boundary), + // yet the entry would be stamped with the queried version; and since the + // freshness knobs carry no affectQueryResult annotation, that entry would + // share its cache key with freshness-free runs of the same statement, + // poisoning their exact hits and incremental merges with the missing tail + // rows. So such a query must be a pure cache consumer: the write-back is + // suppressed at init, and nothing may land in the cache even after blocks + // flow through. + config::deploy_mode = "cloud"; + state->set_query_freshness_tolerance_ms(5000); + + sink = std::make_unique(); + TQueryCacheParam cache_param; + cache_param.node_id = 0; + cache_param.digest = "test_digest"; + cache_param.output_slot_mapping[0] = 0; + cache_param.tablet_to_range.insert({42, "test"}); + cache_param.force_refresh_query_cache = false; + cache_param.entry_max_bytes = 1024 * 1024; + cache_param.entry_max_rows = 1000; + source = std::make_unique( + &pool, /*plan_node_id=*/0, /*operator_id=*/0, cache_param, + std::make_shared(cache_param, query_cache)); + EXPECT_TRUE(source->set_child(child_op)); + child_op->_mock_row_desc.reset( + new MockRowDescriptor {{std::make_shared()}, &pool}); + create_local_state(); + + EXPECT_FALSE(source_local_state->_need_insert_cache); + { + auto block = ColumnHelper::create_block({1, 2, 3, 4, 5}); + auto st = sink->sink(state.get(), &block, true); + EXPECT_TRUE(st.ok()) << st.msg(); + } + { + Block block; + bool eos = false; + auto st = source->get_block(state.get(), &block, &eos); + EXPECT_TRUE(st.ok()) << st.msg(); + EXPECT_TRUE(eos); + EXPECT_EQ(block.rows(), 5); + EXPECT_EQ(query_cache->get_element_count(), 0); + } +} + +TEST_F(QueryCacheOperatorTest, test_prefer_cached_fill_does_not_write_back) { + // Prefer-cached-rowset (cloud) may OVERSHOOT the queried version: unlike + // the plain capture, its version-graph walk never clips an edge whose end + // lies past the requested window, so a warmed compaction rowset spanning + // the queried version drags the read beyond it while the entry would + // still be stamped with the queried version. Sharing the cache key with + // plain runs (no affectQueryResult annotation), such an entry would serve + // ahead-of-version rows on an exact hit and double-count the overshot + // range on an incremental merge. So a prefer-cached query must not write + // back either. + config::deploy_mode = "cloud"; + state->set_enable_prefer_cached_rowset(true); + + sink = std::make_unique(); + TQueryCacheParam cache_param; + cache_param.node_id = 0; + cache_param.digest = "test_digest"; + cache_param.output_slot_mapping[0] = 0; + cache_param.tablet_to_range.insert({42, "test"}); + cache_param.force_refresh_query_cache = false; + cache_param.entry_max_bytes = 1024 * 1024; + cache_param.entry_max_rows = 1000; + source = std::make_unique( + &pool, /*plan_node_id=*/0, /*operator_id=*/0, cache_param, + std::make_shared(cache_param, query_cache)); + EXPECT_TRUE(source->set_child(child_op)); + child_op->_mock_row_desc.reset( + new MockRowDescriptor {{std::make_shared()}, &pool}); + create_local_state(); + + EXPECT_FALSE(source_local_state->_need_insert_cache); + { + auto block = ColumnHelper::create_block({1, 2, 3, 4, 5}); + auto st = sink->sink(state.get(), &block, true); + EXPECT_TRUE(st.ok()) << st.msg(); + } + { + Block block; + bool eos = false; + auto st = source->get_block(state.get(), &block, &eos); + EXPECT_TRUE(st.ok()) << st.msg(); + EXPECT_TRUE(eos); + EXPECT_EQ(block.rows(), 5); + EXPECT_EQ(query_cache->get_element_count(), 0); + } +} + +TEST_F(QueryCacheOperatorTest, test_write_back_kept_on_cloud_without_freshness) { + // The freshness suppression must not clip an ordinary cloud query: with + // the tolerance unset, a cloud MISS still writes its result back. + config::deploy_mode = "cloud"; + + sink = std::make_unique(); + TQueryCacheParam cache_param; + cache_param.node_id = 0; + cache_param.digest = "test_digest"; + cache_param.output_slot_mapping[0] = 0; + cache_param.tablet_to_range.insert({42, "test"}); + cache_param.force_refresh_query_cache = false; + cache_param.entry_max_bytes = 1024 * 1024; + cache_param.entry_max_rows = 1000; + source = std::make_unique( + &pool, /*plan_node_id=*/0, /*operator_id=*/0, cache_param, + std::make_shared(cache_param, query_cache)); + EXPECT_TRUE(source->set_child(child_op)); + child_op->_mock_row_desc.reset( + new MockRowDescriptor {{std::make_shared()}, &pool}); + create_local_state(); + + EXPECT_TRUE(source_local_state->_need_insert_cache); +} + +TEST_F(QueryCacheOperatorTest, test_write_back_kept_on_local_with_freshness_set) { + // Freshness tolerance is inert on local storage (the scan clamps it to + // disabled), so a local deployment must keep writing back even when the + // session carries a tolerance value: the suppression is cloud-only. + state->set_query_freshness_tolerance_ms(5000); + + sink = std::make_unique(); + TQueryCacheParam cache_param; + cache_param.node_id = 0; + cache_param.digest = "test_digest"; + cache_param.output_slot_mapping[0] = 0; + cache_param.tablet_to_range.insert({42, "test"}); + cache_param.force_refresh_query_cache = false; + cache_param.entry_max_bytes = 1024 * 1024; + cache_param.entry_max_rows = 1000; + source = std::make_unique( + &pool, /*plan_node_id=*/0, /*operator_id=*/0, cache_param, + std::make_shared(cache_param, query_cache)); + EXPECT_TRUE(source->set_child(child_op)); + child_op->_mock_row_desc.reset( + new MockRowDescriptor {{std::make_shared()}, &pool}); + create_local_state(); + + EXPECT_TRUE(source_local_state->_need_insert_cache); +} + } // namespace doris diff --git a/be/test/exec/pipeline/query_cache_test.cpp b/be/test/exec/pipeline/query_cache_test.cpp index 4f79f4e8943146..bfd2146a88381e 100644 --- a/be/test/exec/pipeline/query_cache_test.cpp +++ b/be/test/exec/pipeline/query_cache_test.cpp @@ -28,9 +28,12 @@ #include #include #include +#include #include #include +#include "cloud/cloud_storage_engine.h" +#include "cloud/cloud_tablet.h" #include "cloud/config.h" #include "common/config.h" #include "common/metrics/doris_metrics.h" @@ -40,6 +43,7 @@ #include "json2pb/json_to_pb.h" #include "storage/data_dir.h" #include "storage/options.h" +#include "storage/rowset/rowset_factory.h" #include "storage/rowset/rowset_meta.h" #include "storage/storage_engine.h" #include "storage/tablet/base_tablet.h" @@ -585,29 +589,6 @@ TEST_F(QueryCacheTest, take_delta_read_source) { EXPECT_EQ(decision->take_delta_read_source(42), nullptr); } -TEST_F(QueryCacheTest, runtime_decision_stale_incremental_cloud_mode) { - std::unique_ptr cache(QueryCache::create_global_cache(1024 * 1024)); - auto scan_ranges = make_scan_ranges(42, "100"); - auto cache_param = make_cache_param(42); - cache_param.__set_allow_incremental(true); - - std::string cache_key; - int64_t version = 0; - EXPECT_TRUE(QueryCache::build_cache_key(scan_ranges, cache_param, &cache_key, &version).ok()); - insert_entry(cache.get(), cache_key, 50, 0); - - // Incremental merge only supports local storage for now. - std::string saved_deploy_mode = config::deploy_mode; - config::deploy_mode = "cloud"; - QueryCacheRuntime runtime(cache_param, cache.get()); - auto decision = runtime.get_or_make_decision(scan_ranges); - config::deploy_mode = saved_deploy_mode; - - EXPECT_EQ(decision->mode, QueryCacheInstanceDecision::Mode::MISS); - EXPECT_TRUE(decision->key_valid); - EXPECT_EQ(decision->incremental_fallback_reason, "cloud mode"); -} - // Exercises the per-tablet part of the incremental decision against a real // (metadata-only) tablet registered in a real storage engine: capturing the // delta read source never touches segment files, so no data is needed. @@ -1058,13 +1039,28 @@ TEST_F(QueryCacheIncrementalTest, fallback_on_partial_capture) { EXPECT_EQ(decision->incremental_fallback_reason, "delta versions not capturable"); } +TEST_F(QueryCacheIncrementalTest, mismatched_engine_falls_back) { + // is_cloud_mode() can flip on a live local deployment (cloud_unique_id is + // a mutable config): ExecEnv::get_tablet then still hands out a plain + // local Tablet, and the decision must degrade the misconfiguration to a + // full recompute instead of dereferencing the failed CloudTablet cast. + create_tablet(TKeysType::DUP_KEYS, false, {{0, 50}, {51, 100}}); + std::string saved_deploy_mode = config::deploy_mode; + config::deploy_mode = "cloud"; + auto decision = make_stale_decision(); + config::deploy_mode = saved_deploy_mode; + EXPECT_EQ(decision->mode, QueryCacheInstanceDecision::Mode::MISS); + EXPECT_TRUE(decision->key_valid); + EXPECT_EQ(decision->incremental_fallback_reason, "tablet is not a cloud tablet"); +} + TEST_F(QueryCacheIncrementalTest, concurrent_racers_share_one_decision) { // The candidate decision is built outside the runtime lock (a stale - // merge-on-write decision may scan a large delete bitmap there), so two - // operators of the same instance can race: the sync point parks both - // threads right before publication, forcing two live candidates. Exactly - // one must win, both callers must observe the winner, and the metrics - // must be settled once. + // decision may scan a large delete bitmap there, and on cloud tablets + // first sync the view from the meta service), so two operators of the + // same instance can race: the sync point parks both threads right before + // publication, forcing two live candidates. Exactly one must win, both + // callers must observe the winner, and the metrics must be settled once. create_tablet(TKeysType::DUP_KEYS, false, {{0, 50}, {51, 100}}); auto scan_ranges = make_scan_ranges(kTabletId, "100"); auto cache_param = make_cache_param(kTabletId); @@ -1140,4 +1136,639 @@ TEST_F(QueryCacheIncrementalTest, concurrent_racers_settle_fallback_once) { fallback_before + 1); } +// Exercises the incremental decision against a real CloudTablet materialized +// through the regular cache-miss load of ExecEnv::get_tablet, with the two +// meta-service RPCs intercepted through sync points (the same pattern as +// cloud_schema_change_job_test): get_tablet_meta serves a local meta and the +// loader's first sync_tablet_rowsets installs the requested rowsets. The +// decision's own sync is then either short-cut by the local max version +// covering query_version, or (as the second sync call) answered with an +// injected result without changing the view. Capturing the delta read source +// never touches rowset data, so none exists. +class QueryCacheCloudIncrementalTest : public testing::Test { +protected: + static constexpr int64_t kTabletId = 15673; + + void SetUp() override { + _saved_deploy_mode = config::deploy_mode; + config::deploy_mode = "cloud"; + auto engine = std::make_unique(EngineOptions {}); + _engine = engine.get(); + ExecEnv::GetInstance()->set_storage_engine(std::move(engine)); + _cache.reset(QueryCache::create_global_cache(1024 * 1024)); + _sp = SyncPoint::get_instance(); + _sp->enable_processing(); + } + + void TearDown() override { + _sp->disable_processing(); + _sp->clear_all_call_backs(); + _cache.reset(); + ExecEnv::GetInstance()->set_storage_engine(nullptr); + _engine = nullptr; + config::deploy_mode = _saved_deploy_mode; + } + + // Builds metadata-only rowsets for `versions` and installs them into the + // tablet's view, exactly like a production sync would. Also records them + // in _installed_rowsets for tests that stamp delete-bitmap entries onto + // specific rowsets. + void install_rowsets(CloudTablet* tablet, + const std::vector>& versions) { + std::vector rowsets; + rowsets.reserve(versions.size()); + for (auto [start, end] : versions) { + auto rs_meta = std::make_shared(); + rs_meta->set_rowset_type(BETA_ROWSET); + rs_meta->set_version({start, end}); + rs_meta->set_rowset_id(_engine->next_rowset_id()); + rs_meta->set_num_segments(1); + rs_meta->set_tablet_schema(tablet->tablet_meta()->tablet_schema()); + RowsetSharedPtr rowset; + Status st = RowsetFactory::create_rowset(nullptr, "", rs_meta, &rowset); + EXPECT_TRUE(st.ok()) << st; + if (st.ok()) { + rowsets.push_back(std::move(rowset)); + } + } + _installed_rowsets.insert(_installed_rowsets.end(), rowsets.begin(), rowsets.end()); + std::unique_lock lock(tablet->get_header_lock()); + tablet->add_rowsets(std::move(rowsets), /*version_overlap=*/false, lock); + } + + void intercept_tablet_load( + std::vector> versions, Status decision_sync_result, + TKeysType::type keys_type = TKeysType::DUP_KEYS, bool enable_merge_on_write = false, + std::vector> decision_sync_installs = {}, + // Delete-bitmap marks {index into _installed_rowsets, version} the + // decision's sync stamps AFTER installing its rowsets, but ONLY when + // that sync carried sync_delete_bitmap (mirroring the real body). + // Lets a test both exercise cloud sync-completeness and GUARD the + // flag: a rewrite entry the loader's sync never carried, that only a + // sync_delete_bitmap=true decision sync brings, then feeds the + // history check. + std::vector> decision_sync_bitmap_marks = {}) { + _sp->set_call_back("CloudMetaMgr::get_tablet_meta", + [keys_type, enable_merge_on_write](auto&& args) { + TTabletSchema schema; + schema.keys_type = keys_type; + auto meta = std::make_shared( + 1, 2, kTabletId, 15674, 4, 5, schema, 6, + std::unordered_map {{7, 8}}, + UniqueId(9, 10), TTabletType::TABLET_TYPE_DISK, + TCompressionType::LZ4F, 0, enable_merge_on_write); + // Pin the RUNNING state (also the ctor default) explicitly: a + // tablet that is not RUNNING takes sync_if_not_running(), which + // clears the just-installed view and re-syncs through the same + // interception, breaking both the version view and _sync_calls. + meta->set_tablet_state(TABLET_RUNNING); + *try_any_cast(args[1]) = std::move(meta); + try_any_cast_ret(args)->second = true; + }); + _sp->set_call_back( + "CloudMetaMgr::sync_tablet_rowsets", + [this, versions = std::move(versions), decision_sync_result, + decision_sync_installs = std::move(decision_sync_installs), + decision_sync_bitmap_marks = + std::move(decision_sync_bitmap_marks)](auto&& args) { + auto* tablet = try_any_cast(args[0]); + const auto* options = try_any_cast(args[1]); + auto* ret = try_any_cast_ret(args); + if (_sync_calls.fetch_add(1) == 0) { + // The loader's sync: install the requested rowsets. + install_rowsets(tablet, versions); + } else { + // The incremental decision's sync: install the extra + // rowsets, if any, stamp any delete-bitmap marks it is + // responsible for bringing, and inject the result. + install_rowsets(tablet, decision_sync_installs); + // Gate the marks on sync_delete_bitmap exactly as the real + // sync body does (cloud_meta_mgr.cpp): a sync that did not + // carry the flag brings no delete bitmap. This makes the + // mock flag-faithful, so a test can prove the pre-sync set + // the flag -- flip it to false and the mark is dropped. + if (options->sync_delete_bitmap) { + for (auto [rs_idx, ver] : decision_sync_bitmap_marks) { + // .at(): a mistyped index in a future test should + // fail loudly, not be silent out-of-bounds UB. + tablet->tablet_meta()->delete_bitmap().add( + {_installed_rowsets.at(rs_idx)->rowset_id(), 0, ver}, 7); + } + } + ret->first = decision_sync_result; + } + ret->second = true; + }); + } + + // A stale entry of version 50 while the query reads version 100 with + // allow_incremental set, mirroring the local-storage fixture. + std::shared_ptr make_stale_decision() { + auto scan_ranges = make_scan_ranges(kTabletId, "100"); + auto cache_param = make_cache_param(kTabletId); + cache_param.__set_allow_incremental(true); + std::string cache_key; + int64_t version = 0; + EXPECT_TRUE( + QueryCache::build_cache_key(scan_ranges, cache_param, &cache_key, &version).ok()); + insert_entry(_cache.get(), cache_key, 50, 0); + QueryCacheRuntime runtime(cache_param, _cache.get()); + return runtime.get_or_make_decision(scan_ranges); + } + + std::string _saved_deploy_mode; + CloudStorageEngine* _engine = nullptr; + std::unique_ptr _cache; + SyncPoint* _sp = nullptr; + std::atomic _sync_calls {0}; + // The rowsets the loader's sync installed, for tests that stamp + // delete-bitmap entries onto specific rowsets. + std::vector _installed_rowsets; +}; + +TEST_F(QueryCacheCloudIncrementalTest, incremental_success_on_synced_view) { + // The loader installs a view that already covers query_version 100, so + // the decision's sync_rowsets() takes its no-op shortcut (the sync point + // sees exactly one call) and the decision merges incrementally, exactly + // like on local storage. + intercept_tablet_load({{0, 50}, {51, 100}}, Status::OK()); + int64_t stale_hits_before = DorisMetrics::instance()->query_cache_stale_hit_total->value(); + auto decision = make_stale_decision(); + EXPECT_EQ(DorisMetrics::instance()->query_cache_stale_hit_total->value(), + stale_hits_before + 1); + EXPECT_EQ(decision->mode, QueryCacheInstanceDecision::Mode::INCREMENTAL); + EXPECT_EQ(decision->cached_version, 50); + EXPECT_TRUE(decision->incremental_fallback_reason.empty()); + EXPECT_EQ(_sync_calls.load(), 1); + auto source = decision->take_delta_read_source(kTabletId); + ASSERT_NE(source, nullptr); + EXPECT_EQ(source->rs_splits.size(), 1); + EXPECT_EQ(source->rs_splits.front().rs_reader->rowset()->start_version(), 51); + EXPECT_EQ(source->rs_splits.front().rs_reader->rowset()->end_version(), 100); +} + +TEST_F(QueryCacheCloudIncrementalTest, fallback_on_sync_failure) { + // The local view (max 50) does not cover query_version 100, so the + // decision must sync from the meta service first; the injected failure + // falls the query back to a full recompute. + intercept_tablet_load({{0, 50}}, Status::InternalError("injected sync failure")); + int64_t fallbacks_before = + DorisMetrics::instance()->query_cache_incremental_fallback_total->value(); + auto decision = make_stale_decision(); + EXPECT_EQ(DorisMetrics::instance()->query_cache_incremental_fallback_total->value(), + fallbacks_before + 1); + EXPECT_EQ(decision->mode, QueryCacheInstanceDecision::Mode::MISS); + EXPECT_EQ(decision->incremental_fallback_reason, "cloud rowset sync failed"); + EXPECT_EQ(_sync_calls.load(), 2); +} + +TEST_F(QueryCacheCloudIncrementalTest, incremental_after_decision_sync_installs_delta) { + // The flagship cloud composition: the local view (max 50) does not cover + // query_version 100, so the decision's sync really goes to the meta + // service; the injected response installs the missing delta rowset, and + // the capture on the freshly synced view then merges incrementally. + intercept_tablet_load({{0, 50}}, Status::OK(), TKeysType::DUP_KEYS, + /*enable_merge_on_write=*/false, + /*decision_sync_installs=*/ {{51, 100}}); + int64_t stale_hits_before = DorisMetrics::instance()->query_cache_stale_hit_total->value(); + auto decision = make_stale_decision(); + EXPECT_EQ(DorisMetrics::instance()->query_cache_stale_hit_total->value(), + stale_hits_before + 1); + EXPECT_EQ(decision->mode, QueryCacheInstanceDecision::Mode::INCREMENTAL); + EXPECT_TRUE(decision->incremental_fallback_reason.empty()); + EXPECT_EQ(_sync_calls.load(), 2); + auto source = decision->take_delta_read_source(kTabletId); + ASSERT_NE(source, nullptr); + EXPECT_EQ(source->rs_splits.size(), 1); + EXPECT_EQ(source->rs_splits.front().rs_reader->rowset()->start_version(), 51); + EXPECT_EQ(source->rs_splits.front().rs_reader->rowset()->end_version(), 100); +} + +TEST_F(QueryCacheCloudIncrementalTest, fallback_when_synced_view_misses_delta) { + // The decision's sync succeeds (injected OK) but leaves the view at max + // 50, so the delta versions (51, 100] are not on the local version + // graph: the quiet capture returns an empty read source and the decision + // falls back instead of silently dropping the delta rows. + intercept_tablet_load({{0, 50}}, Status::OK()); + auto decision = make_stale_decision(); + EXPECT_EQ(decision->mode, QueryCacheInstanceDecision::Mode::MISS); + EXPECT_EQ(decision->incremental_fallback_reason, "delta versions not capturable"); + EXPECT_EQ(_sync_calls.load(), 2); +} + +TEST_F(QueryCacheCloudIncrementalTest, mow_pure_append_incremental) { + // A merge-on-write UNIQUE cloud tablet whose delta window carries no + // delete-bitmap entry (the hourly-append pattern) merges incrementally, + // exactly like on local storage: the history-rewrite check runs on the + // freshly synced bitmap and finds nothing. + intercept_tablet_load({{0, 50}, {51, 100}}, Status::OK(), TKeysType::UNIQUE_KEYS, + /*enable_merge_on_write=*/true); + auto decision = make_stale_decision(); + EXPECT_EQ(decision->mode, QueryCacheInstanceDecision::Mode::INCREMENTAL); + EXPECT_TRUE(decision->incremental_fallback_reason.empty()); + EXPECT_EQ(_sync_calls.load(), 1); + auto source = decision->take_delta_read_source(kTabletId); + ASSERT_NE(source, nullptr); + EXPECT_EQ(source->rs_splits.size(), 1); +} + +TEST_F(QueryCacheCloudIncrementalTest, mow_history_rewrite_falls_back) { + // A delete-bitmap entry inside the delta window that targets the + // baseline rowset (a backfill rewrote a key predating the cached + // version): the same history-rewrite fallback as on local storage, on a + // bitmap the cloud sync is responsible for keeping complete. + intercept_tablet_load({{0, 50}, {51, 100}}, Status::OK(), TKeysType::UNIQUE_KEYS, + /*enable_merge_on_write=*/true); + auto tablet_res = ExecEnv::get_tablet(kTabletId); + ASSERT_TRUE(tablet_res.has_value()) << tablet_res.error(); + ASSERT_EQ(_installed_rowsets.size(), 2); + tablet_res.value()->tablet_meta()->delete_bitmap().add( + {_installed_rowsets[0]->rowset_id(), 0, 60}, 7); + + int64_t fallbacks_before = + DorisMetrics::instance()->query_cache_incremental_fallback_total->value(); + auto decision = make_stale_decision(); + EXPECT_EQ(DorisMetrics::instance()->query_cache_incremental_fallback_total->value(), + fallbacks_before + 1); + EXPECT_EQ(decision->mode, QueryCacheInstanceDecision::Mode::MISS); + EXPECT_EQ(decision->incremental_fallback_reason, "delta rewrites history rows"); + // The loader already brought the view up to 100, so the decision's sync + // took the no-op shortcut: still exactly one meta-service round trip. + EXPECT_EQ(_sync_calls.load(), 1); +} + +TEST_F(QueryCacheCloudIncrementalTest, not_incremental_skips_decision_sync) { + // allow_incremental unset: the stale entry is a plain miss and the + // decision never reaches the per-tablet path, so it must not load the + // tablet or talk to the meta service at all (no hidden RPC cost when the + // feature is off). + intercept_tablet_load({{0, 50}}, Status::OK()); + auto scan_ranges = make_scan_ranges(kTabletId, "100"); + auto cache_param = make_cache_param(kTabletId); + std::string cache_key; + int64_t version = 0; + EXPECT_TRUE(QueryCache::build_cache_key(scan_ranges, cache_param, &cache_key, &version).ok()); + insert_entry(_cache.get(), cache_key, 50, 0); + QueryCacheRuntime runtime(cache_param, _cache.get()); + auto decision = runtime.get_or_make_decision(scan_ranges); + EXPECT_EQ(decision->mode, QueryCacheInstanceDecision::Mode::MISS); + EXPECT_TRUE(decision->incremental_fallback_reason.empty()); + EXPECT_EQ(_sync_calls.load(), 0); +} + +TEST_F(QueryCacheCloudIncrementalTest, agg_keys_rejected_before_decision_sync) { + // The keys-type check precedes the cloud sync, so an AGG tablet is + // rejected after the loader's round trip but without a second one: no + // meta-service RPC is spent on a tablet that can never merge. + intercept_tablet_load({{0, 50}}, Status::OK(), TKeysType::AGG_KEYS); + int64_t fallbacks_before = + DorisMetrics::instance()->query_cache_incremental_fallback_total->value(); + auto decision = make_stale_decision(); + EXPECT_EQ(DorisMetrics::instance()->query_cache_incremental_fallback_total->value(), + fallbacks_before + 1); + EXPECT_EQ(decision->mode, QueryCacheInstanceDecision::Mode::MISS); + EXPECT_EQ(decision->incremental_fallback_reason, "keys type not append-only"); + EXPECT_EQ(_sync_calls.load(), 1); +} + +TEST_F(QueryCacheCloudIncrementalTest, newer_entry_rejected_before_tablet_access) { + // The cached entry (version 200) is newer than the queried version 100: + // rejected before the per-tablet loop, so the tablet is never loaded and + // the meta service is never contacted. + intercept_tablet_load({{0, 50}}, Status::OK()); + auto scan_ranges = make_scan_ranges(kTabletId, "100"); + auto cache_param = make_cache_param(kTabletId); + cache_param.__set_allow_incremental(true); + std::string cache_key; + int64_t version = 0; + EXPECT_TRUE(QueryCache::build_cache_key(scan_ranges, cache_param, &cache_key, &version).ok()); + insert_entry(_cache.get(), cache_key, 200, 0); + QueryCacheRuntime runtime(cache_param, _cache.get()); + auto decision = runtime.get_or_make_decision(scan_ranges); + EXPECT_EQ(decision->mode, QueryCacheInstanceDecision::Mode::MISS); + EXPECT_EQ(decision->incremental_fallback_reason, "cached entry is newer"); + EXPECT_EQ(_sync_calls.load(), 0); +} + +TEST_F(QueryCacheCloudIncrementalTest, presync_tablet_load_failure_falls_back) { + // The pre-sync fan-out issues the first ExecEnv::get_tablet; a meta-service + // failure injected there fails that load, so _presync_cloud_delta_tablets + // takes its !tablet_res skip (records no reason, leaving the miss for the + // capture loop to report), and the capture loop's own get_tablet then falls + // the query back to a full recompute with "tablet not found". This exercises + // the one pre-sync branch the other cloud tests never reach: they all load + // the tablet successfully before syncing. + std::atomic meta_loads {0}; + _sp->set_call_back("CloudMetaMgr::get_tablet_meta", [&meta_loads](auto&& args) { + meta_loads.fetch_add(1); + auto* ret = try_any_cast_ret(args); + ret->first = Status::InternalError("injected: tablet meta unavailable"); + ret->second = true; + }); + int64_t fallbacks_before = + DorisMetrics::instance()->query_cache_incremental_fallback_total->value(); + auto decision = make_stale_decision(); + EXPECT_EQ(DorisMetrics::instance()->query_cache_incremental_fallback_total->value(), + fallbacks_before + 1); + EXPECT_EQ(decision->mode, QueryCacheInstanceDecision::Mode::MISS); + EXPECT_EQ(decision->incremental_fallback_reason, "tablet not found"); + // The pre-sync get_tablet failed before any sync, so no meta-service rowset + // sync RPC was ever issued. + EXPECT_EQ(_sync_calls.load(), 0); + // Discriminating witness that the PRE-SYNC skip branch actually ran (not + // that the pre-sync was bypassed and only the capture loop reported "tablet + // not found", which would give the same mode/reason/_sync_calls): the + // pre-sync fan-out and the capture loop each attempt their own get_tablet, + // so the load is tried twice. A bypassed pre-sync would attempt it once. + EXPECT_EQ(meta_loads.load(), 2); +} + +TEST_F(QueryCacheCloudIncrementalTest, unique_non_mow_rejected_before_decision_sync) { + // A UNIQUE_KEYS tablet WITHOUT merge-on-write (the legacy merge-on-read + // unique table) is not append-only either, so the keys-type check rejects + // it before the cloud sync, exactly like an AGG tablet. Exercises the + // enable_unique_key_merge_on_write()==false leg of the append-only test: + // the merge-on-write cases take its true leg, AGG never evaluates it (short + // -circuited by the keys-type compare), so this is the one input that drives + // that leg false. + intercept_tablet_load({{0, 50}}, Status::OK(), TKeysType::UNIQUE_KEYS, + /*enable_merge_on_write=*/false); + int64_t fallbacks_before = + DorisMetrics::instance()->query_cache_incremental_fallback_total->value(); + auto decision = make_stale_decision(); + EXPECT_EQ(DorisMetrics::instance()->query_cache_incremental_fallback_total->value(), + fallbacks_before + 1); + EXPECT_EQ(decision->mode, QueryCacheInstanceDecision::Mode::MISS); + EXPECT_EQ(decision->incremental_fallback_reason, "keys type not append-only"); + // Rejected at the keys-type check, after the loader's single round trip and + // without a second (decision) sync. + EXPECT_EQ(_sync_calls.load(), 1); +} + +TEST_F(QueryCacheCloudIncrementalTest, mow_decision_sync_brings_bitmap_detects_rewrite) { + // Cloud sync-completeness, GUARDING options.sync_delete_bitmap=true: the + // loader view stops at 50 with no delta and no delete-bitmap entry, so the + // decision's sync really runs (_sync_calls==2). The fixture mock is + // flag-faithful -- it stamps the injected rewrite mark only when that sync + // carried sync_delete_bitmap=true (the CloudMetaMgr::sync_tablet_rowsets sync + // point forwards the SyncOptions, mirroring the real body's gate). Because the + // pre-sync sets the flag, the mark on the baseline rowset (version 60, inside + // the delta window) lands, the history-rewrite check reads that freshly-synced + // bitmap, and the query falls back. If a refactor flipped the flag to false, + // the mock would drop the mark just as a real meta-service would drop the + // bitmap sync, the rewrite would go undetected, and this test would fail + // (INCREMENTAL instead of MISS) -- so the load-bearing MoW invariant is + // UT-guarded, not just documented. mow_history_rewrite_falls_back cannot show + // this: there the loader carried the view to 100, so the decision sync was a + // no-op and the mark was hand-placed. + intercept_tablet_load({{0, 50}}, Status::OK(), TKeysType::UNIQUE_KEYS, + /*enable_merge_on_write=*/true, + /*decision_sync_installs=*/{{51, 100}}, + /*decision_sync_bitmap_marks=*/{{0, 60}}); + int64_t fallbacks_before = + DorisMetrics::instance()->query_cache_incremental_fallback_total->value(); + auto decision = make_stale_decision(); + EXPECT_EQ(DorisMetrics::instance()->query_cache_incremental_fallback_total->value(), + fallbacks_before + 1); + EXPECT_EQ(decision->mode, QueryCacheInstanceDecision::Mode::MISS); + EXPECT_EQ(decision->incremental_fallback_reason, "delta rewrites history rows"); + // The loader's view (max 50) did not cover the query, so the decision's sync + // really ran -- two round trips -- and it is that second sync that brought + // the rewrite evidence the classification tripped on. + EXPECT_EQ(_sync_calls.load(), 2); +} + +TEST_F(QueryCacheCloudIncrementalTest, mow_cloud_cached_side_not_pinnable_falls_back) { + // The cloud analogue of mow_cached_side_not_pinnable_falls_back, exercising a + // DEFENSIVE branch that is unreachable after a real cloud sync: the meta + // service serves a gapless committed set, so a synced [0,max] view never has + // a hole below the cached version and [0,cached] is always pinnable. The test + // FORGES the hole ([41,50] missing) with install_rowsets; since the loader + // already carries the view to 100 the decision sync short-cuts (_sync_calls + // ==1), so the pin runs against that loader-installed forged view. It + // confirms the pin-and-endpoint guard is wired on the cloud path (the block + // is now shared, not cloud-early-returned) and would fire if the gapless + // invariant were ever broken: the cached-side pin walks only [0,40] and the + // endpoint check -- short of cached version 50 -- rejects it (not emptiness). + intercept_tablet_load({{0, 40}, {51, 80}, {81, 100}}, Status::OK(), + TKeysType::UNIQUE_KEYS, /*enable_merge_on_write=*/true); + int64_t fallbacks_before = + DorisMetrics::instance()->query_cache_incremental_fallback_total->value(); + auto decision = make_stale_decision(); + EXPECT_EQ(DorisMetrics::instance()->query_cache_incremental_fallback_total->value(), + fallbacks_before + 1); + EXPECT_EQ(decision->mode, QueryCacheInstanceDecision::Mode::MISS); + EXPECT_EQ(decision->incremental_fallback_reason, "cached versions not pinnable"); + // The loader carried the view to 100, so the decision's sync took its no-op + // shortcut: one meta-service round trip. + EXPECT_EQ(_sync_calls.load(), 1); +} + +TEST_F(QueryCacheCloudIncrementalTest, mow_decision_sync_clean_bitmap_stays_incremental) { + // Positive complement (and non-vacuousness proof) of + // mow_decision_sync_brings_bitmap_detects_rewrite: identical shape -- loader + // view 50, the decision's real sync (_sync_calls==2) installs the delta + // (50,100] on a merge-on-write tablet -- but with NO rewrite mark. The + // history-rewrite check runs on the freshly synced bitmap, finds nothing, and + // the merge stays INCREMENTAL. This shows the sibling's fallback is driven by + // its injected mark and not by the decision-sync-on-MoW path itself: same + // path, no mark, no fallback. + intercept_tablet_load({{0, 50}}, Status::OK(), TKeysType::UNIQUE_KEYS, + /*enable_merge_on_write=*/true, + /*decision_sync_installs=*/{{51, 100}}); + auto decision = make_stale_decision(); + EXPECT_EQ(decision->mode, QueryCacheInstanceDecision::Mode::INCREMENTAL); + EXPECT_TRUE(decision->incremental_fallback_reason.empty()); + EXPECT_EQ(_sync_calls.load(), 2); + auto source = decision->take_delta_read_source(kTabletId); + ASSERT_NE(source, nullptr); + EXPECT_EQ(source->rs_splits.size(), 1); + EXPECT_EQ(source->rs_splits.front().rs_reader->rowset()->start_version(), 51); + EXPECT_EQ(source->rs_splits.front().rs_reader->rowset()->end_version(), 100); +} + +TEST_F(QueryCacheCloudIncrementalTest, presync_fans_out_across_tablets_and_merges_reason) { + // Two append-only tablets in one instance drive the parallel pre-sync + // fan-out (_presync_cloud_delta_tablets) at N>1 -- every other cloud test + // here exercises it only at N=1. T1's loader view already covers the queried + // version, so its decision sync short-cuts; T2's stops short, so its decision + // sync runs and is injected to fail. The fan-out must sync both tablets, the + // post-join merge must surface T2's failure keyed by its tablet id, and the + // capture loop must fall the whole decision back on it -- exercising the + // multi-tablet fan-out, the per-index reason merge, and the capture-loop + // consumption together, none of which the single-tablet cases reach. + constexpr int64_t kTabletId2 = 15680; + + // get_tablet_meta serves a DUP_KEYS meta stamped with the REQUESTED tablet + // id (args[0]), so both scan ranges materialize as distinct tablets. It is + // stateless, so the concurrent fan-out loads need no synchronization here. + _sp->set_call_back("CloudMetaMgr::get_tablet_meta", [](auto&& args) { + auto tid = try_any_cast(args[0]); + TTabletSchema schema; + schema.keys_type = TKeysType::DUP_KEYS; + auto meta = std::make_shared( + 1, 2, tid, 15674, 4, 5, schema, 6, + std::unordered_map {{7, 8}}, UniqueId(9, 10), + TTabletType::TABLET_TYPE_DISK, TCompressionType::LZ4F, 0, false); + meta->set_tablet_state(TABLET_RUNNING); + *try_any_cast(args[1]) = std::move(meta); + try_any_cast_ret(args)->second = true; + }); + + // sync_tablet_rowsets fires concurrently across the fan-out bthreads; a mutex + // guards the per-tablet call count and the metadata-only install (which draws + // from the shared engine's rowset-id generator). Per tablet the first call is + // the loader: T1 gets a view covering 100 (its later decision sync short-cuts, + // so it never calls back a second time), T2 stops at 50 so its decision sync + // runs as the second call and is failed there. add_rowsets takes each + // tablet's own header lock, so the two installs never contend. + std::mutex mu; + std::unordered_map calls; + _sp->set_call_back("CloudMetaMgr::sync_tablet_rowsets", [&](auto&& args) { + auto* tablet = try_any_cast(args[0]); + auto* ret = try_any_cast_ret(args); + std::lock_guard lk(mu); + if (calls[tablet->tablet_id()]++ == 0) { + std::vector> versions = + tablet->tablet_id() == kTabletId + ? std::vector> {{0, 50}, {51, 100}} + : std::vector> {{0, 50}}; + std::vector rowsets; + for (auto [start, end] : versions) { + auto rs_meta = std::make_shared(); + rs_meta->set_rowset_type(BETA_ROWSET); + rs_meta->set_version({start, end}); + rs_meta->set_rowset_id(_engine->next_rowset_id()); + rs_meta->set_num_segments(1); + rs_meta->set_tablet_schema(tablet->tablet_meta()->tablet_schema()); + RowsetSharedPtr rowset; + if (RowsetFactory::create_rowset(nullptr, "", rs_meta, &rowset).ok()) { + rowsets.push_back(std::move(rowset)); + } + } + std::unique_lock lock(tablet->get_header_lock()); + tablet->add_rowsets(std::move(rowsets), /*version_overlap=*/false, lock); + } else { + // T2's decision sync (T1's short-cuts and never reaches here). + ret->first = Status::InternalError("injected sync failure"); + } + ret->second = true; + }); + + // A stale entry (version 50) keyed over both tablets, queried at 100. + std::vector scan_ranges = make_scan_ranges(kTabletId, "100"); + scan_ranges.push_back(make_scan_ranges(kTabletId2, "100").front()); + auto cache_param = make_cache_param(kTabletId); + cache_param.tablet_to_range.insert({kTabletId2, "range"}); + cache_param.__set_allow_incremental(true); + std::string cache_key; + int64_t version = 0; + EXPECT_TRUE(QueryCache::build_cache_key(scan_ranges, cache_param, &cache_key, &version).ok()); + insert_entry(_cache.get(), cache_key, 50, 0); + QueryCacheRuntime runtime(cache_param, _cache.get()); + + int64_t fallbacks_before = + DorisMetrics::instance()->query_cache_incremental_fallback_total->value(); + auto decision = runtime.get_or_make_decision(scan_ranges); + EXPECT_EQ(DorisMetrics::instance()->query_cache_incremental_fallback_total->value(), + fallbacks_before + 1); + EXPECT_EQ(decision->mode, QueryCacheInstanceDecision::Mode::MISS); + EXPECT_EQ(decision->incremental_fallback_reason, "cloud rowset sync failed"); + // Both tablets fanned out and were synced: T1 once (loader; decision + // short-cut) and T2 twice (loader + the failed decision sync). + std::lock_guard lk(mu); + EXPECT_EQ(calls[kTabletId], 1); + EXPECT_EQ(calls[kTabletId2], 2); +} + +TEST_F(QueryCacheCloudIncrementalTest, decision_sync_timeout_falls_back) { + // A meta-service brownout: the decision's rowset sync stalls past the + // fast-fail budget (query_cache_decision_sync_timeout_ms). The decision must + // abandon the wait and fall the query back to a full recompute rather than + // hold its (bounded) admission thread for the RPC retry budget. Made + // deterministic with a gate that keeps the decision sync blocked until AFTER + // the assertion, so the decision's wait_for is guaranteed to expire first; + // the test then releases it and waits for the detached sync to fully exit + // its callback before teardown, so the callback never runs after the fixture + // (and the std::function it runs in) is torn down. + int32_t saved_timeout = config::query_cache_decision_sync_timeout_ms; + config::query_cache_decision_sync_timeout_ms = 50; + Defer restore_timeout {[&] { config::query_cache_decision_sync_timeout_ms = saved_timeout; }}; + + std::atomic release_gate {false}; + std::atomic decision_sync_returned {false}; + _sp->set_call_back("CloudMetaMgr::get_tablet_meta", [](auto&& args) { + TTabletSchema schema; + schema.keys_type = TKeysType::DUP_KEYS; + auto meta = std::make_shared( + 1, 2, kTabletId, 15674, 4, 5, schema, 6, + std::unordered_map {{7, 8}}, UniqueId(9, 10), + TTabletType::TABLET_TYPE_DISK, TCompressionType::LZ4F, 0, false); + meta->set_tablet_state(TABLET_RUNNING); + *try_any_cast(args[1]) = std::move(meta); + try_any_cast_ret(args)->second = true; + }); + _sp->set_call_back("CloudMetaMgr::sync_tablet_rowsets", + [this, &release_gate, &decision_sync_returned](auto&& args) { + auto* tablet = try_any_cast(args[0]); + auto* ret = try_any_cast_ret(args); + ret->second = true; + if (_sync_calls.fetch_add(1) == 0) { + // Loader sync: install a view that does NOT cover + // query 100, so the decision sync really runs. + install_rowsets(tablet, {{0, 50}}); + return; + } + // The decision's sync stalls until the test releases + // the gate (after it has asserted the fast-fail). + while (!release_gate.load()) { + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + } + // Last statement: signals the callback body is done, + // so the test can wait for it before teardown. + decision_sync_returned = true; + }); + + int64_t fallbacks_before = + DorisMetrics::instance()->query_cache_incremental_fallback_total->value(); + int64_t decision_sync_ms_before = + DorisMetrics::instance()->query_cache_decision_sync_time_ms->value(); + auto decision = make_stale_decision(); + EXPECT_EQ(decision->mode, QueryCacheInstanceDecision::Mode::MISS); + EXPECT_EQ(decision->incremental_fallback_reason, "cloud rowset sync timed out"); + EXPECT_EQ(DorisMetrics::instance()->query_cache_incremental_fallback_total->value(), + fallbacks_before + 1); + // The timeout path still records the wall time it blocked before giving up + // (steady_clock taken before the fan-out, read again after wait_for expires), + // so the sync-time counter advances by at least the budget. This is the one + // test that proves the metric is wired on the fast-fail path; wait_for is + // guaranteed not to return before its duration elapses, so >= budget is not + // a timing bet. + EXPECT_GE(DorisMetrics::instance()->query_cache_decision_sync_time_ms->value() - + decision_sync_ms_before, + config::query_cache_decision_sync_timeout_ms); + + // Release the stalled sync and wait for its callback to fully exit before + // teardown clears the sync points (destroying the callback while it runs is + // UB). The detached fan-out task reaches its decision sync eventually even + // on a slow host, so this converges. + release_gate = true; + for (int i = 0; i < 2000 && !decision_sync_returned.load(); ++i) { + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + } + ASSERT_TRUE(decision_sync_returned.load()); + // Both syncs ran once the fan-out joined: the loader and the (stalled) + // decision sync. The decision just did not wait for the second to finish. + EXPECT_EQ(_sync_calls.load(), 2); + // The decision deliberately abandons its fan-out future, so there is no + // handle to join the detached sync on. This margin covers the microsecond + // stack unwind of the callback and its SyncPoint shell after the last store + // above, plus the task's tail (releasing a non-last CloudTablet ref -- the + // tablet cache still holds one, so nothing is destroyed here -- and the + // fork-join bookkeeping). None of that tail touches fixture state, so a + // 200ms margin over a microsecond window is a safe cushion, not a bet on + // timing being fast enough. + std::this_thread::sleep_for(std::chrono::milliseconds(200)); +} + } // namespace doris diff --git a/be/test/testutil/mock/mock_runtime_state.h b/be/test/testutil/mock/mock_runtime_state.h index 82c7d788943e4f..961c00f5daef14 100644 --- a/be/test/testutil/mock/mock_runtime_state.h +++ b/be/test/testutil/mock/mock_runtime_state.h @@ -69,6 +69,14 @@ class MockRuntimeState : public RuntimeState { void set_enable_strict_cast(bool enable) { _query_options.__set_enable_strict_cast(enable); } + void set_query_freshness_tolerance_ms(int64_t ms) { + _query_options.__set_query_freshness_tolerance_ms(ms); + } + + void set_enable_prefer_cached_rowset(bool enable) { + _query_options.__set_enable_prefer_cached_rowset(enable); + } + bool enable_local_exchange() const override { return true; } WorkloadGroupPtr workload_group() override { return _workload_group; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/normalize/QueryCacheNormalizer.java b/fe/fe-core/src/main/java/org/apache/doris/planner/normalize/QueryCacheNormalizer.java index 5212c5a5fd6533..c73e19cff9da6f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/normalize/QueryCacheNormalizer.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/normalize/QueryCacheNormalizer.java @@ -179,6 +179,33 @@ private boolean computeAllowIncremental(CachePoint cachePoint, SessionVariable s if (!sessionVariable.getEnableQueryCacheIncremental()) { return false; } + // Freshness tolerance and prefer-cached-rowset (both cloud-only in + // effect) trade exactness for speed and locality: the scan may read a + // warmed-up layout that stops below the queried version (freshness + // halts at the warmed boundary) or reaches beyond it (neither walk + // clips an edge spanning it). An incremental merge would defeat both knobs, + // because the delta capture always targets the exact queried version + // (it must, to keep the merged entry correct): under freshness + // tolerance it would force the wait for un-warmed data the query + // explicitly chose to skip, and under prefer-cached-rowset it would + // ignore the layout preference the user set. So exclude incremental + // for such queries and let them take their cheap path. Correctness + // against their version-inexact reads does NOT rest on this per-query + // gate (an entry filled by such a read could still be reused by a + // different, knob-free query sharing the cache key): on cloud, the + // only mode where these reads occur, the BE suppresses their cache + // write-back, so no entry whose content mismatches its version stamp + // exists in the first place. These knobs are inert on local storage, + // so gating them in any mode only forgoes incremental for a query + // that opted into a cloud trade-off; the gate stays mode-agnostic on + // purpose, because a mode-conditioned gate cannot be exercised by the + // local FE unit-test harness (planning under a flipped cloud flag casts + // SystemInfoService to CloudSystemInfoService and fails), so it would + // ship an untestable branch for no correctness gain. + if (sessionVariable.getQueryFreshnessToleranceMs() > 0 + || sessionVariable.getEnablePreferCachedRowset()) { + return false; + } // The cache point is always an aggregation node (see doComputeCachePoint). if (((AggregationNode) cachePoint.cacheRoot).isNeedsFinalize()) { return false; diff --git a/fe/fe-core/src/test/java/org/apache/doris/planner/QueryCacheNormalizerTest.java b/fe/fe-core/src/test/java/org/apache/doris/planner/QueryCacheNormalizerTest.java index d7cb0e7d94c434..3882f3e37ff9b6 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/planner/QueryCacheNormalizerTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/planner/QueryCacheNormalizerTest.java @@ -465,6 +465,37 @@ public void testAllowIncremental() throws Exception { + " (select k1, count(*) cnt from db1.non_part group by k1) x" + " group by cnt"); Assertions.assertFalse(nestedAgg.allow_incremental); + + // Query freshness tolerance trades exactness for speed; an + // incremental merge would force the wait for un-warmed data the + // query chose to skip (the BE separately suppresses such a query's + // cache write-back on cloud, which is where correctness lives). The + // gate is mode-agnostic (it forgoes incremental in any mode for a + // query that set the knob), so it must reject incremental for an + // otherwise-eligible DUP_KEYS query here in the local harness too. + connectContext.getSessionVariable().queryFreshnessToleranceMs = 5000; + try { + TQueryCacheParam withFreshness = getQueryCacheParam( + "select k2, sum(v1) as v from db1.part1 group by k2"); + Assertions.assertFalse(withFreshness.allow_incremental); + } finally { + connectContext.getSessionVariable().queryFreshnessToleranceMs = -1; + } + + // Prefer-cached-rowset may overshoot the queried version (its walk + // never clips an edge spanning it), and the incremental delta + // capture always targets the exact queried version anyway (the BE + // separately suppresses such a query's cache write-back on cloud); + // the same mode-agnostic gate rejects incremental for the same + // otherwise-eligible query. + connectContext.getSessionVariable().enablePreferCachedRowset = true; + try { + TQueryCacheParam withPreferCached = getQueryCacheParam( + "select k2, sum(v1) as v from db1.part1 group by k2"); + Assertions.assertFalse(withPreferCached.allow_incremental); + } finally { + connectContext.getSessionVariable().enablePreferCachedRowset = false; + } } finally { connectContext.getSessionVariable().setEnableQueryCacheIncremental(false); } diff --git a/regression-test/suites/query_p0/cache/query_cache_incremental.groovy b/regression-test/suites/query_p0/cache/query_cache_incremental.groovy index 9ee2d876550e3c..1b3d18723cc6c2 100644 --- a/regression-test/suites/query_p0/cache/query_cache_incremental.groovy +++ b/regression-test/suites/query_p0/cache/query_cache_incremental.groovy @@ -20,14 +20,17 @@ // delta rowsets since the cached version and merging them with the cached // partial aggregation blocks. Every query below is checked against the same // query with the cache disabled, so results stay verified even where -// incremental merge falls back to a full recompute (e.g. cloud mode). On -// local storage the suite additionally proves through the BE metrics that -// the incremental path really fires: the early stale queries must increase -// query_cache_stale_hit_total, and the two designed fallback phases (a -// delete predicate in the delta, a merge-on-write backfill that rewrites -// history) must increase query_cache_incremental_fallback_total. The final -// phase covers a long-dormant entry whose first reuse faces a large, -// many-rowset delta that goes through the parallel scanner builder. +// incremental merge falls back to a full recompute (e.g. when the delta was +// merged away by a compaction). The suite additionally proves through the BE +// metrics that the incremental path really fires: the early stale queries +// must increase query_cache_stale_hit_total, and the two designed fallback +// phases (a delete predicate in the delta, a merge-on-write backfill that +// rewrites history) must increase query_cache_incremental_fallback_total. +// The final phase covers a long-dormant entry whose first reuse faces a +// large, many-rowset delta that goes through the parallel scanner builder. +// This holds in cloud mode too: the decision first brings the tablet view up +// to the queried version (usually a no-op, the scan would sync anyway) and +// then takes the same incremental path as local storage. suite("query_cache_incremental") { def querySql = """ SELECT @@ -72,20 +75,26 @@ suite("query_cache_incremental") { return total } - // Besides result consistency, prove on local storage that the stale entry - // was really merged incrementally (Mode::INCREMENTAL), not recomputed: - // only the incremental path increases query_cache_stale_hit_total, and no + // Besides result consistency, prove that the stale entry was really + // merged incrementally (Mode::INCREMENTAL), not recomputed: only the + // incremental path increases query_cache_stale_hit_total, and no // concurrent suite touches it because the session switch defaults to off. - // Cloud mode always falls back by design, so it only checks consistency. // Residual caveat (here and in checkIncrementalFallback): a // memory-pressure prune evicting the entry inside the tiny fill-to-assert // window surfaces as a plain MISS and would fail the assertion; accepted - // as rare, and a rerun re-establishes the counters from fresh deltas. + // as rare, and a rerun re-establishes the counters from fresh deltas. The + // one channel that is NOT left to chance is compaction: every table here + // sets disable_auto_compaction, so no background compaction can merge the + // delta away mid-window and turn the expected incremental hit into a + // fallback (this matters most on cloud, where compaction is driven + // externally rather than by the querying BE). + // On cloud two more channels surface the same way and fail only + // checkStaleIncremental (checkIncrementalFallback is immune: any + // non-empty reason still counts): a transient meta-service hiccup during + // the decision's view sync, and the tablet rebalancer routing the stale + // query to a BE that holds no entry. Both equally rare inside this window, + // same remedy. def checkStaleIncremental = { String sqlText -> - if (isCloudMode()) { - checkConsistency(sqlText) - return - } def before = sumBeMetric("query_cache_stale_hit_total") checkConsistency(sqlText) def after = sumBeMetric("query_cache_stale_hit_total") @@ -95,10 +104,6 @@ suite("query_cache_incremental") { // Prove that a designed fallback phase really took the fallback path. def checkIncrementalFallback = { String sqlText -> - if (isCloudMode()) { - checkConsistency(sqlText) - return - } def before = sumBeMetric("query_cache_incremental_fallback_total") checkConsistency(sqlText) def after = sumBeMetric("query_cache_incremental_fallback_total") @@ -131,7 +136,8 @@ suite("query_cache_incremental") { DISTRIBUTED BY HASH(user_id) BUCKETS 3 PROPERTIES ( - "replication_num" = "1" + "replication_num" = "1", + "disable_auto_compaction" = "true" ) """ @@ -168,13 +174,15 @@ suite("query_cache_incremental") { ('2026-01-13',${100 + i},'/inc',${10 * i}) """ if (i == 1) { - // The hot partition holds just two data rowsets here, far from - // both the merge-count threshold and any compaction, so on local - // storage this stale query must take the incremental path. + // The hot partition holds just two data rowsets here, well below + // the merge-count threshold, and auto-compaction is disabled on the + // table, so this stale query must take the incremental path on both + // deployment modes. checkStaleIncremental(querySql) } else { - // Later rounds may legitimately recompute in full (merge-count - // threshold, background compaction), so assert correctness only. + // Later rounds may legitimately recompute in full (the merge-count + // threshold forces one every query_cache_max_incremental_merge_count + // merges), so assert correctness only. checkConsistency(querySql) } } @@ -215,7 +223,8 @@ suite("query_cache_incremental") { PROPERTIES ( "replication_num" = "1", - "enable_unique_key_merge_on_write" = "true" + "enable_unique_key_merge_on_write" = "true", + "disable_auto_compaction" = "true" ) """ def uniqueQuerySql = """ @@ -249,9 +258,11 @@ suite("query_cache_incremental") { // entry. sql "INSERT INTO test_query_cache_incremental_mow VALUES ('2026-01-01',1,100)" checkIncrementalFallback(uniqueQuerySql) - // After the re-base, pure appends take the incremental path again (not - // asserted through the metric: by now enough rowsets piled up that a - // background compaction could legitimately force a full recompute). + // After the re-base, pure appends take the incremental path again. Left as + // a correctness-only check (not metric-asserted): the earlier rounds + // already pin the incremental path through the metric, and on cloud the + // residual meta-service/rebalance channels noted above could still flip a + // single stale-hit assertion to a spurious MISS. sql "INSERT INTO test_query_cache_incremental_mow VALUES ('2026-01-06',200,7)" checkConsistency(uniqueQuerySql) order_qt_mow_final "${uniqueQuerySql}" @@ -285,7 +296,8 @@ suite("query_cache_incremental") { DISTRIBUTED BY HASH(user_id) BUCKETS 1 PROPERTIES ( - "replication_num" = "1" + "replication_num" = "1", + "disable_auto_compaction" = "true" ) """ def dormantQuerySql = """ @@ -312,10 +324,10 @@ suite("query_cache_incremental") { sql "INSERT INTO test_query_cache_incremental_dormant VALUES ${values}" } // The first query after the idle stretch must still take the incremental - // path on local storage and agree with the uncached result. The loads - // landed moments ago, so a boundary-crossing base compaction inside this - // window is as unlikely as in the first-round assertions above (an - // in-window cumulative compaction keeps the capture, and therefore the - // assertion, intact). + // path on both deployment modes and agree with the uncached result. + // Auto-compaction is disabled on the table, so the 30-rowset delta stays + // intact and no boundary-crossing compaction can turn this into a fallback; + // on cloud the residual rare meta-service/rebalance channels noted at + // checkStaleIncremental still apply and are handled the same way (rerun). checkStaleIncremental(dormantQuerySql) } From 90896ddbf09d6b61df902215a95814e18e141eef Mon Sep 17 00:00:00 2001 From: Benedict Jin Date: Sun, 19 Jul 2026 23:06:26 +0800 Subject: [PATCH 2/4] [fix](query-cache) Harden cloud incremental merge per review feedback Follow-up on PR #65788 addressing the review of the cloud incremental merge change. No storage format or symbol change; the only wire change is one optional, appended thrift field. Ownership and bound of the cloud decision pre-sync: - The per-tablet decision syncs no longer run on raw detached bthreads. They run on a dedicated, bounded, engine-owned ThreadPool (query_cache_delta_sync_pool). stop() drains it first, before the compaction and delete-bitmap pools its sync_rowsets callbacks depend on and before ~CloudStorageEngine destroys _meta_mgr/_tablet_mgr, so a task that outlived a timed-out query always runs against a live engine and none can survive it. - The pool has a fixed width and a bounded queue (config query_cache_delta_sync_ thread / query_cache_delta_sync_max_pending_tasks, both validated at startup), so a meta-service brownout can neither spawn an unbounded fanout nor grow the backlog without bound. Its workers are pre-started at engine construction and verified present with a CHECK on the worker count (ThreadPool::init swallows per-thread start failures, so a successful build alone proves nothing): submit is therefore pure enqueue and never creates a thread on the query admission path. - One absolute deadline, anchored at the fan-out start, covers submission plus the wait, so nothing can re-grant the budget. A shared abandoned flag, set when the wait expires, lets a not-yet-started task skip its RPC so the queue drains fast. A non-positive timeout launches nothing at all. - Each slot is claimed via an atomic exchange BEFORE any sync RPC, so exactly one of the task and the inline submit-failure path owns the RPC, the result slot, and the latch count-down; an enqueue-then-fail task a later worker picks up loses the claim and cannot fire a sync for an already-settled query. - The capture loop consumes each tablet's recorded fallback reason BEFORE its own get_tablet, and a worker-side tablet-load failure publishes a reason instead of leaving its slot empty, so no fallback route can push the synchronous cache-miss meta-service load onto the admission thread outside the deadline. Cloud MOW write-back and mixed-version: - BE no longer keys write-back on the raw prefer-cached knob for cloud MOW. FE resolves the selected index MOW state before the knob gate and reports it via a new optional field TQueryCacheParam.is_merge_on_write, and BE keeps write-back for the version-exact MOW read. The optional field is the mixed-version guard: old FE to new BE reads its default false and takes the conservative branch, new FE to old BE ignores it. Tests and regression determinism: - The regression stale-hit and fallback metric checks are gated to the shared-nothing path, sampling included; an aggregated BE-local counter cannot prove a specific cloud query used a stale entry when routing is topology-dependent. Cloud correctness is proven by result consistency plus deterministic BE unit tests. - New BE tests: pool-rejects-submit fallback, rejection-plus-cache-miss proving zero admission-thread meta loads, worker-load-failure proving a single worker-side load, non-cloud-engine fallback, engine-stopped fallback, decision-sync timeout fallback, cloud MOW prefer write-back, and cloud sync-completeness history-rewrite detection. --- be/src/cloud/cloud_storage_engine.cpp | 62 +++ be/src/cloud/cloud_storage_engine.h | 15 + be/src/cloud/config.cpp | 2 + be/src/cloud/config.h | 23 +- .../exec/operator/cache_source_operator.cpp | 15 +- be/src/runtime/query_cache/query_cache.cpp | 371 ++++++++++++------ be/src/runtime/query_cache/query_cache.h | 8 + .../operator/query_cache_operator_test.cpp | 63 +++ be/test/exec/pipeline/query_cache_test.cpp | 240 +++++++++-- .../normalize/QueryCacheNormalizer.java | 118 ++++-- .../planner/QueryCacheNormalizerTest.java | 23 ++ gensrc/thrift/QueryCache.thrift | 11 + .../cache/query_cache_incremental.groovy | 76 ++-- 13 files changed, 801 insertions(+), 226 deletions(-) diff --git a/be/src/cloud/cloud_storage_engine.cpp b/be/src/cloud/cloud_storage_engine.cpp index 0dc8321c2c4e3a..5b917528fa60ae 100644 --- a/be/src/cloud/cloud_storage_engine.cpp +++ b/be/src/cloud/cloud_storage_engine.cpp @@ -103,6 +103,54 @@ CloudStorageEngine::CloudStorageEngine(const EngineOptions& options) _cumulative_compaction_policies[CUMULATIVE_TIME_SERIES_POLICY] = std::make_shared(); _startup_timepoint = std::chrono::system_clock::now(); + // Build the dedicated, bounded query-cache delta pre-sync pool up front (see + // the member declaration): a decision fan-out must never find it null, and the + // destructor's stop() drains it before _meta_mgr/_tablet_mgr die. A build + // failure here means the process cannot create threads at startup, already + // fatal, so CHECK rather than limp on with a null pool. + // + // Report a misconfigured sizing loudly instead of silently coercing it: an + // operator who set an invalid value must see why the BE refused to start rather + // than run with a capacity that differs from the config (invalid state fails, it + // never silently continues). A max of >= 1 is required (a zero-width pool can run + // nothing) and the queue bound must be non-negative. + int32_t qc_sync_threads = config::query_cache_delta_sync_thread; + int32_t qc_sync_queue = config::query_cache_delta_sync_max_pending_tasks; + CHECK_GE(qc_sync_threads, 1) << "query_cache_delta_sync_thread must be >= 1, got " + << qc_sync_threads; + CHECK_GE(qc_sync_queue, 0) << "query_cache_delta_sync_max_pending_tasks must be >= 0, got " + << qc_sync_queue; + // Pre-start all workers here (min == max), at engine construction, OFF the query + // admission path. The decision fan-out submits from operator init, which runs on + // the bounded light_work_pool, and ThreadPool creates workers synchronously + // inside submit_func when the pool is short of the demand (creation can take + // hundreds of ms). A lazily grown pool (min = 0) would move that creation latency + // onto the query's critical path and, worse, could hand the fan-out a submit that + // enqueued a task yet returned an error at zero live threads. A fixed pool sized + // up front keeps submit_func pure enqueue (it never creates a thread, so it never + // blocks), so the fast-fail budget is spent on the sync RPCs rather than on + // spawning threads. The cost is a fixed set of mostly-idle workers on every cloud + // BE even when this opt-in feature is unused; that steady footprint is the + // deliberate trade for predictable, non-blocking admission. + Status qc_pool_st = ThreadPoolBuilder("QueryCacheDeltaSyncThreadPool") + .set_min_threads(qc_sync_threads) + .set_max_threads(qc_sync_threads) + .set_max_queue_size(qc_sync_queue) + .build(&_query_cache_delta_sync_pool); + CHECK(qc_pool_st.ok()) << "failed to build QueryCacheDeltaSyncThreadPool: " << qc_pool_st; + // A successful build does NOT prove the workers exist: ThreadPool::init() + // deliberately ignores per-thread creation failures (a worker can be created + // later on submit). Verify the fixed worker set actually started and fail + // construction if not, so the fan-out's "submit never creates a thread" property + // is a guarantee, not a hope -- without this a startup thread shortage would + // silently push synchronous thread creation back onto the query admission path, + // exactly the latency this fixed pool exists to avoid. num_threads() counts + // started plus pending-start workers, so it equals the request precisely when + // every worker was spawned. + CHECK_EQ(_query_cache_delta_sync_pool->num_threads(), qc_sync_threads) + << "QueryCacheDeltaSyncThreadPool started " + << _query_cache_delta_sync_pool->num_threads() << " of " << qc_sync_threads + << " workers"; } CloudStorageEngine::~CloudStorageEngine() { @@ -274,6 +322,20 @@ void CloudStorageEngine::stop() { } } + // Drain the query-cache delta pre-sync fan-out FIRST, before the pools its + // sync_rowsets callbacks depend on: a merge-on-write delete-bitmap sync + // submits file-read work to _sync_delete_bitmap_thread_pool / + // _calc_tablet_delete_bitmap_task_thread_pool and blocks on it. Shutting a + // dependency pool down first would discard its queued callbacks and leave a + // query-cache worker blocked forever, hanging this drain and BE shutdown. + // Draining this pool first lets its in-flight workers finish against the + // still-live dependency pools. stop() also runs before any member is + // destroyed (~CloudStorageEngine calls it first), so those tasks see live + // _meta_mgr/_tablet_mgr as well. + if (_query_cache_delta_sync_pool) { + _query_cache_delta_sync_pool->shutdown(); + } + if (_base_compaction_thread_pool) { _base_compaction_thread_pool->shutdown(); } diff --git a/be/src/cloud/cloud_storage_engine.h b/be/src/cloud/cloud_storage_engine.h index b939bbf094a1d5..9ae1edabb5aac8 100644 --- a/be/src/cloud/cloud_storage_engine.h +++ b/be/src/cloud/cloud_storage_engine.h @@ -176,6 +176,11 @@ class CloudStorageEngine final : public BaseStorageEngine { return *_sync_load_for_tablets_thread_pool; } + // Dedicated bounded pool for the query-cache incremental decision's per-tablet + // rowset pre-sync. Always non-null (built in the constructor), so callers need + // no null check. See the member declaration and cloud/config.h. + ThreadPool& query_cache_delta_sync_pool() const { return *_query_cache_delta_sync_pool; } + ThreadPool& warmup_cache_async_thread_pool() const { return *_warmup_cache_async_thread_pool; } Status register_compaction_stop_token(CloudTabletSPtr tablet, int64_t initiator); @@ -234,6 +239,16 @@ class CloudStorageEngine final : public BaseStorageEngine { std::shared_ptr _cloud_warm_up_manager; std::unique_ptr _tablet_hotspot; std::unique_ptr _sync_load_for_tablets_thread_pool; + // Dedicated bounded pool for the query-cache incremental decision's per-tablet + // rowset pre-sync (QueryCacheRuntime::_presync_cloud_delta_tablets). Built in + // the constructor -- not open() -- so a decision fan-out never dereferences a + // null pool and unit fixtures that construct the engine without open() have + // it; drained in stop() (which the destructor calls first) so every in-flight + // task joins before _meta_mgr/_tablet_mgr are torn down. Declared after those + // managers so reverse-order member destruction is a second guarantee of the + // same ordering. Kept separate from the FE-warmup SyncLoadForTabletsThreadPool + // so a meta-service brownout cannot couple the two paths. + std::unique_ptr _query_cache_delta_sync_pool; std::unique_ptr _warmup_cache_async_thread_pool; std::unique_ptr _cloud_snapshot_mgr; diff --git a/be/src/cloud/config.cpp b/be/src/cloud/config.cpp index 54008ddc2fb655..3735b44ca855f4 100644 --- a/be/src/cloud/config.cpp +++ b/be/src/cloud/config.cpp @@ -39,6 +39,8 @@ DEFINE_mInt32(tablet_sync_interval_s, "1800"); DEFINE_mInt32(init_scanner_sync_rowsets_parallelism, "10"); DEFINE_mInt32(sync_rowsets_slow_threshold_ms, "1000"); DEFINE_mInt32(query_cache_decision_sync_timeout_ms, "2000"); +DEFINE_Int32(query_cache_delta_sync_thread, "16"); +DEFINE_Int32(query_cache_delta_sync_max_pending_tasks, "2048"); DEFINE_mInt64(min_compaction_failure_interval_ms, "5000"); DEFINE_mInt64(base_compaction_freeze_interval_s, "1800"); diff --git a/be/src/cloud/config.h b/be/src/cloud/config.h index 02c0c1a6de38de..28417701d20e29 100644 --- a/be/src/cloud/config.h +++ b/be/src/cloud/config.h @@ -80,14 +80,25 @@ DECLARE_mInt32(sync_rowsets_slow_threshold_ms); // falls back to one full recompute (the scan node's own async sync still brings // the view up for the actual scan). A healthy sync is milliseconds, far under // this, so it only trips under real meta-service degradation and leaves the -// steady-state incremental path unchanged. A value <= 0 makes the wait expire -// immediately, so the decision falls back to a full recompute whenever a sync -// is still in flight (a fan-out that already finished -- every tablet a no-op -// -- still proceeds correctly): that disables cloud incremental merge under any -// real sync latency rather than blocking, a fail-safe, not a useful setting. -// Cloud only. +// steady-state incremental path unchanged. A value <= 0 disables cloud +// incremental merge outright: the decision skips the pre-sync fan-out entirely +// (launching nothing) and falls every scanned tablet back to a full recompute, +// a fail-safe rather than a useful setting. Cloud only. DECLARE_mInt32(query_cache_decision_sync_timeout_ms); +// The dedicated, bounded thread pool that runs the query-cache incremental +// decision's per-tablet rowset pre-sync (CloudStorageEngine owns it, created at +// construction and drained in stop()). It is deliberately NOT the shared +// SyncLoadForTabletsThreadPool: that pool is on the FE-driven warmup path, and +// giving the decision fan-out its own pool keeps a meta-service brownout from +// coupling the two. `query_cache_delta_sync_thread` bounds the concurrent +// syncs; `query_cache_delta_sync_max_pending_tasks` caps the queue so a brownout +// (every sync stalls in retry_rpc while new stale queries keep enqueuing) cannot +// grow the backlog without bound -- a submit that would exceed the cap fails +// fast and that tablet falls back to a full recompute. Cloud only. +DECLARE_Int32(query_cache_delta_sync_thread); +DECLARE_Int32(query_cache_delta_sync_max_pending_tasks); + // Cloud compaction config DECLARE_mInt64(min_compaction_failure_interval_ms); DECLARE_mBool(enable_new_tablet_do_compaction); diff --git a/be/src/exec/operator/cache_source_operator.cpp b/be/src/exec/operator/cache_source_operator.cpp index 83936946cc394d..a56ae61150ab75 100644 --- a/be/src/exec/operator/cache_source_operator.cpp +++ b/be/src/exec/operator/cache_source_operator.cpp @@ -112,9 +112,20 @@ Status CacheSourceLocalState::init(RuntimeState* state, LocalStateInfo& info) { // where it overshot. Suppressing the write-back keeps these reads pure // consumers: they may still serve an exact HIT filled by a plain run // (precise data, no worse than asked). + // + // Carve-out for the prefer knob on cloud merge-on-write UNIQUE tables: + // CloudTablet::capture_consistent_versions_unlocked honors + // enable_prefer_cached_rowset only for non-MOW tables (it guards on + // !enable_unique_key_merge_on_write()), so a MOW query that set prefer but + // not freshness still reads the exact queried version. That fill is + // version-exact and cacheable; suppressing it would keep a cloud MOW table + // with prefer set from ever populating the cache. Freshness tolerance has + // no such MOW guard (it is honored for MOW too), so it still forces + // suppression regardless of table type. const bool inexact_version_fill = - config::is_cloud_mode() && (state->enable_query_freshness_tolerance() || - state->enable_prefer_cached_rowset()); + config::is_cloud_mode() && + (state->enable_query_freshness_tolerance() || + (state->enable_prefer_cached_rowset() && !cache_param.is_merge_on_write)); _need_insert_cache = _cache_decision->key_valid && !hit_cache && _cache_decision->write_back_feasible && !inexact_version_fill; _insert_delta_count = _is_incremental ? _cache_decision->cached_delta_count + 1 : 0; diff --git a/be/src/runtime/query_cache/query_cache.cpp b/be/src/runtime/query_cache/query_cache.cpp index ccbb6ca66c79d3..6ecdb8fe4dbfeb 100644 --- a/be/src/runtime/query_cache/query_cache.cpp +++ b/be/src/runtime/query_cache/query_cache.cpp @@ -20,13 +20,13 @@ #include #include #include -#include #include #include #include #include #include "cloud/cloud_meta_mgr.h" +#include "cloud/cloud_storage_engine.h" #include "cloud/cloud_tablet.h" #include "cloud/config.h" #include "common/logging.h" @@ -37,6 +37,9 @@ #include "storage/rowset/rowset_reader.h" #include "storage/tablet/base_tablet.h" #include "storage/tablet/tablet_meta.h" +#include "util/countdown_latch.h" +#include "util/defer_op.h" +#include "util/threadpool.h" namespace doris { @@ -268,8 +271,7 @@ bool QueryCacheRuntime::_try_prepare_incremental(const std::vector presync_reasons; if (config::is_cloud_mode()) { - presync_reasons = - _presync_cloud_delta_tablets(scan_ranges, decision->current_version); + presync_reasons = _presync_cloud_delta_tablets(scan_ranges, decision->current_version); } for (const auto& scan_range : scan_ranges) { @@ -297,48 +299,189 @@ bool QueryCacheRuntime::_try_prepare_incremental(const std::vector QueryCacheRuntime::_presync_cloud_delta_tablets( const std::vector& scan_ranges, int64_t current_version) { - // Index-aligned so the fork-joined tasks write disjoint slots without a - // lock; an empty slot means synced (or skipped as non-append-only), a - // non-empty slot is the fallback reason. Held behind a shared_ptr because - // the fan-out below is waited on with a fast-fail deadline: on a timeout - // this frame returns while the still-running tasks keep writing their - // slots, so the storage must outlive the frame -- the shared_ptr each task - // captured keeps it alive until the last one finishes. Only read on the - // completed (non-timeout) path, where every task is done. + std::unordered_map fallback_reasons; + + // (c) A non-positive budget disables cloud incremental merge: return WITHOUT + // launching any work. The previous code launched the fan-out and abandoned + // it on an instantly expired wait, spawning tasks whose only effect was to + // be orphaned. Fall every scanned tablet back so the whole instance + // recomputes in full. Checked before touching the engine so the semantics + // (and its unit test) do not depend on engine state. + if (config::query_cache_decision_sync_timeout_ms <= 0) { + for (const auto& scan_range : scan_ranges) { + fallback_reasons[scan_range.scan_range.palo_scan_range.tablet_id] = + "cloud incremental sync disabled"; + } + return fallback_reasons; + } + + // Reached only under config::is_cloud_mode() (the caller gates on it), but + // cloud_unique_id is a mutable config, so is_cloud_mode() can flip true on a + // live LOCAL deployment whose engine stays a local StorageEngine. Degrade + // gracefully rather than aborting in the hard CHECK inside to_cloud(): fall + // every tablet back, mirroring the per-tablet cloud-cast fallback below. + auto* engine = dynamic_cast(&ExecEnv::GetInstance()->storage_engine()); + if (engine == nullptr) { + for (const auto& scan_range : scan_ranges) { + fallback_reasons[scan_range.scan_range.palo_scan_range.tablet_id] = + "storage engine is not cloud"; + } + return fallback_reasons; + } + + // (a) The BE is tearing down: do not add new sync work. Fall every scanned + // tablet back rather than race teardown. + if (engine->stopped()) { + for (const auto& scan_range : scan_ranges) { + fallback_reasons[scan_range.scan_range.palo_scan_range.tablet_id] = + "be is stopping, sync skipped"; + } + return fallback_reasons; + } + + // Index-aligned so the fan-out tasks write disjoint slots without a lock; an + // empty slot means synced (or skipped as non-append-only), a non-empty slot + // is the fallback reason. Held (with the latch and per-slot count guard) + // behind a shared_ptr because the fan-out is waited on with a fast-fail + // deadline: on a timeout this frame returns while still-running tasks keep + // writing their slots, so the storage must outlive the frame -- the shared_ptr + // each task captured keeps it alive until the last one finishes. Only read on + // the completed path. auto per_range_reason = std::make_shared>(scan_ranges.size()); - std::vector> tasks; - tasks.reserve(scan_ranges.size()); + auto fanout_done = std::make_shared(static_cast(scan_ranges.size())); + // Single-owner claim per slot. Two parties can target the same slot: the task, + // and -- on the narrow ThreadPool path where do_submit enqueues the task and + // THEN returns an error (thread creation failed with zero live workers) -- the + // inline submit-failure path, which then runs concurrently with the very task it + // failed to schedule. Each party claims the slot with an atomic exchange BEFORE + // touching it: the winner runs the sync (or the inline fallback), writes + // per_range_reason[idx] via publish_slot, and counts the latch down; the loser + // returns having touched neither. Claiming before the RPC (an exchange, not a + // check-then-act load) is what stops the enqueue-then-fail task from firing a + // sync for an already-settled query, and it keeps the caller, once the latch + // settles, the sole reader of a slot no task is still writing. (Value-initialized + // to false.) + auto slot_counted = std::make_shared>>(scan_ranges.size()); + // Set true when the caller abandons the wait (its deadline passed): a not-yet- + // started task then skips its sync RPC instead of spending the full retry + // budget, so a meta-service brownout drains the bounded queue fast rather than + // running abandoned work to completion and starving the pool. + auto abandoned = std::make_shared>(false); + // Called ONLY by whoever already won the exclusive slot claim (the slot_counted + // exchange below), so it is the sole writer of the slot: publish its reason (an + // empty reason means synced OK / skipped, leaving the default-empty slot) and + // release the latch exactly once. The claim winner counting down is what lets the + // caller move the slot out the instant the latch settles. + auto publish_slot = [fanout_done, per_range_reason](size_t idx, std::string reason) { + if (!reason.empty()) { + (*per_range_reason)[idx] = std::move(reason); + } + fanout_done->count_down(); + }; + + // Run the per-tablet syncs on the engine-owned, bounded query-cache delta + // pre-sync pool -- a DEDICATED pool, not the shared SyncLoadForTablets warmup + // pool, so a meta-service brownout cannot couple the two paths. Properties + // this relies on: (1) fixed width + a bounded queue (cloud/config.h + // query_cache_delta_sync_*), so a brownout can neither spawn unbounded + // concurrent syncs nor grow the backlog without bound (a submit past the queue + // cap fails fast and that tablet falls back); (2) CloudStorageEngine::stop() + // drains this pool (joins running tasks, discards queued ones) and the + // destructor calls stop() before _meta_mgr/_tablet_mgr are destroyed, so a + // task that outlived its timed-out query still runs against a live engine and + // none can survive it. The raw detached bthreads this replaced were joined by + // nothing, so one sleeping in retry_rpc could wake after the engine was freed. + // + // Fast-fail is preserved: this runs in operator init on the bounded query + // admission pool (light_work_pool, "must be light, not locked"). submit_func + // never creates a worker -- the pool's fixed worker set (min == max) is + // pre-started AND verified present in the engine ctor (a CHECK_EQ on num_threads, + // because ThreadPool::init swallows creation failures), so submit is pure enqueue: + // it returns immediately at capacity/shutdown and never blocks on thread creation, + // keeping thread-creation latency off this critical path. The whole fan-out plus + // wait is then bounded by ONE absolute deadline (query_cache_decision_sync_ + // timeout_ms from the start below), which the pure-enqueue submit loop cannot + // exhaust before the wait even begins. A healthy sync is milliseconds, far under + // the budget, so the steady-state path is unchanged; this only trips under real + // meta-service degradation. Every task records its own sync error into its slot so + // no failure aborts the others. + // + // The tasks do no explicit MemTracker/ResourceContext attach: this is a + // detached engine-layer sync with no live query context to charge to (the + // query may have already timed out and returned, so attaching to it would be + // both unavailable here -- this is a static engine path -- and wrong). Their + // allocations fall to the worker thread's default accounting, matching the + // same-shaped sync on the same kind of pool by CloudStorageEngine's other + // sync_rowsets caller, CloudBackendService::sync_load_for_tablets (the FE + // warmup path), which likewise does not attach. + auto sync_fanout_start = std::chrono::steady_clock::now(); + auto& pool = engine->query_cache_delta_sync_pool(); + // Safe to capture the engine by raw pointer: stop() drains this pool before + // the engine (and _meta_mgr/_tablet_mgr) is destroyed, so a running task + // always sees a live engine. + CloudStorageEngine* engine_ptr = engine; for (size_t i = 0; i < scan_ranges.size(); ++i) { int64_t tablet_id = scan_ranges[i].scan_range.palo_scan_range.tablet_id; - tasks.emplace_back([tablet_id, current_version, per_range_reason, i]() -> Status { + Status submit_st = pool.submit_func([tablet_id, current_version, abandoned, i, engine_ptr, + publish_slot, slot_counted]() { + // Claim this slot atomically BEFORE any external work. If the inline + // submit-failure path already claimed it -- the narrow enqueue-then-fail + // case where do_submit queued this task but then returned an error at zero + // live workers, and the caller finished the slot inline -- bail before the + // sync RPC: the caller owns the slot and the count-down, and this task's + // result would be discarded. Claiming before the RPC (an exchange, not a + // plain load that a concurrent inline claim could race between the read + // and the RPC) is what keeps an enqueue-then-fail task that a later worker + // picks up from firing a meta-service call for an already-settled query. + if ((*slot_counted)[i].exchange(true, std::memory_order_acq_rel)) { + return; + } + // This task now exclusively owns the slot and MUST publish it exactly once + // on every exit path so the caller's bounded wait can settle. Empty reason + // means synced OK (or skipped as non-append-only); a non-empty reason + // falls this tablet back. + std::string reason; + Defer done([&] { publish_slot(i, std::move(reason)); }); + // (a) Shutdown began, or the caller already abandoned the wait (its + // deadline passed): bail before a fresh sync RPC so the pool drains + // quickly instead of spending the full RPC retry budget on work whose + // result no one will read. (An abandoned task's reason is never read -- + // the caller took the timeout path -- but recording it is harmless.) + if (engine_ptr->stopped() || abandoned->load(std::memory_order_acquire)) { + reason = "be is stopping, sync skipped"; + return; + } auto tablet_res = ExecEnv::get_tablet(tablet_id); if (!tablet_res) { - // _capture_tablet_delta reports "tablet not found" from its own - // get_tablet; nothing to sync. Should that later get_tablet - // succeed instead (a transient failure here), it takes the - // cache-miss load, which itself syncs rowsets AND the delete - // bitmap up to the visible version -- that load-bearing - // invariant (plus the endpoint check downstream) is what keeps - // a tablet that skipped this pre-sync safe to capture. - return Status::OK(); + // Publish the failure instead of leaving the slot empty: an + // empty reason tells _capture_tablet_delta the view is synced, + // so its own get_tablet would RETRY this very cache-miss + // meta-service load synchronously on the admission thread, + // outside the fast-fail deadline -- during a brownout (the one + // situation where this load fails slowly) that repeats the + // slow RPC on the pool this budget exists to protect. Falling + // back is also never wrong here: this worker-side load is + // strictly conservative (a transient failure only costs the + // incremental merge, never correctness). + reason = "cloud tablet load failed"; + return; } BaseTabletSPtr tablet = std::move(tablet_res.value()); - const bool append_only = - tablet->keys_type() == KeysType::DUP_KEYS || - (tablet->keys_type() == KeysType::UNIQUE_KEYS && - tablet->enable_unique_key_merge_on_write()); + const bool append_only = tablet->keys_type() == KeysType::DUP_KEYS || + (tablet->keys_type() == KeysType::UNIQUE_KEYS && + tablet->enable_unique_key_merge_on_write()); if (!append_only) { // Skip the RPC entirely: _capture_tablet_delta rejects this // tablet at its keys-type check before it would consume a sync. - return Status::OK(); + return; } auto cloud_tablet = std::dynamic_pointer_cast(std::move(tablet)); if (cloud_tablet == nullptr) { // is_cloud_mode() flipped on a live local deployment // (cloud_unique_id is a mutable config): degrade to a full // recompute rather than dereference the failed cast downstream. - (*per_range_reason)[i] = "tablet is not a cloud tablet"; - return Status::OK(); + reason = "tablet is not a cloud tablet"; + return; } SyncOptions options; options.query_version = current_version; @@ -362,89 +505,60 @@ std::unordered_map QueryCacheRuntime::_presync_cloud_delta LOG_EVERY_N(WARNING, 100) << "query cache incremental merge falls back, cloud rowset sync failed" << ", tablet_id=" << tablet_id << ", status=" << st.to_string(); - (*per_range_reason)[i] = "cloud rowset sync failed"; + reason = "cloud rowset sync failed"; } - return Status::OK(); }); + if (!submit_st.ok()) { + // Pool shutting down or queue at capacity: do not block. Claim the slot + // for the inline fallback. do_submit rejects shutdown and capacity before + // enqueue, so the closure will not run; on the narrow path where it did + // enqueue before failing (thread creation at zero live workers), the + // exchange makes exactly one of this path and the queued task own the slot + // -- if the task claimed it first it will publish, so do nothing here. + if (!(*slot_counted)[i].exchange(true, std::memory_order_acq_rel)) { + publish_slot(i, "cloud rowset sync not scheduled"); + } + } } - // Fan the syncs out asynchronously and wait on the result with a fast-fail - // budget instead of blocking unconditionally. This decision runs in - // operator init on a bounded query-admission pool (the BE light_work_pool, - // contractually "must be light, not locked"), so a meta-service brownout - // that stalls these RPCs must not hold that thread for the full RPC retry - // budget (tens of seconds): sustained, it would exhaust the pool and reject - // query admission cluster-wide. If the fan-out does not finish within - // query_cache_decision_sync_timeout_ms the decision abandons the wait and - // falls the whole instance back to a full recompute; the still-running - // tasks are correctness-harmless and bounded: on a slow-but-healthy sync - // each merely advances the tablet view early (work the scan node's own - // async sync would otherwise do), and on a failing sync each is at worst one - // extra bounded sync attempt that changes no result; either way they stay - // cheap while pending (small per-task captures; the brpc I/O yields the - // pthread rather than pinning it) and self-limiting once the meta service - // recovers, and keep their slot storage alive through the shared_ptr they - // captured. A healthy sync is - // milliseconds, far under the budget, so this only trips under real - // meta-service degradation and leaves the steady-state incremental path - // unchanged. - // - // Parallelism reuses init_scanner_sync_rowsets_parallelism on purpose: this - // is the same per-tablet rowset-sync fan-out the scan node runs, and one - // shared knob keeps the two fan-outs' budgets aligned rather than letting a - // cache-only config silently drift apart from it. Every task swallows its - // own error into its slot and returns OK, which is load-bearing: - // bthread_fork_join stops DISPATCHING the tasks queued after the first one - // that returns non-OK (see cloud_meta_mgr.cpp), so a task that propagated - // its sync error would leave the tablets queued behind it un-synced; - // recording the failure into the slot keeps every tablet's sync attempted. - // The blocking time (bounded by the budget) is folded into - // query_cache_decision_sync_time_ms. - auto sync_fanout_start = std::chrono::steady_clock::now(); - std::future fanout_done; - // Clamp the shared parallelism knob to >= 1: bthread_fork_join waits for a - // free slot before dispatching its first task (count 0 >= concurrency 0), - // so a misconfigured 0/negative would park the driver bthread forever with - // nothing to notify it. On the scan node that surfaces as a hung query; on - // this fast-fail path the caller's wait_for still expires and falls back, so - // the deadlock would instead leak the driver bthread and its task closures - // silently, once per incremental candidate. A local floor avoids that. - Status launch_st = cloud::bthread_fork_join( - std::move(tasks), std::max(1, config::init_scanner_sync_rowsets_parallelism), - &fanout_done); - bool timed_out = false; - if (!launch_st.ok()) { - // Could not even spawn the fan-out driver bthread: fall back rather than - // block, the same as a timeout (its future would never be fulfilled). - timed_out = true; - } else if (fanout_done.wait_for(std::chrono::milliseconds( - config::query_cache_decision_sync_timeout_ms)) != - std::future_status::ready) { - timed_out = true; - } else { - // Completed within budget. Every task returns OK by construction (each - // swallows its own error into its slot), so the join status is always - // OK; assert it rather than discard, so a future refactor that lets a - // task propagate an error trips here instead of silently leaving the - // tablets queued behind it un-synced. - Status join_st = fanout_done.get(); - DCHECK(join_st.ok()) << join_st; + + // ONE absolute deadline across the whole operation, anchored at the fan-out + // start above rather than restarted here. With the pre-started pool the submit + // loop is pure enqueue and returns in microseconds, so remaining is essentially + // the full budget; pinning the wait to the same start keeps the TOTAL budget at + // query_cache_decision_sync_timeout_ms even if a degraded pool ever made + // submission slow, instead of granting the full timeout again after submission + // already consumed part of it. + auto sync_deadline = sync_fanout_start + + std::chrono::milliseconds(config::query_cache_decision_sync_timeout_ms); + auto sync_now = std::chrono::steady_clock::now(); + // Keep full clock precision here -- do NOT truncate to whole milliseconds. + // Truncating would shave the sub-millisecond submit time off the budget and let + // the total blocked time dip just under query_cache_decision_sync_timeout_ms. + // wait_for never returns before its duration elapses, so with the raw remaining + // the total time from sync_fanout_start stays >= the budget. + auto remaining = sync_deadline > sync_now ? (sync_deadline - sync_now) + : std::chrono::steady_clock::duration::zero(); + bool timed_out = !fanout_done->wait_for(remaining); + if (timed_out) { + // Tell not-yet-started tasks to skip their sync RPC so the pool drains + // fast instead of running abandoned work to completion. + abandoned->store(true, std::memory_order_release); } DorisMetrics::instance()->query_cache_decision_sync_time_ms->increment( - std::chrono::duration_cast( - std::chrono::steady_clock::now() - sync_fanout_start) + std::chrono::duration_cast(std::chrono::steady_clock::now() - + sync_fanout_start) .count()); - std::unordered_map fallback_reasons; if (timed_out) { // The fan-out outran its fast-fail budget. Do NOT read per_range_reason - // (the detached tasks are still writing it); fall every scanned tablet - // back so the whole instance recomputes in full. A distinct reason from - // a hard sync failure so the profile tells a slow meta service apart - // from a failing one. Throttled like the sync-failure log above: a - // brownout trips this for every stale tablet of every query at once, and - // this timeout is the primary operator-visible symptom of that brownout, - // so a sampled line correlates it without a storm (the per-query reason - // still reaches the user via the profile). + // (still-running tasks are writing it); fall every scanned tablet back so + // the whole instance recomputes in full. A distinct reason from a hard + // sync failure so the profile tells a slow meta service apart from a + // failing one. Throttled like the sync-failure log above: a brownout + // trips this for every stale tablet of every query at once, and this + // timeout is the primary operator-visible symptom of that brownout, so a + // sampled line correlates it without a storm (the per-query reason still + // reaches the user via the profile). LOG_EVERY_N(WARNING, 100) << "query cache incremental merge falls back, cloud rowset sync did not finish" << " within query_cache_decision_sync_timeout_ms=" @@ -476,27 +590,6 @@ bool QueryCacheRuntime::_capture_tablet_delta( int64_t tablet_id, int64_t cached_version, const std::unordered_map& presync_reasons, QueryCacheInstanceDecision* decision) { - auto tablet_res = ExecEnv::get_tablet(tablet_id); - if (!tablet_res) { - decision->incremental_fallback_reason = "tablet not found"; - return false; - } - BaseTabletSPtr tablet = std::move(tablet_res.value()); - // Defensive re-check of what FE promised with allow_incremental: - // "cached snapshot + delta rowsets == new snapshot" holds unconditionally - // for append-only DUP_KEYS data, and for merge-on-write UNIQUE data as - // long as the delta did not rewrite any row that predates the cached - // version (verified below once the delta rowsets are known). It never - // holds for merge-on-read UNIQUE data (duplicates are resolved by merging - // across rowsets at read time, so a delta-only scan cannot stand alone) - // nor for AGG tables (rows merge in the storage layer likewise). - const bool merge_on_write = tablet->keys_type() == KeysType::UNIQUE_KEYS && - tablet->enable_unique_key_merge_on_write(); - if (tablet->keys_type() != KeysType::DUP_KEYS && !merge_on_write) { - decision->incremental_fallback_reason = "keys type not append-only"; - return false; - } - if (config::is_cloud_mode()) { // A CloudTablet keeps its rowset list (and for merge-on-write tables // its delete bitmap) as a lazily synced copy of the meta service, and @@ -507,8 +600,14 @@ bool QueryCacheRuntime::_capture_tablet_delta( // which is the single sync per tablet; this loop only consumes its // outcome. A recorded reason means the sync could not vouch for the // view (a failed cast on a misconfigured deployment, an infrastructure - // sync failure, or the fast-fail budget expiring on a slow meta - // service), so fall back to a full recompute. No reason means the view + // sync failure, the fast-fail budget expiring on a slow meta service, + // or the pool refusing the sync outright), so fall back to a full + // recompute. Consumed BEFORE get_tablet below, because that reason can + // also mean the tablet was never loaded here (a refused submit loads + // nothing): get_tablet would then take the synchronous cache-miss + // meta-service load on this admission thread -- the very blocking + // window the fast-fail budget exists to cap -- to fetch a tablet whose + // decision is already known to be a fallback. No reason means the view // is synced up to query_version, and the history-rewrite check below // reads the tablet's delete bitmap as the sync brought it to that // version (sync_delete_bitmap=true) -- exactly as complete as the @@ -525,6 +624,26 @@ bool QueryCacheRuntime::_capture_tablet_delta( return false; } } + auto tablet_res = ExecEnv::get_tablet(tablet_id); + if (!tablet_res) { + decision->incremental_fallback_reason = "tablet not found"; + return false; + } + BaseTabletSPtr tablet = std::move(tablet_res.value()); + // Defensive re-check of what FE promised with allow_incremental: + // "cached snapshot + delta rowsets == new snapshot" holds unconditionally + // for append-only DUP_KEYS data, and for merge-on-write UNIQUE data as + // long as the delta did not rewrite any row that predates the cached + // version (verified below once the delta rowsets are known). It never + // holds for merge-on-read UNIQUE data (duplicates are resolved by merging + // across rowsets at read time, so a delta-only scan cannot stand alone) + // nor for AGG tables (rows merge in the storage layer likewise). + const bool merge_on_write = tablet->keys_type() == KeysType::UNIQUE_KEYS && + tablet->enable_unique_key_merge_on_write(); + if (tablet->keys_type() != KeysType::DUP_KEYS && !merge_on_write) { + decision->incremental_fallback_reason = "keys type not append-only"; + return false; + } // quiet: on a version-graph miss (the delta merged away by compaction, an // expected per-query situation) this option makes the capture API neither diff --git a/be/src/runtime/query_cache/query_cache.h b/be/src/runtime/query_cache/query_cache.h index e726a79937e0a0..0127763f07fa5e 100644 --- a/be/src/runtime/query_cache/query_cache.h +++ b/be/src/runtime/query_cache/query_cache.h @@ -361,6 +361,14 @@ class QueryCacheRuntime { std::lock_guard lock(_lock); _decisions[cache_key] = std::move(decision); } + + // Exercise the cloud presync fan-out (a private static) directly, so unit + // tests can assert the <= 0 skip-launch and the engine-stopped bail without + // standing up a full fragment. + static std::unordered_map presync_cloud_delta_tablets_for_test( + const std::vector& scan_ranges, int64_t current_version) { + return _presync_cloud_delta_tablets(scan_ranges, current_version); + } #endif private: diff --git a/be/test/exec/operator/query_cache_operator_test.cpp b/be/test/exec/operator/query_cache_operator_test.cpp index 943d38e4d2a2f0..96f23024a27ae4 100644 --- a/be/test/exec/operator/query_cache_operator_test.cpp +++ b/be/test/exec/operator/query_cache_operator_test.cpp @@ -1175,4 +1175,67 @@ TEST_F(QueryCacheOperatorTest, test_write_back_kept_on_local_with_freshness_set) EXPECT_TRUE(source_local_state->_need_insert_cache); } +TEST_F(QueryCacheOperatorTest, test_prefer_cached_fill_on_cloud_mow_writes_back) { + // Prefer-cached-rowset is ignored by cloud storage for a merge-on-write + // UNIQUE table (CloudTablet::capture_consistent_versions_unlocked guards the + // prefer branch on !enable_unique_key_merge_on_write()), so such a read + // still lands on the exact queried version. Its fill is therefore + // version-exact and safe to cache; FE reports the table type through + // TQueryCacheParam.is_merge_on_write so BE keeps the write-back rather than + // suppressing it. Mirrors test_prefer_cached_fill_does_not_write_back with + // the MOW flag set: the flag is what flips the outcome. + config::deploy_mode = "cloud"; + state->set_enable_prefer_cached_rowset(true); + + sink = std::make_unique(); + TQueryCacheParam cache_param; + cache_param.node_id = 0; + cache_param.digest = "test_digest"; + cache_param.output_slot_mapping[0] = 0; + cache_param.tablet_to_range.insert({42, "test"}); + cache_param.force_refresh_query_cache = false; + cache_param.entry_max_bytes = 1024 * 1024; + cache_param.entry_max_rows = 1000; + cache_param.__set_is_merge_on_write(true); + source = std::make_unique( + &pool, /*plan_node_id=*/0, /*operator_id=*/0, cache_param, + std::make_shared(cache_param, query_cache)); + EXPECT_TRUE(source->set_child(child_op)); + child_op->_mock_row_desc.reset( + new MockRowDescriptor {{std::make_shared()}, &pool}); + create_local_state(); + + EXPECT_TRUE(source_local_state->_need_insert_cache); +} + +TEST_F(QueryCacheOperatorTest, test_freshness_fill_on_cloud_mow_still_suppressed) { + // The MOW carve-out is prefer-only: freshness tolerance has no MOW guard on + // the storage side (CloudTablet honors it for every table type), so a + // freshness read can still stop below or overshoot the queried version even + // on a merge-on-write table. Its fill must stay suppressed regardless of the + // is_merge_on_write flag -- proving the carve-out did not widen to freshness. + config::deploy_mode = "cloud"; + state->set_query_freshness_tolerance_ms(5000); + + sink = std::make_unique(); + TQueryCacheParam cache_param; + cache_param.node_id = 0; + cache_param.digest = "test_digest"; + cache_param.output_slot_mapping[0] = 0; + cache_param.tablet_to_range.insert({42, "test"}); + cache_param.force_refresh_query_cache = false; + cache_param.entry_max_bytes = 1024 * 1024; + cache_param.entry_max_rows = 1000; + cache_param.__set_is_merge_on_write(true); + source = std::make_unique( + &pool, /*plan_node_id=*/0, /*operator_id=*/0, cache_param, + std::make_shared(cache_param, query_cache)); + EXPECT_TRUE(source->set_child(child_op)); + child_op->_mock_row_desc.reset( + new MockRowDescriptor {{std::make_shared()}, &pool}); + create_local_state(); + + EXPECT_FALSE(source_local_state->_need_insert_cache); +} + } // namespace doris diff --git a/be/test/exec/pipeline/query_cache_test.cpp b/be/test/exec/pipeline/query_cache_test.cpp index bfd2146a88381e..9dc9598a4ecb0f 100644 --- a/be/test/exec/pipeline/query_cache_test.cpp +++ b/be/test/exec/pipeline/query_cache_test.cpp @@ -53,6 +53,7 @@ #include "testutil/column_helper.h" #include "util/debug_points.h" #include "util/defer_op.h" +#include "util/threadpool.h" #include "util/uid_util.h" namespace doris { @@ -417,6 +418,151 @@ TEST_F(QueryCacheTest, runtime_decision_hit) { fallback_before); } +// (c) query_cache_decision_sync_timeout_ms <= 0 disables cloud incremental +// merge: the presync must spawn NO sync work and fall every scanned tablet back. +// No storage engine is installed, so the <= 0 guard returning cleanly (before +// to_cloud()) is itself proof nothing launched -- any launch would have +// dereferenced the absent engine. The sync-point counter double-confirms zero +// sync_rowsets calls. +TEST_F(QueryCacheTest, presync_skips_launch_when_timeout_non_positive) { + auto* sp = SyncPoint::get_instance(); + sp->enable_processing(); + std::atomic sync_calls {0}; + sp->set_call_back("CloudMetaMgr::sync_tablet_rowsets", + [&](auto&&) { sync_calls.fetch_add(1); }); + Defer sp_defer {[sp] { + sp->disable_processing(); + sp->clear_all_call_backs(); + }}; + + auto saved = config::query_cache_decision_sync_timeout_ms; + config::query_cache_decision_sync_timeout_ms = 0; + Defer cfg_defer {[&] { config::query_cache_decision_sync_timeout_ms = saved; }}; + + auto scan_ranges = make_scan_ranges(101, "100"); + auto reasons = QueryCacheRuntime::presync_cloud_delta_tablets_for_test(scan_ranges, 100); + + ASSERT_EQ(reasons.count(101), 1); + EXPECT_EQ(reasons[101], "cloud incremental sync disabled"); + EXPECT_EQ(sync_calls.load(), 0); +} + +// (a) A stopped engine must bail before launching: positive budget, but +// stopped() is true, so no sync_rowsets runs and every tablet falls back. The +// stopped() check precedes the pool access, so an un-opened engine's null pool +// is never touched. +TEST_F(QueryCacheTest, presync_bails_when_engine_stopped) { + auto engine = std::make_unique(EngineOptions {}); + engine->stop(); // sets _stopped = true; safe on an un-opened engine. + ExecEnv::GetInstance()->set_storage_engine(std::move(engine)); + Defer eng_defer {[] { ExecEnv::GetInstance()->set_storage_engine(nullptr); }}; + + auto* sp = SyncPoint::get_instance(); + sp->enable_processing(); + std::atomic sync_calls {0}; + sp->set_call_back("CloudMetaMgr::sync_tablet_rowsets", + [&](auto&&) { sync_calls.fetch_add(1); }); + Defer sp_defer {[sp] { + sp->disable_processing(); + sp->clear_all_call_backs(); + }}; + + auto saved = config::query_cache_decision_sync_timeout_ms; + config::query_cache_decision_sync_timeout_ms = 2000; // positive: would launch if not stopped. + Defer cfg_defer {[&] { config::query_cache_decision_sync_timeout_ms = saved; }}; + + auto scan_ranges = make_scan_ranges(42, "100"); + auto reasons = QueryCacheRuntime::presync_cloud_delta_tablets_for_test(scan_ranges, 100); + + ASSERT_EQ(reasons.count(42), 1); + EXPECT_EQ(reasons[42], "be is stopping, sync skipped"); + EXPECT_EQ(sync_calls.load(), 0); +} + +// (finding 1) is_cloud_mode() can flip true on a live LOCAL deployment (the +// mutable cloud_unique_id), leaving a local StorageEngine installed. Reaching +// the fan-out then must degrade gracefully, not abort in the hard CHECK inside +// to_cloud(): the engine-type dynamic_cast fails, so every tablet falls back +// with "storage engine is not cloud" and no sync is issued. +TEST_F(QueryCacheTest, presync_falls_back_on_non_cloud_engine) { + auto saved_mode = config::deploy_mode; + config::deploy_mode = "cloud"; // is_cloud_mode() true while the engine stays local + ExecEnv::GetInstance()->set_storage_engine(std::make_unique(EngineOptions {})); + Defer eng_defer {[&] { + ExecEnv::GetInstance()->set_storage_engine(nullptr); + config::deploy_mode = saved_mode; + }}; + + auto* sp = SyncPoint::get_instance(); + sp->enable_processing(); + std::atomic sync_calls {0}; + sp->set_call_back("CloudMetaMgr::sync_tablet_rowsets", + [&](auto&&) { sync_calls.fetch_add(1); }); + Defer sp_defer {[sp] { + sp->disable_processing(); + sp->clear_all_call_backs(); + }}; + + auto saved = config::query_cache_decision_sync_timeout_ms; + config::query_cache_decision_sync_timeout_ms = + 2000; // positive: would launch on a cloud engine. + Defer cfg_defer {[&] { config::query_cache_decision_sync_timeout_ms = saved; }}; + + auto scan_ranges = make_scan_ranges(42, "100"); + auto reasons = QueryCacheRuntime::presync_cloud_delta_tablets_for_test(scan_ranges, 100); + + ASSERT_EQ(reasons.count(42), 1); + EXPECT_EQ(reasons[42], "storage engine is not cloud"); + EXPECT_EQ(sync_calls.load(), 0); +} + +// The dedicated pre-sync pool refuses work (here: shut down, the same rejection a +// full bounded queue gives) while the engine itself is still live, so the fan-out +// is reached rather than short-cut by the stopped() guard. Every submit_func then +// fails, and the inline submit-failure path must fall each tablet back AND release +// its own latch slot via finish_slot so the bounded wait still settles instead of +// hanging on a slot no task will ever count down. Two tablets prove the latch +// reaches N purely through the inline claim. No sync RPC is issued (rejected before +// enqueue, so the closure never runs). +TEST_F(QueryCacheTest, presync_falls_back_when_pool_rejects_submit) { + auto saved_mode = config::deploy_mode; + config::deploy_mode = "cloud"; + auto engine = std::make_unique(EngineOptions {}); + auto* engine_ptr = engine.get(); + ExecEnv::GetInstance()->set_storage_engine(std::move(engine)); + Defer eng_defer {[&] { + ExecEnv::GetInstance()->set_storage_engine(nullptr); + config::deploy_mode = saved_mode; + }}; + // Shut the pool so do_submit rejects before enqueue; the engine stays live + // (not stopped()), so the fan-out is entered and hits the inline fallback. + engine_ptr->query_cache_delta_sync_pool().shutdown(); + + auto* sp = SyncPoint::get_instance(); + sp->enable_processing(); + std::atomic sync_calls {0}; + sp->set_call_back("CloudMetaMgr::sync_tablet_rowsets", + [&](auto&&) { sync_calls.fetch_add(1); }); + Defer sp_defer {[sp] { + sp->disable_processing(); + sp->clear_all_call_backs(); + }}; + + auto saved = config::query_cache_decision_sync_timeout_ms; + config::query_cache_decision_sync_timeout_ms = 2000; // positive: a healthy pool would launch. + Defer cfg_defer {[&] { config::query_cache_decision_sync_timeout_ms = saved; }}; + + auto scan_ranges = make_scan_ranges(42, "100"); + scan_ranges.push_back(make_scan_ranges(43, "100").front()); + auto reasons = QueryCacheRuntime::presync_cloud_delta_tablets_for_test(scan_ranges, 100); + + ASSERT_EQ(reasons.count(42), 1); + ASSERT_EQ(reasons.count(43), 1); + EXPECT_EQ(reasons[42], "cloud rowset sync not scheduled"); + EXPECT_EQ(reasons[43], "cloud rowset sync not scheduled"); + EXPECT_EQ(sync_calls.load(), 0); +} + TEST_F(QueryCacheTest, runtime_decision_force_refresh) { std::unique_ptr cache(QueryCache::create_global_cache(1024 * 1024)); auto scan_ranges = make_scan_ranges(42, "100"); @@ -1040,10 +1186,12 @@ TEST_F(QueryCacheIncrementalTest, fallback_on_partial_capture) { } TEST_F(QueryCacheIncrementalTest, mismatched_engine_falls_back) { - // is_cloud_mode() can flip on a live local deployment (cloud_unique_id is - // a mutable config): ExecEnv::get_tablet then still hands out a plain - // local Tablet, and the decision must degrade the misconfiguration to a - // full recompute instead of dereferencing the failed CloudTablet cast. + // is_cloud_mode() can flip on a live local deployment (cloud_unique_id is a + // mutable config) while the installed engine stays a local StorageEngine. The + // decision must degrade the misconfiguration to a full recompute instead of + // aborting in the hard CHECK inside to_cloud(): the pre-sync's engine-type + // dynamic_cast fails first, so every scanned tablet falls back with "storage + // engine is not cloud" (earlier and more precise than the per-tablet cast). create_tablet(TKeysType::DUP_KEYS, false, {{0, 50}, {51, 100}}); std::string saved_deploy_mode = config::deploy_mode; config::deploy_mode = "cloud"; @@ -1051,7 +1199,7 @@ TEST_F(QueryCacheIncrementalTest, mismatched_engine_falls_back) { config::deploy_mode = saved_deploy_mode; EXPECT_EQ(decision->mode, QueryCacheInstanceDecision::Mode::MISS); EXPECT_TRUE(decision->key_valid); - EXPECT_EQ(decision->incremental_fallback_reason, "tablet is not a cloud tablet"); + EXPECT_EQ(decision->incremental_fallback_reason, "storage engine is not cloud"); } TEST_F(QueryCacheIncrementalTest, concurrent_racers_share_one_decision) { @@ -1229,8 +1377,7 @@ class QueryCacheCloudIncrementalTest : public testing::Test { "CloudMetaMgr::sync_tablet_rowsets", [this, versions = std::move(versions), decision_sync_result, decision_sync_installs = std::move(decision_sync_installs), - decision_sync_bitmap_marks = - std::move(decision_sync_bitmap_marks)](auto&& args) { + decision_sync_bitmap_marks = std::move(decision_sync_bitmap_marks)](auto&& args) { auto* tablet = try_any_cast(args[0]); const auto* options = try_any_cast(args[1]); auto* ret = try_any_cast_ret(args); @@ -1452,10 +1599,12 @@ TEST_F(QueryCacheCloudIncrementalTest, newer_entry_rejected_before_tablet_access TEST_F(QueryCacheCloudIncrementalTest, presync_tablet_load_failure_falls_back) { // The pre-sync fan-out issues the first ExecEnv::get_tablet; a meta-service - // failure injected there fails that load, so _presync_cloud_delta_tablets - // takes its !tablet_res skip (records no reason, leaving the miss for the - // capture loop to report), and the capture loop's own get_tablet then falls - // the query back to a full recompute with "tablet not found". This exercises + // failure injected there fails that load on the WORKER thread, and the task + // publishes "cloud tablet load failed" into its slot. The capture loop must + // consume that reason and fall back WITHOUT retrying the cache-miss load on + // the admission thread: during a brownout (the one situation where this + // load fails slowly) a capture-side retry would repeat the slow RPC on the + // bounded admission pool, outside the fast-fail deadline. This exercises // the one pre-sync branch the other cloud tests never reach: they all load // the tablet successfully before syncing. std::atomic meta_loads {0}; @@ -1471,16 +1620,15 @@ TEST_F(QueryCacheCloudIncrementalTest, presync_tablet_load_failure_falls_back) { EXPECT_EQ(DorisMetrics::instance()->query_cache_incremental_fallback_total->value(), fallbacks_before + 1); EXPECT_EQ(decision->mode, QueryCacheInstanceDecision::Mode::MISS); - EXPECT_EQ(decision->incremental_fallback_reason, "tablet not found"); + EXPECT_EQ(decision->incremental_fallback_reason, "cloud tablet load failed"); // The pre-sync get_tablet failed before any sync, so no meta-service rowset // sync RPC was ever issued. EXPECT_EQ(_sync_calls.load(), 0); - // Discriminating witness that the PRE-SYNC skip branch actually ran (not - // that the pre-sync was bypassed and only the capture loop reported "tablet - // not found", which would give the same mode/reason/_sync_calls): the - // pre-sync fan-out and the capture loop each attempt their own get_tablet, - // so the load is tried twice. A bypassed pre-sync would attempt it once. - EXPECT_EQ(meta_loads.load(), 2); + // Exactly ONE load: the worker's. Two would mean the capture loop retried + // the failed load on the admission thread (the pre-fix behavior); the + // published reason above is the second witness of the same ordering (a + // capture-side retry would report its own "tablet not found" instead). + EXPECT_EQ(meta_loads.load(), 1); } TEST_F(QueryCacheCloudIncrementalTest, unique_non_mow_rejected_before_decision_sync) { @@ -1523,8 +1671,8 @@ TEST_F(QueryCacheCloudIncrementalTest, mow_decision_sync_brings_bitmap_detects_r // no-op and the mark was hand-placed. intercept_tablet_load({{0, 50}}, Status::OK(), TKeysType::UNIQUE_KEYS, /*enable_merge_on_write=*/true, - /*decision_sync_installs=*/{{51, 100}}, - /*decision_sync_bitmap_marks=*/{{0, 60}}); + /*decision_sync_installs=*/ {{51, 100}}, + /*decision_sync_bitmap_marks=*/ {{0, 60}}); int64_t fallbacks_before = DorisMetrics::instance()->query_cache_incremental_fallback_total->value(); auto decision = make_stale_decision(); @@ -1550,8 +1698,8 @@ TEST_F(QueryCacheCloudIncrementalTest, mow_cloud_cached_side_not_pinnable_falls_ // is now shared, not cloud-early-returned) and would fire if the gapless // invariant were ever broken: the cached-side pin walks only [0,40] and the // endpoint check -- short of cached version 50 -- rejects it (not emptiness). - intercept_tablet_load({{0, 40}, {51, 80}, {81, 100}}, Status::OK(), - TKeysType::UNIQUE_KEYS, /*enable_merge_on_write=*/true); + intercept_tablet_load({{0, 40}, {51, 80}, {81, 100}}, Status::OK(), TKeysType::UNIQUE_KEYS, + /*enable_merge_on_write=*/true); int64_t fallbacks_before = DorisMetrics::instance()->query_cache_incremental_fallback_total->value(); auto decision = make_stale_decision(); @@ -1575,7 +1723,7 @@ TEST_F(QueryCacheCloudIncrementalTest, mow_decision_sync_clean_bitmap_stays_incr // path, no mark, no fallback. intercept_tablet_load({{0, 50}}, Status::OK(), TKeysType::UNIQUE_KEYS, /*enable_merge_on_write=*/true, - /*decision_sync_installs=*/{{51, 100}}); + /*decision_sync_installs=*/ {{51, 100}}); auto decision = make_stale_decision(); EXPECT_EQ(decision->mode, QueryCacheInstanceDecision::Mode::INCREMENTAL); EXPECT_TRUE(decision->incremental_fallback_reason.empty()); @@ -1607,9 +1755,8 @@ TEST_F(QueryCacheCloudIncrementalTest, presync_fans_out_across_tablets_and_merge TTabletSchema schema; schema.keys_type = TKeysType::DUP_KEYS; auto meta = std::make_shared( - 1, 2, tid, 15674, 4, 5, schema, 6, - std::unordered_map {{7, 8}}, UniqueId(9, 10), - TTabletType::TABLET_TYPE_DISK, TCompressionType::LZ4F, 0, false); + 1, 2, tid, 15674, 4, 5, schema, 6, std::unordered_map {{7, 8}}, + UniqueId(9, 10), TTabletType::TABLET_TYPE_DISK, TCompressionType::LZ4F, 0, false); meta->set_tablet_state(TABLET_RUNNING); *try_any_cast(args[1]) = std::move(meta); try_any_cast_ret(args)->second = true; @@ -1681,6 +1828,39 @@ TEST_F(QueryCacheCloudIncrementalTest, presync_fans_out_across_tablets_and_merge EXPECT_EQ(calls[kTabletId2], 2); } +TEST_F(QueryCacheCloudIncrementalTest, pool_rejection_skips_admission_tablet_load) { + // Composition of two degradations: the dedicated pre-sync pool refuses the + // sync (here: shut down, the same inline rejection a full bounded queue + // gives) AND the tablet is absent from the local tablet cache (nothing -- + // loader or pre-sync, whose closure never ran -- ever loaded it). The + // decision must consume the recorded "not scheduled" reason BEFORE touching + // the tablet: the capture loop's get_tablet would otherwise take the + // synchronous cache-miss meta-service load on the admission thread -- the + // blocking window the fast-fail budget exists to cap -- to fetch a tablet + // whose decision is already a known fallback. The meta-load counter proves + // the admission path stayed load-free; a regression in the consumption + // order shows up twice (a nonzero count, and the weaker "tablet not found" + // reason from the failed load displacing the pool's). + std::atomic meta_loads {0}; + _sp->set_call_back("CloudMetaMgr::get_tablet_meta", [&](auto&& args) { + meta_loads.fetch_add(1); + auto* ret = try_any_cast_ret(args); + ret->first = Status::InternalError("meta service unreachable"); + ret->second = true; + }); + _engine->query_cache_delta_sync_pool().shutdown(); + + int64_t fallbacks_before = + DorisMetrics::instance()->query_cache_incremental_fallback_total->value(); + auto decision = make_stale_decision(); + EXPECT_EQ(DorisMetrics::instance()->query_cache_incremental_fallback_total->value(), + fallbacks_before + 1); + EXPECT_EQ(decision->mode, QueryCacheInstanceDecision::Mode::MISS); + EXPECT_EQ(decision->incremental_fallback_reason, "cloud rowset sync not scheduled"); + EXPECT_EQ(meta_loads.load(), 0); + EXPECT_EQ(_sync_calls.load(), 0); +} + TEST_F(QueryCacheCloudIncrementalTest, decision_sync_timeout_falls_back) { // A meta-service brownout: the decision's rowset sync stalls past the // fast-fail budget (query_cache_decision_sync_timeout_ms). The decision must @@ -1700,10 +1880,10 @@ TEST_F(QueryCacheCloudIncrementalTest, decision_sync_timeout_falls_back) { _sp->set_call_back("CloudMetaMgr::get_tablet_meta", [](auto&& args) { TTabletSchema schema; schema.keys_type = TKeysType::DUP_KEYS; - auto meta = std::make_shared( - 1, 2, kTabletId, 15674, 4, 5, schema, 6, - std::unordered_map {{7, 8}}, UniqueId(9, 10), - TTabletType::TABLET_TYPE_DISK, TCompressionType::LZ4F, 0, false); + auto meta = std::make_shared(1, 2, kTabletId, 15674, 4, 5, schema, 6, + std::unordered_map {{7, 8}}, + UniqueId(9, 10), TTabletType::TABLET_TYPE_DISK, + TCompressionType::LZ4F, 0, false); meta->set_tablet_state(TABLET_RUNNING); *try_any_cast(args[1]) = std::move(meta); try_any_cast_ret(args)->second = true; diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/normalize/QueryCacheNormalizer.java b/fe/fe-core/src/main/java/org/apache/doris/planner/normalize/QueryCacheNormalizer.java index c73e19cff9da6f..8ddd0617709b78 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/normalize/QueryCacheNormalizer.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/normalize/QueryCacheNormalizer.java @@ -100,6 +100,7 @@ private Optional setQueryCacheParam( queryCacheParam.setEntryMaxBytes(sessionVariable.getQueryCacheEntryMaxBytes()); queryCacheParam.setEntryMaxRows(sessionVariable.getQueryCacheEntryMaxRows()); queryCacheParam.setAllowIncremental(computeAllowIncremental(cachePoint, sessionVariable)); + queryCacheParam.setIsMergeOnWrite(computeIsMergeOnWrite(cachePoint)); queryCacheParam.setOutputSlotMapping( cachePoint.cacheRoot.getOutputTupleIds() @@ -179,33 +180,6 @@ private boolean computeAllowIncremental(CachePoint cachePoint, SessionVariable s if (!sessionVariable.getEnableQueryCacheIncremental()) { return false; } - // Freshness tolerance and prefer-cached-rowset (both cloud-only in - // effect) trade exactness for speed and locality: the scan may read a - // warmed-up layout that stops below the queried version (freshness - // halts at the warmed boundary) or reaches beyond it (neither walk - // clips an edge spanning it). An incremental merge would defeat both knobs, - // because the delta capture always targets the exact queried version - // (it must, to keep the merged entry correct): under freshness - // tolerance it would force the wait for un-warmed data the query - // explicitly chose to skip, and under prefer-cached-rowset it would - // ignore the layout preference the user set. So exclude incremental - // for such queries and let them take their cheap path. Correctness - // against their version-inexact reads does NOT rest on this per-query - // gate (an entry filled by such a read could still be reused by a - // different, knob-free query sharing the cache key): on cloud, the - // only mode where these reads occur, the BE suppresses their cache - // write-back, so no entry whose content mismatches its version stamp - // exists in the first place. These knobs are inert on local storage, - // so gating them in any mode only forgoes incremental for a query - // that opted into a cloud trade-off; the gate stays mode-agnostic on - // purpose, because a mode-conditioned gate cannot be exercised by the - // local FE unit-test harness (planning under a flipped cloud flag casts - // SystemInfoService to CloudSystemInfoService and fails), so it would - // ship an untestable branch for no correctness gain. - if (sessionVariable.getQueryFreshnessToleranceMs() > 0 - || sessionVariable.getEnablePreferCachedRowset()) { - return false; - } // The cache point is always an aggregation node (see doComputeCachePoint). if (((AggregationNode) cachePoint.cacheRoot).isNeedsFinalize()) { return false; @@ -225,20 +199,86 @@ private boolean computeAllowIncremental(CachePoint cachePoint, SessionVariable s long selectIndexId = scanNode.getSelectedIndexId() == -1 ? olapTable.getBaseIndexId() : scanNode.getSelectedIndexId(); - // Note: judged on the selected index, not the base table: a DUP table - // may serve the query from an aggregated materialized view, whose data - // is no longer append-only. + // The scanned index must be append-only, guaranteeing "cached snapshot + + // delta rowsets == new snapshot". Note: judged on the selected index, not + // the base table -- a DUP table may serve the query from an aggregated + // materialized view, whose data is no longer append-only. + // + // DUP_KEYS is always append-only. A merge-on-write UNIQUE index is + // append-only as long as a load does not touch pre-existing keys, which + // covers the common "hourly append plus occasional backfill" pattern: BE + // verifies per tablet through the delete bitmap of the delta window and + // falls back to a full recompute for the rare load that rewrites history. + // A merge-on-read UNIQUE index resolves duplicates by merging across + // rowsets at read time, so a delta-only scan cannot stand alone there; + // AGG tables merge rows inside the storage layer likewise. KeysType keysType = olapTable.getKeysTypeByIndexId(selectIndexId); - if (keysType == KeysType.DUP_KEYS) { - return true; + boolean mergeOnWrite = + keysType == KeysType.UNIQUE_KEYS && olapTable.getEnableUniqueKeyMergeOnWrite(); + if (keysType != KeysType.DUP_KEYS && !mergeOnWrite) { + return false; + } + // Freshness tolerance and prefer-cached-rowset (both cloud-only in + // effect) trade exactness for speed and locality: the scan may read a + // warmed-up layout that stops below the queried version (freshness halts + // at the warmed boundary) or reaches beyond it (neither walk clips an + // edge spanning it). An incremental merge would defeat such a knob, + // because the delta capture always targets the exact queried version (it + // must, to keep the merged entry correct), forcing the un-warmed reads + // the query explicitly chose to skip. Correctness against version-inexact + // reads does NOT rest on this per-query gate (an entry filled by such a + // read could still be reused by a different, knob-free query sharing the + // cache key): on cloud, the only mode where these reads occur, the BE + // suppresses their cache write-back, so no entry whose content mismatches + // its version stamp exists in the first place. The gate stays mode- + // agnostic (no is_cloud_mode): the knobs are inert on local storage, so + // excluding incremental there only forgoes it for a query that opted into + // a cloud trade-off, and a mode-conditioned gate cannot be exercised by + // the local FE unit-test harness (planning under a flipped cloud flag + // casts SystemInfoService to CloudSystemInfoService and fails). + if (sessionVariable.getQueryFreshnessToleranceMs() > 0) { + return false; + } + // Prefer-cached-rowset is honored by the storage layer only for non-MOW + // tables: CloudTablet::capture_consistent_versions_unlocked guards the + // prefer branch on !enable_unique_key_merge_on_write(), so a MOW query + // reads the exact queried version regardless of the knob. Its delta + // capture is therefore version-exact and safe to merge incrementally, so + // only non-MOW tables are excluded here. This carve-out keys on table + // type, not mode, so the FE unit test can exercise it directly; it is + // sound in both modes because the read is version-exact for MOW whether + // the knob is inert (local) or explicitly ignored (cloud). Freshness + // above has no such MOW guard on the storage side, so it excludes + // incremental for every table type. + if (sessionVariable.getEnablePreferCachedRowset() && !mergeOnWrite) { + return false; } - // A merge-on-write UNIQUE index is append-only as long as a load does - // not touch pre-existing keys, which covers the common "hourly append - // plus occasional backfill" pattern: BE verifies per tablet through - // the delete bitmap of the delta window and falls back to a full - // recompute for the rare load that rewrites history. A merge-on-read - // UNIQUE index resolves duplicates by merging across rowsets at read - // time, so a delta-only scan cannot stand alone there. + return true; + } + + // Whether the selected index of the cache point's scan is a merge-on-write + // UNIQUE table. BE consumes this in the cloud cache write-back gate: cloud + // ignores enable_prefer_cached_rowset for a MOW table, so a prefer-only MOW + // read is version-exact and its fill must not be suppressed. Reported + // independently of allow_incremental because the write-back gate also applies + // to a MISS (a MOW MISS under prefer must still be cached). Determined on the + // selected index, mirroring computeAllowIncremental. False when the cache + // point has no direct OlapScanNode child (a nested-agg shape, never + // incrementally cacheable), which leaves BE at its safe suppress default: such + // a shape over a MOW scan also forgoes the prefer-cached-rowset write-back, + // a deliberate over-suppression (the safe direction, it never caches a + // version-inexact entry) rather than threading scan resolution through the + // aggregation chain for a cache point whose incremental merge is already off. + private boolean computeIsMergeOnWrite(CachePoint cachePoint) { + if (!(cachePoint.cacheRoot.getChild(0) instanceof OlapScanNode)) { + return false; + } + OlapScanNode scanNode = (OlapScanNode) cachePoint.cacheRoot.getChild(0); + OlapTable olapTable = scanNode.getOlapTable(); + long selectIndexId = scanNode.getSelectedIndexId() == -1 + ? olapTable.getBaseIndexId() + : scanNode.getSelectedIndexId(); + KeysType keysType = olapTable.getKeysTypeByIndexId(selectIndexId); return keysType == KeysType.UNIQUE_KEYS && olapTable.getEnableUniqueKeyMergeOnWrite(); } diff --git a/fe/fe-core/src/test/java/org/apache/doris/planner/QueryCacheNormalizerTest.java b/fe/fe-core/src/test/java/org/apache/doris/planner/QueryCacheNormalizerTest.java index 3882f3e37ff9b6..f5ec8a6d94fde7 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/planner/QueryCacheNormalizerTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/planner/QueryCacheNormalizerTest.java @@ -420,6 +420,9 @@ public void testAllowIncremental() throws Exception { TQueryCacheParam dupTwoPhase = getQueryCacheParam( "select k2, sum(v1) as v from db1.part1 group by k2"); Assertions.assertTrue(dupTwoPhase.allow_incremental); + // A DUP table is not merge-on-write: the flag BE reads for its cloud + // write-back gate stays false. + Assertions.assertFalse(dupTwoPhase.is_merge_on_write); // One-phase aggregation finalizes inside the cached fragment: its // output has no downstream merge, so emitting cached and delta @@ -436,12 +439,17 @@ public void testAllowIncremental() throws Exception { TQueryCacheParam uniqueMow = getQueryCacheParam( "select v1, count(*) as v from db1.uniq_mow group by v1"); Assertions.assertTrue(uniqueMow.allow_incremental); + // FE reports the merge-on-write table type so BE can keep the cloud + // write-back for a prefer-only MOW read (which stays version-exact). + Assertions.assertTrue(uniqueMow.is_merge_on_write); // UNIQUE merge-on-read resolves duplicates by merging across // rowsets at read time: a delta-only scan cannot stand alone. TQueryCacheParam uniqueMor = getQueryCacheParam( "select v1, count(*) as v from db1.uniq_mor group by v1"); Assertions.assertFalse(uniqueMor.allow_incremental); + // Merge-on-read UNIQUE is not merge-on-write: the flag is false. + Assertions.assertFalse(uniqueMor.is_merge_on_write); // AGG_KEYS merges rows inside the storage layer. TQueryCacheParam aggKeys = getQueryCacheParam( @@ -478,6 +486,11 @@ public void testAllowIncremental() throws Exception { TQueryCacheParam withFreshness = getQueryCacheParam( "select k2, sum(v1) as v from db1.part1 group by k2"); Assertions.assertFalse(withFreshness.allow_incremental); + // Freshness has no merge-on-write carve-out (cloud honors it for + // every table type), so it blocks incremental for MOW as well. + TQueryCacheParam mowWithFreshness = getQueryCacheParam( + "select v1, count(*) as v from db1.uniq_mow group by v1"); + Assertions.assertFalse(mowWithFreshness.allow_incremental); } finally { connectContext.getSessionVariable().queryFreshnessToleranceMs = -1; } @@ -493,6 +506,16 @@ public void testAllowIncremental() throws Exception { TQueryCacheParam withPreferCached = getQueryCacheParam( "select k2, sum(v1) as v from db1.part1 group by k2"); Assertions.assertFalse(withPreferCached.allow_incremental); + // Cloud ignores prefer-cached-rowset for a merge-on-write table + // (its read stays version-exact), so incremental remains allowed + // for MOW under prefer -- the carve-out keys on table type, and + // being mode-agnostic it holds in the local harness too. The + // is_merge_on_write flag is what BE also uses to keep the fill's + // write-back for such a query. + TQueryCacheParam mowWithPreferCached = getQueryCacheParam( + "select v1, count(*) as v from db1.uniq_mow group by v1"); + Assertions.assertTrue(mowWithPreferCached.allow_incremental); + Assertions.assertTrue(mowWithPreferCached.is_merge_on_write); } finally { connectContext.getSessionVariable().enablePreferCachedRowset = false; } diff --git a/gensrc/thrift/QueryCache.thrift b/gensrc/thrift/QueryCache.thrift index 67c31cd4bc6120..eaab618ae10e7b 100644 --- a/gensrc/thrift/QueryCache.thrift +++ b/gensrc/thrift/QueryCache.thrift @@ -77,4 +77,15 @@ struct TQueryCacheParam { // cannot be captured (e.g. merged away by compaction), when the delta // contains delete predicates, or when the delta rewrites history rows. 8: optional bool allow_incremental + + // Whether the scanned index is a merge-on-write UNIQUE table. BE needs this + // for the cloud write-back gate: enable_prefer_cached_rowset lets the storage + // layer read a warmed-up layout that may not match the queried version, so BE + // suppresses the cache write-back of such a read. But on cloud that knob is + // ignored for a merge-on-write table (CloudTablet::capture_consistent_ + // versions_unlocked guards it on !enable_unique_key_merge_on_write()), so a + // MOW query still reads the exact version and its fill is safe to cache. FE + // reports the table type here (on the selected index) so BE can keep the + // write-back for a prefer-only MOW query instead of forgoing its cache. + 9: optional bool is_merge_on_write } diff --git a/regression-test/suites/query_p0/cache/query_cache_incremental.groovy b/regression-test/suites/query_p0/cache/query_cache_incremental.groovy index 1b3d18723cc6c2..831e303650ddc5 100644 --- a/regression-test/suites/query_p0/cache/query_cache_incremental.groovy +++ b/regression-test/suites/query_p0/cache/query_cache_incremental.groovy @@ -79,22 +79,38 @@ suite("query_cache_incremental") { // merged incrementally (Mode::INCREMENTAL), not recomputed: only the // incremental path increases query_cache_stale_hit_total, and no // concurrent suite touches it because the session switch defaults to off. - // Residual caveat (here and in checkIncrementalFallback): a - // memory-pressure prune evicting the entry inside the tiny fill-to-assert - // window surfaces as a plain MISS and would fail the assertion; accepted - // as rare, and a rerun re-establishes the counters from fresh deltas. The - // one channel that is NOT left to chance is compaction: every table here - // sets disable_auto_compaction, so no background compaction can merge the - // delta away mid-window and turn the expected incremental hit into a - // fallback (this matters most on cloud, where compaction is driven - // externally rather than by the querying BE). - // On cloud two more channels surface the same way and fail only - // checkStaleIncremental (checkIncrementalFallback is immune: any - // non-empty reason still counts): a transient meta-service hiccup during - // the decision's view sync, and the tablet rebalancer routing the stale - // query to a BE that holds no entry. Both equally rare inside this window, - // same remedy. + // Compaction is kept out of the way in every mode: every table here sets + // disable_auto_compaction, so no background compaction can merge the delta + // away mid-window and turn the expected incremental hit into a fallback + // (this matters most on cloud, where compaction is driven externally). + // + // The stale_hit assertion runs on non-cloud only. Summed across BEs it is a + // deterministic INCREMENTAL signal solely when the fill and the reuse land + // on the same BE, which holds on the single-BE local cluster but not on + // multi-BE cloud. Two cloud channels flip it with no product-correctness + // meaning: a transient meta-service hiccup during the decision's view sync + // turns the expected stale hit into a full-recompute fallback (stale_hit + // stays flat, the fallback counter moves instead), and because the entry is + // BE-local a query routed to a BE that does not hold it recomputes in full + // there. A retry repairs neither: a fallback re-bases nothing, and a + // no-entry BE fills at the current version so its later runs are exact hits, + // never the stale hit. Cloud still exercises the whole incremental flow + // through checkConsistency (end-to-end result correctness on a real cloud + // cluster); that the cloud decision truly reaches Mode::INCREMENTAL is + // proven deterministically by the BE unit tests (QueryCacheCloudIncremental + // Test), which drive the cloud sync-then-capture path without cluster + // routing in play. A local memory-pressure prune evicting the entry inside + // the tiny fill-to-assert window is the one accepted residual (rare; a rerun + // re-establishes the counters). def checkStaleIncremental = { String sqlText -> + // On cloud the metric endpoints are not even sampled: a slow or briefly + // unreachable BE metrics port would hang or throw in sumBeMetric before + // a merely gated assertion, re-introducing the very topology-dependent + // flakiness this gate exists to remove (and the samples would go unused). + if (isCloudMode()) { + checkConsistency(sqlText) + return + } def before = sumBeMetric("query_cache_stale_hit_total") checkConsistency(sqlText) def after = sumBeMetric("query_cache_stale_hit_total") @@ -102,8 +118,21 @@ suite("query_cache_incremental") { "expected a stale incremental cache hit, but query_cache_stale_hit_total stayed at ${before}") } - // Prove that a designed fallback phase really took the fallback path. + // Prove that a designed fallback phase really took the fallback path. Like the + // stale_hit assertion, the metric check is restricted to non-cloud: the entry + // is BE-local, so on a multi-BE cloud cluster the query can route to a BE that + // does not hold the stale entry, take a plain MISS (which is not a fallback, + // so the counter does not move), and spuriously fail. Cloud still verifies the + // fallback phase's result correctness through checkConsistency; that the cloud + // fallback classification itself fires is proven deterministically by the BE + // unit tests (QueryCacheCloudIncrementalTest.mow_history_rewrite_falls_back, + // fallback_on_sync_failure, and the delete-predicate/version-gap cases). def checkIncrementalFallback = { String sqlText -> + // Cloud skips the sampling too, for the same reason as above. + if (isCloudMode()) { + checkConsistency(sqlText) + return + } def before = sumBeMetric("query_cache_incremental_fallback_total") checkConsistency(sqlText) def after = sumBeMetric("query_cache_incremental_fallback_total") @@ -259,10 +288,10 @@ suite("query_cache_incremental") { sql "INSERT INTO test_query_cache_incremental_mow VALUES ('2026-01-01',1,100)" checkIncrementalFallback(uniqueQuerySql) // After the re-base, pure appends take the incremental path again. Left as - // a correctness-only check (not metric-asserted): the earlier rounds - // already pin the incremental path through the metric, and on cloud the - // residual meta-service/rebalance channels noted above could still flip a - // single stale-hit assertion to a spurious MISS. + // a correctness-only check (not metric-asserted): the earlier rounds already + // pin the incremental path through the metric on non-cloud, so repeating a + // stale_hit assertion here would add nothing while carrying the same cloud + // caveats that already make checkStaleIncremental result-only there. sql "INSERT INTO test_query_cache_incremental_mow VALUES ('2026-01-06',200,7)" checkConsistency(uniqueQuerySql) order_qt_mow_final "${uniqueQuerySql}" @@ -326,8 +355,9 @@ suite("query_cache_incremental") { // The first query after the idle stretch must still take the incremental // path on both deployment modes and agree with the uncached result. // Auto-compaction is disabled on the table, so the 30-rowset delta stays - // intact and no boundary-crossing compaction can turn this into a fallback; - // on cloud the residual rare meta-service/rebalance channels noted at - // checkStaleIncremental still apply and are handled the same way (rerun). + // intact and no boundary-crossing compaction can turn this into a fallback. + // checkStaleIncremental asserts the stale_hit metric on non-cloud (where it + // is deterministic) and verifies result consistency on cloud, so no + // cluster-routing or transient-sync flake reaches this assertion. checkStaleIncremental(dormantQuerySql) } From 53d7f268fe708aad27cb31c0f8c7c7ed416d88f0 Mon Sep 17 00:00:00 2001 From: Benedict Jin Date: Mon, 20 Jul 2026 12:16:22 +0800 Subject: [PATCH 3/4] [fix](query-cache) Single-flight cloud presync and mirror FE knob gate on BE Address the second-round review on the cloud incremental merge: 1. Cross-fragment stampede (new P1): QueryCacheRuntime is per fragment, so N concurrent identical stale queries each submitted their own per-tablet sync fan-out into the bounded engine pool. Add a single-flight registry on the BE-global QueryCache keyed by (cache_key, version): the first runtime owns the ONE fan-out, later identical runtimes join as waiters on the shared completion latch, each under its own decision-sync deadline. The registry entry is reaped when the fan-out drains, by the last waiter or, if every waiter already timed out, by the final task; a last leaver that exits early marks the flight abandoned so queued tasks skip their RPC, and keeps the entry as a tombstone so a later identical arrival joins the draining flight instead of re-submitting a duplicate. 2. Mirror the FE incremental knob gate on the BE at the single point where the fragment-wide runtime is born, closing the rolling-upgrade window where an older FE still requests incremental while a freshness/prefer-cached knob is active (cloud only, clear only). 3. Charge the flight's per-tablet buffers (reasons, tablet ids) to the stable query-cache MemTracker via allocator-aware DorisVector with a deleter that releases on the same limiter, instead of the transient per-query tracker the flight outlives. Reasons are static literals stored as pointers, so publishing a slot never allocates and is safe on the bad_alloc unwind path. 4. Add a query_cache_presync_inflight gauge for live registry entries. 5. Tests: coalescing with an unrelated key proceeding, reversed scan order reason mapping, MoW coalescing, post-timeout arrival joining the draining flight, tombstone reap with and without a later arrival, deterministic per-role deadlines, and a MemTracker balance check. --- be/src/common/metrics/doris_metrics.cpp | 2 + be/src/common/metrics/doris_metrics.h | 5 + .../pipeline/pipeline_fragment_context.cpp | 26 +- be/src/runtime/query_cache/query_cache.cpp | 620 +++++++++--- be/src/runtime/query_cache/query_cache.h | 137 ++- be/test/exec/pipeline/query_cache_test.cpp | 919 +++++++++++++++++- 6 files changed, 1542 insertions(+), 167 deletions(-) diff --git a/be/src/common/metrics/doris_metrics.cpp b/be/src/common/metrics/doris_metrics.cpp index 6b789749c5c75d..2cec7cce201de5 100644 --- a/be/src/common/metrics/doris_metrics.cpp +++ b/be/src/common/metrics/doris_metrics.cpp @@ -60,6 +60,7 @@ DEFINE_COUNTER_METRIC_PROTOTYPE_3ARG(query_cache_write_back_total, MetricUnit::R DEFINE_COUNTER_METRIC_PROTOTYPE_3ARG(query_cache_decision_sync_time_ms, MetricUnit::MILLISECONDS, "Cumulative wall time the cloud incremental-merge decision " "spent blocking on its pre-sync rowset fan-out."); +DEFINE_GAUGE_CORE_METRIC_PROTOTYPE_2ARG(query_cache_presync_inflight, MetricUnit::NOUNIT); DEFINE_COUNTER_METRIC_PROTOTYPE_5ARG(push_requests_success_total, MetricUnit::REQUESTS, "", push_requests_total, Labels({{"status", "SUCCESS"}})); DEFINE_COUNTER_METRIC_PROTOTYPE_5ARG(push_requests_fail_total, MetricUnit::REQUESTS, "", @@ -307,6 +308,7 @@ DorisMetrics::DorisMetrics() : _metric_registry(_s_registry_name) { INT_COUNTER_METRIC_REGISTER(_server_metric_entity, query_cache_incremental_fallback_total); INT_COUNTER_METRIC_REGISTER(_server_metric_entity, query_cache_write_back_total); INT_COUNTER_METRIC_REGISTER(_server_metric_entity, query_cache_decision_sync_time_ms); + INT_GAUGE_METRIC_REGISTER(_server_metric_entity, query_cache_presync_inflight); INT_COUNTER_METRIC_REGISTER(_server_metric_entity, push_requests_success_total); INT_COUNTER_METRIC_REGISTER(_server_metric_entity, push_requests_fail_total); diff --git a/be/src/common/metrics/doris_metrics.h b/be/src/common/metrics/doris_metrics.h index 47c9806dc3cf1b..be1e6fda6f4ad3 100644 --- a/be/src/common/metrics/doris_metrics.h +++ b/be/src/common/metrics/doris_metrics.h @@ -69,6 +69,11 @@ class DorisMetrics { // inside a profiled scan node, so without this counter its (potentially // meta-service-bound) latency is invisible. IntCounter* query_cache_decision_sync_time_ms = nullptr; + // Live count of cloud pre-sync single-flight entries currently registered + // (owned or draining). Steady state is near zero; a persistently rising value + // would flag a registry leak or a stuck fan-out. Also makes the coalescing + // mechanism observable (how many concurrent identical fan-outs are in flight). + IntGauge* query_cache_presync_inflight = nullptr; IntCounter* push_requests_success_total = nullptr; IntCounter* push_requests_fail_total = nullptr; diff --git a/be/src/exec/pipeline/pipeline_fragment_context.cpp b/be/src/exec/pipeline/pipeline_fragment_context.cpp index def2cf473aea77..ef069881471a1c 100644 --- a/be/src/exec/pipeline/pipeline_fragment_context.cpp +++ b/be/src/exec/pipeline/pipeline_fragment_context.cpp @@ -1605,8 +1605,32 @@ Status PipelineFragmentContext::_create_operator(ObjectPool* pool, const TPlanNo auto cache_node_id = _params.local_params[0].per_node_scan_ranges.begin()->first; auto cache_source_id = next_operator_id(); if (_query_cache_runtime == nullptr) { + TQueryCacheParam runtime_param = _params.fragment.query_cache_param; + // BE-side mirror of the FE incremental knob gates: a same-version + // FE already cleared allow_incremental under an active freshness/ + // prefer-cached knob, making this a no-op; it closes the rolling- + // upgrade window where an older FE without those gates still + // requests incremental while a knob is active, which would defeat + // the knob's warmed-read intent (never correctness: the delta + // capture is version-exact either way). Applied here, the single + // point where the fragment-wide runtime is born, so the cache + // source and the scan can never disagree on it; cloud-only since + // both knobs are inert on local storage. The whole decision is a + // pure function (unit-tested) so this stays trivial wiring; + // deliberately NOT part of the cache key (key identity must not + // depend on who asks). + runtime_param.allow_incremental = + QueryCacheRuntime::gate_allow_incremental_for_cloud_knobs( + runtime_param.allow_incremental, runtime_param.is_merge_on_write, + config::is_cloud_mode(), + _runtime_state->enable_query_freshness_tolerance(), + _runtime_state->enable_prefer_cached_rowset()); + // `runtime_param` is a throwaway built solely to carry the gated + // flag into the runtime, so move it in: the by-value constructor + // then move-constructs `_param` and no second O(tablets) deep copy + // of TQueryCacheParam happens on this admission path. _query_cache_runtime = - std::make_shared(_params.fragment.query_cache_param); + std::make_shared(std::move(runtime_param)); } op = std::make_shared(pool, cache_node_id, cache_source_id, _params.fragment.query_cache_param, diff --git a/be/src/runtime/query_cache/query_cache.cpp b/be/src/runtime/query_cache/query_cache.cpp index 6ecdb8fe4dbfeb..3ba976e2759222 100644 --- a/be/src/runtime/query_cache/query_cache.cpp +++ b/be/src/runtime/query_cache/query_cache.cpp @@ -271,7 +271,8 @@ bool QueryCacheRuntime::_try_prepare_incremental(const std::vector presync_reasons; if (config::is_cloud_mode()) { - presync_reasons = _presync_cloud_delta_tablets(scan_ranges, decision->current_version); + presync_reasons = _presync_cloud_delta_tablets(_cache, decision->cache_key, scan_ranges, + decision->current_version); } for (const auto& scan_range : scan_ranges) { @@ -298,6 +299,7 @@ bool QueryCacheRuntime::_try_prepare_incremental(const std::vector QueryCacheRuntime::_presync_cloud_delta_tablets( + QueryCache* cache, const std::string& cache_key, const std::vector& scan_ranges, int64_t current_version) { std::unordered_map fallback_reasons; @@ -339,44 +341,284 @@ std::unordered_map QueryCacheRuntime::_presync_cloud_delta return fallback_reasons; } - // Index-aligned so the fan-out tasks write disjoint slots without a lock; an - // empty slot means synced (or skipped as non-append-only), a non-empty slot - // is the fallback reason. Held (with the latch and per-slot count guard) - // behind a shared_ptr because the fan-out is waited on with a fast-fail - // deadline: on a timeout this frame returns while still-running tasks keep - // writing their slots, so the storage must outlive the frame -- the shared_ptr - // each task captured keeps it alive until the last one finishes. Only read on - // the completed path. - auto per_range_reason = std::make_shared>(scan_ranges.size()); - auto fanout_done = std::make_shared(static_cast(scan_ranges.size())); - // Single-owner claim per slot. Two parties can target the same slot: the task, - // and -- on the narrow ThreadPool path where do_submit enqueues the task and - // THEN returns an error (thread creation failed with zero live workers) -- the - // inline submit-failure path, which then runs concurrently with the very task it - // failed to schedule. Each party claims the slot with an atomic exchange BEFORE - // touching it: the winner runs the sync (or the inline fallback), writes - // per_range_reason[idx] via publish_slot, and counts the latch down; the loser - // returns having touched neither. Claiming before the RPC (an exchange, not a - // check-then-act load) is what stops the enqueue-then-fail task from firing a - // sync for an already-settled query, and it keeps the caller, once the latch - // settles, the sole reader of a slot no task is still writing. (Value-initialized - // to false.) - auto slot_counted = std::make_shared>>(scan_ranges.size()); - // Set true when the caller abandons the wait (its deadline passed): a not-yet- - // started task then skips its sync RPC instead of spending the full retry - // budget, so a meta-service brownout drains the bounded queue fast rather than - // running abandoned work to completion and starving the pool. - auto abandoned = std::make_shared>(false); + // Coalesce identical concurrent fan-outs through the cache's single-flight + // registry. Runtimes are private per fragment context, so without this, N + // concurrent identical queries hitting the same stale key would each submit + // the full per-tablet fan-out: the duplicates waste the bounded queue's + // capacity (rejecting tablets that would have fit as one fan-out) and, + // during a meta-service brownout, park pool workers on the same per-tablet + // _sync_meta_lock, denying admission to unrelated keys. The first arrival + // (the flight owner) submits the fan-out; later identical arrivals join as + // waiters on the SAME completion latch, each under its own deadline. The + // flight's shared pieces are exactly what a private fan-out uses; a null + // cache (test-only) skips the registry and keeps the fan-out private. + bool flight_owner = true; + std::shared_ptr flight; + std::string flight_key; + auto make_flight = [&] { + // Attribute the flight's DOMINANT O(tablets) buffers to the STABLE query-cache + // MemTracker rather than to the transient per-query tracker of whichever + // admission thread happened to create the flight (the flight outlives that + // query: coalesced waiters and still-running fan-out tasks keep its shared + // pieces alive past the query's teardown). + // + // IMPORTANT -- how memory is charged in Doris BE: a MemTrackerLimiter is + // charged either by an allocator-aware container (CustomStdAllocator -> + // Allocator::consume_memory, which is what DorisVector does) or by an explicit + // manual consume()/release() side-channel (what the LRU layer does with its + // tracking_bytes, see QueryCache::insert above). There is NO global new/malloc + // hook (be/src/runtime/memory/jemalloc_hook.cpp aliases malloc/free straight to + // jemalloc with zero tracker interaction, and there is no global ::operator new + // override), so a plain `new` / std::vector / std::string that does neither is + // invisible to every MemTrackerLimiter and appears only in process RSS. Hence + // the two hot buffers + // below are DorisVector, whose CustomStdAllocator explicitly consumes/releases + // the CURRENT thread's limiter. The SCOPED_SWITCH makes that current limiter the + // query-cache one at allocation; the custom deleter re-enters the SAME limiter + // at free (on whichever pool/waiter thread drops the last ref), so the + // DorisVector buffer is charged and released on the same stable limiter -- + // balanced. Pinning BOTH sides is required: a switch at construction alone would + // credit the cross-thread free elsewhere and drift this limiter. The deleter + // captures the limiter by shared_ptr so it stays alive until the last buffer is + // gone. + // + // Everything else the flight owns -- the PresyncFlight object, the six shared_ptr + // CONTROL BLOCKS, `slot_counted`, the `key` string, and the registry map node -- + // is plain `new`, so it is NOT charged to ANY MemTrackerLimiter (not this one and + // not the per-query one): RSS-only on both alloc and free. It therefore cannot + // strand as an orphan on a destroyed query tracker, nor drift this limiter -- the + // switch is irrelevant to it since a plain `new` never reaches consume_memory. + // These bytes are bounded per flight and the live-flight COUNT is observable via + // the query_cache_presync_inflight gauge; the same untracked O(tablets) + // metadata shape already ships on the sibling warmup path, whose pool closure + // captures a plain std::vector of tablet ids by value + // (CloudBackendService::sync_load_for_tablets). The two DorisVectors carry 16 + // of the roughly 33 bytes per tablet a live flight holds (the rest: the two + // 8-byte-per-tablet key copies, a claim flag, plus each waiter's transient + // flight_key), so the charged and uncharged halves are comparable -- charging + // the DorisVectors buys stable attribution of the data-path buffers, not full + // byte coverage. If full metadata attribution is ever wanted, the LRU layer's + // manual tracking_bytes consume()/release() idiom (QueryCache::insert above) + // is the cheap in-tree way to add it. + auto qc_limiter = ExecEnv::GetInstance()->query_cache_mem_tracker(); + auto tracked_delete = [qc_limiter](auto* p) { + SCOPED_SWITCH_THREAD_MEM_TRACKER_LIMITER(qc_limiter); + delete p; + }; + SCOPED_SWITCH_THREAD_MEM_TRACKER_LIMITER(qc_limiter); + std::shared_ptr f(new QueryCache::PresyncFlight(), + tracked_delete); + // Index-aligned so the fan-out tasks write disjoint slots without a + // lock; an empty (nullptr) slot means synced (or skipped as + // non-append-only), a non-empty slot is the fallback reason (a static + // literal). Each piece sits behind its own shared_ptr because the fan-out + // is waited on with a fast-fail deadline: on a timeout a waiter returns + // while still-running tasks keep writing their slots, so the storage must + // outlive every frame -- the shared_ptr each task captured keeps it alive + // until the last one finishes. Only read on the completed path. + f->per_range_reason = std::shared_ptr>( + new DorisVector(scan_ranges.size()), tracked_delete); + f->fanout_done = std::shared_ptr( + new CountDownLatch(static_cast(scan_ranges.size())), tracked_delete); + // Per-slot single-owner claim flags; see the claim comments at the + // submit loop below. (Value-initialized to false.) + f->slot_counted = std::shared_ptr>>( + new std::vector>(scan_ranges.size()), tracked_delete); + // Set true when the LAST interested waiter abandons the flight (its + // deadline passed): a not-yet-started task then skips its sync RPC + // instead of spending the full retry budget, so a meta-service brownout + // drains the bounded queue fast rather than running abandoned work to + // completion and starving the pool. + f->abandoned = + std::shared_ptr>(new std::atomic(false), tracked_delete); + // Snapshot the owner's tablet ids in slot order. Every waiter (this + // owner and any that coalesce onto the flight) builds its fallback map + // from THESE ids, not from its own scan-range order: build_cache_key + // sorts tablet ids into the key, so two runtimes whose scan ranges hold + // the same tablets in a different order share this flight, and keying + // the shared index-aligned slots off a joiner's own order would pin the + // owner's per-slot reason to the wrong tablet. + std::shared_ptr> ids(new DorisVector(), tracked_delete); + ids->reserve(scan_ranges.size()); + for (const auto& scan_range : scan_ranges) { + ids->push_back(scan_range.scan_range.palo_scan_range.tablet_id); + } + f->tablet_ids = std::move(ids); + // The flight's own registry key. A plain std::string, so its buffer is + // RSS-only like the other non-DorisVector pieces (see the header comment + // above); on the null-cache test path flight_key is empty, so this is a + // harmless no-op there. + f->key = flight_key; + f->slots = scan_ranges.size(); + f->waiters = 1; + return f; + }; + if (cache != nullptr) { + // flight_key embeds the digest and the SORTED tablet set + version, and + // is injective: build_cache_key appends the version as decimal digits, so + // this '@' is the LAST one in the string (no digit is '@') and splitting + // at the final '@' recovers exactly (cache_key, version) even though the + // binary cache_key may itself contain '@' bytes. Equal flight_key + // therefore means the identical tablet set at the identical version -- + // correct to coalesce, never a cross-key alias. + flight_key = cache_key; + flight_key += '@'; + flight_key += std::to_string(current_version); + // Double-checked registration. Check under the lock FIRST (the common + // coalescing case joins without allocating anything); only on a miss do we + // build the O(tablets) candidate -- OUTSIDE the lock, because holding the + // BE-global registry lock across that allocation would serialize every + // cloud incremental query, and a stampeding follower must not build a + // candidate it immediately discards -- then re-lock and re-check, since a + // concurrent owner may have inserted in the gap (we then join theirs and + // free the candidate). A found flight always covers the identical tablet + // set and slot count (the key invariant above), so assert rather than + // branch; joining a flight whose original waiters all already timed out is + // intentional -- it rides the draining fan-out instead of submitting a + // DUPLICATE one, which is what keeps a sustained brownout from recreating + // wave after wave of identical syncs. The entry is reaped only once the + // latch drains -- by the last waiter (leave_flight) or, if none is left, + // by the final task (publish_slot) -- so it never lingers. + { + std::lock_guard reg_lock(cache->_presync_flights_lock); + auto it = cache->_presync_flights.find(flight_key); + if (it != cache->_presync_flights.end()) { + DCHECK_EQ(it->second->slots, scan_ranges.size()); + flight = it->second; + ++flight->waiters; + flight_owner = false; + } + } + if (flight_owner) { + auto candidate = make_flight(); + std::lock_guard reg_lock(cache->_presync_flights_lock); + // try_emplace does the re-check and the insert in ONE hash of this long + // key (digest + sorted tablet set + version): it inserts `candidate` + // only when no concurrent owner beat us to the key in the unlocked gap, + // otherwise it leaves the map untouched and hands back the existing + // flight to join (and `candidate` is dropped). This replaces a find() + // plus a second operator[] that would hash the key twice under the + // BE-global registry lock. + auto [it, inserted] = cache->_presync_flights.try_emplace(flight_key, candidate); + if (inserted) { + flight = candidate; + DorisMetrics::instance()->query_cache_presync_inflight->increment(1); + } else { + DCHECK_EQ(it->second->slots, scan_ranges.size()); + flight = it->second; + ++flight->waiters; + flight_owner = false; + } + } + } else { + flight = make_flight(); + } + auto per_range_reason = flight->per_range_reason; + auto fanout_done = flight->fanout_done; + auto slot_counted = flight->slot_counted; + auto abandoned = flight->abandoned; + // The owner's tablet ids in slot order; every waiter (owner and coalesced + // joiners) keys the merged reasons off THESE, never off its own scan-range + // order (build_cache_key sorts, so a joiner's order may differ). Shared for + // read only, so a plain copy of the shared_ptr is enough. + auto flight_tablet_ids = flight->tablet_ids; + // Leave the flight on EVERY exit path -- normal return AND an exception + // unwinding out of the submit loop or the merge below -- via RAII, so a + // decrement can never be skipped and strand the registry entry. Two rules: + // (1) Reap the entry only once the fan-out has actually DRAINED (latch at + // 0), NOT merely when the last waiter's deadline expired. Erasing while + // tasks are still mid-RPC would let the next identical query find no + // entry, become a fresh owner, and submit a DUPLICATE fan-out; under a + // sustained meta-service brownout that recreates wave after wave of + // duplicate syncs -- the exact bounded-pool pressure single-flight + // exists to prevent. So while tasks remain, KEEP the entry: post-timeout + // arrivals then join this draining flight (see the join above) instead + // of piling on. If a waiter is present when the latch drains it reaps + // here; if every waiter already left (a timed-out tombstone), the FINAL + // task reaps in publish_slot when it drives the latch to 0 and sees zero + // waiters -- so a drained tombstone is never stranded, even with no + // future identical query. Every slot is published exactly once (run, + // skip, inline-reject, or submit-throw fallback), so the latch always + // reaches 0 in bounded time (except at pool shutdown, which discards + // queued-but-unstarted tasks; that strand is a process-teardown residual + // reclaimed when the cache is destroyed, not runtime growth), and a + // drained flight makes wait_for return immediately for a joiner. + // (2) A last leaver that leaves while tasks still run -- its deadline + // expired, OR an exception (bad_alloc) unwound the owner mid-submit -- + // sets `abandoned` UNDER the registry lock (so a joiner cannot observe + // the entry pre-abandon), telling not-yet-started tasks to skip so the + // pool drains fast instead of running work no one will read. NOT + // per-waiter: a timed-out waiter must not cancel work a still-waiting + // sibling needs. A private (null-cache) flight has exactly one waiter, + // so this degenerates to plain abandon-if-tasks-remain. + bool timed_out = false; + Defer leave_flight([&] { + if (cache != nullptr) { + std::lock_guard reg_lock(cache->_presync_flights_lock); + bool last = (--flight->waiters == 0); + bool drained = (fanout_done->count() == 0); + if (last && drained) { + // Guard `it->second == flight`: a prior tombstone for this key may + // already have been reaped and a fresh flight inserted; erase only + // our own. + auto it = cache->_presync_flights.find(flight_key); + if (it != cache->_presync_flights.end() && it->second == flight) { + cache->_presync_flights.erase(it); + DorisMetrics::instance()->query_cache_presync_inflight->increment(-1); + } + } else if (last) { + // Last waiter leaving with the fan-out still running (timeout or an + // owner exception): keep the tombstone and abandon so queued tasks + // skip; the final task reaps it in publish_slot once it drains. + abandoned->store(true, std::memory_order_release); + } + } else if (fanout_done->count() != 0) { + // Private (null-cache) flight, tasks still running: abandon so they + // skip. There is no registry entry to reap. + abandoned->store(true, std::memory_order_release); + } + }); // Called ONLY by whoever already won the exclusive slot claim (the slot_counted - // exchange below), so it is the sole writer of the slot: publish its reason (an - // empty reason means synced OK / skipped, leaving the default-empty slot) and - // release the latch exactly once. The claim winner counting down is what lets the - // caller move the slot out the instant the latch settles. - auto publish_slot = [fanout_done, per_range_reason](size_t idx, std::string reason) { - if (!reason.empty()) { - (*per_range_reason)[idx] = std::move(reason); + // exchange below), so it is the sole writer of the slot: publish its reason (a + // null/empty reason means synced OK / skipped, leaving the default-empty slot) + // and release the latch exactly once. The claim winner counting down is what + // lets the caller move the slot out the instant the latch settles. + // + // `reason` is a `const char*` (a static literal), NOT a std::string, and the + // slot is a `const char*` too, so storing it is a plain pointer write that + // CANNOT allocate or throw. This matters because this also runs on the + // bad_alloc unwind (publish_unsubmitted below): a throwing store inside that + // Defer would std::terminate. A null/empty reason means synced OK / skipped and + // leaves the default-null slot. The latch count_down and the reap always run, + // so a slot never strands the flight; the tablet-id map materializes the + // reason strings once, after the latch settles. + auto publish_slot = [fanout_done, per_range_reason, cache, flight](size_t idx, + const char* reason) { + if (reason != nullptr && reason[0] != '\0') { + (*per_range_reason)[idx] = reason; } fanout_done->count_down(); + // Reap-on-drain by the FINAL publisher when no waiter is left. If this + // count_down drained the latch and every interested waiter has already + // left (a timed-out tombstone, or an owner that unwound on bad_alloc), no + // leave_flight will run to reap it -- so reap here. The count()==0 + // pre-check takes only the latch's own lock, NOT the BE-global registry + // lock (the latch saturates at 0, so once true it stays true, and every + // slot publishes exactly once); only the settling publisher takes the + // registry lock, where it re-reads waiters and erases only its own entry. + // A joiner that raced in (++waiters under the same lock) makes waiters != 0, + // so this skips and that joiner reaps on its own leave. `cache` is null on + // the test-only private path (no registry). + if (cache != nullptr && fanout_done->count() == 0) { + std::lock_guard reg_lock(cache->_presync_flights_lock); + if (flight->waiters == 0) { + auto it = cache->_presync_flights.find(flight->key); + if (it != cache->_presync_flights.end() && it->second == flight) { + cache->_presync_flights.erase(it); + DorisMetrics::instance()->query_cache_presync_inflight->increment(-1); + } + } + } }; // Run the per-tablet syncs on the engine-owned, bounded query-cache delta @@ -415,109 +657,153 @@ std::unordered_map QueryCacheRuntime::_presync_cloud_delta // sync_rowsets caller, CloudBackendService::sync_load_for_tablets (the FE // warmup path), which likewise does not attach. auto sync_fanout_start = std::chrono::steady_clock::now(); - auto& pool = engine->query_cache_delta_sync_pool(); - // Safe to capture the engine by raw pointer: stop() drains this pool before - // the engine (and _meta_mgr/_tablet_mgr) is destroyed, so a running task - // always sees a live engine. - CloudStorageEngine* engine_ptr = engine; - for (size_t i = 0; i < scan_ranges.size(); ++i) { - int64_t tablet_id = scan_ranges[i].scan_range.palo_scan_range.tablet_id; - Status submit_st = pool.submit_func([tablet_id, current_version, abandoned, i, engine_ptr, - publish_slot, slot_counted]() { - // Claim this slot atomically BEFORE any external work. If the inline - // submit-failure path already claimed it -- the narrow enqueue-then-fail - // case where do_submit queued this task but then returned an error at zero - // live workers, and the caller finished the slot inline -- bail before the - // sync RPC: the caller owns the slot and the count-down, and this task's - // result would be discarded. Claiming before the RPC (an exchange, not a - // plain load that a concurrent inline claim could race between the read - // and the RPC) is what keeps an enqueue-then-fail task that a later worker - // picks up from firing a meta-service call for an already-settled query. - if ((*slot_counted)[i].exchange(true, std::memory_order_acq_rel)) { - return; - } - // This task now exclusively owns the slot and MUST publish it exactly once - // on every exit path so the caller's bounded wait can settle. Empty reason - // means synced OK (or skipped as non-append-only); a non-empty reason - // falls this tablet back. - std::string reason; - Defer done([&] { publish_slot(i, std::move(reason)); }); - // (a) Shutdown began, or the caller already abandoned the wait (its - // deadline passed): bail before a fresh sync RPC so the pool drains - // quickly instead of spending the full RPC retry budget on work whose - // result no one will read. (An abandoned task's reason is never read -- - // the caller took the timeout path -- but recording it is harmless.) - if (engine_ptr->stopped() || abandoned->load(std::memory_order_acquire)) { - reason = "be is stopping, sync skipped"; - return; - } - auto tablet_res = ExecEnv::get_tablet(tablet_id); - if (!tablet_res) { - // Publish the failure instead of leaving the slot empty: an - // empty reason tells _capture_tablet_delta the view is synced, - // so its own get_tablet would RETRY this very cache-miss - // meta-service load synchronously on the admission thread, - // outside the fast-fail deadline -- during a brownout (the one - // situation where this load fails slowly) that repeats the - // slow RPC on the pool this budget exists to protect. Falling - // back is also never wrong here: this worker-side load is - // strictly conservative (a transient failure only costs the - // incremental merge, never correctness). - reason = "cloud tablet load failed"; - return; - } - BaseTabletSPtr tablet = std::move(tablet_res.value()); - const bool append_only = tablet->keys_type() == KeysType::DUP_KEYS || - (tablet->keys_type() == KeysType::UNIQUE_KEYS && - tablet->enable_unique_key_merge_on_write()); - if (!append_only) { - // Skip the RPC entirely: _capture_tablet_delta rejects this - // tablet at its keys-type check before it would consume a sync. - return; - } - auto cloud_tablet = std::dynamic_pointer_cast(std::move(tablet)); - if (cloud_tablet == nullptr) { - // is_cloud_mode() flipped on a live local deployment - // (cloud_unique_id is a mutable config): degrade to a full - // recompute rather than dereference the failed cast downstream. - reason = "tablet is not a cloud tablet"; - return; - } - SyncOptions options; - options.query_version = current_version; - options.merge_schema = true; - // The history-rewrite check reads the delete bitmap this sync merges, - // so pin the dependency explicitly rather than inherit the struct - // default (which a refactor could flip with no compile error): a - // merge-on-write delta cannot be classified on a bitmap the sync did - // not bring up to the queried version. - options.sync_delete_bitmap = true; - Status st = cloud_tablet->sync_rowsets(options); - if (!st.ok()) { - // Unlike the other fallback reasons (expected data shapes), a - // failed sync is an infrastructure error: log the status so the - // profile reason correlates to a cause. Throttled with - // LOG_EVERY_N because a meta-service brownout fails this for - // every stale tablet of every query at once; the underlying RPC - // errors are already logged per retry inside retry_rpc, so a - // sampled line here is enough to correlate without a storm, and - // the exact reason still reaches the user via the query profile. - LOG_EVERY_N(WARNING, 100) - << "query cache incremental merge falls back, cloud rowset sync failed" - << ", tablet_id=" << tablet_id << ", status=" << st.to_string(); - reason = "cloud rowset sync failed"; + if (!flight_owner) { + // Joined an in-flight fan-out for the same (cache_key, version): the + // owner's tasks fill the shared slots; just wait on the shared latch + // below under this waiter's own deadline. + TEST_SYNC_POINT("QueryCacheRuntime::_presync_cloud_delta_tablets.follower_joined"); + } else { + auto& pool = engine->query_cache_delta_sync_pool(); + // Safe to capture the engine by raw pointer: stop() drains this pool + // before the engine (and _meta_mgr/_tablet_mgr) is destroyed, so a + // running task always sees a live engine. + CloudStorageEngine* engine_ptr = engine; + // Guarantee the latch always settles even if the submit loop unwinds + // partway (submit_func can throw bad_alloc heap-allocating a task closure + // under memory pressure). `submitted` counts the slots the loop has fully + // handled (a task was enqueued, or the inline-reject path published it); + // on an unwind this Defer publishes exactly the slots NOT yet reached, so + // the CountDownLatch still reaches 0. It must publish only [submitted, N): + // slots already submitted are owned by their (possibly still-queued) tasks + // and stealing their claim would falsely report a scheduled sync as + // unscheduled -- so on the normal path (submitted == N) this publishes + // nothing. A settling latch is what lets the leave Defer's reap-on-drain + // rule eventually remove the registry entry instead of stranding it. + size_t submitted = 0; + Defer publish_unsubmitted([&] { + for (size_t j = submitted; j < scan_ranges.size(); ++j) { + if (!(*slot_counted)[j].exchange(true, std::memory_order_acq_rel)) { + publish_slot(j, "cloud rowset sync not scheduled"); + } } }); - if (!submit_st.ok()) { - // Pool shutting down or queue at capacity: do not block. Claim the slot - // for the inline fallback. do_submit rejects shutdown and capacity before - // enqueue, so the closure will not run; on the narrow path where it did - // enqueue before failing (thread creation at zero live workers), the - // exchange makes exactly one of this path and the queued task own the slot - // -- if the task claimed it first it will publish, so do nothing here. - if (!(*slot_counted)[i].exchange(true, std::memory_order_acq_rel)) { - publish_slot(i, "cloud rowset sync not scheduled"); + for (size_t i = 0; i < scan_ranges.size(); ++i) { + int64_t tablet_id = scan_ranges[i].scan_range.palo_scan_range.tablet_id; + Status submit_st = pool.submit_func([tablet_id, current_version, abandoned, i, + engine_ptr, publish_slot, slot_counted]() { + // Claim this slot atomically BEFORE any external work. If the inline + // submit-failure path already claimed it -- the narrow enqueue-then-fail + // case where do_submit queued this task but then returned an error at zero + // live workers, and the caller finished the slot inline -- bail before the + // sync RPC: the caller owns the slot and the count-down, and this task's + // result would be discarded. Claiming before the RPC (an exchange, not a + // plain load that a concurrent inline claim could race between the read + // and the RPC) is what keeps an enqueue-then-fail task that a later worker + // picks up from firing a meta-service call for an already-settled query. + if ((*slot_counted)[i].exchange(true, std::memory_order_acq_rel)) { + return; + } + // This task now exclusively owns the slot and MUST publish it exactly once + // on every exit path so the caller's bounded wait can settle. Empty reason + // means synced OK (or skipped as non-append-only); a non-empty reason + // falls this tablet back. + // A static literal (const char*), never an owning std::string, so + // publishing it in `done` cannot allocate on the bad_alloc unwind + // path (see publish_slot's contract above). + const char* reason = nullptr; + Defer done([&] { publish_slot(i, reason); }); + // (a) Shutdown began, or the interested waiters all abandoned the + // wait (their deadlines passed, or an owner unwound on bad_alloc): + // bail before a fresh sync RPC so the pool drains quickly instead of + // spending the full RPC retry budget on work the timed-out waiters + // will not read. A coalesced joiner MAY later reuse this slot (it + // rides the draining tombstone), so the reason must be accurate: + // keep "be is stopping" for a genuine teardown, but give the + // abandonment case a distinct reason (covering BOTH the timeout and + // the owner-unwind causes) so a joiner on a HEALTHY BE never sees a + // false shutdown diagnosis in its profile. + if (engine_ptr->stopped()) { + reason = "be is stopping, sync skipped"; + return; + } + if (abandoned->load(std::memory_order_acquire)) { + reason = "cloud rowset sync abandoned (all waiters left)"; + return; + } + auto tablet_res = ExecEnv::get_tablet(tablet_id); + if (!tablet_res) { + // Publish the failure instead of leaving the slot empty: an + // empty reason tells _capture_tablet_delta the view is synced, + // so its own get_tablet would RETRY this very cache-miss + // meta-service load synchronously on the admission thread, + // outside the fast-fail deadline -- during a brownout (the one + // situation where this load fails slowly) that repeats the + // slow RPC on the pool this budget exists to protect. Falling + // back is also never wrong here: this worker-side load is + // strictly conservative (a transient failure only costs the + // incremental merge, never correctness). + reason = "cloud tablet load failed"; + return; + } + BaseTabletSPtr tablet = std::move(tablet_res.value()); + const bool append_only = tablet->keys_type() == KeysType::DUP_KEYS || + (tablet->keys_type() == KeysType::UNIQUE_KEYS && + tablet->enable_unique_key_merge_on_write()); + if (!append_only) { + // Skip the RPC entirely: _capture_tablet_delta rejects this + // tablet at its keys-type check before it would consume a sync. + return; + } + auto cloud_tablet = std::dynamic_pointer_cast(std::move(tablet)); + if (cloud_tablet == nullptr) { + // is_cloud_mode() flipped on a live local deployment + // (cloud_unique_id is a mutable config): degrade to a full + // recompute rather than dereference the failed cast downstream. + reason = "tablet is not a cloud tablet"; + return; + } + SyncOptions options; + options.query_version = current_version; + options.merge_schema = true; + // The history-rewrite check reads the delete bitmap this sync merges, + // so pin the dependency explicitly rather than inherit the struct + // default (which a refactor could flip with no compile error): a + // merge-on-write delta cannot be classified on a bitmap the sync did + // not bring up to the queried version. + options.sync_delete_bitmap = true; + Status st = cloud_tablet->sync_rowsets(options); + if (!st.ok()) { + // Unlike the other fallback reasons (expected data shapes), a + // failed sync is an infrastructure error: log the status so the + // profile reason correlates to a cause. Throttled with + // LOG_EVERY_N because a meta-service brownout fails this for + // every stale tablet of every query at once; the underlying RPC + // errors are already logged per retry inside retry_rpc, so a + // sampled line here is enough to correlate without a storm, and + // the exact reason still reaches the user via the query profile. + LOG_EVERY_N(WARNING, 100) + << "query cache incremental merge falls back, cloud rowset sync failed" + << ", tablet_id=" << tablet_id << ", status=" << st.to_string(); + reason = "cloud rowset sync failed"; + } + }); + if (!submit_st.ok()) { + // Pool shutting down or queue at capacity: do not block. Claim the + // slot for the inline fallback. do_submit rejects shutdown and + // capacity before enqueue, so the closure will not run; on the + // narrow path where it did enqueue before failing (thread creation + // at zero live workers), the exchange makes exactly one of this + // path and the queued task own the slot -- if the task claimed it + // first it will publish, so do nothing here. + if (!(*slot_counted)[i].exchange(true, std::memory_order_acq_rel)) { + publish_slot(i, "cloud rowset sync not scheduled"); + } } + // Slot i is now fully handled (a task was enqueued above, or the + // inline-reject path published it); advance the safety-net floor so + // the unwind Defer never re-touches it. Reached only when submit_func + // did NOT throw -- a throw skips this and leaves slot i to the Defer. + submitted = i + 1; } } @@ -528,8 +814,19 @@ std::unordered_map QueryCacheRuntime::_presync_cloud_delta // query_cache_decision_sync_timeout_ms even if a degraded pool ever made // submission slow, instead of granting the full timeout again after submission // already consumed part of it. - auto sync_deadline = sync_fanout_start + - std::chrono::milliseconds(config::query_cache_decision_sync_timeout_ms); + int64_t decision_sync_timeout_ms = config::query_cache_decision_sync_timeout_ms; + // Test seam: lets a test pin a deterministic deadline per coalescing role. When + // two runtimes share one flight, a test that wants to observe the intermediate + // "first waiter left, second still waiting" state needs the leave order fixed by + // construction (give the owner a shorter timeout than the joining follower) + // rather than by wall-clock start staggering, which a descheduled CI runner could + // invert. `flight_owner` is the role discriminator (true for the runtime that + // created the flight, false for one that joined it). A no-op in production (no + // callback registered); this function is static, so the role, not `this`, is what + // a test can key on. + TEST_SYNC_POINT_CALLBACK("QueryCacheRuntime::_presync_cloud_delta_tablets.deadline_ms", + flight_owner, &decision_sync_timeout_ms); + auto sync_deadline = sync_fanout_start + std::chrono::milliseconds(decision_sync_timeout_ms); auto sync_now = std::chrono::steady_clock::now(); // Keep full clock precision here -- do NOT truncate to whole milliseconds. // Truncating would shave the sub-millisecond submit time off the budget and let @@ -538,12 +835,14 @@ std::unordered_map QueryCacheRuntime::_presync_cloud_delta // the total time from sync_fanout_start stays >= the budget. auto remaining = sync_deadline > sync_now ? (sync_deadline - sync_now) : std::chrono::steady_clock::duration::zero(); - bool timed_out = !fanout_done->wait_for(remaining); - if (timed_out) { - // Tell not-yet-started tasks to skip their sync RPC so the pool drains - // fast instead of running abandoned work to completion. - abandoned->store(true, std::memory_order_release); - } + // The leave-the-flight bookkeeping (decrement waiters, reap on drain, mark + // abandoned) runs in the `leave_flight` Defer established above, so it fires + // on every exit including an unwind. That Defer keys its reap/abandon decision + // off the latch count (`drained`), NOT off `timed_out` -- a last leaver can + // exit with `timed_out==false` (an owner unwinding on bad_alloc mid-submit) + // yet with tasks still running, and only the latch count captures that. Record + // the timeout result here; `timed_out` is read only by the fallback below. + timed_out = !fanout_done->wait_for(remaining); DorisMetrics::instance()->query_cache_decision_sync_time_ms->increment( std::chrono::duration_cast(std::chrono::steady_clock::now() - sync_fanout_start) @@ -570,17 +869,30 @@ std::unordered_map QueryCacheRuntime::_presync_cloud_delta return fallback_reasons; } // Merge the index-aligned slots into a tablet-id map for the capture loop. - // Safe to read now: the fan-out completed, so no task is still writing. - // Scan ranges carry distinct tablet ids in practice (each tablet is scanned - // once per query), but were one to appear twice the fan-out would only - // issue an idempotent redundant sync (serialized inside sync_rowsets by - // _sync_meta_lock) and this map, keyed by tablet id, would still resolve to - // one consistent reason for it -- so a duplicate is harmless, not something - // to guard against with an unreachable de-dup branch. + // Safe to read now: the fan-out completed, so no task is still writing. COPY + // rather than move: the slots are shared with every other runtime coalesced + // onto this flight, and each builds its own map from them. Key each slot off + // the OWNER's tablet id (flight_tablet_ids), NOT this waiter's scan_ranges: + // build_cache_key sorts tablet ids, so a joiner whose scan ranges differ + // from the owner's only in order still shares this flight, and reading a + // slot through the joiner's own position would misassign the owner's + // per-slot reason to the wrong tablet (a failed tablet's reason lands on a + // sibling, and the truly failed tablet, left with no reason, is treated as + // synced -- so _capture_tablet_delta reissues a synchronous get_tablet on + // the admission thread outside the fast-fail budget, exactly the brownout + // stall the presync exists to avoid). Scan ranges carry distinct tablet ids + // in practice (each tablet is scanned once per query), but were one to + // appear twice the fan-out would only issue an idempotent redundant sync + // (serialized inside sync_rowsets by _sync_meta_lock) and this map, keyed by + // tablet id, would still resolve to one consistent reason for it -- so a + // duplicate is harmless, not something to guard against with an unreachable + // de-dup branch. for (size_t i = 0; i < per_range_reason->size(); ++i) { - if (!(*per_range_reason)[i].empty()) { - fallback_reasons[scan_ranges[i].scan_range.palo_scan_range.tablet_id] = - std::move((*per_range_reason)[i]); + // A null slot is a synced-OK / skipped tablet; a non-null slot is a static + // reason literal (publish_slot only stores non-empty ones). Materialize the + // literal into the tablet-id map here, once, now that the fan-out has drained. + if ((*per_range_reason)[i] != nullptr) { + fallback_reasons[(*flight_tablet_ids)[i]] = (*per_range_reason)[i]; } } return fallback_reasons; diff --git a/be/src/runtime/query_cache/query_cache.h b/be/src/runtime/query_cache/query_cache.h index 0127763f07fa5e..6d38d03098166c 100644 --- a/be/src/runtime/query_cache/query_cache.h +++ b/be/src/runtime/query_cache/query_cache.h @@ -34,11 +34,13 @@ #include "common/config.h" #include "common/status.h" #include "core/block/block.h" +#include "core/custom_allocator.h" #include "io/fs/file_system.h" #include "io/fs/path.h" #include "runtime/exec_env.h" #include "runtime/memory/lru_cache_policy.h" #include "runtime/memory/mem_tracker.h" +#include "util/countdown_latch.h" #include "util/lru_cache.h" #include "util/slice.h" #include "util/time.h" @@ -257,6 +259,65 @@ class QueryCache : public LRUCachePolicy { void insert(const CacheKey& key, int64_t version, CacheResult& result, const std::vector& solt_orders, int64_t cache_size, int64_t delta_count = 0); + + // ---- cloud presync single-flight registry (internal to QueryCacheRuntime) ---- + // Fragment runtimes are private per PipelineFragmentContext, so without this + // registry N concurrent identical queries hitting the same stale entry would + // each submit the full per-tablet sync fan-out to the (bounded) engine pool: + // duplicates waste queue capacity and, during a meta-service brownout, park + // workers on the same per-tablet sync lock, denying admission to unrelated + // keys. The registry, keyed by (cache_key, version), lets the first arriving + // runtime own the fan-out while later identical runtimes wait on the SAME + // completion latch under their own deadlines. It lives here because this + // cache object is the one BE-global thing every runtime already shares. + struct PresyncFlight { + // The same shared pieces a private fan-out uses (see + // _presync_cloud_delta_tablets); sharing the struct just widens who may + // wait on them. Tasks capture the individual shared_ptrs, so the pieces + // outlive both the registry entry and every waiter. + // Per-slot fallback reason, or nullptr/"" for a synced-OK (or skipped) + // tablet. A `const char*` to a static string literal, NEVER an owning + // std::string: publishing a slot is then a plain pointer store that cannot + // allocate, so it is safe on the bad_alloc unwind path (publish_unsubmitted) + // where constructing a std::string could throw again inside a Defer and + // std::terminate. The merged tablet-id map materializes the strings once, + // after the fan-out latch settles. Allocator-aware (DorisVector) so this + // O(tablets) buffer is charged to a MemTracker (see make_flight, which pins + // it to the stable query-cache limiter). + std::shared_ptr> per_range_reason; + std::shared_ptr fanout_done; + std::shared_ptr>> slot_counted; + std::shared_ptr> abandoned; + // The owner's tablet ids in slot order, so the index-aligned slots map + // back to tablets by the OWNER's positions, which every waiter shares. + // A joining runtime must NOT key the merged reasons off its own + // scan-range order: build_cache_key sorts tablet ids, so two runtimes + // whose scan ranges differ only in order share a flight, and reading a + // slot through the joiner's own positions would misassign the owner's + // per-slot reason to the wrong tablet. Written once at creation, read + // (never mutated) by every waiter, so no lock is needed. Allocator-aware + // (DorisVector) so this O(tablets) buffer is charged to a MemTracker. + std::shared_ptr> tablet_ids; + // This flight's own registry key (cache_key + '@' + version). Stored so + // the fan-out's final publisher can reap a drained-but-waiterless flight + // by key without capturing the string into every task closure -- see the + // reap-on-drain rule below. Written once at creation, then read-only. + std::string key; + // Slot count, to sanity-check that a joining runtime's scan-range set + // matches. cache-key equality implies it (the key embeds the sorted + // tablet set), so it is an invariant, DCHECK_EQ'd at the join site. + size_t slots = 0; + // Interested waiters (owner included), guarded by _presync_flights_lock. + // The registry entry is reaped only once the fan-out has DRAINED (latch + // at 0): either by the last waiter to leave (its leave_flight Defer) or, + // if every waiter left before the fan-out drained (a timed-out tombstone), + // by the final task that drives the latch to 0 when it observes zero + // waiters. A last leaver that leaves while tasks still run keeps the entry + // as a tombstone and sets `abandoned` so not-yet-started tasks skip. + int waiters = 0; + }; + std::mutex _presync_flights_lock; + std::unordered_map> _presync_flights; }; // The per-fragment-instance decision of how the query cache participates in the @@ -337,8 +398,13 @@ class QueryCacheRuntime { public: // `cache` is injectable for tests; production callers pass nullptr and the // global instance is used. - explicit QueryCacheRuntime(const TQueryCacheParam& param, QueryCache* cache = nullptr) - : _param(param), _cache(cache != nullptr ? cache : QueryCache::instance()) {} + // Takes the param BY VALUE and moves it into `_param`, so a caller that no + // longer needs its copy (the fragment-context gate builds a mutated + // `runtime_param` solely to pass here) can `std::move` it in and avoid a second + // O(tablets) deep copy of TQueryCacheParam; an lvalue caller (the tests) still + // copies exactly once, as before. + explicit QueryCacheRuntime(TQueryCacheParam param, QueryCache* cache = nullptr) + : _param(std::move(param)), _cache(cache != nullptr ? cache : QueryCache::instance()) {} QueryCache* cache() const { return _cache; } @@ -347,6 +413,46 @@ class QueryCacheRuntime { // threaded, before any local state init), so no locking is needed. void disable_for_binlog_scan() { _binlog_scan = true; } + // BE-side mirror of the FE incremental knob gates (QueryCacheNormalizer. + // computeAllowIncremental): freshness tolerance defeats an incremental merge + // for every table (the delta capture is version-exact by design, forcing the + // un-warmed reads the query chose to skip), and prefer-cached-rowset does so + // for every non-merge-on-write index (a MOW read is version-exact regardless + // of the knob, see CloudTablet::capture_consistent_versions_unlocked). A + // same-version FE already clears allow_incremental in these cases, making + // this re-derivation a no-op; it exists for the rolling-upgrade window where + // an older FE without the knob gates still sets allow_incremental while a + // knob is active. Intent, not correctness: the merged entry such a query + // would produce is version-exact either way; the gate preserves the knob's + // "skip un-warmed reads" promise. With the optional is_merge_on_write field + // absent (that same older FE), prefer suppresses MOW too, the conservative + // direction. Pure so the truth table is unit-testable; the caller (the + // fragment context, which has the session variables) applies it cloud-only, + // since both knobs are inert on local storage. + static bool cloud_knobs_suppress_incremental(bool freshness_tolerance_active, + bool prefer_cached_rowset_active, + bool is_merge_on_write) { + return freshness_tolerance_active || (prefer_cached_rowset_active && !is_merge_on_write); + } + + // The whole gate as a pure function, so the wiring (cloud-only, gated by the + // FE's request and the table type) is unit-testable without standing up a + // fragment and constructing operators: returns the effective + // allow_incremental. The fragment context passes config::is_cloud_mode() and + // its session variables. The gate is a no-op off cloud (the knobs are inert + // on local storage) and only ever clears a request, never grants one. + static bool gate_allow_incremental_for_cloud_knobs(bool requested_allow_incremental, + bool is_merge_on_write, bool cloud_mode, + bool freshness_tolerance_active, + bool prefer_cached_rowset_active) { + if (cloud_mode && requested_allow_incremental && + cloud_knobs_suppress_incremental(freshness_tolerance_active, + prefer_cached_rowset_active, is_merge_on_write)) { + return false; + } + return requested_allow_incremental; + } + // Idempotent: the first call for a given instance (identified by the cache // key derived from its scan ranges) makes the decision, later calls return // the same decision object. Never returns nullptr. @@ -364,10 +470,13 @@ class QueryCacheRuntime { // Exercise the cloud presync fan-out (a private static) directly, so unit // tests can assert the <= 0 skip-launch and the engine-stopped bail without - // standing up a full fragment. + // standing up a full fragment. The default null cache skips the single- + // flight registry (a private, uncoalesced fan-out), which is exactly what + // these early-path tests need. static std::unordered_map presync_cloud_delta_tablets_for_test( - const std::vector& scan_ranges, int64_t current_version) { - return _presync_cloud_delta_tablets(scan_ranges, current_version); + const std::vector& scan_ranges, int64_t current_version, + QueryCache* cache = nullptr, const std::string& cache_key = "") { + return _presync_cloud_delta_tablets(cache, cache_key, scan_ranges, current_version); } #endif @@ -386,13 +495,19 @@ class QueryCacheRuntime { // scan node's own tablet-sync fan-out) before the serial per-tablet capture // below runs, instead of issuing those RPCs one tablet at a time on the // shared prepare thread. This is the single sync per tablet: _capture_ - // tablet_delta consumes the result here and does not sync again. Returns - // the per-tablet fallback reasons the sync produced (keyed by tablet id, - // only failures present): a cast failure ("tablet is not a cloud tablet") - // or an infrastructure sync failure ("cloud rowset sync failed"). Tablets - // that are not append-only are skipped (no wasted RPC); _capture_tablet_ - // delta rejects them at its own keys-type check. + // tablet_delta consumes the result here and does not sync again. Concurrent + // identical runtimes are coalesced through the cache's single-flight + // registry keyed by (cache_key, current_version): the first arrival owns + // the fan-out, later ones wait on the same completion latch under their own + // deadlines (a null `cache` skips coalescing, test-only). Returns the + // per-tablet fallback reasons the sync produced (keyed by tablet id, only + // failures present): a cast failure ("tablet is not a cloud tablet"), a + // worker-side load failure ("cloud tablet load failed"), or an + // infrastructure sync failure ("cloud rowset sync failed"). Tablets that + // are not append-only are skipped (no wasted RPC); _capture_tablet_delta + // rejects them at its own keys-type check. static std::unordered_map _presync_cloud_delta_tablets( + QueryCache* cache, const std::string& cache_key, const std::vector& scan_ranges, int64_t current_version); // Validate one tablet for incremental merge and capture its delta read diff --git a/be/test/exec/pipeline/query_cache_test.cpp b/be/test/exec/pipeline/query_cache_test.cpp index 9dc9598a4ecb0f..688849ed387342 100644 --- a/be/test/exec/pipeline/query_cache_test.cpp +++ b/be/test/exec/pipeline/query_cache_test.cpp @@ -520,7 +520,7 @@ TEST_F(QueryCacheTest, presync_falls_back_on_non_cloud_engine) { // full bounded queue gives) while the engine itself is still live, so the fan-out // is reached rather than short-cut by the stopped() guard. Every submit_func then // fails, and the inline submit-failure path must fall each tablet back AND release -// its own latch slot via finish_slot so the bounded wait still settles instead of +// its own latch slot via publish_slot so the bounded wait still settles instead of // hanging on a slot no task will ever count down. Two tablets prove the latch // reaches N purely through the inline claim. No sync RPC is issued (rejected before // enqueue, so the closure never runs). @@ -563,6 +563,63 @@ TEST_F(QueryCacheTest, presync_falls_back_when_pool_rejects_submit) { EXPECT_EQ(sync_calls.load(), 0); } +// The BE-side mirror of the FE incremental knob gates, applied where the +// fragment context creates the shared runtime. A same-version FE already +// cleared allow_incremental in these cases; the mirror exists for the +// rolling-upgrade window where an older FE without the knob gates still +// requests incremental while a knob is active. The truth table is the FE +// gate's, verbatim: freshness suppresses for every table type, prefer-cached +// only for non-merge-on-write (a MOW read is version-exact regardless of the +// knob), no active knob suppresses nothing. +TEST_F(QueryCacheTest, cloud_knobs_suppress_incremental_truth_table) { + // Freshness tolerance active: suppresses regardless of the other inputs. + EXPECT_TRUE(QueryCacheRuntime::cloud_knobs_suppress_incremental(true, false, false)); + EXPECT_TRUE(QueryCacheRuntime::cloud_knobs_suppress_incremental(true, false, true)); + EXPECT_TRUE(QueryCacheRuntime::cloud_knobs_suppress_incremental(true, true, false)); + EXPECT_TRUE(QueryCacheRuntime::cloud_knobs_suppress_incremental(true, true, true)); + // Prefer-cached-rowset active: suppresses only non-MOW. + EXPECT_TRUE(QueryCacheRuntime::cloud_knobs_suppress_incremental(false, true, false)); + EXPECT_FALSE(QueryCacheRuntime::cloud_knobs_suppress_incremental(false, true, true)); + // No knob active: never suppresses. + EXPECT_FALSE(QueryCacheRuntime::cloud_knobs_suppress_incremental(false, false, false)); + EXPECT_FALSE(QueryCacheRuntime::cloud_knobs_suppress_incremental(false, false, true)); +} + +// The whole gate the fragment context applies at runtime creation, as a pure +// function. Covers the two wrappers the truth table above does not: the +// cloud-only guard, and that the gate only ever clears an FE request (never +// grants one). The fragment context's own call site is trivial wiring on top of +// this (it passes config::is_cloud_mode() and the session getters). +TEST_F(QueryCacheTest, gate_allow_incremental_for_cloud_knobs) { + // Cloud + a suppressing knob (freshness) + FE requested it: cleared. This is + // the rolling-upgrade case an older FE without the gates produces. + EXPECT_FALSE(QueryCacheRuntime::gate_allow_incremental_for_cloud_knobs( + /*requested=*/true, /*is_mow=*/false, /*cloud=*/true, /*freshness=*/true, + /*prefer=*/false)); + // Same knob, but LOCAL storage: the knobs are inert, so the gate must not + // fire (local incremental keeps working under an inert freshness setting). + EXPECT_TRUE(QueryCacheRuntime::gate_allow_incremental_for_cloud_knobs( + /*requested=*/true, /*is_mow=*/false, /*cloud=*/false, /*freshness=*/true, + /*prefer=*/false)); + // Cloud + prefer-cached, but the index is MOW: prefer does not apply to MOW, + // so the request survives (the MOW read is version-exact regardless). + EXPECT_TRUE(QueryCacheRuntime::gate_allow_incremental_for_cloud_knobs( + /*requested=*/true, /*is_mow=*/true, /*cloud=*/true, /*freshness=*/false, + /*prefer=*/true)); + // The gate never grants: FE did not request incremental, so a knob (or its + // absence) leaves it off. + EXPECT_FALSE(QueryCacheRuntime::gate_allow_incremental_for_cloud_knobs( + /*requested=*/false, /*is_mow=*/false, /*cloud=*/true, /*freshness=*/true, + /*prefer=*/false)); + EXPECT_FALSE(QueryCacheRuntime::gate_allow_incremental_for_cloud_knobs( + /*requested=*/false, /*is_mow=*/false, /*cloud=*/true, /*freshness=*/false, + /*prefer=*/false)); + // Cloud, request set, but no knob active: passes through untouched. + EXPECT_TRUE(QueryCacheRuntime::gate_allow_incremental_for_cloud_knobs( + /*requested=*/true, /*is_mow=*/false, /*cloud=*/true, /*freshness=*/false, + /*prefer=*/false)); +} + TEST_F(QueryCacheTest, runtime_decision_force_refresh) { std::unique_ptr cache(QueryCache::create_global_cache(1024 * 1024)); auto scan_ranges = make_scan_ranges(42, "100"); @@ -1309,6 +1366,19 @@ class QueryCacheCloudIncrementalTest : public testing::Test { } void TearDown() override { + // Drain the dedicated delta-sync pool BEFORE tearing anything down: a + // task's publish_slot reap now touches _cache->_presync_flights_lock (the + // round-2 tombstone reap), so a worker that resumes after _cache is freed + // would use-after-free the registry lock (the 200ms cushions the tests add + // do not guarantee a descheduled worker finished). wait() blocks while the + // cache, engine, and SyncPoint callbacks are all still alive; a task's + // remaining publish_slot work triggers no SyncPoint callback, so waiting + // with the callbacks still installed is safe. Only then clear callbacks and + // destroy the cache/engine -- mirroring production, where + // CloudStorageEngine::stop() drains this pool before the cache is torn down. + if (_engine != nullptr) { + _engine->query_cache_delta_sync_pool().wait(); + } _sp->disable_processing(); _sp->clear_all_call_backs(); _cache.reset(); @@ -1861,6 +1931,853 @@ TEST_F(QueryCacheCloudIncrementalTest, pool_rejection_skips_admission_tablet_loa EXPECT_EQ(_sync_calls.load(), 0); } +TEST_F(QueryCacheCloudIncrementalTest, presync_single_flight_coalesces_and_unrelated_key_proceeds) { + // Two fragment runtimes over the SAME stale key must coalesce onto ONE + // presync fan-out through the cache's single-flight registry (runtimes are + // private per fragment context, so without the registry each would submit + // the full fan-out), while a runtime over an UNRELATED key builds its own + // flight and completes even though the first flight is parked on a slow + // meta service. Deterministic: the shared flight's decision sync parks on a + // gate, the second runtime's join is witnessed by the follower_joined sync + // point, and only then does the unrelated key run and the gate open. + constexpr int64_t kOtherTablet = 15690; + _sp->set_call_back("CloudMetaMgr::get_tablet_meta", [](auto&& args) { + auto tid = try_any_cast(args[0]); + TTabletSchema schema; + schema.keys_type = TKeysType::DUP_KEYS; + auto meta = std::make_shared( + 1, 2, tid, 15674, 4, 5, schema, 6, std::unordered_map {{7, 8}}, + UniqueId(9, 10), TTabletType::TABLET_TYPE_DISK, TCompressionType::LZ4F, 0, false); + meta->set_tablet_state(TABLET_RUNNING); + *try_any_cast(args[1]) = std::move(meta); + try_any_cast_ret(args)->second = true; + }); + std::mutex mu; + std::unordered_map calls; + std::atomic gate_open {false}; + std::atomic gated_sync_parked {false}; + _sp->set_call_back("CloudMetaMgr::sync_tablet_rowsets", [&](auto&& args) { + auto* tablet = try_any_cast(args[0]); + auto* ret = try_any_cast_ret(args); + ret->second = true; + int n; + { + std::lock_guard lk(mu); + n = calls[tablet->tablet_id()]++; + } + if (n == 0) { + // Loader sync: a view that stops short of query version 100, so + // the decision sync really runs. Serialized under the test mutex + // (install draws from the shared engine rowset-id generator). + std::lock_guard lk(mu); + install_rowsets(tablet, {{0, 50}}); + return; + } + if (tablet->tablet_id() == kTabletId) { + gated_sync_parked = true; + while (!gate_open.load()) { + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + } + } + std::lock_guard lk(mu); + install_rowsets(tablet, {{51, 100}}); + }); + + auto ranges_a = make_scan_ranges(kTabletId, "100"); + auto param_a = make_cache_param(kTabletId); + param_a.__set_allow_incremental(true); + std::string key_a; + int64_t version = 0; + EXPECT_TRUE(QueryCache::build_cache_key(ranges_a, param_a, &key_a, &version).ok()); + insert_entry(_cache.get(), key_a, 50, 0); + auto ranges_c = make_scan_ranges(kOtherTablet, "100"); + auto param_c = make_cache_param(kOtherTablet); + param_c.__set_allow_incremental(true); + std::string key_c; + EXPECT_TRUE(QueryCache::build_cache_key(ranges_c, param_c, &key_c, &version).ok()); + insert_entry(_cache.get(), key_c, 50, 0); + + QueryCacheRuntime runtime_a1(param_a, _cache.get()); + QueryCacheRuntime runtime_a2(param_a, _cache.get()); + QueryCacheRuntime runtime_c(param_c, _cache.get()); + + std::atomic follower_joined {false}; + _sp->set_call_back("QueryCacheRuntime::_presync_cloud_delta_tablets.follower_joined", + [&](auto&&) { follower_joined = true; }); + + std::shared_ptr d1; + std::shared_ptr d2; + std::thread t1; + std::thread t2; + // A fatal ASSERT below returns while a spawned thread is still joinable; its + // std::thread destructor would std::terminate the whole UT binary and mask + // the real failure. Open the gate (so a parked worker can finish) and join + // both on every exit, mirroring race_two_callers. + Defer thread_guard {[&] { + gate_open = true; + if (t1.joinable()) { + t1.join(); + } + if (t2.joinable()) { + t2.join(); + } + }}; + + t1 = std::thread([&] { d1 = runtime_a1.get_or_make_decision(ranges_a); }); + for (int i = 0; i < 10000 && !gated_sync_parked.load(); ++i) { + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + } + ASSERT_TRUE(gated_sync_parked.load()); + + t2 = std::thread([&] { d2 = runtime_a2.get_or_make_decision(ranges_a); }); + for (int i = 0; i < 10000 && !follower_joined.load(); ++i) { + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + } + ASSERT_TRUE(follower_joined.load()); + + // With the shared flight parked, an unrelated key runs end to end on its + // own flight: registry entries do not interfere across keys. + auto d3 = runtime_c.get_or_make_decision(ranges_c); + EXPECT_EQ(d3->mode, QueryCacheInstanceDecision::Mode::INCREMENTAL); + + gate_open = true; + t1.join(); + t2.join(); + EXPECT_EQ(d1->mode, QueryCacheInstanceDecision::Mode::INCREMENTAL); + EXPECT_EQ(d2->mode, QueryCacheInstanceDecision::Mode::INCREMENTAL); + { + std::lock_guard lk(mu); + // ONE shared fan-out for the coalesced key: the loader plus the single + // gated decision sync. The joined runtime added no sync of its own; the + // unrelated tablet ran its own loader + decision pair. + EXPECT_EQ(calls[kTabletId], 2); + EXPECT_EQ(calls[kOtherTablet], 2); + } + // All waiters left, so the registry drained (BE UT compiles with + // -fno-access-control, so peek at the internals directly). + { + std::lock_guard lk(_cache->_presync_flights_lock); + EXPECT_TRUE(_cache->_presync_flights.empty()); + } +} + +TEST_F(QueryCacheCloudIncrementalTest, + presync_single_flight_last_leaver_keeps_tombstone_reaped_on_drain) { + // Two coalesced waiters expire their OWN deadlines while the shared decision + // sync is parked on a slow meta service. Three properties: + // (1) The FIRST leaver must NOT abandon/erase -- a timed-out waiter cannot + // cancel work the still-waiting sibling is entitled to. + // (2) The LAST leaver, seeing the fan-out still un-drained (task parked), + // marks the flight abandoned but KEEPS the registry entry as a + // tombstone. Erasing on timeout would let the next identical query + // become a fresh owner and submit a DUPLICATE fan-out during the very + // brownout single-flight exists to smooth. + // (3) The tombstone is reaped only once the fan-out DRAINS: after the parked + // task returns (latch hits 0), a later identical arrival joins the + // drained flight, reuses its settled slots, and erases it on leave. + // To observe the intermediate "first left, second still waiting" state the two + // waiters are given DETERMINISTIC per-runtime deadlines via the + // ..._presync_cloud_delta_tablets.deadline_ms test seam (below): the owner gets + // 2000ms, the follower 4000ms. Because the follower cannot even start until the + // owner's task is already parked (its sync_fanout_start is strictly later), the + // owner's deadline (t_owner + 2000) is ALWAYS earlier than the follower's + // (t_follower + 4000 >= t_owner + 4000), so the owner times out first by + // construction -- no wall-clock staggering, and no CI deschedule can invert the + // order. The 2000ms owner deadline also gives the follower the suite's standard + // 2000ms of scheduling headroom to REGISTER while the owner is still waiting + // (the two-waiter overlap the intermediate assertions observe) -- the same + // headroom every other parked-coalescing test gets from the 2000ms default + // timeout, so this test is not the flakiest link. (The prior versions staggered + // starts by a fixed 500ms sleep, then used 500/1000ms deadlines whose 500ms + // registration window was tighter than the suite convention.) + int32_t saved_timeout = config::query_cache_decision_sync_timeout_ms; + // Kept > 0 so the early enable check passes; the per-runtime seam overrides the + // actual deadline for both waiters, so this nominal value is otherwise unused here. + config::query_cache_decision_sync_timeout_ms = 2000; + Defer restore_timeout {[&] { config::query_cache_decision_sync_timeout_ms = saved_timeout; }}; + + _sp->set_call_back("CloudMetaMgr::get_tablet_meta", [](auto&& args) { + TTabletSchema schema; + schema.keys_type = TKeysType::DUP_KEYS; + auto meta = std::make_shared(1, 2, kTabletId, 15674, 4, 5, schema, 6, + std::unordered_map {{7, 8}}, + UniqueId(9, 10), TTabletType::TABLET_TYPE_DISK, + TCompressionType::LZ4F, 0, false); + meta->set_tablet_state(TABLET_RUNNING); + *try_any_cast(args[1]) = std::move(meta); + try_any_cast_ret(args)->second = true; + }); + std::mutex mu; + std::atomic syncs {0}; + std::atomic gate_open {false}; + std::atomic gated_sync_parked {false}; + std::atomic gated_sync_returned {false}; + _sp->set_call_back("CloudMetaMgr::sync_tablet_rowsets", [&](auto&& args) { + auto* tablet = try_any_cast(args[0]); + auto* ret = try_any_cast_ret(args); + ret->second = true; + if (syncs.fetch_add(1) == 0) { + std::lock_guard lk(mu); + install_rowsets(tablet, {{0, 50}}); + return; + } + gated_sync_parked = true; + while (!gate_open.load()) { + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + } + // On release the parked decision sync completes the view to the queried + // version, so the later reaping arrival can go INCREMENTAL off the + // settled slot -- proving the reuse path, not just the erase. + { + std::lock_guard lk(mu); + install_rowsets(tablet, {{51, 100}}); + } + gated_sync_returned = true; + }); + + auto ranges = make_scan_ranges(kTabletId, "100"); + auto param = make_cache_param(kTabletId); + param.__set_allow_incremental(true); + std::string key; + int64_t version = 0; + EXPECT_TRUE(QueryCache::build_cache_key(ranges, param, &key, &version).ok()); + insert_entry(_cache.get(), key, 50, 0); + QueryCacheRuntime runtime_a(param, _cache.get()); + QueryCacheRuntime runtime_b(param, _cache.get()); + + // Deterministic deadlines by coalescing role: the flight owner gets 2000ms and a + // joining follower 4000ms, so the owner times out first by construction (see the + // header comment), independent of scheduling jitter. runtime_a starts first and + // owns the flight; runtime_b joins it. + _sp->set_call_back("QueryCacheRuntime::_presync_cloud_delta_tablets.deadline_ms", + [](auto&& args) { + bool is_owner = try_any_cast(args[0]); + *try_any_cast(args[1]) = is_owner ? 2000 : 4000; + }); + + std::atomic follower_joined {false}; + _sp->set_call_back("QueryCacheRuntime::_presync_cloud_delta_tablets.follower_joined", + [&](auto&&) { follower_joined = true; }); + + std::shared_ptr d1; + std::shared_ptr d2; + std::thread t1; + std::thread t2; + // A fatal ASSERT below returns while a spawned thread is still joinable; its + // std::thread destructor would std::terminate the whole UT binary and mask + // the real failure. Open the gate (so the parked worker can finish) and join + // both on every exit, mirroring race_two_callers. + Defer thread_guard {[&] { + gate_open = true; + if (t1.joinable()) { + t1.join(); + } + if (t2.joinable()) { + t2.join(); + } + }}; + + t1 = std::thread([&] { d1 = runtime_a.get_or_make_decision(ranges); }); + for (int i = 0; i < 10000 && !gated_sync_parked.load(); ++i) { + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + } + ASSERT_TRUE(gated_sync_parked.load()); + + // The follower can start as soon as the owner's task has parked: its longer + // per-runtime deadline (4000ms vs the owner's 2000ms) guarantees the owner times + // out first regardless of when exactly the follower joins, so no start stagger. + t2 = std::thread([&] { d2 = runtime_b.get_or_make_decision(ranges); }); + for (int i = 0; i < 10000 && !follower_joined.load(); ++i) { + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + } + ASSERT_TRUE(follower_joined.load()); + + // Capture the shared flight while both wait: present, two waiters, not abandoned. + std::shared_ptr flight; + { + std::lock_guard lk(_cache->_presync_flights_lock); + ASSERT_EQ(_cache->_presync_flights.size(), 1); + flight = _cache->_presync_flights.begin()->second; + EXPECT_EQ(flight->waiters, 2); + } + EXPECT_FALSE(flight->abandoned->load()); + + // The owner times out first (it started ~500ms earlier). As the FIRST leaver + // it must NOT abandon or erase: exactly one waiter left, entry still + // registered, flag still clear. + t1.join(); + EXPECT_EQ(d1->mode, QueryCacheInstanceDecision::Mode::MISS); + EXPECT_EQ(d1->incremental_fallback_reason, "cloud rowset sync timed out"); + { + std::lock_guard lk(_cache->_presync_flights_lock); + EXPECT_EQ(_cache->_presync_flights.size(), 1); + EXPECT_EQ(flight->waiters, 1); + } + EXPECT_FALSE(flight->abandoned->load()); + + // The follower times out next. As the LAST leaver, with the fan-out still + // parked (un-drained), it marks the flight abandoned but KEEPS the tombstone + // -- it is NOT erased on timeout (that is the codex-flagged premature-erase + // regression this design fixes). + t2.join(); + EXPECT_EQ(d2->mode, QueryCacheInstanceDecision::Mode::MISS); + EXPECT_EQ(d2->incremental_fallback_reason, "cloud rowset sync timed out"); + EXPECT_TRUE(flight->abandoned->load()); + { + std::lock_guard lk(_cache->_presync_flights_lock); + ASSERT_EQ(_cache->_presync_flights.size(), 1); + EXPECT_EQ(_cache->_presync_flights.begin()->second, flight); + EXPECT_EQ(flight->waiters, 0); + } + + // Release the parked task and let it fully return. As it drives the latch to 0 + // with ZERO waiters remaining (both timed out into the tombstone above), the + // FINAL task reaps the tombstone itself in publish_slot -- no later arrival is + // needed (the round-2 reap-by-final-task fix; the round-1 code left this + // waiterless tombstone stranded). NOTE: an identical arrival here would NOT + // join and reuse the slot -- it would find the registry already reaped by the + // final task and re-own, so the two-waiter tombstone's final-task reap is what + // this phase proves (join-not-reown is proved by + // presync_single_flight_post_timeout_arrival_joins_not_reowns). + gate_open = true; + for (int i = 0; i < 10000 && !gated_sync_returned.load(); ++i) { + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + } + ASSERT_TRUE(gated_sync_returned.load()); + bool reaped = false; + for (int i = 0; i < 10000; ++i) { + { + std::lock_guard lk(_cache->_presync_flights_lock); + if (_cache->_presync_flights.empty()) { + reaped = true; + break; + } + } + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + } + EXPECT_TRUE(reaped) << "the final task must reap the waiterless tombstone on drain"; + EXPECT_EQ(flight->fanout_done->count(), 0u); + // The parked task published exactly once (loader + the single decision sync); + // no leaver re-submitted a duplicate fan-out. + EXPECT_EQ(syncs.load(), 2); + // Cushion for the callback shell's unwind after its last store, mirroring + // decision_sync_timeout_falls_back, before TearDown clears the sync points. + std::this_thread::sleep_for(std::chrono::milliseconds(200)); +} + +TEST_F(QueryCacheCloudIncrementalTest, + presync_single_flight_post_timeout_arrival_joins_not_reowns) { + // The codex-flagged brownout regression, directly: after every original + // waiter times out on a parked (still-running) fan-out, a NEW identical query + // must JOIN that draining flight, NOT become a fresh owner and submit a + // DUPLICATE fan-out (which would double the bounded-pool pressure wave after + // wave during a sustained meta-service brownout). The owner times out while + // its single decision sync is parked, leaving an abandoned-but-kept tombstone; + // a second runtime then arrives, joins, and adds NO sync of its own -- proven + // by the sync-call count staying at loader + the one parked decision sync. + int32_t saved_timeout = config::query_cache_decision_sync_timeout_ms; + config::query_cache_decision_sync_timeout_ms = 500; + Defer restore_timeout {[&] { config::query_cache_decision_sync_timeout_ms = saved_timeout; }}; + + _sp->set_call_back("CloudMetaMgr::get_tablet_meta", [](auto&& args) { + TTabletSchema schema; + schema.keys_type = TKeysType::DUP_KEYS; + auto meta = std::make_shared(1, 2, kTabletId, 15674, 4, 5, schema, 6, + std::unordered_map {{7, 8}}, + UniqueId(9, 10), TTabletType::TABLET_TYPE_DISK, + TCompressionType::LZ4F, 0, false); + meta->set_tablet_state(TABLET_RUNNING); + *try_any_cast(args[1]) = std::move(meta); + try_any_cast_ret(args)->second = true; + }); + std::mutex mu; + std::atomic syncs {0}; + std::atomic gate_open {false}; + std::atomic gated_sync_parked {false}; + std::atomic gated_sync_returned {false}; + _sp->set_call_back("CloudMetaMgr::sync_tablet_rowsets", [&](auto&& args) { + auto* tablet = try_any_cast(args[0]); + auto* ret = try_any_cast_ret(args); + ret->second = true; + if (syncs.fetch_add(1) == 0) { + std::lock_guard lk(mu); + install_rowsets(tablet, {{0, 50}}); + return; + } + gated_sync_parked = true; + while (!gate_open.load()) { + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + } + { + std::lock_guard lk(mu); + install_rowsets(tablet, {{51, 100}}); + } + gated_sync_returned = true; + }); + + auto ranges = make_scan_ranges(kTabletId, "100"); + auto param = make_cache_param(kTabletId); + param.__set_allow_incremental(true); + std::string key; + int64_t version = 0; + EXPECT_TRUE(QueryCache::build_cache_key(ranges, param, &key, &version).ok()); + insert_entry(_cache.get(), key, 50, 0); + QueryCacheRuntime runtime_owner(param, _cache.get()); + QueryCacheRuntime runtime_late(param, _cache.get()); + + std::atomic late_joined {false}; + _sp->set_call_back("QueryCacheRuntime::_presync_cloud_delta_tablets.follower_joined", + [&](auto&&) { late_joined = true; }); + + std::shared_ptr d_owner; + std::shared_ptr d_late; + std::thread t_owner; + std::thread t_late; + Defer thread_guard {[&] { + gate_open = true; + if (t_owner.joinable()) { + t_owner.join(); + } + if (t_late.joinable()) { + t_late.join(); + } + }}; + + t_owner = std::thread([&] { d_owner = runtime_owner.get_or_make_decision(ranges); }); + for (int i = 0; i < 10000 && !gated_sync_parked.load(); ++i) { + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + } + ASSERT_TRUE(gated_sync_parked.load()); + + // The owner times out while the decision sync is still parked, leaving an + // abandoned tombstone (the fan-out is un-drained, so it is NOT erased). + t_owner.join(); + EXPECT_EQ(d_owner->mode, QueryCacheInstanceDecision::Mode::MISS); + EXPECT_EQ(d_owner->incremental_fallback_reason, "cloud rowset sync timed out"); + { + std::lock_guard lk(_cache->_presync_flights_lock); + ASSERT_EQ(_cache->_presync_flights.size(), 1); + EXPECT_TRUE(_cache->_presync_flights.begin()->second->abandoned->load()); + } + + // A NEW identical query arrives while the fan-out is still parked. It must + // JOIN the tombstone (witnessed by the follower_joined sync point), not + // re-own and submit a second fan-out. + t_late = std::thread([&] { d_late = runtime_late.get_or_make_decision(ranges); }); + for (int i = 0; i < 10000 && !late_joined.load(); ++i) { + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + } + ASSERT_TRUE(late_joined.load()); + + // Release the parked task; the late arrival (waiting on the shared latch) + // wakes, reuses the completed view, goes INCREMENTAL, and reaps the flight. + gate_open = true; + t_late.join(); + EXPECT_EQ(d_late->mode, QueryCacheInstanceDecision::Mode::INCREMENTAL); + // The crux: exactly ONE fan-out ran (loader + the single parked decision + // sync). The late arrival added no sync -- it joined instead of re-owning. + EXPECT_EQ(syncs.load(), 2); + { + std::lock_guard lk(_cache->_presync_flights_lock); + EXPECT_TRUE(_cache->_presync_flights.empty()); + } + for (int i = 0; i < 10000 && !gated_sync_returned.load(); ++i) { + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + } + ASSERT_TRUE(gated_sync_returned.load()); + std::this_thread::sleep_for(std::chrono::milliseconds(200)); +} + +TEST_F(QueryCacheCloudIncrementalTest, + presync_single_flight_tombstone_reaped_on_drain_without_arrival) { + // The round-2 tombstone-leak fix: a flight whose sole/last waiter times out + // while a task is still parked is KEPT as a tombstone, but once the fan-out + // DRAINS it must be reaped by the FINAL task itself, even if NO later + // identical query ever arrives to join and reap it. Without this the entry + // leaks for the BE lifetime on exactly the brownout path the feature targets. + // A single owner times out; the parked task is then released and drains the + // latch with ZERO waiters; the registry must end empty with no reaping + // arrival. (Pre-fix -- reap only in leave_flight -- this asserts a permanent + // leak: the final _presync_flights.empty() never holds.) + int32_t saved_timeout = config::query_cache_decision_sync_timeout_ms; + config::query_cache_decision_sync_timeout_ms = 500; + Defer restore_timeout {[&] { config::query_cache_decision_sync_timeout_ms = saved_timeout; }}; + + _sp->set_call_back("CloudMetaMgr::get_tablet_meta", [](auto&& args) { + TTabletSchema schema; + schema.keys_type = TKeysType::DUP_KEYS; + auto meta = std::make_shared(1, 2, kTabletId, 15674, 4, 5, schema, 6, + std::unordered_map {{7, 8}}, + UniqueId(9, 10), TTabletType::TABLET_TYPE_DISK, + TCompressionType::LZ4F, 0, false); + meta->set_tablet_state(TABLET_RUNNING); + *try_any_cast(args[1]) = std::move(meta); + try_any_cast_ret(args)->second = true; + }); + std::mutex mu; + std::atomic syncs {0}; + std::atomic gate_open {false}; + std::atomic gated_sync_parked {false}; + std::atomic gated_sync_returned {false}; + _sp->set_call_back("CloudMetaMgr::sync_tablet_rowsets", [&](auto&& args) { + auto* tablet = try_any_cast(args[0]); + auto* ret = try_any_cast_ret(args); + ret->second = true; + if (syncs.fetch_add(1) == 0) { + std::lock_guard lk(mu); + install_rowsets(tablet, {{0, 50}}); + return; + } + gated_sync_parked = true; + while (!gate_open.load()) { + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + } + gated_sync_returned = true; + }); + + auto ranges = make_scan_ranges(kTabletId, "100"); + auto param = make_cache_param(kTabletId); + param.__set_allow_incremental(true); + std::string key; + int64_t version = 0; + EXPECT_TRUE(QueryCache::build_cache_key(ranges, param, &key, &version).ok()); + insert_entry(_cache.get(), key, 50, 0); + // Baseline the query-cache MemTracker AFTER the cache entry is in (that charge is + // a constant offset for the rest of the test). The flight's two DorisVector buffers + // (per_range_reason, tablet_ids) are charged to this SAME stable limiter, so + // consumption must rise while the flight is live and return to EXACTLY this baseline + // once the flight is reaped -- the balance that proves those buffers are stably + // charged and fully released. Scope note: ONLY the DorisVectors are tracked here; + // the flight's plain-`new` pieces (the object, the shared_ptr control blocks, + // slot_counted, the key string, the registry map node) are RSS-only and invisible + // to every MemTrackerLimiter (Doris has no global new/malloc hook -- see + // make_flight), so they never move this counter. The byte-exact EQ below also + // depends on this fixture keeping the query-cache tracker otherwise quiescent + // between baseline and assertion (one tiny cache entry, no eviction, no second + // flight); do not add cache traffic in between without switching to a tolerance + // band. + const int64_t qc_mem_baseline = + ExecEnv::GetInstance()->query_cache_mem_tracker()->consumption(); + QueryCacheRuntime runtime_owner(param, _cache.get()); + + std::shared_ptr d; + std::thread t; + Defer thread_guard {[&] { + gate_open = true; + if (t.joinable()) { + t.join(); + } + }}; + + t = std::thread([&] { d = runtime_owner.get_or_make_decision(ranges); }); + for (int i = 0; i < 10000 && !gated_sync_parked.load(); ++i) { + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + } + ASSERT_TRUE(gated_sync_parked.load()); + + // The owner times out while its task is parked, leaving an abandoned tombstone + // with zero waiters and the fan-out not yet drained. + t.join(); + EXPECT_EQ(d->mode, QueryCacheInstanceDecision::Mode::MISS); + EXPECT_EQ(d->incremental_fallback_reason, "cloud rowset sync timed out"); + std::shared_ptr flight; + { + std::lock_guard lk(_cache->_presync_flights_lock); + ASSERT_EQ(_cache->_presync_flights.size(), 1); + flight = _cache->_presync_flights.begin()->second; + EXPECT_EQ(flight->waiters, 0); + } + EXPECT_TRUE(flight->abandoned->load()); + // The live flight's buffers are charged to the stable query-cache MemTracker. + EXPECT_GT(ExecEnv::GetInstance()->query_cache_mem_tracker()->consumption(), qc_mem_baseline) + << "the live flight's buffers must be charged to the query-cache MemTracker"; + + // Release the parked task; with zero waiters, the FINAL task reaps the + // tombstone itself -- no later identical query is ever issued. + gate_open = true; + for (int i = 0; i < 10000 && !gated_sync_returned.load(); ++i) { + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + } + ASSERT_TRUE(gated_sync_returned.load()); + bool reaped = false; + for (int i = 0; i < 10000; ++i) { + { + std::lock_guard lk(_cache->_presync_flights_lock); + if (_cache->_presync_flights.empty()) { + reaped = true; + break; + } + } + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + } + EXPECT_TRUE(reaped) << "the final task must reap the drained tombstone with no arrival"; + // Drain the pool so the parked task's closure (a holder of the flight's shared + // pieces) is destroyed, then drop the test's own captured reference. With the + // registry entry reaped and no waiters, the flight and all its buffers are now + // freed, so the query-cache MemTracker must return to EXACTLY the pre-flight + // baseline -- proving the O(tablets) buffers were charged to the stable limiter + // and fully released on reap (no leak, no over-release). This also removes the + // former unwind cushion: pool.wait() hard-synchronizes the callback shell's exit. + _engine->query_cache_delta_sync_pool().wait(); + flight.reset(); + EXPECT_EQ(ExecEnv::GetInstance()->query_cache_mem_tracker()->consumption(), qc_mem_baseline) + << "reaping the flight must release its charge back to the query-cache MemTracker"; +} + +TEST_F(QueryCacheCloudIncrementalTest, + presync_single_flight_reversed_order_maps_reasons_by_tablet) { + // The single-flight coalescing hazard the shared slots must survive: + // build_cache_key sorts tablet ids, so two runtimes whose scan ranges hold + // the same two tablets in a DIFFERENT order share one flight. The owner's + // fan-out writes the index-aligned slots in the OWNER's order; a follower + // that keyed the merged reasons off its OWN (reversed) order would misassign + // the failed tablet's reason to its sibling, leave the truly failed tablet + // unreasoned, and so drive a synchronous admission-thread get_tablet for it + // outside the fast-fail budget -- the exact brownout stall the presync + // exists to avoid. With the reasons keyed off the owner's tablet ids, the + // follower must reach the SAME decision as the owner: MISS, keyed to the + // failed tablet. (Pre-fix this test fails: the follower's fallback reason is + // a capture-side one for the sibling, not the owner's sync-failure reason.) + constexpr int64_t kTabletId2 = 15680; + + _sp->set_call_back("CloudMetaMgr::get_tablet_meta", [](auto&& args) { + auto tid = try_any_cast(args[0]); + TTabletSchema schema; + schema.keys_type = TKeysType::DUP_KEYS; + auto meta = std::make_shared( + 1, 2, tid, 15674, 4, 5, schema, 6, std::unordered_map {{7, 8}}, + UniqueId(9, 10), TTabletType::TABLET_TYPE_DISK, TCompressionType::LZ4F, 0, false); + meta->set_tablet_state(TABLET_RUNNING); + *try_any_cast(args[1]) = std::move(meta); + try_any_cast_ret(args)->second = true; + }); + std::mutex mu; + std::unordered_map calls; + std::atomic gate_open {false}; + std::atomic gated_sync_parked {false}; + _sp->set_call_back("CloudMetaMgr::sync_tablet_rowsets", [&](auto&& args) { + auto* tablet = try_any_cast(args[0]); + auto* ret = try_any_cast_ret(args); + ret->second = true; + int n; + { + std::lock_guard lk(mu); + n = calls[tablet->tablet_id()]++; + } + if (n == 0) { + // Loader: a view stopping short of 100 so the decision sync runs. + std::lock_guard lk(mu); + install_rowsets(tablet, {{0, 50}}); + return; + } + // Decision sync. The failing tablet parks first, so the follower joins + // the still-live flight before the fan-out settles; then it fails. The + // other tablet's decision sync succeeds. + if (tablet->tablet_id() == kTabletId2) { + gated_sync_parked = true; + while (!gate_open.load()) { + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + } + ret->first = Status::InternalError("injected sync failure"); + return; + } + std::lock_guard lk(mu); + install_rowsets(tablet, {{51, 100}}); + }); + + // One stale entry keyed over both tablets, queried at 100. The owner scans + // [kTabletId, kTabletId2]; the follower scans the SAME tablets REVERSED. + auto ranges_owner = make_scan_ranges(kTabletId, "100"); + ranges_owner.push_back(make_scan_ranges(kTabletId2, "100").front()); + auto ranges_follower = make_scan_ranges(kTabletId2, "100"); + ranges_follower.push_back(make_scan_ranges(kTabletId, "100").front()); + auto param = make_cache_param(kTabletId); + param.tablet_to_range.insert({kTabletId2, "range"}); + param.__set_allow_incremental(true); + std::string key; + int64_t version = 0; + EXPECT_TRUE(QueryCache::build_cache_key(ranges_owner, param, &key, &version).ok()); + // Sanity: the reversed order yields the identical key (so they coalesce). + std::string key_reversed; + EXPECT_TRUE(QueryCache::build_cache_key(ranges_follower, param, &key_reversed, &version).ok()); + ASSERT_EQ(key, key_reversed); + insert_entry(_cache.get(), key, 50, 0); + + QueryCacheRuntime runtime_owner(param, _cache.get()); + QueryCacheRuntime runtime_follower(param, _cache.get()); + + std::atomic follower_joined {false}; + _sp->set_call_back("QueryCacheRuntime::_presync_cloud_delta_tablets.follower_joined", + [&](auto&&) { follower_joined = true; }); + + std::shared_ptr d_owner; + std::shared_ptr d_follower; + std::thread t_owner; + std::thread t_follower; + Defer thread_guard {[&] { + gate_open = true; + if (t_owner.joinable()) { + t_owner.join(); + } + if (t_follower.joinable()) { + t_follower.join(); + } + }}; + + t_owner = std::thread([&] { d_owner = runtime_owner.get_or_make_decision(ranges_owner); }); + for (int i = 0; i < 10000 && !gated_sync_parked.load(); ++i) { + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + } + ASSERT_TRUE(gated_sync_parked.load()); + + t_follower = std::thread( + [&] { d_follower = runtime_follower.get_or_make_decision(ranges_follower); }); + for (int i = 0; i < 10000 && !follower_joined.load(); ++i) { + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + } + ASSERT_TRUE(follower_joined.load()); + + gate_open = true; + t_owner.join(); + t_follower.join(); + + // Both decisions fall the whole instance back, keyed to the SAME failed + // tablet: the owner surfaces kTabletId2's sync failure, and the reversed + // follower -- reading the owner-ordered slots by the owner's tablet ids -- + // surfaces the identical reason rather than misattributing it. + EXPECT_EQ(d_owner->mode, QueryCacheInstanceDecision::Mode::MISS); + EXPECT_EQ(d_owner->incremental_fallback_reason, "cloud rowset sync failed"); + EXPECT_EQ(d_follower->mode, QueryCacheInstanceDecision::Mode::MISS); + EXPECT_EQ(d_follower->incremental_fallback_reason, "cloud rowset sync failed"); + { + std::lock_guard lk(mu); + // ONE shared fan-out: each tablet synced its loader plus its single + // decision sync. The coalesced follower added no sync of its own. + EXPECT_EQ(calls[kTabletId], 2); + EXPECT_EQ(calls[kTabletId2], 2); + } + { + std::lock_guard lk(_cache->_presync_flights_lock); + EXPECT_TRUE(_cache->_presync_flights.empty()); + } +} + +TEST_F(QueryCacheCloudIncrementalTest, presync_single_flight_coalesces_mow_tablet) { + // Coalescing over a MERGE-ON-WRITE tablet: the owner's presync carries + // options.sync_delete_bitmap=true (a MoW read is delete-bitmap-sensitive), + // and a coalesced follower REUSES that one sync rather than issuing its own. + // Both must reach the SAME decision off the shared, bitmap-complete view. + // Covers the MoW x single-flight combination the DUP_KEYS coalesce tests do + // not: the DUP tests prove the registry mechanics, the single-runtime MoW + // tests prove the delete-bitmap capture, and this proves a follower riding + // the owner's bitmap sync classifies identically (clean bitmap + pure-append + // delta -> both INCREMENTAL). + _sp->set_call_back("CloudMetaMgr::get_tablet_meta", [](auto&& args) { + auto tid = try_any_cast(args[0]); + TTabletSchema schema; + schema.keys_type = TKeysType::UNIQUE_KEYS; + auto meta = std::make_shared( + 1, 2, tid, 15674, 4, 5, schema, 6, std::unordered_map {{7, 8}}, + UniqueId(9, 10), TTabletType::TABLET_TYPE_DISK, TCompressionType::LZ4F, 0, + /*enable_merge_on_write=*/true); + meta->set_tablet_state(TABLET_RUNNING); + *try_any_cast(args[1]) = std::move(meta); + try_any_cast_ret(args)->second = true; + }); + std::mutex mu; + std::unordered_map calls; + std::atomic gate_open {false}; + std::atomic gated_sync_parked {false}; + std::atomic saw_sync_delete_bitmap {false}; + _sp->set_call_back("CloudMetaMgr::sync_tablet_rowsets", [&](auto&& args) { + auto* tablet = try_any_cast(args[0]); + const auto* options = try_any_cast(args[1]); + auto* ret = try_any_cast_ret(args); + ret->second = true; + int n; + { + std::lock_guard lk(mu); + n = calls[tablet->tablet_id()]++; + } + if (n == 0) { + std::lock_guard lk(mu); + install_rowsets(tablet, {{0, 50}}); + return; + } + // Decision sync on a MoW tablet must carry the delete-bitmap sync. + if (options != nullptr && options->sync_delete_bitmap) { + saw_sync_delete_bitmap = true; + } + gated_sync_parked = true; + while (!gate_open.load()) { + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + } + std::lock_guard lk(mu); + install_rowsets(tablet, {{51, 100}}); + }); + + auto ranges = make_scan_ranges(kTabletId, "100"); + auto param = make_cache_param(kTabletId); + param.__set_allow_incremental(true); + std::string key; + int64_t version = 0; + EXPECT_TRUE(QueryCache::build_cache_key(ranges, param, &key, &version).ok()); + insert_entry(_cache.get(), key, 50, 0); + QueryCacheRuntime runtime_a(param, _cache.get()); + QueryCacheRuntime runtime_b(param, _cache.get()); + + std::atomic follower_joined {false}; + _sp->set_call_back("QueryCacheRuntime::_presync_cloud_delta_tablets.follower_joined", + [&](auto&&) { follower_joined = true; }); + + std::shared_ptr d1; + std::shared_ptr d2; + std::thread t1; + std::thread t2; + Defer thread_guard {[&] { + gate_open = true; + if (t1.joinable()) { + t1.join(); + } + if (t2.joinable()) { + t2.join(); + } + }}; + + t1 = std::thread([&] { d1 = runtime_a.get_or_make_decision(ranges); }); + for (int i = 0; i < 10000 && !gated_sync_parked.load(); ++i) { + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + } + ASSERT_TRUE(gated_sync_parked.load()); + t2 = std::thread([&] { d2 = runtime_b.get_or_make_decision(ranges); }); + for (int i = 0; i < 10000 && !follower_joined.load(); ++i) { + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + } + ASSERT_TRUE(follower_joined.load()); + + gate_open = true; + t1.join(); + t2.join(); + // Clean bitmap + pure-append delta: both coalesced MoW waiters go INCREMENTAL + // and, riding the same sync, reach the identical decision. + EXPECT_EQ(d1->mode, QueryCacheInstanceDecision::Mode::INCREMENTAL); + EXPECT_EQ(d2->mode, QueryCacheInstanceDecision::Mode::INCREMENTAL); + EXPECT_EQ(d1->mode, d2->mode); + EXPECT_EQ(d1->incremental_fallback_reason, d2->incremental_fallback_reason); + // The owner's shared decision sync carried the MoW delete-bitmap sync. + EXPECT_TRUE(saw_sync_delete_bitmap.load()); + { + std::lock_guard lk(mu); + // ONE shared fan-out: loader + the single gated decision sync. The + // follower rode the owner's MoW sync, adding none of its own. + EXPECT_EQ(calls[kTabletId], 2); + } + { + std::lock_guard lk(_cache->_presync_flights_lock); + EXPECT_TRUE(_cache->_presync_flights.empty()); + } +} + TEST_F(QueryCacheCloudIncrementalTest, decision_sync_timeout_falls_back) { // A meta-service brownout: the decision's rowset sync stalls past the // fast-fail budget (query_cache_decision_sync_timeout_ms). The decision must From da0b255173f8f71fdd9a19b820fdbee69097d526 Mon Sep 17 00:00:00 2001 From: Benedict Jin Date: Mon, 20 Jul 2026 21:57:56 +0800 Subject: [PATCH 4/4] [fix](query-cache) Harden cloud presync: fallback safety, width-aware wait cap, cache-only decision load Follow-up hardening for the cloud stale-entry incremental merge, from a full re-review of the presync decision path. BE: - Wrap the per-tablet presync task body in a try/catch (doris::Exception before std::exception): a single-flight std::system_error from get_tablet or an allocation failure inside sync_rowsets now converts to a per-tablet fallback instead of escaping the pool worker and terminating the BE. - Charge the flight's O(tablets) buffers through a noexcept consume/release on the query-cache tracker, removing a shared_ptr deleter that switched the thread tracker and could throw inside a noexcept destructor. - Bound the decision-sync waiters that block the query-admission light pool to the smaller of query_cache_max_concurrent_decision_sync and half the actual light pool width. brpc_light_work_pool_threads is configurable, so a fixed cap would not bind on a shrunk pool, and a one-thread pool now admits no waiter. - Make the decision capture-site get_tablet cache-only (force_use_only_cached): an evicted presynced tablet falls back rather than reissuing a synchronous meta-service load on the light admission thread. - Correct the delete-bitmap comment. A full-compaction submission does load a MOW tablet with sync_delete_bitmap=false (task_worker_pool.cpp and cloud_compaction_action.cpp), so the classification inherits the same pre-existing incomplete-bitmap window a normal MOW scan faces; the storage-side fix is left as a follow-up. FE: - Gate the warmed-read knobs (query_freshness_tolerance, prefer_cached_rowset) behind cloud mode through a pure, unit-testable helper; they are inert on shared-nothing storage, so suppressing incremental there only forgoes it. - Resolve the scan table type through the unary chain for the write-back gate, so a nested-agg-over-MOW query is reported as merge-on-write and can populate the cache under prefer, while incremental stays gated on a direct-child scan. Tests cover each guard with red/green verification. --- be/src/cloud/config.cpp | 1 + be/src/cloud/config.h | 20 + be/src/runtime/query_cache/query_cache.cpp | 526 ++++++++++++------ be/src/runtime/query_cache/query_cache.h | 70 ++- be/test/exec/pipeline/query_cache_test.cpp | 331 ++++++++++- .../normalize/QueryCacheNormalizer.java | 116 ++-- .../planner/QueryCacheNormalizerTest.java | 106 +++- 7 files changed, 896 insertions(+), 274 deletions(-) diff --git a/be/src/cloud/config.cpp b/be/src/cloud/config.cpp index 3735b44ca855f4..5c5376c49a703b 100644 --- a/be/src/cloud/config.cpp +++ b/be/src/cloud/config.cpp @@ -41,6 +41,7 @@ DEFINE_mInt32(sync_rowsets_slow_threshold_ms, "1000"); DEFINE_mInt32(query_cache_decision_sync_timeout_ms, "2000"); DEFINE_Int32(query_cache_delta_sync_thread, "16"); DEFINE_Int32(query_cache_delta_sync_max_pending_tasks, "2048"); +DEFINE_mInt32(query_cache_max_concurrent_decision_sync, "32"); DEFINE_mInt64(min_compaction_failure_interval_ms, "5000"); DEFINE_mInt64(base_compaction_freeze_interval_s, "1800"); diff --git a/be/src/cloud/config.h b/be/src/cloud/config.h index 28417701d20e29..887bb60b5d8dfe 100644 --- a/be/src/cloud/config.h +++ b/be/src/cloud/config.h @@ -99,6 +99,26 @@ DECLARE_mInt32(query_cache_decision_sync_timeout_ms); DECLARE_Int32(query_cache_delta_sync_thread); DECLARE_Int32(query_cache_delta_sync_max_pending_tasks); +// Upper bound on how many query-cache incremental decisions may block in the +// decision-sync wait at the same time. That wait runs on brpc's light work pool +// (query admission; "must be light, not locked"), and single-flight coalesces only +// IDENTICAL cache keys, so under a meta-service brownout a wave of DISTINCT stale +// keys could otherwise park every light-pool worker for the full decision-sync +// timeout and starve unrelated fragment admission (the paired scan and cache-source +// operators can each park one on a first-call race, so the pressure is per operator, +// not just per query). The effective bound is the smaller of this value and half the +// ACTUAL light-pool width (the configured brpc_light_work_pool_threads, or +// max(128, 4*cores) when left at -1), so even a shrunk pool keeps spare workers for +// that admission -- this value only tightens below that floor. A query arriving over +// the bound skips incremental merge this round and recomputes in full (always correct, +// the right posture for an optimization under load). A non-positive value does not +// disable the guard; it falls back to the width-derived half. The fully non-blocking +// alternative is to drive the decision +// off a pipeline dependency, mirroring OlapScanLocalState::_cloud_tablet_dependency; +// that is a larger cross-operator change left as a follow-up. Mutable so it can be +// retuned online. Cloud only. +DECLARE_mInt32(query_cache_max_concurrent_decision_sync); + // Cloud compaction config DECLARE_mInt64(min_compaction_failure_interval_ms); DECLARE_mBool(enable_new_tablet_do_compaction); diff --git a/be/src/runtime/query_cache/query_cache.cpp b/be/src/runtime/query_cache/query_cache.cpp index 3ba976e2759222..f6ce1e4963b754 100644 --- a/be/src/runtime/query_cache/query_cache.cpp +++ b/be/src/runtime/query_cache/query_cache.cpp @@ -29,6 +29,7 @@ #include "cloud/cloud_storage_engine.h" #include "cloud/cloud_tablet.h" #include "cloud/config.h" +#include "common/exception.h" #include "common/logging.h" #include "common/metrics/doris_metrics.h" #include "cpp/sync_point.h" @@ -38,6 +39,7 @@ #include "storage/tablet/base_tablet.h" #include "storage/tablet/tablet_meta.h" #include "util/countdown_latch.h" +#include "util/cpu_info.h" #include "util/defer_op.h" #include "util/threadpool.h" @@ -298,6 +300,24 @@ bool QueryCacheRuntime::_try_prepare_incremental(const std::vector QueryCacheRuntime::_presync_active_waiters {0}; + +// Refund the flight's one-lump memory charge on its final teardown. release() is a +// plain atomic subtract, so this destructor never allocates and never throws (safe on +// any thread that drops the last ref -- a pool worker, a waiter, or a losing candidate +// being discarded). Skipped when the flight was never armed (mem_tracker null). +QueryCache::PresyncFlight::~PresyncFlight() { + if (mem_tracker != nullptr) { + mem_tracker->release(tracked_bytes); + } +} + std::unordered_map QueryCacheRuntime::_presync_cloud_delta_tablets( QueryCache* cache, const std::string& cache_key, const std::vector& scan_ranges, int64_t current_version) { @@ -341,6 +361,53 @@ std::unordered_map QueryCacheRuntime::_presync_cloud_delta return fallback_reasons; } + // Bound how many decision-sync waits may block the query-admission light pool at + // once. The fan-out wait below runs on that pool ("must be light, not locked"), and + // single-flight coalesces only IDENTICAL keys, so under a meta-service brownout a + // wave of DISTINCT stale keys (and, on a first-call race, the paired scan and cache- + // source operators each waiting) could otherwise park every admission worker for the + // full timeout and starve unrelated fragments. A soft counting cap keeps spare + // workers free: a query over the cap skips incremental this round and recomputes in + // full (always correct). + // + // The ceiling is the SMALLER of the tunable (query_cache_max_concurrent_decision_sync, + // default 32) and half the ACTUAL light-pool width. The width must be figured in, not + // just the config: brpc_light_work_pool_threads is operator-configurable, and on a + // pool shrunk to e.g. 16 a fixed 32 would never bind -- every worker could still park. + // The width is recomputed with the same formula internal_service.cpp sizes the pool by + // (the configured thread count, or max(128, 4*cores) when left at -1), so any pool + // keeps a worker reserve. The config only tightens below that floor; a non-positive + // config does NOT disable the guard (which would reopen the exact starvation this + // prevents) but falls back to the width-derived half. The fetch_add-then-back-out on a + // miss allows a brief boundary overshoot (harmless for a pool guard) while reading the + // mutable configs each call. Placed before the registry and fan-out so a rejected + // query does no work at all, and the Defer releases the slot on every admitted exit + // path (including the timeout and unwind paths below). + const int light_pool_width = config::brpc_light_work_pool_threads != -1 + ? config::brpc_light_work_pool_threads + : std::max(128, CpuInfo::num_cores() * 4); + // Integer half: a degenerate 1-thread light pool floors to 0, so the guard then + // admits nobody and every stale query falls back rather than parking the sole worker + // for the decision timeout (the worker-reserve guarantee would otherwise be violated + // exactly where it matters most). + const int width_cap = light_pool_width / 2; + int wait_cap = config::query_cache_max_concurrent_decision_sync; + if (wait_cap <= 0 || wait_cap > width_cap) { + wait_cap = width_cap; + } + const bool wait_admitted = + _presync_active_waiters.fetch_add(1, std::memory_order_acq_rel) < wait_cap; + if (!wait_admitted) { + _presync_active_waiters.fetch_sub(1, std::memory_order_acq_rel); + for (const auto& scan_range : scan_ranges) { + fallback_reasons[scan_range.scan_range.palo_scan_range.tablet_id] = + "cloud decision sync at capacity"; + } + return fallback_reasons; + } + Defer release_wait_slot( + [&] { _presync_active_waiters.fetch_sub(1, std::memory_order_acq_rel); }); + // Coalesce identical concurrent fan-outs through the cache's single-flight // registry. Runtimes are private per fragment context, so without this, N // concurrent identical queries hitting the same stale key would each submit @@ -356,101 +423,107 @@ std::unordered_map QueryCacheRuntime::_presync_cloud_delta std::shared_ptr flight; std::string flight_key; auto make_flight = [&] { - // Attribute the flight's DOMINANT O(tablets) buffers to the STABLE query-cache - // MemTracker rather than to the transient per-query tracker of whichever - // admission thread happened to create the flight (the flight outlives that - // query: coalesced waiters and still-running fan-out tasks keep its shared - // pieces alive past the query's teardown). + // Attribute the flight's retained memory to the STABLE query-cache MemTracker + // rather than to the transient per-query tracker of whichever admission thread + // happened to create the flight (the flight outlives that query: coalesced + // waiters and still-running fan-out tasks keep its shared pieces alive past the + // query's teardown). // - // IMPORTANT -- how memory is charged in Doris BE: a MemTrackerLimiter is - // charged either by an allocator-aware container (CustomStdAllocator -> - // Allocator::consume_memory, which is what DorisVector does) or by an explicit - // manual consume()/release() side-channel (what the LRU layer does with its - // tracking_bytes, see QueryCache::insert above). There is NO global new/malloc - // hook (be/src/runtime/memory/jemalloc_hook.cpp aliases malloc/free straight to - // jemalloc with zero tracker interaction, and there is no global ::operator new - // override), so a plain `new` / std::vector / std::string that does neither is - // invisible to every MemTrackerLimiter and appears only in process RSS. Hence - // the two hot buffers - // below are DorisVector, whose CustomStdAllocator explicitly consumes/releases - // the CURRENT thread's limiter. The SCOPED_SWITCH makes that current limiter the - // query-cache one at allocation; the custom deleter re-enters the SAME limiter - // at free (on whichever pool/waiter thread drops the last ref), so the - // DorisVector buffer is charged and released on the same stable limiter -- - // balanced. Pinning BOTH sides is required: a switch at construction alone would - // credit the cross-thread free elsewhere and drift this limiter. The deleter - // captures the limiter by shared_ptr so it stays alive until the last buffer is - // gone. + // The whole flight's estimated retained bytes are charged as ONE lump via the + // manual consume()/release() side-channel on the query-cache MemTrackerLimiter + // -- the SAME idiom the LRU layer uses for its tracking_bytes + // (QueryCache::insert above). consume()/release() are plain atomic counter ops: + // they NEVER allocate and NEVER throw, so the matching release runs safely in + // ~PresyncFlight (a noexcept destructor) on whatever pool/waiter thread drops + // the last ref. An earlier version charged the two hot buffers through a + // DorisVector whose deleter had to SWITCH the thread's MemTrackerLimiter, but + // that switch pushes onto a std::vector (attach_limiter_tracker) and can throw + // bad_alloc under memory pressure, and a throw from a shared_ptr deleter runs + // inside a noexcept destructor and std::terminates the BE. The single manual + // lump keeps stable attribution without that terminate risk, lets every buffer + // be a plain make_shared, and (unlike the earlier two-buffer-only charge) covers + // ALL of the flight's significant allocations -- the object, the + // per_range_reason / tablet_ids / slot_counted O(tablets) buffers, the control + // blocks, and the registry key -- so the query-cache tracker does not + // underreport retained RSS while flights and tombstones accumulate during a + // meta-service brownout. (BE has no global new/malloc hook -- + // be/src/runtime/memory/jemalloc_hook.cpp aliases malloc/free straight to + // jemalloc with no tracker interaction, and there is no global ::operator new + // override -- so absent this manual charge a plain new/std::vector/std::string + // is invisible to every MemTrackerLimiter and shows up only in process RSS.) + // The lifetimes coincide: publish_slot (captured by every task) holds the + // flight shared_ptr, so the flight object outlives every buffer, and + // ~PresyncFlight releases exactly when its last ref -- and thus all the buffers + // -- is gone. // - // Everything else the flight owns -- the PresyncFlight object, the six shared_ptr - // CONTROL BLOCKS, `slot_counted`, the `key` string, and the registry map node -- - // is plain `new`, so it is NOT charged to ANY MemTrackerLimiter (not this one and - // not the per-query one): RSS-only on both alloc and free. It therefore cannot - // strand as an orphan on a destroyed query tracker, nor drift this limiter -- the - // switch is irrelevant to it since a plain `new` never reaches consume_memory. - // These bytes are bounded per flight and the live-flight COUNT is observable via - // the query_cache_presync_inflight gauge; the same untracked O(tablets) - // metadata shape already ships on the sibling warmup path, whose pool closure - // captures a plain std::vector of tablet ids by value - // (CloudBackendService::sync_load_for_tablets). The two DorisVectors carry 16 - // of the roughly 33 bytes per tablet a live flight holds (the rest: the two - // 8-byte-per-tablet key copies, a claim flag, plus each waiter's transient - // flight_key), so the charged and uncharged halves are comparable -- charging - // the DorisVectors buys stable attribution of the data-path buffers, not full - // byte coverage. If full metadata attribution is ever wanted, the LRU layer's - // manual tracking_bytes consume()/release() idiom (QueryCache::insert above) - // is the cheap in-tree way to add it. + // This is a tracking charge, not a process reservation. The O(tablets) pieces + // are small metadata (a pointer / an int64 / an atomic per tablet), doubly + // bounded by the concurrency cap (query_cache_max_concurrent_decision_sync) and + // the fan-out pool's fixed queue, and are dwarfed by the delta scan data this + // decision gates -- a query near the process limit is tipped over by that scan, + // not by these vectors. If an allocation here does fail under pressure it + // surfaces as an ordinary per-query bad_alloc (the admission caller fails that + // one query; a fan-out task's catch below turns its own into a fallback), never + // data corruption or a BE crash, so a try_reserve for metadata this small and + // this well-bounded is deliberately not taken. auto qc_limiter = ExecEnv::GetInstance()->query_cache_mem_tracker(); - auto tracked_delete = [qc_limiter](auto* p) { - SCOPED_SWITCH_THREAD_MEM_TRACKER_LIMITER(qc_limiter); - delete p; - }; - SCOPED_SWITCH_THREAD_MEM_TRACKER_LIMITER(qc_limiter); - std::shared_ptr f(new QueryCache::PresyncFlight(), - tracked_delete); - // Index-aligned so the fan-out tasks write disjoint slots without a - // lock; an empty (nullptr) slot means synced (or skipped as - // non-append-only), a non-empty slot is the fallback reason (a static - // literal). Each piece sits behind its own shared_ptr because the fan-out - // is waited on with a fast-fail deadline: on a timeout a waiter returns - // while still-running tasks keep writing their slots, so the storage must - // outlive every frame -- the shared_ptr each task captured keeps it alive - // until the last one finishes. Only read on the completed path. - f->per_range_reason = std::shared_ptr>( - new DorisVector(scan_ranges.size()), tracked_delete); - f->fanout_done = std::shared_ptr( - new CountDownLatch(static_cast(scan_ranges.size())), tracked_delete); - // Per-slot single-owner claim flags; see the claim comments at the - // submit loop below. (Value-initialized to false.) - f->slot_counted = std::shared_ptr>>( - new std::vector>(scan_ranges.size()), tracked_delete); - // Set true when the LAST interested waiter abandons the flight (its - // deadline passed): a not-yet-started task then skips its sync RPC - // instead of spending the full retry budget, so a meta-service brownout - // drains the bounded queue fast rather than running abandoned work to - // completion and starving the pool. - f->abandoned = - std::shared_ptr>(new std::atomic(false), tracked_delete); - // Snapshot the owner's tablet ids in slot order. Every waiter (this - // owner and any that coalesce onto the flight) builds its fallback map - // from THESE ids, not from its own scan-range order: build_cache_key - // sorts tablet ids into the key, so two runtimes whose scan ranges hold - // the same tablets in a different order share this flight, and keying - // the shared index-aligned slots off a joiner's own order would pin the - // owner's per-slot reason to the wrong tablet. - std::shared_ptr> ids(new DorisVector(), tracked_delete); + const int64_t slot_n = static_cast(scan_ranges.size()); + + auto f = std::make_shared(); + // Index-aligned so the fan-out tasks write disjoint slots without a lock; an + // empty (nullptr) slot means synced (or skipped as non-append-only), a + // non-empty slot is the fallback reason (a static literal). Each piece sits + // behind its own shared_ptr because the fan-out is waited on with a fast-fail + // deadline: on a timeout a waiter returns while still-running tasks keep writing + // their slots, so the storage must outlive every frame -- the shared_ptr each + // task captured keeps it alive until the last one finishes. Only read on the + // completed path. + f->per_range_reason = std::make_shared>(scan_ranges.size()); + f->fanout_done = std::make_shared(static_cast(scan_ranges.size())); + // Per-slot single-owner claim flags; see the claim comments at the submit loop + // below. (Value-initialized to false.) + f->slot_counted = std::make_shared>>(scan_ranges.size()); + // Set true when the LAST interested waiter abandons the flight (its deadline + // passed): a not-yet-started task then skips its sync RPC instead of spending + // the full retry budget, so a meta-service brownout drains the bounded queue + // fast rather than running abandoned work to completion and starving the pool. + f->abandoned = std::make_shared>(false); + // Snapshot the owner's tablet ids in slot order. Every waiter (this owner and + // any that coalesce onto the flight) builds its fallback map from THESE ids, not + // from its own scan-range order: build_cache_key sorts tablet ids into the key, + // so two runtimes whose scan ranges hold the same tablets in a different order + // share this flight, and keying the shared index-aligned slots off a joiner's + // own order would pin the owner's per-slot reason to the wrong tablet. + auto ids = std::make_shared>(); ids->reserve(scan_ranges.size()); for (const auto& scan_range : scan_ranges) { ids->push_back(scan_range.scan_range.palo_scan_range.tablet_id); } f->tablet_ids = std::move(ids); - // The flight's own registry key. A plain std::string, so its buffer is - // RSS-only like the other non-DorisVector pieces (see the header comment - // above); on the null-cache test path flight_key is empty, so this is a - // harmless no-op there. + // The flight's own registry key (a plain std::string); on the null-cache test + // path flight_key is empty, so this is a harmless no-op there. f->key = flight_key; f->slots = scan_ranges.size(); f->waiters = 1; + // Estimate the flight's retained bytes and charge them as one lump. Covered: + // the object; the three O(tablets) buffers (per_range_reason 8B, tablet_ids 8B, + // slot_counted ~1B per tablet); BOTH copies of the registry key (PresyncFlight:: + // key and the registry map node's own key are separate std::string buffers); the + // N queued fan-out task closures (submit_func heap-wraps each lambda, which + // captures the shared pieces -- a handful of shared_ptrs plus small scalars -- so + // a per-task slack covers each); and a fixed slack for the ~6 shared_ptr control + // blocks and the latch/atomic bodies. Arm the release (mem_tracker + + // tracked_bytes) BEFORE the consume so any later unwind still balances; + // ~PresyncFlight releases the same tracked_bytes off the same tracker. + constexpr int64_t kPerTaskClosureBytes = 128; // ~a handful of shared_ptrs + scalars + const int64_t flight_tracked_bytes = + static_cast(sizeof(QueryCache::PresyncFlight)) + + slot_n * static_cast(sizeof(const char*) + sizeof(int64_t) + + sizeof(std::atomic) + kPerTaskClosureBytes) + + 2 * static_cast(f->key.size()) + 6 * 64; + f->tracked_bytes = flight_tracked_bytes; + f->mem_tracker = qc_limiter; + qc_limiter->consume(flight_tracked_bytes); return f; }; if (cache != nullptr) { @@ -648,14 +721,18 @@ std::unordered_map QueryCacheRuntime::_presync_cloud_delta // meta-service degradation. Every task records its own sync error into its slot so // no failure aborts the others. // - // The tasks do no explicit MemTracker/ResourceContext attach: this is a - // detached engine-layer sync with no live query context to charge to (the - // query may have already timed out and returned, so attaching to it would be - // both unavailable here -- this is a static engine path -- and wrong). Their + // The tasks do no explicit MemTracker/ResourceContext attach: this is a detached + // engine-layer sync with no live query context to charge to (the query may have + // already timed out and returned, so attaching to it would be both unavailable + // here -- this is a static engine path -- and WRONG: sync_rowsets' allocations + // land in the tablet cache's own structures (rowset metas, delete bitmap) that + // outlive this decision, so charging them to the query-cache or a per-query + // tracker would mis-attribute tablet-cache memory to the wrong owner). Their // allocations fall to the worker thread's default accounting, matching the // same-shaped sync on the same kind of pool by CloudStorageEngine's other - // sync_rowsets caller, CloudBackendService::sync_load_for_tablets (the FE - // warmup path), which likewise does not attach. + // sync_rowsets caller, CloudBackendService::sync_load_for_tablets (the FE warmup + // path, be/src/cloud/cloud_backend_service.cpp), which likewise submits the sync + // to its pool with no attach. auto sync_fanout_start = std::chrono::steady_clock::now(); if (!flight_owner) { // Joined an in-flight fan-out for the same (cache_key, version): the @@ -712,79 +789,173 @@ std::unordered_map QueryCacheRuntime::_presync_cloud_delta // path (see publish_slot's contract above). const char* reason = nullptr; Defer done([&] { publish_slot(i, reason); }); - // (a) Shutdown began, or the interested waiters all abandoned the - // wait (their deadlines passed, or an owner unwound on bad_alloc): - // bail before a fresh sync RPC so the pool drains quickly instead of - // spending the full RPC retry budget on work the timed-out waiters - // will not read. A coalesced joiner MAY later reuse this slot (it - // rides the draining tombstone), so the reason must be accurate: - // keep "be is stopping" for a genuine teardown, but give the - // abandonment case a distinct reason (covering BOTH the timeout and - // the owner-unwind causes) so a joiner on a HEALTHY BE never sees a - // false shutdown diagnosis in its profile. - if (engine_ptr->stopped()) { - reason = "be is stopping, sync skipped"; - return; - } - if (abandoned->load(std::memory_order_acquire)) { - reason = "cloud rowset sync abandoned (all waiters left)"; - return; - } - auto tablet_res = ExecEnv::get_tablet(tablet_id); - if (!tablet_res) { - // Publish the failure instead of leaving the slot empty: an - // empty reason tells _capture_tablet_delta the view is synced, - // so its own get_tablet would RETRY this very cache-miss - // meta-service load synchronously on the admission thread, - // outside the fast-fail deadline -- during a brownout (the one - // situation where this load fails slowly) that repeats the - // slow RPC on the pool this budget exists to protect. Falling - // back is also never wrong here: this worker-side load is - // strictly conservative (a transient failure only costs the - // incremental merge, never correctness). - reason = "cloud tablet load failed"; - return; - } - BaseTabletSPtr tablet = std::move(tablet_res.value()); - const bool append_only = tablet->keys_type() == KeysType::DUP_KEYS || - (tablet->keys_type() == KeysType::UNIQUE_KEYS && - tablet->enable_unique_key_merge_on_write()); - if (!append_only) { - // Skip the RPC entirely: _capture_tablet_delta rejects this - // tablet at its keys-type check before it would consume a sync. - return; - } - auto cloud_tablet = std::dynamic_pointer_cast(std::move(tablet)); - if (cloud_tablet == nullptr) { - // is_cloud_mode() flipped on a live local deployment - // (cloud_unique_id is a mutable config): degrade to a full - // recompute rather than dereference the failed cast downstream. - reason = "tablet is not a cloud tablet"; - return; - } - SyncOptions options; - options.query_version = current_version; - options.merge_schema = true; - // The history-rewrite check reads the delete bitmap this sync merges, - // so pin the dependency explicitly rather than inherit the struct - // default (which a refactor could flip with no compile error): a - // merge-on-write delta cannot be classified on a bitmap the sync did - // not bring up to the queried version. - options.sync_delete_bitmap = true; - Status st = cloud_tablet->sync_rowsets(options); - if (!st.ok()) { - // Unlike the other fallback reasons (expected data shapes), a - // failed sync is an infrastructure error: log the status so the - // profile reason correlates to a cause. Throttled with - // LOG_EVERY_N because a meta-service brownout fails this for - // every stale tablet of every query at once; the underlying RPC - // errors are already logged per retry inside retry_rpc, so a - // sampled line here is enough to correlate without a storm, and - // the exact reason still reaches the user via the query profile. + // Guard the whole per-tablet body: get_tablet fans out through + // CloudTabletMgr's single-flight loader, which THROWS + // std::system_error when a duplicate concurrent load's + // CountdownEvent wait returns non-zero (be/src/cloud/ + // cloud_tablet_mgr.cpp), and sync_rowsets can surface a bad_alloc + // (or a callee throw) under memory pressure. FunctionRunnable::run + // invokes this closure with no catch, so an escaped exception on + // this engine-pool worker would std::terminate the whole BE -- + // defeating the per-tablet fallback contract (any sync failure + // must cost only the incremental merge, never the process). The + // catch below converts it to a fallback reason. + try { + // (a) Shutdown began, or the interested waiters all abandoned the + // wait (their deadlines passed, or an owner unwound on bad_alloc): + // bail before a fresh sync RPC so the pool drains quickly instead of + // spending the full RPC retry budget on work the timed-out waiters + // will not read. A coalesced joiner MAY later reuse this slot (it + // rides the draining tombstone), so the reason must be accurate: + // keep "be is stopping" for a genuine teardown, but give the + // abandonment case a distinct reason (covering BOTH the timeout and + // the owner-unwind causes) so a joiner on a HEALTHY BE never sees a + // false shutdown diagnosis in its profile. + if (engine_ptr->stopped()) { + reason = "be is stopping, sync skipped"; + return; + } + if (abandoned->load(std::memory_order_acquire)) { + reason = "cloud rowset sync abandoned (all waiters left)"; + return; + } + auto tablet_res = ExecEnv::get_tablet(tablet_id); + if (!tablet_res) { + // Publish the failure instead of leaving the slot empty: an + // empty reason tells _capture_tablet_delta the view is synced, + // so its own get_tablet would RETRY this very cache-miss + // meta-service load synchronously on the admission thread, + // outside the fast-fail deadline -- during a brownout (the one + // situation where this load fails slowly) that repeats the + // slow RPC on the pool this budget exists to protect. Falling + // back is also never wrong here: this worker-side load is + // strictly conservative (a transient failure only costs the + // incremental merge, never correctness). + reason = "cloud tablet load failed"; + return; + } + BaseTabletSPtr tablet = std::move(tablet_res.value()); + const bool append_only = tablet->keys_type() == KeysType::DUP_KEYS || + (tablet->keys_type() == KeysType::UNIQUE_KEYS && + tablet->enable_unique_key_merge_on_write()); + if (!append_only) { + // Skip the RPC entirely: _capture_tablet_delta rejects this + // tablet at its keys-type check before it would consume a sync. + return; + } + auto cloud_tablet = std::dynamic_pointer_cast(std::move(tablet)); + if (cloud_tablet == nullptr) { + // is_cloud_mode() flipped on a live local deployment + // (cloud_unique_id is a mutable config): degrade to a full + // recompute rather than dereference the failed cast downstream. + reason = "tablet is not a cloud tablet"; + return; + } + SyncOptions options; + options.query_version = current_version; + options.merge_schema = true; + // The history-rewrite check reads the delete bitmap this sync merges, + // so request it explicitly rather than inherit the struct default (which + // a refactor could flip with no compile error): a merge-on-write delta + // cannot be classified on a bitmap the sync did not bring up to the + // queried version. This is the IDENTICAL sync shape the normal cloud MOW + // scan issues (olap_scan_operator.cpp sets query_version + merge_schema + // and leaves sync_delete_bitmap at its true default, cloud_tablet.h), and + // the classification then reads the very same tablet_meta()->delete_ + // bitmap() with no lazy per-rowset re-fetch, so it is no more exposed than + // the scan whose result it caches. + // + // It is NOT fully immune, but the gap is pre-existing and shared with + // that scan, not introduced here. sync_rowsets' fast path (_max_version + // >= query_version, cloud_tablet.cpp) returns without consulting + // sync_delete_bitmap, so if the tablet is already resident with rowsets + // synced to the queried version while its bitmap was left behind, this + // sync no-ops and the check reads the short bitmap. A full-compaction + // SUBMISSION creates exactly that state: it cache-miss-loads the tablet + // with sync_delete_bitmap=false (task_worker_pool.cpp and cloud_ + // compaction_action.cpp both pass false for FULL_COMPACTION), advancing + // _max_version ahead of the bitmap for the span of an admin-triggered + // full compaction that lands on a query-serving BE (with + // enable_compaction_rw_separation compaction and queries never share a + // BE's tablet cache, so it cannot happen). In that window a plain MOW + // scan already returns un-deduplicated rows; the only thing this feature + // adds is that a mis-classified delta is written back, so the wrong + // result persists in the entry instead of being a one-shot -- bounded by + // the same window, since committing the compaction swaps the rowsets and + // rebases the entry. The clean repair is in the storage layer (have the + // fast path require bitmap coverage too, or keep a sync_delete_bitmap= + // false load out of the query-visible cache), which fixes the scan and + // this path together; it is left as a follow-up rather than widened into + // this change. A delta rowset a compaction folded away in the meantime is + // a separate, benign case -- the capture below fails to find it and falls + // the tablet back to a full recompute. + options.sync_delete_bitmap = true; + // Re-check abandonment/shutdown right before the sync: a task that + // passed the earlier checks may have sat here while its waiters all + // timed out. sync_rowsets serializes same-tablet syncs on + // _sync_meta_lock and early-returns once one succeeds, but on a + // persistent brownout each DISTINCT-key task that reaches it re-runs + // the full retry budget (distinct keys do not share this flight's + // `abandoned` flag); skipping here when nobody will read the result + // keeps abandoned work from occupying the bounded pool and prolonging + // teardown. This narrows the window to the common case (abandonment + // during the wait, before the sync starts); it cannot be fully closed + // at this layer, because a task already inside sync_rowsets blocked on + // _sync_meta_lock does not re-check -- threading cancellation into + // sync_rowsets is a storage-layer change left as a follow-up. + if (engine_ptr->stopped()) { + reason = "be is stopping, sync skipped"; + return; + } + if (abandoned->load(std::memory_order_acquire)) { + reason = "cloud rowset sync abandoned (all waiters left)"; + return; + } + Status st = cloud_tablet->sync_rowsets(options); + if (!st.ok()) { + // Unlike the other fallback reasons (expected data shapes), a + // failed sync is an infrastructure error: log the status so the + // profile reason correlates to a cause. Throttled with + // LOG_EVERY_N because a meta-service brownout fails this for + // every stale tablet of every query at once; the underlying RPC + // errors are already logged per retry inside retry_rpc, so a + // sampled line here is enough to correlate without a storm, and + // the exact reason still reaches the user via the query profile. + LOG_EVERY_N(WARNING, 100) + << "query cache incremental merge falls back, cloud rowset sync " + "failed" + << ", tablet_id=" << tablet_id << ", status=" << st.to_string(); + reason = "cloud rowset sync failed"; + } + } catch (const doris::Exception& e) { + // A Doris-specific throw (e.g. surfaced by a nested THROW_IF_ERROR): + // catch it BEFORE the generic std::exception so the Doris error code + // survives into the log, per the standard catch order + // doris::Exception -> std::exception -> ... (be/src/common/AGENTS.md). + // Same fallback as below: the tablet recomputes in full rather than + // the escaped exception terminating the BE. + reason = "cloud tablet sync raised"; + LOG_EVERY_N(WARNING, 100) + << "query cache incremental merge falls back, cloud tablet sync raised" + << ", tablet_id=" << tablet_id << ", error=" << e.code() << ": " + << e.to_string(); + } catch (const std::exception& e) { + // Any other escaped exception (single-flight std::system_error, a + // bad_alloc, a callee throw) falls this tablet back instead of + // terminating the BE. `done` then publishes this reason, so the + // empty-slot "synced OK" path (which would make + // _capture_tablet_delta reissue the load synchronously) is never + // taken on a throw. Throttled like the sync-failed log because a + // brownout raises it for many tablets of many queries at once. + reason = "cloud tablet sync raised"; LOG_EVERY_N(WARNING, 100) - << "query cache incremental merge falls back, cloud rowset sync failed" - << ", tablet_id=" << tablet_id << ", status=" << st.to_string(); - reason = "cloud rowset sync failed"; + << "query cache incremental merge falls back, cloud tablet sync raised" + << ", tablet_id=" << tablet_id << ", what=" << e.what(); + } catch (...) { + reason = "cloud tablet sync raised"; + LOG_EVERY_N(WARNING, 100) << "query cache incremental merge falls back, cloud " + "tablet sync raised (non-std exception)" + << ", tablet_id=" << tablet_id; } }); if (!submit_st.ok()) { @@ -914,12 +1085,13 @@ bool QueryCacheRuntime::_capture_tablet_delta( // view (a failed cast on a misconfigured deployment, an infrastructure // sync failure, the fast-fail budget expiring on a slow meta service, // or the pool refusing the sync outright), so fall back to a full - // recompute. Consumed BEFORE get_tablet below, because that reason can - // also mean the tablet was never loaded here (a refused submit loads - // nothing): get_tablet would then take the synchronous cache-miss - // meta-service load on this admission thread -- the very blocking - // window the fast-fail budget exists to cap -- to fetch a tablet whose - // decision is already known to be a fallback. No reason means the view + // recompute. Consumed BEFORE get_tablet below for two reasons: a refused + // submit loads nothing, so this recorded reason is more precise than the + // generic cache-only miss get_tablet would report; and a sync that + // FAILED may still leave a stale copy cached from an earlier query, which + // the cache-only get_tablet would return -- classifying on that unsynced + // view would be wrong, so the reason must veto it first. No reason means + // the view // is synced up to query_version, and the history-rewrite check below // reads the tablet's delete bitmap as the sync brought it to that // version (sync_delete_bitmap=true) -- exactly as complete as the @@ -936,7 +1108,23 @@ bool QueryCacheRuntime::_capture_tablet_delta( return false; } } - auto tablet_res = ExecEnv::get_tablet(tablet_id); + // Cache-only lookup (force_use_only_cached=true): the presync fan-out just + // synced this tablet, so in the common case it is still cached here and this + // is a hit with no RPC. If capacity pressure evicted it in the narrow window + // between that sync and this consume, a plain get_tablet would take the + // synchronous cache-miss meta-service load on THIS light admission thread -- + // the very blocking window the presync fast-fail budget caps, reopened after + // the fact. Forcing cache-only turns that miss into an immediate error that + // falls the tablet back to a full recompute instead; the fallback is never + // wrong (a missed incremental costs only recompute, never correctness), and + // the scan node's own later get_tablet still performs any genuine reload off + // this thread. The "tablet not found" reason is deliberately shared with a + // genuinely absent tablet id: both are simply "not resident locally", and + // this path cannot (nor needs to) tell an eviction from a never-loaded id. + // In non-cloud mode the flag is ignored (StorageEngine::get_tablet reads the + // resident tablet map, which never evicts), so this is a no-op there. + auto tablet_res = ExecEnv::get_tablet(tablet_id, /*sync_stats=*/nullptr, + /*force_use_only_cached=*/true); if (!tablet_res) { decision->incremental_fallback_reason = "tablet not found"; return false; diff --git a/be/src/runtime/query_cache/query_cache.h b/be/src/runtime/query_cache/query_cache.h index 6d38d03098166c..a2a66f2d6f4016 100644 --- a/be/src/runtime/query_cache/query_cache.h +++ b/be/src/runtime/query_cache/query_cache.h @@ -34,7 +34,6 @@ #include "common/config.h" #include "common/status.h" #include "core/block/block.h" -#include "core/custom_allocator.h" #include "io/fs/file_system.h" #include "io/fs/path.h" #include "runtime/exec_env.h" @@ -281,10 +280,11 @@ class QueryCache : public LRUCachePolicy { // allocate, so it is safe on the bad_alloc unwind path (publish_unsubmitted) // where constructing a std::string could throw again inside a Defer and // std::terminate. The merged tablet-id map materializes the strings once, - // after the fan-out latch settles. Allocator-aware (DorisVector) so this - // O(tablets) buffer is charged to a MemTracker (see make_flight, which pins - // it to the stable query-cache limiter). - std::shared_ptr> per_range_reason; + // after the fan-out latch settles. A plain std::vector; its O(tablets) buffer + // is covered by the flight's single lump charge to the query-cache MemTracker + // (see make_flight and ~PresyncFlight), not an allocator-aware container, so + // nothing in its teardown switches trackers or can throw. + std::shared_ptr> per_range_reason; std::shared_ptr fanout_done; std::shared_ptr>> slot_counted; std::shared_ptr> abandoned; @@ -295,9 +295,10 @@ class QueryCache : public LRUCachePolicy { // whose scan ranges differ only in order share a flight, and reading a // slot through the joiner's own positions would misassign the owner's // per-slot reason to the wrong tablet. Written once at creation, read - // (never mutated) by every waiter, so no lock is needed. Allocator-aware - // (DorisVector) so this O(tablets) buffer is charged to a MemTracker. - std::shared_ptr> tablet_ids; + // (never mutated) by every waiter, so no lock is needed. A plain + // std::vector whose O(tablets) buffer is covered by the flight's single lump + // charge to the query-cache MemTracker (see make_flight and ~PresyncFlight). + std::shared_ptr> tablet_ids; // This flight's own registry key (cache_key + '@' + version). Stored so // the fan-out's final publisher can reap a drained-but-waiterless flight // by key without capturing the string into every task closure -- see the @@ -315,6 +316,19 @@ class QueryCache : public LRUCachePolicy { // waiters. A last leaver that leaves while tasks still run keeps the entry // as a tombstone and sets `abandoned` so not-yet-started tasks skip. int waiters = 0; + + // The query-cache MemTracker and the estimated retained bytes make_flight + // charged as one lump for THIS flight (object + all O(tablets) buffers + + // control blocks + key). The destructor releases exactly this many bytes off + // this tracker, so a flight is charged on creation and refunded on its final + // teardown (whether it wins the registry, is dropped as a losing candidate, or + // is reaped as a tombstone). release() is a plain atomic op, so the destructor + // never allocates and never throws. Null tracker means unaccounted (never armed + // -- e.g. an exception before make_flight finished), in which case the refund is + // skipped and nothing was consumed. + std::shared_ptr mem_tracker; + int64_t tracked_bytes = 0; + ~PresyncFlight(); }; std::mutex _presync_flights_lock; std::unordered_map> _presync_flights; @@ -478,6 +492,22 @@ class QueryCacheRuntime { QueryCache* cache = nullptr, const std::string& cache_key = "") { return _presync_cloud_delta_tablets(cache, cache_key, scan_ranges, current_version); } + + // Drives the concurrency-cap counter so a test can simulate the cap being + // reached without standing up real concurrent parked waiters. + static std::atomic& presync_active_waiters_for_test() { return _presync_active_waiters; } + + // Drives _capture_tablet_delta (a private member) directly so a test can + // prove the capture-site get_tablet is cache-only: with no recorded presync + // reason but the tablet absent from the cloud tablet cache (evicted after a + // successful presync), the classification must fall back WITHOUT reissuing a + // synchronous meta-service load on this admission thread. + bool capture_tablet_delta_for_test( + int64_t tablet_id, int64_t cached_version, + const std::unordered_map& presync_reasons, + QueryCacheInstanceDecision* decision) { + return _capture_tablet_delta(tablet_id, cached_version, presync_reasons, decision); + } #endif private: @@ -499,13 +529,18 @@ class QueryCacheRuntime { // identical runtimes are coalesced through the cache's single-flight // registry keyed by (cache_key, current_version): the first arrival owns // the fan-out, later ones wait on the same completion latch under their own - // deadlines (a null `cache` skips coalescing, test-only). Returns the - // per-tablet fallback reasons the sync produced (keyed by tablet id, only - // failures present): a cast failure ("tablet is not a cloud tablet"), a - // worker-side load failure ("cloud tablet load failed"), or an - // infrastructure sync failure ("cloud rowset sync failed"). Tablets that - // are not append-only are skipped (no wasted RPC); _capture_tablet_delta - // rejects them at its own keys-type check. + // deadlines (a null `cache` skips coalescing, test-only). The number of + // decisions that may block in the fan-out wait at once is soft-capped + // (config::query_cache_max_concurrent_decision_sync) so a meta-service brownout + // cannot park the whole light admission pool; a decision arriving over the cap + // returns every tablet as "cloud decision sync at capacity" and recomputes in + // full. Returns the per-tablet fallback reasons the sync produced (keyed by + // tablet id, only failures present): the over-capacity reason above, a cast + // failure ("tablet is not a cloud tablet"), a worker-side load failure ("cloud + // tablet load failed"), an infrastructure sync failure ("cloud rowset sync + // failed"), or a raised exception ("cloud tablet sync raised"). Tablets that are + // not append-only are skipped (no wasted RPC); _capture_tablet_delta rejects them + // at its own keys-type check. static std::unordered_map _presync_cloud_delta_tablets( QueryCache* cache, const std::string& cache_key, const std::vector& scan_ranges, int64_t current_version); @@ -538,6 +573,11 @@ class QueryCacheRuntime { // Shared by every instance whose cache key cannot be built (see // get_or_make_decision): one immutable MISS decision, one log line. std::shared_ptr _invalid_decision; + + // BE-global soft cap counter for concurrent decision-sync waiters (see + // _presync_cloud_delta_tablets). Static because the bound is process-wide across + // every fragment runtime, not per instance. + static std::atomic _presync_active_waiters; }; } // namespace doris diff --git a/be/test/exec/pipeline/query_cache_test.cpp b/be/test/exec/pipeline/query_cache_test.cpp index 688849ed387342..958f6c199ff3a4 100644 --- a/be/test/exec/pipeline/query_cache_test.cpp +++ b/be/test/exec/pipeline/query_cache_test.cpp @@ -36,6 +36,7 @@ #include "cloud/cloud_tablet.h" #include "cloud/config.h" #include "common/config.h" +#include "common/exception.h" #include "common/metrics/doris_metrics.h" #include "core/data_type/data_type_number.h" #include "cpp/sync_point.h" @@ -563,6 +564,173 @@ TEST_F(QueryCacheTest, presync_falls_back_when_pool_rejects_submit) { EXPECT_EQ(sync_calls.load(), 0); } +TEST_F(QueryCacheTest, presync_over_concurrency_cap_falls_back) { + // The decision-sync wait runs on the query-admission light pool. When the + // concurrent-waiter soft cap (config::query_cache_max_concurrent_decision_sync) is + // already reached, a new decision must NOT enter that wait: doing so would park + // yet another admission worker, and under a brownout of DISTINCT (non-coalescing) + // keys that is exactly what would starve the whole pool. It falls every scanned + // tablet back to a full recompute instead. Simulate the cap being reached by + // presetting the global counter, then assert the fan-out is never launched and + // every tablet carries the over-capacity reason. Pre-fix (no cap) this same call + // proceeds into the fan-out and returns scheduling reasons, not this reason. + auto saved_mode = config::deploy_mode; + config::deploy_mode = "cloud"; + auto engine = std::make_unique(EngineOptions {}); + ExecEnv::GetInstance()->set_storage_engine(std::move(engine)); + Defer eng_defer {[&] { + ExecEnv::GetInstance()->set_storage_engine(nullptr); + config::deploy_mode = saved_mode; + }}; + + auto* sp = SyncPoint::get_instance(); + sp->enable_processing(); + std::atomic sync_calls {0}; + sp->set_call_back("CloudMetaMgr::sync_tablet_rowsets", + [&](auto&&) { sync_calls.fetch_add(1); }); + Defer sp_defer {[sp] { + sp->disable_processing(); + sp->clear_all_call_backs(); + }}; + + auto saved_timeout = config::query_cache_decision_sync_timeout_ms; + config::query_cache_decision_sync_timeout_ms = 2000; // positive: a healthy pool would launch. + auto saved_cap = config::query_cache_max_concurrent_decision_sync; + config::query_cache_max_concurrent_decision_sync = 1; + Defer cfg_defer {[&] { + config::query_cache_decision_sync_timeout_ms = saved_timeout; + config::query_cache_max_concurrent_decision_sync = saved_cap; + }}; + + // Simulate the single slot already held by one parked waiter. + QueryCacheRuntime::presync_active_waiters_for_test().store(1, std::memory_order_release); + Defer waiter_defer {[] { + QueryCacheRuntime::presync_active_waiters_for_test().store(0, std::memory_order_release); + }}; + + auto scan_ranges = make_scan_ranges(42, "100"); + scan_ranges.push_back(make_scan_ranges(43, "100").front()); + auto reasons = QueryCacheRuntime::presync_cloud_delta_tablets_for_test(scan_ranges, 100); + + ASSERT_EQ(reasons.count(42), 1); + ASSERT_EQ(reasons.count(43), 1); + EXPECT_EQ(reasons[42], "cloud decision sync at capacity"); + EXPECT_EQ(reasons[43], "cloud decision sync at capacity"); + // The cap gate returns BEFORE the registry and fan-out, so no sync RPC is issued, + // and the rejected path takes no slot (fetch_add then immediate fetch_sub), so the + // counter is left exactly as preset. + EXPECT_EQ(sync_calls.load(), 0); + EXPECT_EQ(QueryCacheRuntime::presync_active_waiters_for_test().load(), 1); +} + +TEST_F(QueryCacheTest, presync_cap_clamps_to_configured_light_pool_width) { + // The waiter cap must derive from the ACTUAL light-pool width, not just the config. + // brpc_light_work_pool_threads is operator-configurable; shrunk to 16, the effective + // cap is width/2 = 8, so the config default of 32 does NOT bind and the 9th concurrent + // waiter still falls back. Pre-fix (a fixed 32) the counter at 8 is under the cap, so + // this same call would ADMIT and enter the fan-out instead of rejecting -- exactly the + // starvation a small configured pool would suffer. + auto saved_mode = config::deploy_mode; + config::deploy_mode = "cloud"; + auto engine = std::make_unique(EngineOptions {}); + ExecEnv::GetInstance()->set_storage_engine(std::move(engine)); + Defer eng_defer {[&] { + ExecEnv::GetInstance()->set_storage_engine(nullptr); + config::deploy_mode = saved_mode; + }}; + + auto* sp = SyncPoint::get_instance(); + sp->enable_processing(); + std::atomic sync_calls {0}; + sp->set_call_back("CloudMetaMgr::sync_tablet_rowsets", + [&](auto&&) { sync_calls.fetch_add(1); }); + Defer sp_defer {[sp] { + sp->disable_processing(); + sp->clear_all_call_backs(); + }}; + + auto saved_timeout = config::query_cache_decision_sync_timeout_ms; + config::query_cache_decision_sync_timeout_ms = 2000; // positive: a healthy pool would launch. + auto saved_cap = config::query_cache_max_concurrent_decision_sync; + auto saved_pool = config::brpc_light_work_pool_threads; + // A 16-thread pool caps effective waiters at 8, below the untouched config of 32. + config::brpc_light_work_pool_threads = 16; + config::query_cache_max_concurrent_decision_sync = 32; + Defer cfg_defer {[&] { + config::query_cache_decision_sync_timeout_ms = saved_timeout; + config::query_cache_max_concurrent_decision_sync = saved_cap; + config::brpc_light_work_pool_threads = saved_pool; + }}; + + // Exactly the width-derived cap (16/2) already held: the next waiter is over it even + // though the config cap (32) is not. + QueryCacheRuntime::presync_active_waiters_for_test().store(8, std::memory_order_release); + Defer waiter_defer {[] { + QueryCacheRuntime::presync_active_waiters_for_test().store(0, std::memory_order_release); + }}; + + auto scan_ranges = make_scan_ranges(42, "100"); + auto reasons = QueryCacheRuntime::presync_cloud_delta_tablets_for_test(scan_ranges, 100); + + ASSERT_EQ(reasons.count(42), 1); + EXPECT_EQ(reasons[42], "cloud decision sync at capacity"); + EXPECT_EQ(sync_calls.load(), 0); + EXPECT_EQ(QueryCacheRuntime::presync_active_waiters_for_test().load(), 8); +} + +TEST_F(QueryCacheTest, presync_single_thread_light_pool_never_parks_sole_worker) { + // A 1-thread light pool floors the width-derived cap (1/2) to 0, so NO decision-sync + // waiter may be admitted: parking the only worker for the timeout would starve all + // unrelated fragment admission. Every stale query falls back immediately, even with + // the counter at 0. Pre-fix (max(1, width/2)) the cap was 1, so the first waiter was + // admitted and parked the sole worker. + auto saved_mode = config::deploy_mode; + config::deploy_mode = "cloud"; + auto engine = std::make_unique(EngineOptions {}); + ExecEnv::GetInstance()->set_storage_engine(std::move(engine)); + Defer eng_defer {[&] { + ExecEnv::GetInstance()->set_storage_engine(nullptr); + config::deploy_mode = saved_mode; + }}; + + auto* sp = SyncPoint::get_instance(); + sp->enable_processing(); + std::atomic sync_calls {0}; + sp->set_call_back("CloudMetaMgr::sync_tablet_rowsets", + [&](auto&&) { sync_calls.fetch_add(1); }); + Defer sp_defer {[sp] { + sp->disable_processing(); + sp->clear_all_call_backs(); + }}; + + auto saved_timeout = config::query_cache_decision_sync_timeout_ms; + config::query_cache_decision_sync_timeout_ms = 2000; + auto saved_cap = config::query_cache_max_concurrent_decision_sync; + auto saved_pool = config::brpc_light_work_pool_threads; + config::brpc_light_work_pool_threads = 1; // width/2 floors to 0 -> admit nobody + config::query_cache_max_concurrent_decision_sync = 32; + Defer cfg_defer {[&] { + config::query_cache_decision_sync_timeout_ms = saved_timeout; + config::query_cache_max_concurrent_decision_sync = saved_cap; + config::brpc_light_work_pool_threads = saved_pool; + }}; + + // Even the FIRST waiter (counter 0) is rejected on a 1-thread pool. + QueryCacheRuntime::presync_active_waiters_for_test().store(0, std::memory_order_release); + Defer waiter_defer {[] { + QueryCacheRuntime::presync_active_waiters_for_test().store(0, std::memory_order_release); + }}; + + auto scan_ranges = make_scan_ranges(42, "100"); + auto reasons = QueryCacheRuntime::presync_cloud_delta_tablets_for_test(scan_ranges, 100); + + ASSERT_EQ(reasons.count(42), 1); + EXPECT_EQ(reasons[42], "cloud decision sync at capacity"); + EXPECT_EQ(sync_calls.load(), 0); + // The rejected fetch_add was immediately backed out, so the counter is left at 0. + EXPECT_EQ(QueryCacheRuntime::presync_active_waiters_for_test().load(), 0); +} + // The BE-side mirror of the FE incremental knob gates, applied where the // fragment context creates the shared runtime. A same-version FE already // cleared allow_incremental in these cases; the mirror exists for the @@ -1701,6 +1869,136 @@ TEST_F(QueryCacheCloudIncrementalTest, presync_tablet_load_failure_falls_back) { EXPECT_EQ(meta_loads.load(), 1); } +TEST_F(QueryCacheCloudIncrementalTest, presync_decision_sync_raise_falls_back) { + // The worker-side task body calls get_tablet and sync_rowsets. get_tablet fans out + // through CloudTabletMgr's single-flight loader, which THROWS (std::system_error) + // when a duplicate concurrent load's CountdownEvent wait fails, and sync_rowsets + // can surface a bad_alloc under memory pressure. FunctionRunnable::run invokes the + // task with no catch, so pre-fix an escaped exception std::terminates the BE and + // defeats the fallback contract (a sync failure must cost only the incremental + // merge, never the process). The task-body guard must convert ANY throw into a + // per-tablet fallback. Inject the throw at the decision's sync: it propagates + // uncaught through CloudTablet::sync_rowsets (no try/catch there, its RAII locks + // release on unwind) into the guard, exactly as the single-flight throw would. + _sp->set_call_back("CloudMetaMgr::get_tablet_meta", [](auto&& args) { + TTabletSchema schema; + schema.keys_type = TKeysType::DUP_KEYS; + auto meta = std::make_shared(1, 2, kTabletId, 15674, 4, 5, schema, 6, + std::unordered_map {{7, 8}}, + UniqueId(9, 10), TTabletType::TABLET_TYPE_DISK, + TCompressionType::LZ4F, 0, false); + meta->set_tablet_state(TABLET_RUNNING); + *try_any_cast(args[1]) = std::move(meta); + try_any_cast_ret(args)->second = true; + }); + _sp->set_call_back("CloudMetaMgr::sync_tablet_rowsets", [this](auto&& args) { + auto* tablet = try_any_cast(args[0]); + if (_sync_calls.fetch_add(1) == 0) { + // The loader's sync brings a stale view (< the queried version 100), so the + // decision's own sync below actually runs and can raise. + install_rowsets(tablet, {{0, 50}}); + try_any_cast_ret(args)->second = true; + return; + } + // The decision's sync, inside the guarded task body: raise. + throw std::runtime_error("injected: decision sync raised"); + }); + int64_t fallbacks_before = + DorisMetrics::instance()->query_cache_incremental_fallback_total->value(); + auto decision = make_stale_decision(); + EXPECT_EQ(DorisMetrics::instance()->query_cache_incremental_fallback_total->value(), + fallbacks_before + 1); + EXPECT_EQ(decision->mode, QueryCacheInstanceDecision::Mode::MISS); + EXPECT_EQ(decision->incremental_fallback_reason, "cloud tablet sync raised"); + // Two syncs entered: the loader's (installed the stale view) and the decision's + // (raised, then converted to a fallback by the guard rather than escaping to + // terminate the BE). + EXPECT_EQ(_sync_calls.load(), 2); +} + +TEST_F(QueryCacheCloudIncrementalTest, presync_decision_sync_raises_doris_exception_falls_back) { + // Same guard, but the decision sync raises a doris::Exception (what a nested + // THROW_IF_ERROR surfaces). The task body catches doris::Exception BEFORE the + // generic std::exception (the standard catch order per be/src/common/AGENTS.md), so + // the Doris error code reaches the log, and converts it to the same per-tablet + // fallback rather than letting it escape and terminate the BE. + _sp->set_call_back("CloudMetaMgr::get_tablet_meta", [](auto&& args) { + TTabletSchema schema; + schema.keys_type = TKeysType::DUP_KEYS; + auto meta = std::make_shared(1, 2, kTabletId, 15674, 4, 5, schema, 6, + std::unordered_map {{7, 8}}, + UniqueId(9, 10), TTabletType::TABLET_TYPE_DISK, + TCompressionType::LZ4F, 0, false); + meta->set_tablet_state(TABLET_RUNNING); + *try_any_cast(args[1]) = std::move(meta); + try_any_cast_ret(args)->second = true; + }); + _sp->set_call_back("CloudMetaMgr::sync_tablet_rowsets", [this](auto&& args) { + auto* tablet = try_any_cast(args[0]); + if (_sync_calls.fetch_add(1) == 0) { + install_rowsets(tablet, {{0, 50}}); + try_any_cast_ret(args)->second = true; + return; + } + throw doris::Exception(ErrorCode::INTERNAL_ERROR, + "injected: decision sync doris exception"); + }); + int64_t fallbacks_before = + DorisMetrics::instance()->query_cache_incremental_fallback_total->value(); + auto decision = make_stale_decision(); + EXPECT_EQ(DorisMetrics::instance()->query_cache_incremental_fallback_total->value(), + fallbacks_before + 1); + EXPECT_EQ(decision->mode, QueryCacheInstanceDecision::Mode::MISS); + EXPECT_EQ(decision->incremental_fallback_reason, "cloud tablet sync raised"); + EXPECT_EQ(_sync_calls.load(), 2); +} + +TEST_F(QueryCacheCloudIncrementalTest, capture_site_get_tablet_is_cache_only_on_eviction) { + // The presync fan-out synced this tablet, but capacity pressure evicted it + // from the cloud tablet cache before the serial capture consumed the result. + // With no recorded presync reason, _capture_tablet_delta's capture-site + // get_tablet must be cache-only (force_use_only_cached=true): a miss becomes + // an immediate fallback instead of a synchronous meta-service reload on this + // light admission thread (the very blocking window the presync fast-fail + // budget caps, which a plain reload would reopen after the fact). Both + // callbacks below serve a valid tablet, so a regression to a plain get_tablet + // would SUCCEED the reload and fail this test on the load count rather than + // crash, keeping the revert a clean red. + std::atomic meta_loads {0}; + _sp->set_call_back("CloudMetaMgr::get_tablet_meta", [&meta_loads](auto&& args) { + ++meta_loads; + TTabletSchema schema; + schema.keys_type = TKeysType::DUP_KEYS; + auto meta = std::make_shared(1, 2, kTabletId, 15674, 4, 5, schema, 6, + std::unordered_map {{7, 8}}, + UniqueId(9, 10), TTabletType::TABLET_TYPE_DISK, + TCompressionType::LZ4F, 0, false); + meta->set_tablet_state(TABLET_RUNNING); + *try_any_cast(args[1]) = std::move(meta); + try_any_cast_ret(args)->second = true; + }); + _sp->set_call_back("CloudMetaMgr::sync_tablet_rowsets", [this](auto&& args) { + auto* tablet = try_any_cast(args[0]); + install_rowsets(tablet, {{0, 50}, {51, 100}}); + try_any_cast_ret(args)->second = true; + }); + + QueryCacheRuntime runtime(make_cache_param(kTabletId), _cache.get()); + QueryCacheInstanceDecision decision; + decision.current_version = 100; + decision.cached_version = 50; + // Empty presync reasons + a tablet absent from the cloud tablet cache (never + // loaded here) reproduce "synced OK by presync, then evicted". + bool incremental = runtime.capture_tablet_delta_for_test(kTabletId, 50, {}, &decision); + EXPECT_FALSE(incremental); + // Shared with a genuinely absent tablet: both are "not resident locally". + EXPECT_EQ(decision.incremental_fallback_reason, "tablet not found"); + // The load-bearing assertion: the cache-only lookup issued NO meta-service + // RPC. A plain get_tablet would reload the evicted tablet here (meta_loads + // == 1), which is exactly the admission-thread blocking this fix removes. + EXPECT_EQ(meta_loads.load(), 0); +} + TEST_F(QueryCacheCloudIncrementalTest, unique_non_mow_rejected_before_decision_sync) { // A UNIQUE_KEYS tablet WITHOUT merge-on-write (the legacy merge-on-read // unique table) is not append-only either, so the keys-type check rejects @@ -2260,9 +2558,10 @@ TEST_F(QueryCacheCloudIncrementalTest, // The parked task published exactly once (loader + the single decision sync); // no leaver re-submitted a duplicate fan-out. EXPECT_EQ(syncs.load(), 2); - // Cushion for the callback shell's unwind after its last store, mirroring - // decision_sync_timeout_falls_back, before TearDown clears the sync points. - std::this_thread::sleep_for(std::chrono::milliseconds(200)); + // Deterministically drain the callback shell's unwind after its last store, mirroring + // decision_sync_timeout_falls_back, before TearDown clears the sync points: pool.wait() + // hard-synchronizes the task's exit instead of betting a fixed sleep outlasts it. + _engine->query_cache_delta_sync_pool().wait(); } TEST_F(QueryCacheCloudIncrementalTest, @@ -2385,7 +2684,9 @@ TEST_F(QueryCacheCloudIncrementalTest, std::this_thread::sleep_for(std::chrono::milliseconds(1)); } ASSERT_TRUE(gated_sync_returned.load()); - std::this_thread::sleep_for(std::chrono::milliseconds(200)); + // Drain the task's tail after its last store deterministically (pool.wait() hard- + // synchronizes the callback shell's exit) before TearDown clears the sync points. + _engine->query_cache_delta_sync_pool().wait(); } TEST_F(QueryCacheCloudIncrementalTest, @@ -2443,15 +2744,13 @@ TEST_F(QueryCacheCloudIncrementalTest, EXPECT_TRUE(QueryCache::build_cache_key(ranges, param, &key, &version).ok()); insert_entry(_cache.get(), key, 50, 0); // Baseline the query-cache MemTracker AFTER the cache entry is in (that charge is - // a constant offset for the rest of the test). The flight's two DorisVector buffers - // (per_range_reason, tablet_ids) are charged to this SAME stable limiter, so + // a constant offset for the rest of the test). make_flight charges the whole + // flight's estimated retained bytes (object + the per_range_reason / tablet_ids / + // slot_counted O(tablets) buffers + control blocks + key) to this SAME stable + // limiter as ONE lump, and ~PresyncFlight releases exactly that lump, so // consumption must rise while the flight is live and return to EXACTLY this baseline - // once the flight is reaped -- the balance that proves those buffers are stably - // charged and fully released. Scope note: ONLY the DorisVectors are tracked here; - // the flight's plain-`new` pieces (the object, the shared_ptr control blocks, - // slot_counted, the key string, the registry map node) are RSS-only and invisible - // to every MemTrackerLimiter (Doris has no global new/malloc hook -- see - // make_flight), so they never move this counter. The byte-exact EQ below also + // once the flight is reaped and destroyed -- the balance that proves the charge is + // stable and fully refunded. The byte-exact EQ below also // depends on this fixture keeping the query-cache tracker otherwise quiescent // between baseline and assertion (one tiny cache entry, no eviction, no second // flight); do not add cache traffic in between without switching to a tolerance @@ -2862,10 +3161,10 @@ TEST_F(QueryCacheCloudIncrementalTest, decision_sync_timeout_falls_back) { // stack unwind of the callback and its SyncPoint shell after the last store // above, plus the task's tail (releasing a non-last CloudTablet ref -- the // tablet cache still holds one, so nothing is destroyed here -- and the - // fork-join bookkeeping). None of that tail touches fixture state, so a - // 200ms margin over a microsecond window is a safe cushion, not a bet on - // timing being fast enough. - std::this_thread::sleep_for(std::chrono::milliseconds(200)); + // fork-join bookkeeping). Drain it deterministically with pool.wait(), which hard- + // synchronizes the callback shell's exit, rather than betting a fixed sleep outlasts + // that tail before TearDown clears the sync points. + _engine->query_cache_delta_sync_pool().wait(); } } // namespace doris diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/normalize/QueryCacheNormalizer.java b/fe/fe-core/src/main/java/org/apache/doris/planner/normalize/QueryCacheNormalizer.java index 8ddd0617709b78..a522dcb991a685 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/normalize/QueryCacheNormalizer.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/normalize/QueryCacheNormalizer.java @@ -24,6 +24,7 @@ import org.apache.doris.catalog.KeysType; import org.apache.doris.catalog.OlapTable; import org.apache.doris.catalog.Tablet; +import org.apache.doris.common.Config; import org.apache.doris.common.Pair; import org.apache.doris.planner.AggregationNode; import org.apache.doris.planner.ExchangeNode; @@ -218,62 +219,83 @@ private boolean computeAllowIncremental(CachePoint cachePoint, SessionVariable s if (keysType != KeysType.DUP_KEYS && !mergeOnWrite) { return false; } - // Freshness tolerance and prefer-cached-rowset (both cloud-only in - // effect) trade exactness for speed and locality: the scan may read a - // warmed-up layout that stops below the queried version (freshness halts - // at the warmed boundary) or reaches beyond it (neither walk clips an - // edge spanning it). An incremental merge would defeat such a knob, - // because the delta capture always targets the exact queried version (it - // must, to keep the merged entry correct), forcing the un-warmed reads - // the query explicitly chose to skip. Correctness against version-inexact - // reads does NOT rest on this per-query gate (an entry filled by such a - // read could still be reused by a different, knob-free query sharing the - // cache key): on cloud, the only mode where these reads occur, the BE - // suppresses their cache write-back, so no entry whose content mismatches - // its version stamp exists in the first place. The gate stays mode- - // agnostic (no is_cloud_mode): the knobs are inert on local storage, so - // excluding incremental there only forgoes it for a query that opted into - // a cloud trade-off, and a mode-conditioned gate cannot be exercised by - // the local FE unit-test harness (planning under a flipped cloud flag - // casts SystemInfoService to CloudSystemInfoService and fails). - if (sessionVariable.getQueryFreshnessToleranceMs() > 0) { + // Freshness tolerance and prefer-cached-rowset (both cloud-only in effect) + // trade exactness for speed and locality: the scan may read a warmed-up layout + // that stops below the queried version (freshness halts at the warmed boundary) + // or reaches beyond it (neither walk clips an edge spanning it). An incremental + // merge would defeat such a knob, because the delta capture always targets the + // exact queried version (it must, to keep the merged entry correct), forcing the + // un-warmed reads the query chose to skip. The suppression is mode-gated + // (Config.isCloudMode()) through a pure helper: these knobs are INERT on local + // storage, so suppressing incremental there would only forgo it for no + // correctness benefit (the local read is version-exact regardless). Correctness + // against version-inexact reads does NOT rest on this per-query gate anyway (an + // entry filled by such a read could still be reused by a knob-free query sharing + // the cache key): on cloud the BE suppresses their write-back, so no entry whose + // content mismatches its version stamp exists. Extracting the pure, + // mode-parameterized helper is what lets the unit test exercise BOTH modes + // directly, without planning under a flipped cloud flag (which casts + // SystemInfoService to CloudSystemInfoService and fails). + if (cloudKnobsSuppressIncremental(Config.isCloudMode(), + sessionVariable.getQueryFreshnessToleranceMs(), + sessionVariable.getEnablePreferCachedRowset(), mergeOnWrite)) { return false; } - // Prefer-cached-rowset is honored by the storage layer only for non-MOW - // tables: CloudTablet::capture_consistent_versions_unlocked guards the - // prefer branch on !enable_unique_key_merge_on_write(), so a MOW query - // reads the exact queried version regardless of the knob. Its delta - // capture is therefore version-exact and safe to merge incrementally, so - // only non-MOW tables are excluded here. This carve-out keys on table - // type, not mode, so the FE unit test can exercise it directly; it is - // sound in both modes because the read is version-exact for MOW whether - // the knob is inert (local) or explicitly ignored (cloud). Freshness - // above has no such MOW guard on the storage side, so it excludes - // incremental for every table type. - if (sessionVariable.getEnablePreferCachedRowset() && !mergeOnWrite) { + return true; + } + + // Whether the cloud warmed-read knobs suppress incremental merge for this query. + // Pure and mode-parameterized (see the call site for why): only cloud mode honors + // freshness-tolerance / prefer-cached-rowset, so in shared-nothing mode they are + // inert and never suppress. Mirrors the BE gate + // QueryCacheRuntime::cloud_knobs_suppress_incremental, plus the cloud-mode guard. + @VisibleForTesting + public static boolean cloudKnobsSuppressIncremental(boolean cloudMode, + long queryFreshnessToleranceMs, boolean enablePreferCachedRowset, + boolean mergeOnWrite) { + if (!cloudMode) { return false; } - return true; + // Freshness tolerance may read a layout below the queried version, so an + // incremental merge (which targets the exact version) would defeat it: suppress + // for every table type. + if (queryFreshnessToleranceMs > 0) { + return true; + } + // Prefer-cached-rowset is honored by the storage layer only for non-MOW tables + // (CloudTablet::capture_consistent_versions_unlocked guards on + // !enable_unique_key_merge_on_write()), so a MOW query reads the exact queried + // version regardless of the knob and stays incrementally mergeable; only non-MOW + // is suppressed. + return enablePreferCachedRowset && !mergeOnWrite; } - // Whether the selected index of the cache point's scan is a merge-on-write - // UNIQUE table. BE consumes this in the cloud cache write-back gate: cloud - // ignores enable_prefer_cached_rowset for a MOW table, so a prefer-only MOW - // read is version-exact and its fill must not be suppressed. Reported - // independently of allow_incremental because the write-back gate also applies - // to a MISS (a MOW MISS under prefer must still be cached). Determined on the - // selected index, mirroring computeAllowIncremental. False when the cache - // point has no direct OlapScanNode child (a nested-agg shape, never - // incrementally cacheable), which leaves BE at its safe suppress default: such - // a shape over a MOW scan also forgoes the prefer-cached-rowset write-back, - // a deliberate over-suppression (the safe direction, it never caches a - // version-inexact entry) rather than threading scan resolution through the - // aggregation chain for a cache point whose incremental merge is already off. + // Whether the selected index of the cache point's underlying scan is a merge-on- + // write UNIQUE table. BE consumes this in the cloud cache write-back gate: cloud + // ignores enable_prefer_cached_rowset for a MOW table, so a prefer-only MOW read is + // version-exact and its fill must not be suppressed. Reported independently of + // allow_incremental because the write-back gate also applies to a MISS (a MOW MISS + // under prefer must still be cached). Determined on the selected index, mirroring + // computeAllowIncremental. + // + // Resolves the scan through the (possibly multi-level) unary chain, not just the + // direct child: incremental merge requires a DIRECT-child scan (a nested agg is not + // mergeable, see computeAllowIncremental), but this write-back gate only needs the + // scan's TABLE TYPE, which is well-defined for a nested-agg cache point too. Reading + // it through the chain lets a nested-agg-over-MOW query populate the cache under + // prefer (its read is still version-exact) instead of being over-suppressed just + // because the scan is not the direct child. Returns false if the chain branches or + // bottoms out at a non-scan (no single underlying olap scan), leaving BE at its safe + // suppress default. private boolean computeIsMergeOnWrite(CachePoint cachePoint) { - if (!(cachePoint.cacheRoot.getChild(0) instanceof OlapScanNode)) { + PlanNode node = cachePoint.cacheRoot.getChild(0); + while (node != null && !(node instanceof OlapScanNode) && node.getChildren().size() == 1) { + node = node.getChild(0); + } + if (!(node instanceof OlapScanNode)) { return false; } - OlapScanNode scanNode = (OlapScanNode) cachePoint.cacheRoot.getChild(0); + OlapScanNode scanNode = (OlapScanNode) node; OlapTable olapTable = scanNode.getOlapTable(); long selectIndexId = scanNode.getSelectedIndexId() == -1 ? olapTable.getBaseIndexId() diff --git a/fe/fe-core/src/test/java/org/apache/doris/planner/QueryCacheNormalizerTest.java b/fe/fe-core/src/test/java/org/apache/doris/planner/QueryCacheNormalizerTest.java index f5ec8a6d94fde7..2ce51c9e6041cc 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/planner/QueryCacheNormalizerTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/planner/QueryCacheNormalizerTest.java @@ -124,6 +124,19 @@ protected void runBeforeAll() throws Exception { + "distributed by hash(k1) buckets 3\n" + "properties('replication_num' = '1', 'enable_unique_key_merge_on_write' = 'true')"; + // Merge-on-write with a COMPOSITE key so the distribution column k1 is not + // unique: grouping the inner agg by k1 (the distribution column) stays a + // real, colocate (single-phase, no shuffle) aggregation, so the outer agg + // nests above it inside the scan fragment. That produces a genuinely nested + // cache point over a MOW scan, which the nested-MOW test below needs. + String uniqueMowMultiTable = "create table db1.uniq_mow_multi(" + + " k1 int,\n" + + " k2 int,\n" + + " v1 int)\n" + + "UNIQUE KEY(k1, k2)\n" + + "distributed by hash(k1) buckets 3\n" + + "properties('replication_num' = '1', 'enable_unique_key_merge_on_write' = 'true')"; + String uniqueMorTable = "create table db1.uniq_mor(" + " k1 int,\n" + " v1 int)\n" @@ -139,7 +152,7 @@ protected void runBeforeAll() throws Exception { + "properties('replication_num' = '1')"; createTables(nonPart, part1, part2, multiLeveParts, variantTable, uniqueMowTable, - uniqueMorTable, aggTable); + uniqueMowMultiTable, uniqueMorTable, aggTable); connectContext.getSessionVariable().setDisableNereidsRules("PRUNE_EMPTY_PARTITION"); connectContext.getSessionVariable().setEnableQueryCache(true); @@ -474,44 +487,51 @@ public void testAllowIncremental() throws Exception { + " group by cnt"); Assertions.assertFalse(nestedAgg.allow_incremental); - // Query freshness tolerance trades exactness for speed; an - // incremental merge would force the wait for un-warmed data the - // query chose to skip (the BE separately suppresses such a query's - // cache write-back on cloud, which is where correctness lives). The - // gate is mode-agnostic (it forgoes incremental in any mode for a - // query that set the knob), so it must reject incremental for an - // otherwise-eligible DUP_KEYS query here in the local harness too. + // A genuinely nested cache point over a MOW scan. The inner agg groups by + // the distribution column k1 (non-unique here, since the key is (k1, k2)), + // so it is a local single-phase aggregation with no shuffle, and the outer + // agg's partial nests above it inside the scan fragment. The cache point is + // therefore the OUTER agg, whose direct child is the inner agg rather than + // the scan, so incremental is off (nested). But the write-back gate must + // still resolve the scan's table type through the unary chain, so + // is_merge_on_write is true -- letting a nested-agg-over-MOW query populate + // the cache under prefer instead of being over-suppressed just because the + // scan is not the direct child. (Pre-fix, the direct-child-only check + // reported false here.) The direct (non-nested) MOW case, where the cache + // point sits right on the scan and incremental IS allowed, is covered by + // uniqueMow above. + TQueryCacheParam nestedMow = getQueryCacheParam( + "select cnt, count(*) from" + + " (select k1, count(*) cnt from db1.uniq_mow_multi group by k1) x" + + " group by cnt"); + Assertions.assertFalse(nestedMow.allow_incremental); + Assertions.assertTrue(nestedMow.is_merge_on_write); + + // Query freshness tolerance and prefer-cached-rowset are cloud-only knobs; + // in the local (shared-nothing) harness they are inert, so the mode-gated + // suppression no longer forgoes incremental for an otherwise-eligible query + // (the local read is version-exact regardless). The cloud-mode suppression + // itself is verified directly on the pure helper in + // testCloudKnobsSuppressIncremental below, since this planning harness cannot + // flip to cloud mode (the cast to CloudSystemInfoService fails). connectContext.getSessionVariable().queryFreshnessToleranceMs = 5000; try { TQueryCacheParam withFreshness = getQueryCacheParam( "select k2, sum(v1) as v from db1.part1 group by k2"); - Assertions.assertFalse(withFreshness.allow_incremental); - // Freshness has no merge-on-write carve-out (cloud honors it for - // every table type), so it blocks incremental for MOW as well. - TQueryCacheParam mowWithFreshness = getQueryCacheParam( - "select v1, count(*) as v from db1.uniq_mow group by v1"); - Assertions.assertFalse(mowWithFreshness.allow_incremental); + Assertions.assertTrue(withFreshness.allow_incremental); } finally { connectContext.getSessionVariable().queryFreshnessToleranceMs = -1; } - // Prefer-cached-rowset may overshoot the queried version (its walk - // never clips an edge spanning it), and the incremental delta - // capture always targets the exact queried version anyway (the BE - // separately suppresses such a query's cache write-back on cloud); - // the same mode-agnostic gate rejects incremental for the same - // otherwise-eligible query. connectContext.getSessionVariable().enablePreferCachedRowset = true; try { TQueryCacheParam withPreferCached = getQueryCacheParam( "select k2, sum(v1) as v from db1.part1 group by k2"); - Assertions.assertFalse(withPreferCached.allow_incremental); - // Cloud ignores prefer-cached-rowset for a merge-on-write table - // (its read stays version-exact), so incremental remains allowed - // for MOW under prefer -- the carve-out keys on table type, and - // being mode-agnostic it holds in the local harness too. The - // is_merge_on_write flag is what BE also uses to keep the fill's - // write-back for such a query. + Assertions.assertTrue(withPreferCached.allow_incremental); + // The MOW carve-out is mode-independent (a MOW read is version-exact + // whether the knob is inert locally or explicitly ignored on cloud), and + // its is_merge_on_write flag -- BE's write-back gate -- is reported + // regardless of mode. TQueryCacheParam mowWithPreferCached = getQueryCacheParam( "select v1, count(*) as v from db1.uniq_mow group by v1"); Assertions.assertTrue(mowWithPreferCached.allow_incremental); @@ -524,6 +544,38 @@ public void testAllowIncremental() throws Exception { } } + @Test + public void testCloudKnobsSuppressIncremental() { + // Local (shared-nothing) mode: the cloud warmed-read knobs are inert and never + // suppress incremental, whatever their values -- so a local query that happens + // to set them keeps incremental. This is the mode the planning harness above + // runs in; here the mode-gate itself is pinned directly. + Assertions.assertFalse( + QueryCacheNormalizer.cloudKnobsSuppressIncremental(false, 5000, false, false)); + Assertions.assertFalse( + QueryCacheNormalizer.cloudKnobsSuppressIncremental(false, 0, true, false)); + Assertions.assertFalse( + QueryCacheNormalizer.cloudKnobsSuppressIncremental(false, 5000, true, true)); + // Cloud mode, freshness tolerance active: suppresses for every table type. + Assertions.assertTrue( + QueryCacheNormalizer.cloudKnobsSuppressIncremental(true, 5000, false, false)); + Assertions.assertTrue( + QueryCacheNormalizer.cloudKnobsSuppressIncremental(true, 5000, false, true)); + Assertions.assertTrue( + QueryCacheNormalizer.cloudKnobsSuppressIncremental(true, 5000, true, true)); + // Cloud mode, prefer-cached-rowset active: suppresses only non-MOW (a MOW read + // stays version-exact, cloud ignores the knob for it). + Assertions.assertTrue( + QueryCacheNormalizer.cloudKnobsSuppressIncremental(true, 0, true, false)); + Assertions.assertFalse( + QueryCacheNormalizer.cloudKnobsSuppressIncremental(true, 0, true, true)); + // Cloud mode, no knob active: never suppresses. + Assertions.assertFalse( + QueryCacheNormalizer.cloudKnobsSuppressIncremental(true, 0, false, false)); + Assertions.assertFalse( + QueryCacheNormalizer.cloudKnobsSuppressIncremental(true, 0, false, true)); + } + private String getDigest(String sql) throws Exception { return Hex.encodeHexString(getQueryCacheParam(sql).digest); }