From 288aaf325b33939fd1ec671029f22793fd8fa2ba Mon Sep 17 00:00:00 2001 From: SqlRush Date: Thu, 9 Jul 2026 17:27:29 +0800 Subject: [PATCH 1/7] feat(cluster): gcs block dedup eager-reclaim safety primitives + unit tests Add two pure, header-inline decision primitives that carry the eager-reclaim correctness contract for GCS block dedup: - GcsBlockReplyStatusIsReclaimIdempotent(status): in-window reclaim whitelist, default fail-closed (false). Explicit switch so a status is admitted only by an added `case ...: return true;`. Whitelist currently empty. - GcsBlockDedupEntryIsReclaimSafe(entry, now, out_of_window_us): per-entry verdict. in-flight -> never; completed & aged past the out-of-window (2x) -> safe for every status; still in-window -> safe only if the status is whitelisted idempotent. New header-only unit test test_cluster_gcs_block_dedup_reclaim (12 checks, R1-R12): whitelist fail-closed for every enum status, payload/storage-fallback/ pending-x/lost-write/read-image/forward-marker never in-window, in-flight never safe, aged-out safe for all, in-window payload unsafe, strict boundary. Behavioral reclaim path covered in cluster_tap (follow-on). Registered in cluster_unit Makefile (non-simple, links cluster_version.o + common/port srv libs; local ExceptionalCondition stub like siblings). Spec: spec-7.2a-gcs-block-dedup-capacity-gc.md --- src/include/cluster/cluster_gcs_block_dedup.h | 81 +++++++ src/test/cluster_unit/Makefile | 18 +- .../test_cluster_gcs_block_dedup_reclaim.c | 224 ++++++++++++++++++ 3 files changed, 321 insertions(+), 2 deletions(-) create mode 100644 src/test/cluster_unit/test_cluster_gcs_block_dedup_reclaim.c diff --git a/src/include/cluster/cluster_gcs_block_dedup.h b/src/include/cluster/cluster_gcs_block_dedup.h index 5abb7f815e..7cf28ea81f 100644 --- a/src/include/cluster/cluster_gcs_block_dedup.h +++ b/src/include/cluster/cluster_gcs_block_dedup.h @@ -186,6 +186,87 @@ typedef enum GcsBlockDedupResult { } GcsBlockDedupResult; +/* ============================================================ + * Eager reclaim safety decision primitives (spec-7.2a; review r1). + * + * These two pure predicates carry the Rule 8.A correctness contract for + * the spec-7.2a eager reclaim path: a completed dedup entry may only be + * reclaimed under cap pressure when doing so cannot cause a retransmitted + * duplicate to be re-served in a way that breaks de-dup correctness. They + * live in the header (matching the GcsBlockReplyStatus* helpers in + * cluster_gcs_block.h) so the decision logic is unit-testable without + * shmem; the shmem HTAB glue (scan + remove + counters) lives in the .c. + * ============================================================ */ + +/* + * GcsBlockReplyStatusIsReclaimIdempotent -- whether a completed dedup entry + * carrying this reply status may be reclaimed WHILE the sender's retransmit + * budget could still be live (in-window), without breaking retransmit de-dup + * correctness (Rule 8.A). + * + * Default = false (UNSAFE / fail-closed). A status is admitted here ONLY + * after per-site measure-first proves that every master-side install site + * which produces it re-serves with NO master state transition and NO + * side-effect counter, so a duplicate that MISSes after reclaim re-derives a + * bit-for-bit identical verdict. + * + * Statuses proven NOT idempotent (spec-7.2a §3.2, verified against + * cluster_gcs_block.c on stage7-72-integration): + * GRANTED / READ_IMAGE_FROM_XHOLDER payload-bearing re-grant + * *_FROM_HOLDER forward routing marker + * GRANTED_STORAGE_FALLBACK :3305 N->S / :4089 S->X_UPGRADE + * DENIED_PENDING_X :4071 broadcast_invalidate + * DENIED_LOST_WRITE :4393 lost_write_detected_count++ + * + * The whitelist is currently EMPTY: no status is in-window reclaimable until + * its sites are individually proven (follow-on increment, spec-7.2a §11). + * Out-of-window (2x) reclaim, handled by GcsBlockDedupEntryIsReclaimSafe + * below, is safe for every status regardless. Kept as an explicit switch so + * a proven status is added as a `case ...: return true;` here. + */ +static inline bool +GcsBlockReplyStatusIsReclaimIdempotent(GcsBlockReplyStatus status) +{ + switch (status) { + /* (no in-window-idempotent status proven yet — spec-7.2a §11) */ + default: + break; + } + return false; +} + +/* + * GcsBlockDedupEntryIsReclaimSafe -- whether a dedup entry may be reclaimed + * under cap pressure without breaking retransmit de-dup correctness. + * + * out_of_window_us is the 2x total-backoff window (review r1: 1x only + * bounds the sender's last SEND, not the arrival of an in-flight + * retransmit at the master; 2x also covers transit + timing skew — the + * same margin the TTL sweep uses, dedup_expiry_threshold_us()). + * + * in-flight (completed_at_ts == 0) -> never (still processing) + * completed, age > out_of_window_us -> safe for EVERY status + * (spec-7.2a §3.1 theorem) + * completed, age <= out_of_window_us -> safe ONLY if the status is + * site-proven idempotent + */ +static inline bool +GcsBlockDedupEntryIsReclaimSafe(const GcsBlockDedupEntry *entry, TimestampTz now, + int64 out_of_window_us) +{ + int64 age_us; + + if (entry->completed_at_ts == 0) + return false; + + age_us = (int64)(now - entry->completed_at_ts); + if (age_us > out_of_window_us) + return true; + + return GcsBlockReplyStatusIsReclaimIdempotent((GcsBlockReplyStatus)entry->status); +} + + /* ============================================================ * Public API. * ============================================================ */ diff --git a/src/test/cluster_unit/Makefile b/src/test/cluster_unit/Makefile index e9df8745a2..768d293fc0 100644 --- a/src/test/cluster_unit/Makefile +++ b/src/test/cluster_unit/Makefile @@ -38,7 +38,7 @@ TESTS = test_cluster_basic test_cluster_version test_cluster_backend_types \ test_cluster_ic_mock test_cluster_inject test_cluster_pgstat \ test_cluster_debug test_cluster_shared_fs test_cluster_shared_fs_sharedfs test_cluster_shared_fs_block_device test_cluster_smgr \ test_cluster_scn test_cluster_block_format test_cluster_itl_slot \ - test_cluster_buffer_desc test_cluster_pcm_lock test_cluster_bufmgr_pcm_hook test_cluster_gcs_dispatch test_cluster_gcs_block test_cluster_gcs_block_retransmit test_cluster_gcs_block_2way test_cluster_gcs_block_3way test_cluster_gcs_block_lost_write test_cluster_sinval test_cluster_sinval_ack test_cluster_stage2_acceptance test_cluster_tt_status test_cluster_tt_status_hint test_cluster_visibility_fork test_cluster_visibility_decide_scn test_cluster_snapshot_source test_cluster_itl_touch test_cluster_itl_wal test_cluster_uba \ + test_cluster_buffer_desc test_cluster_pcm_lock test_cluster_bufmgr_pcm_hook test_cluster_gcs_dispatch test_cluster_gcs_block test_cluster_gcs_block_retransmit test_cluster_gcs_block_2way test_cluster_gcs_block_3way test_cluster_gcs_block_lost_write test_cluster_gcs_block_dedup_reclaim test_cluster_sinval test_cluster_sinval_ack test_cluster_stage2_acceptance test_cluster_tt_status test_cluster_tt_status_hint test_cluster_visibility_fork test_cluster_visibility_decide_scn test_cluster_snapshot_source test_cluster_itl_touch test_cluster_itl_wal test_cluster_uba \ test_cluster_startup_phase test_cluster_lmon test_cluster_lck test_cluster_diag test_cluster_stats test_cluster_cssd test_cluster_qvotec test_cluster_voting_disk_io test_cluster_quorum_decision \ test_cluster_xlog test_cluster_tt_slot test_cluster_undo_segment \ test_cluster_epoch test_cluster_fence test_cluster_reconfig test_cluster_marker_async \ @@ -181,7 +181,7 @@ test_cluster_backup: test_cluster_backup.c unit_test.h $(CLUSTER_VERSION_O) \ # separate rules because they also link additional cluster_*.o # objects (the test files stub the PG backend symbols those # objects reference). -SIMPLE_TESTS = $(filter-out test_cluster_guc test_cluster_shmem test_cluster_signal test_cluster_views test_cluster_gviews test_cluster_ic test_cluster_conf test_cluster_ic_mock test_cluster_inject test_cluster_pgstat test_cluster_debug test_cluster_shared_fs test_cluster_shared_fs_sharedfs test_cluster_shared_fs_block_device test_cluster_smgr test_cluster_startup_phase test_cluster_lmon test_cluster_lck test_cluster_diag test_cluster_stats test_cluster_cssd test_cluster_qvotec test_cluster_voting_disk_io test_cluster_quorum_decision test_cluster_scn test_cluster_adg test_cluster_epoch test_cluster_fence test_cluster_reconfig test_cluster_ges test_cluster_grd test_cluster_grd_starvation test_cluster_lmd test_cluster_lmd_graph test_cluster_lmd_wait_state test_cluster_cancel_token test_cluster_lmd_probe_collector test_cluster_lock_acquire test_cluster_advisory test_cluster_terminal_authority test_cluster_retention test_cluster_visibility_variants test_cluster_writer_chain test_cluster_tt_2pc test_cluster_stage3_acceptance test_cluster_undo_buf test_cluster_block_apply test_cluster_thread_apply test_cluster_thread_replay test_cluster_thread_driver test_cluster_thread_orchestrator test_cluster_write_fence test_cluster_stage4_acceptance test_cluster_stage5_integrated_acceptance test_cluster_stage5_beta_acceptance test_cluster_ges_mode test_cluster_sequence test_cluster_shared_catalog test_cluster_hw test_cluster_dl test_cluster_extend_gate test_cluster_ir test_cluster_ts test_cluster_ko test_cluster_hw_snapshot test_cluster_cf_authority test_cluster_cf_storage test_cluster_cf_enqueue test_cluster_cf_phase2 test_cluster_cf_stats test_cluster_hang test_cluster_hang_resolve test_cluster_cr_server_policy test_cluster_touched_peers test_cluster_clean_leave test_cluster_membership test_cluster_node_remove test_cluster_resolver_cache test_cluster_backup test_cluster_hang_acceptance test_cluster_gcs_reqid test_cluster_runtime_visibility test_cluster_xid_stripe test_cluster_mxid_stripe test_cluster_bufmgr_pcm_hook test_cluster_cr test_cluster_cr_admit test_cluster_cr_admit_stat test_cluster_cr_cache test_cluster_cr_coordinator test_cluster_cr_key test_cluster_cr_lifecycle test_cluster_cr_pool test_cluster_cr_tuple test_cluster_cr_tuple_stat test_cluster_gcs_block test_cluster_gcs_block_2way test_cluster_gcs_block_3way test_cluster_gcs_block_lost_write test_cluster_gcs_block_retransmit test_cluster_gcs_dispatch test_cluster_ges_handoff test_cluster_heap_lock_tuple test_cluster_hw_lease test_cluster_ic_envelope test_cluster_ic_router test_cluster_itl_cleanout test_cluster_itl_cleanout_perf test_cluster_itl_reader_real_triple test_cluster_itl_touch test_cluster_itl_wal test_cluster_multixact test_cluster_multixact_served test_cluster_pcm_lock test_cluster_perf_gates test_cluster_recovery_merge test_cluster_recovery_plan test_cluster_recovery_worker test_cluster_reverse_key test_cluster_sinval test_cluster_sinval_ack test_cluster_snapshot_source test_cluster_stage2_acceptance test_cluster_stage5_5_cr_acceptance test_cluster_subtrans test_cluster_tt_durable test_cluster_tt_slot_allocator test_cluster_tt_status test_cluster_tt_status_hint test_cluster_uba test_cluster_undo_format test_cluster_undo_lifecycle test_cluster_undo_record test_cluster_visibility_decide_scn test_cluster_visibility_fork test_cluster_visibility_inject test_cluster_wal_state test_cluster_wal_thread test_cluster_xnode_lever test_cluster_xnode_profile test_cluster_pi_shadow test_cluster_oid_lease test_cluster_xid_authority test_cluster_recovery_anchor test_cluster_relmap_authority,$(TESTS)) +SIMPLE_TESTS = $(filter-out test_cluster_guc test_cluster_shmem test_cluster_signal test_cluster_views test_cluster_gviews test_cluster_ic test_cluster_conf test_cluster_ic_mock test_cluster_inject test_cluster_pgstat test_cluster_debug test_cluster_shared_fs test_cluster_shared_fs_sharedfs test_cluster_shared_fs_block_device test_cluster_smgr test_cluster_startup_phase test_cluster_lmon test_cluster_lck test_cluster_diag test_cluster_stats test_cluster_cssd test_cluster_qvotec test_cluster_voting_disk_io test_cluster_quorum_decision test_cluster_scn test_cluster_adg test_cluster_epoch test_cluster_fence test_cluster_reconfig test_cluster_ges test_cluster_grd test_cluster_grd_starvation test_cluster_lmd test_cluster_lmd_graph test_cluster_lmd_wait_state test_cluster_cancel_token test_cluster_lmd_probe_collector test_cluster_lock_acquire test_cluster_advisory test_cluster_terminal_authority test_cluster_retention test_cluster_visibility_variants test_cluster_writer_chain test_cluster_tt_2pc test_cluster_stage3_acceptance test_cluster_undo_buf test_cluster_block_apply test_cluster_thread_apply test_cluster_thread_replay test_cluster_thread_driver test_cluster_thread_orchestrator test_cluster_write_fence test_cluster_stage4_acceptance test_cluster_stage5_integrated_acceptance test_cluster_stage5_beta_acceptance test_cluster_ges_mode test_cluster_sequence test_cluster_shared_catalog test_cluster_hw test_cluster_dl test_cluster_extend_gate test_cluster_ir test_cluster_ts test_cluster_ko test_cluster_hw_snapshot test_cluster_cf_authority test_cluster_cf_storage test_cluster_cf_enqueue test_cluster_cf_phase2 test_cluster_cf_stats test_cluster_hang test_cluster_hang_resolve test_cluster_cr_server_policy test_cluster_touched_peers test_cluster_clean_leave test_cluster_membership test_cluster_node_remove test_cluster_resolver_cache test_cluster_backup test_cluster_hang_acceptance test_cluster_gcs_reqid test_cluster_runtime_visibility test_cluster_xid_stripe test_cluster_mxid_stripe test_cluster_bufmgr_pcm_hook test_cluster_cr test_cluster_cr_admit test_cluster_cr_admit_stat test_cluster_cr_cache test_cluster_cr_coordinator test_cluster_cr_key test_cluster_cr_lifecycle test_cluster_cr_pool test_cluster_cr_tuple test_cluster_cr_tuple_stat test_cluster_gcs_block test_cluster_gcs_block_2way test_cluster_gcs_block_3way test_cluster_gcs_block_lost_write test_cluster_gcs_block_retransmit test_cluster_gcs_block_dedup_reclaim test_cluster_gcs_dispatch test_cluster_ges_handoff test_cluster_heap_lock_tuple test_cluster_hw_lease test_cluster_ic_envelope test_cluster_ic_router test_cluster_itl_cleanout test_cluster_itl_cleanout_perf test_cluster_itl_reader_real_triple test_cluster_itl_touch test_cluster_itl_wal test_cluster_multixact test_cluster_multixact_served test_cluster_pcm_lock test_cluster_perf_gates test_cluster_recovery_merge test_cluster_recovery_plan test_cluster_recovery_worker test_cluster_reverse_key test_cluster_sinval test_cluster_sinval_ack test_cluster_snapshot_source test_cluster_stage2_acceptance test_cluster_stage5_5_cr_acceptance test_cluster_subtrans test_cluster_tt_durable test_cluster_tt_slot_allocator test_cluster_tt_status test_cluster_tt_status_hint test_cluster_uba test_cluster_undo_format test_cluster_undo_lifecycle test_cluster_undo_record test_cluster_visibility_decide_scn test_cluster_visibility_fork test_cluster_visibility_inject test_cluster_wal_state test_cluster_wal_thread test_cluster_xnode_lever test_cluster_xnode_profile test_cluster_pi_shadow test_cluster_oid_lease test_cluster_xid_authority test_cluster_recovery_anchor test_cluster_relmap_authority,$(TESTS)) # spec-2.4 D16: test_cluster_epoch links cluster_epoch.o standalone. # cluster_epoch.c references ShmemInitStruct + cluster_shmem_register_region # + pg_atomic_*; the test stubs all of these and exercises only the @@ -1311,6 +1311,20 @@ test_cluster_gcs_block_retransmit: test_cluster_gcs_block_retransmit.c unit_test $(top_builddir)/src/common/libpgcommon_srv.a \ $(top_builddir)/src/port/libpgport_srv.a -o $@ +# spec-7.2a: test_cluster_gcs_block_dedup_reclaim verifies the eager-reclaim +# safety decision primitives (review r1): in-window reclaim whitelist +# (GcsBlockReplyStatusIsReclaimIdempotent, currently empty/fail-closed) and +# per-entry reclaim verdict (GcsBlockDedupEntryIsReclaimSafe: in-flight never, +# out-of-window 2x safe for all, in-window idempotent-only). Header-only; +# behavioral coverage (cap-full -> reclaim, evict_count, retransmit-dedup +# correctness) in cluster_tap t/NNN_gcs_block_dedup_capacity.pl. +test_cluster_gcs_block_dedup_reclaim: test_cluster_gcs_block_dedup_reclaim.c unit_test.h \ + $(CLUSTER_VERSION_O) + $(CC) $(CFLAGS) $(CPPFLAGS) $< \ + $(CLUSTER_VERSION_O) \ + $(top_builddir)/src/common/libpgcommon_srv.a \ + $(top_builddir)/src/port/libpgport_srv.a -o $@ + # spec-2.35 D17: test_cluster_gcs_block_2way verifies CF 2-way protocol # compile-time invariants (msg_type 16, status 8, GcsBlockForwardPayload # 64B, reply header reserved 重解读 sizeof 不变, dedup FORWARDED_DUPLICATE diff --git a/src/test/cluster_unit/test_cluster_gcs_block_dedup_reclaim.c b/src/test/cluster_unit/test_cluster_gcs_block_dedup_reclaim.c new file mode 100644 index 0000000000..82fb1483e0 --- /dev/null +++ b/src/test/cluster_unit/test_cluster_gcs_block_dedup_reclaim.c @@ -0,0 +1,224 @@ +/*------------------------------------------------------------------------- + * + * test_cluster_gcs_block_dedup_reclaim.c + * Compile-time + pure-logic invariants for the spec-7.2a GCS block dedup + * capacity + eager-GC hardening (review r1). + * + * Covers the two pure reclaim-safety decision primitives that carry the + * Rule 8.A correctness contract: + * - GcsBlockReplyStatusIsReclaimIdempotent() in-window reclaim whitelist + * - GcsBlockDedupEntryIsReclaimSafe() per-entry reclaim verdict + * + * Header-only — no linking of cluster_gcs_block_dedup.o. The behavioral + * reclaim path (cap-full -> reclaim -> register, evict_count accounting, + * retransmit-dedup correctness under drop-reply injection) lives in + * cluster_tap t/NNN_gcs_block_dedup_capacity.pl, which exercises a real + * 2-node ClusterPair. + * + * Tests in this binary: + * R1 whitelist EMPTY: every status 0..19 is in-window UNSAFE (8.A) + * R2 payload GRANTED never in-window reclaimable + * R3 storage-fallback never in-window (:3305 N->S / :4089 S->X_UPGRADE) + * R4 DENIED_PENDING_X never in-window (:4071 broadcast_invalidate) + * R5 DENIED_LOST_WRITE never in-window (:4393 counter side-effect) + * R6 READ_IMAGE_FROM_XHOLDER never in-window (payload) + * R7 *_FROM_HOLDER forward markers never in-window + * R8 in-flight entry (completed_at_ts == 0) never reclaim-safe + * R9 completed entry aged past the out-of-window (2x) is safe for + * EVERY status, including payload GRANTED (§3.1 theorem) + * R10 completed payload GRANTED still in-window is NOT reclaim-safe + * R11 out-of-window boundary is strict (age == window -> in-window path) + * R12 in-window verdict tracks the idempotent whitelist (empty -> unsafe) + * + * + * Portions Copyright (c) 1996-2024, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * Portions Copyright (c) 2026, pgrac contributors + * + * Author: SqlRush + * + * IDENTIFICATION + * src/test/cluster_unit/test_cluster_gcs_block_dedup_reclaim.c + * + * NOTES + * This is a pgrac-original file. + * Spec: spec-7.2a-gcs-block-dedup-capacity-gc.md (APPROVED 2026-07-09) + * + *------------------------------------------------------------------------- + */ +#include "postgres.h" + +#include + +#include "cluster/cluster_gcs_block.h" +#include "cluster/cluster_gcs_block_dedup.h" +#include "unit_test.h" + +UT_DEFINE_GLOBALS(); + +/* + * Assert hook — this header-only test pulls pg_strfromd from libpgport_srv.a + * (via the snprintf family), which references ExceptionalCondition under + * cassert. Provide the standard local stub (matching the sibling unit tests). + */ +void +ExceptionalCondition(const char *conditionName pg_attribute_unused(), + const char *fileName pg_attribute_unused(), + int lineNumber pg_attribute_unused()) +{ + abort(); +} + +/* + * Build a completed dedup entry with the given reply status whose reply was + * produced `age_us` microseconds before `now`. + */ +static GcsBlockDedupEntry +make_completed_entry(GcsBlockReplyStatus status, TimestampTz now, int64 age_us) +{ + GcsBlockDedupEntry e; + + memset(&e, 0, sizeof(e)); + e.status = (uint8) status; + e.registered_at_ts = now - age_us - 1; + e.completed_at_ts = now - age_us; + return e; +} + +/* -------- in-window reclaim whitelist (8.A) -------- */ + +UT_TEST(test_r1_whitelist_empty_all_statuses_in_window_unsafe) +{ + int s; + + /* Every reply status in the enum range must be fail-closed UNSAFE for + * in-window reclaim until its install sites are proven idempotent. */ + for (s = GCS_BLOCK_REPLY_GRANTED; s <= GCS_BLOCK_REPLY_UNDO_VERDICT_RESULT; s++) + UT_ASSERT_EQ(false, GcsBlockReplyStatusIsReclaimIdempotent((GcsBlockReplyStatus) s)); +} + +UT_TEST(test_r2_payload_granted_never_in_window) +{ + UT_ASSERT_EQ(false, GcsBlockReplyStatusIsReclaimIdempotent(GCS_BLOCK_REPLY_GRANTED)); +} + +UT_TEST(test_r3_storage_fallback_never_in_window) +{ + /* :3305 applies N->S, :4089 applies S->X_UPGRADE before returning + * storage-fallback — not integrally idempotent (spec-7.2a §3.2). */ + UT_ASSERT_EQ(false, + GcsBlockReplyStatusIsReclaimIdempotent(GCS_BLOCK_REPLY_GRANTED_STORAGE_FALLBACK)); +} + +UT_TEST(test_r4_pending_x_never_in_window) +{ + /* :4071 broadcast_invalidate + clear_pending_x before deny. */ + UT_ASSERT_EQ(false, GcsBlockReplyStatusIsReclaimIdempotent(GCS_BLOCK_REPLY_DENIED_PENDING_X)); +} + +UT_TEST(test_r5_lost_write_never_in_window) +{ + /* :4393 lost_write_detected_count++ side effect. */ + UT_ASSERT_EQ(false, GcsBlockReplyStatusIsReclaimIdempotent(GCS_BLOCK_REPLY_DENIED_LOST_WRITE)); +} + +UT_TEST(test_r6_read_image_never_in_window) +{ + UT_ASSERT_EQ(false, + GcsBlockReplyStatusIsReclaimIdempotent(GCS_BLOCK_REPLY_READ_IMAGE_FROM_XHOLDER)); +} + +UT_TEST(test_r7_forward_markers_never_in_window) +{ + UT_ASSERT_EQ(false, + GcsBlockReplyStatusIsReclaimIdempotent(GCS_BLOCK_REPLY_GRANTED_FROM_HOLDER)); + UT_ASSERT_EQ(false, + GcsBlockReplyStatusIsReclaimIdempotent(GCS_BLOCK_REPLY_X_GRANTED_FROM_HOLDER)); +} + +/* -------- per-entry reclaim verdict -------- */ + +UT_TEST(test_r8_in_flight_entry_never_reclaim_safe) +{ + GcsBlockDedupEntry e; + TimestampTz now = 1000000; + + memset(&e, 0, sizeof(e)); + e.status = (uint8) GCS_BLOCK_REPLY_GRANTED; + e.registered_at_ts = now - 5000; + e.completed_at_ts = 0; /* in-flight: original request still processing */ + + UT_ASSERT_EQ(false, GcsBlockDedupEntryIsReclaimSafe(&e, now, 1000)); +} + +UT_TEST(test_r9_aged_out_safe_for_every_status) +{ + TimestampTz now = 1000000; + int64 window = 1000; + GcsBlockDedupEntry granted = make_completed_entry(GCS_BLOCK_REPLY_GRANTED, now, 2000); + GcsBlockDedupEntry sfb = + make_completed_entry(GCS_BLOCK_REPLY_GRANTED_STORAGE_FALLBACK, now, 2000); + GcsBlockDedupEntry lost = make_completed_entry(GCS_BLOCK_REPLY_DENIED_LOST_WRITE, now, 2000); + + /* age 2000us > window 1000us: out-of-window, safe for ALL (2x theorem). */ + UT_ASSERT_EQ(true, GcsBlockDedupEntryIsReclaimSafe(&granted, now, window)); + UT_ASSERT_EQ(true, GcsBlockDedupEntryIsReclaimSafe(&sfb, now, window)); + UT_ASSERT_EQ(true, GcsBlockDedupEntryIsReclaimSafe(&lost, now, window)); +} + +UT_TEST(test_r10_in_window_payload_granted_not_safe) +{ + TimestampTz now = 1000000; + int64 window = 5000; + GcsBlockDedupEntry granted = make_completed_entry(GCS_BLOCK_REPLY_GRANTED, now, 500); + + /* age 500us <= window 5000us: in-window; GRANTED is payload -> unsafe. */ + UT_ASSERT_EQ(false, GcsBlockDedupEntryIsReclaimSafe(&granted, now, window)); +} + +UT_TEST(test_r11_out_of_window_boundary_is_strict) +{ + TimestampTz now = 1000000; + int64 window = 1000; + GcsBlockDedupEntry at_boundary = make_completed_entry(GCS_BLOCK_REPLY_GRANTED, now, 1000); + + /* age == window: NOT strictly greater -> still in-window -> GRANTED unsafe. */ + UT_ASSERT_EQ(false, GcsBlockDedupEntryIsReclaimSafe(&at_boundary, now, window)); +} + +UT_TEST(test_r12_in_window_verdict_tracks_whitelist) +{ + TimestampTz now = 1000000; + int64 window = 5000; + int s; + + /* In-window (age 100 << window): verdict must equal the whitelist for + * every status. Whitelist is empty -> all in-window verdicts false. */ + for (s = GCS_BLOCK_REPLY_GRANTED; s <= GCS_BLOCK_REPLY_UNDO_VERDICT_RESULT; s++) + { + GcsBlockDedupEntry e = make_completed_entry((GcsBlockReplyStatus) s, now, 100); + + UT_ASSERT_EQ(GcsBlockReplyStatusIsReclaimIdempotent((GcsBlockReplyStatus) s), + GcsBlockDedupEntryIsReclaimSafe(&e, now, window)); + } +} + +int +main(void) +{ + UT_PLAN(12); + UT_RUN(test_r1_whitelist_empty_all_statuses_in_window_unsafe); + UT_RUN(test_r2_payload_granted_never_in_window); + UT_RUN(test_r3_storage_fallback_never_in_window); + UT_RUN(test_r4_pending_x_never_in_window); + UT_RUN(test_r5_lost_write_never_in_window); + UT_RUN(test_r6_read_image_never_in_window); + UT_RUN(test_r7_forward_markers_never_in_window); + UT_RUN(test_r8_in_flight_entry_never_reclaim_safe); + UT_RUN(test_r9_aged_out_safe_for_every_status); + UT_RUN(test_r10_in_window_payload_granted_not_safe); + UT_RUN(test_r11_out_of_window_boundary_is_strict); + UT_RUN(test_r12_in_window_verdict_tracks_whitelist); + UT_DONE(); + return ut_failed_count == 0 ? 0 : 1; +} From d5646b56c960c7ed8cd4fa8431ae10515beaf0b7 Mon Sep 17 00:00:00 2001 From: SqlRush Date: Thu, 9 Jul 2026 17:35:49 +0800 Subject: [PATCH 2/7] feat(cluster): gcs block dedup MISS-path eager reclaim + capacity bump Wire the eager-reclaim safety primitives into the master-side dedup HTAB so a cap-full request tries to reclaim a reclaim-safe entry before failing closed, and raise the capacity defaults. cluster_gcs_block_dedup.c: - Add evict_count atomic (init + accessor cluster_gcs_block_dedup_get_evict_count). - dedup_reclaim_reclaimable_locked(now, want): bounded probe (GCS_BLOCK_DEDUP_RECLAIM_MAX_PROBE=256, not a full O(cap) scan under the exclusive lock) that removes up to `want` entries GcsBlockDedupEntryIsReclaimSafe approves; hash_seq_term on early break; accounts entry_count-- / evict_count++. - lookup_or_register MISS path: on HASH_ENTER_NULL NULL, reclaim 1 then retry once; still full -> unchanged DENIED_DEDUP_FULL fail-closed (HC92). - TTL sweep also folds removals into evict_count (D5). cluster_guc.c: cluster.gcs_block_dedup_max_entries default 1024->4096, ceiling 16384->65536; description refreshed (entry 8448B, ~34.7MB/~554MB, eager reclaim, PGC_POSTMASTER restart). Reclaim never removes an entry whose retransmitted duplicate could be re-served incorrectly: with the whitelist empty it only removes entries aged past the 2x out-of-window threshold (safe for every status), so it only brings the FULL decision forward in time. Behavioral coverage (cap-full -> reclaim, evict_count, retransmit-dedup correctness) lands in cluster_tap. Spec: spec-7.2a-gcs-block-dedup-capacity-gc.md --- src/backend/cluster/cluster_gcs_block_dedup.c | 89 ++++++++++++++++++- src/backend/cluster/cluster_guc.c | 19 ++-- src/include/cluster/cluster_gcs_block_dedup.h | 1 + 3 files changed, 100 insertions(+), 9 deletions(-) diff --git a/src/backend/cluster/cluster_gcs_block_dedup.c b/src/backend/cluster/cluster_gcs_block_dedup.c index ee2f2bd837..5f2f5f6806 100644 --- a/src/backend/cluster/cluster_gcs_block_dedup.c +++ b/src/backend/cluster/cluster_gcs_block_dedup.c @@ -60,6 +60,7 @@ typedef struct ClusterGcsBlockDedupShared { pg_atomic_uint64 miss_count; /* MISS_REGISTERED paths */ pg_atomic_uint64 collision_count; /* HC91 VALIDATION_FAIL */ pg_atomic_uint64 full_count; /* HC92 FULL */ + pg_atomic_uint64 evict_count; /* spec-7.2a: eager reclaim + TTL sweep removed */ pg_atomic_uint32 entry_count; /* live in-flight + completed entries */ } ClusterGcsBlockDedupShared; @@ -67,6 +68,18 @@ static ClusterGcsBlockDedupShared *cluster_gcs_block_dedup_shared = NULL; static HTAB *cluster_gcs_block_dedup_htab = NULL; static bool dedup_backend_exit_hook_registered = false; +/* + * Upper bound on entries examined per cap-full eager-reclaim probe. We do + * NOT full-scan the HTAB under the exclusive lock on every cap-full MISS + * (spec-7.2a §6 "非全扫"): if no reclaim-safe entry is found within the probe + * budget, fall back to the fail-closed HC92 path. + */ +#define GCS_BLOCK_DEDUP_RECLAIM_MAX_PROBE 256 + +/* Forward declarations for GC helpers used by the MISS-path eager reclaim. */ +static int64 dedup_expiry_threshold_us(void); +static int dedup_reclaim_reclaimable_locked(TimestampTz now, int want); + static int cluster_gcs_block_dedup_effective_entries(void) @@ -125,6 +138,7 @@ cluster_gcs_block_dedup_shmem_init(void) pg_atomic_init_u64(&cluster_gcs_block_dedup_shared->miss_count, 0); pg_atomic_init_u64(&cluster_gcs_block_dedup_shared->collision_count, 0); pg_atomic_init_u64(&cluster_gcs_block_dedup_shared->full_count, 0); + pg_atomic_init_u64(&cluster_gcs_block_dedup_shared->evict_count, 0); pg_atomic_init_u32(&cluster_gcs_block_dedup_shared->entry_count, 0); } @@ -245,6 +259,17 @@ cluster_gcs_block_dedup_lookup_or_register(const GcsBlockDedupKey *key, BufferTa * with cap reached; convert to FULL fail-closed (HC92). */ entry = (GcsBlockDedupEntry *)hash_search(cluster_gcs_block_dedup_htab, key, HASH_ENTER_NULL, &found); + if (entry == NULL) { + /* PGRAC: spec-7.2a D1 — before failing closed, try to reclaim one + * reclaim-safe entry (aged past the 2x window, or a site-proven + * idempotent status) to make room. Reclaim never removes an entry + * whose retransmitted duplicate could still be served incorrectly + * (§3.1); if nothing is safe to reclaim, keep the fail-closed HC92 + * behavior below. */ + if (dedup_reclaim_reclaimable_locked(GetCurrentTimestamp(), 1) > 0) + entry = (GcsBlockDedupEntry *)hash_search(cluster_gcs_block_dedup_htab, key, + HASH_ENTER_NULL, &found); + } if (entry == NULL) { pg_atomic_fetch_add_u64(&cluster_gcs_block_dedup_shared->full_count, 1); LWLockRelease(&cluster_gcs_block_dedup_shared->lock.lock); @@ -381,6 +406,57 @@ dedup_expiry_threshold_us(void) return total_backoff_ms * 1000 * 2; /* × 2 safety margin (HC93) */ } +/* + * dedup_reclaim_reclaimable_locked -- caller MUST hold the dedup lock + * exclusively. Under cap pressure (HASH_ENTER_NULL about to fail), scan the + * HTAB and remove up to `want` reclaim-safe entries so the MISS path can + * register instead of failing closed (spec-7.2a D1). + * + * Only entries GcsBlockDedupEntryIsReclaimSafe() approves are removed: an + * entry aged past the 2x out-of-window threshold (safe for every status, + * §3.1 theorem) or one whose status is site-proven in-window idempotent + * (whitelist currently empty). Reclaim NEVER removes an entry whose + * retransmitted duplicate could be re-served incorrectly — it only ever + * brings the FULL path forward in time, never sacrifices correctness. + * + * The scan is bounded to GCS_BLOCK_DEDUP_RECLAIM_MAX_PROBE entries so a table + * full of in-window entries does not turn every MISS into a full O(cap) scan + * under the exclusive lock (spec-7.2a §6). Returns the number reclaimed. + */ +static int +dedup_reclaim_reclaimable_locked(TimestampTz now, int want) +{ + HASH_SEQ_STATUS seq; + GcsBlockDedupEntry *entry; + int64 out_of_window_us; + int reclaimed = 0; + int probed = 0; + + if (want <= 0) + return 0; + + out_of_window_us = dedup_expiry_threshold_us(); + + hash_seq_init(&seq, cluster_gcs_block_dedup_htab); + while ((entry = (GcsBlockDedupEntry *)hash_seq_search(&seq)) != NULL) { + if (GcsBlockDedupEntryIsReclaimSafe(entry, now, out_of_window_us)) { + (void)hash_search(cluster_gcs_block_dedup_htab, &entry->key, HASH_REMOVE, NULL); + reclaimed++; + } + + if (reclaimed >= want || ++probed >= GCS_BLOCK_DEDUP_RECLAIM_MAX_PROBE) { + hash_seq_term(&seq); /* early break must terminate the scan */ + break; + } + } + + if (reclaimed > 0) { + pg_atomic_fetch_sub_u32(&cluster_gcs_block_dedup_shared->entry_count, (uint32)reclaimed); + pg_atomic_fetch_add_u64(&cluster_gcs_block_dedup_shared->evict_count, (uint64)reclaimed); + } + return reclaimed; +} + void cluster_gcs_block_dedup_sweep_expired(TimestampTz now) { @@ -412,8 +488,11 @@ cluster_gcs_block_dedup_sweep_expired(TimestampTz now) } } - if (removed > 0) + if (removed > 0) { pg_atomic_fetch_sub_u32(&cluster_gcs_block_dedup_shared->entry_count, (uint32)removed); + /* spec-7.2a D5: evict_count aggregates eager reclaim + TTL sweep. */ + pg_atomic_fetch_add_u64(&cluster_gcs_block_dedup_shared->evict_count, (uint64)removed); + } LWLockRelease(&cluster_gcs_block_dedup_shared->lock.lock); } @@ -510,5 +589,13 @@ cluster_gcs_block_dedup_get_in_flight_count(void) : 0; } +uint64 +cluster_gcs_block_dedup_get_evict_count(void) +{ + return cluster_gcs_block_dedup_shared + ? pg_atomic_read_u64(&cluster_gcs_block_dedup_shared->evict_count) + : 0; +} + #endif /* USE_PGRAC_CLUSTER */ diff --git a/src/backend/cluster/cluster_guc.c b/src/backend/cluster/cluster_guc.c index a29528126e..3d6fce3340 100644 --- a/src/backend/cluster/cluster_guc.c +++ b/src/backend/cluster/cluster_guc.c @@ -732,7 +732,7 @@ bool cluster_ic_duty_lazy = true; /* spec-7.2 D1 duty-chain on-demand gating */ * Off keeps the pre-7.1a floor: a TERMINAL remote writer holder fails * closed (SQLSTATE 53R9H) instead of chaining to a sound TM_Result. */ bool cluster_crossnode_write_write = false; -int cluster_gcs_block_dedup_max_entries = 1024; +int cluster_gcs_block_dedup_max_entries = 4096; /* spec-7.2a: raised from 1024 */ /* * PGRAC: spec-4.7 D1 — cluster.gcs_block_recovery_wait_ms. Bounded backend @@ -3946,14 +3946,17 @@ cluster_init_guc(void) DefineCustomIntVariable("cluster.gcs_block_dedup_max_entries", gettext_noop("Master-side GCS block dedup HTAB capacity (entries)."), - gettext_noop("Each entry occupies sizeof(GcsBlockDedupEntry) = 8312B. " - "Default 1024 → ~8.4MB shmem on each node serving as " - "GCS block-ship master; bootstrap/initdb with no " - "configured cluster.node_id does not allocate the HTAB. " - "HASH_ENTER_NULL on cap → " + gettext_noop("Each entry occupies sizeof(GcsBlockDedupEntry) = 8448B. " + "Default 4096 → ~34.7MB shmem on each node serving as " + "GCS block-ship master; ceiling 65536 → ~554MB; " + "bootstrap/initdb with no configured cluster.node_id does " + "not allocate the HTAB. Under cap pressure the master " + "eagerly reclaims reclaim-safe entries before failing " + "closed; HASH_ENTER_NULL on a still-full table → " "DENIED_DEDUP_FULL fail-closed (sender retries via " - "HC96 transient). HC92. PGC_POSTMASTER."), - &cluster_gcs_block_dedup_max_entries, 1024, 256, 16384, PGC_POSTMASTER, + "HC96 transient). HC92. PGC_POSTMASTER (restart to " + "change the fixed HTAB size)."), + &cluster_gcs_block_dedup_max_entries, 4096, 256, 65536, PGC_POSTMASTER, 0, NULL, NULL, NULL); /* diff --git a/src/include/cluster/cluster_gcs_block_dedup.h b/src/include/cluster/cluster_gcs_block_dedup.h index 7cf28ea81f..542644dae1 100644 --- a/src/include/cluster/cluster_gcs_block_dedup.h +++ b/src/include/cluster/cluster_gcs_block_dedup.h @@ -361,6 +361,7 @@ extern uint64 cluster_gcs_block_dedup_get_miss_count(void); extern uint64 cluster_gcs_block_dedup_get_collision_count(void); extern uint64 cluster_gcs_block_dedup_get_full_count(void); extern uint64 cluster_gcs_block_dedup_get_in_flight_count(void); +extern uint64 cluster_gcs_block_dedup_get_evict_count(void); /* spec-7.2a D5 */ /* ============================================================ From 5612a5776fb8ea236ead27bb2590db3016e68c10 Mon Sep 17 00:00:00 2001 From: SqlRush Date: Thu, 9 Jul 2026 22:54:41 +0800 Subject: [PATCH 3/7] =?UTF-8?q?feat(cluster):=20spec-7.2a=20D5/D6=20?= =?UTF-8?q?=E2=80=94=20dedup=20capacity=20observability=20+=20eager-GC=20L?= =?UTF-8?q?OG-once=20+=20count-based=20injection=20+=202-node=20capacity?= =?UTF-8?q?=20TAP?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Observability (D5): - dump_gcs: 3 NEW dedup rows (dedup_entry_count / dedup_evict_count / dedup_max_entries) via cluster_gcs_get_block_dedup_* wrappers - dedup TTL sweep: saturation LOG-once when DENIED_DEDUP_FULL grows past a threshold, lock-free, no hot-path flood Injection framework: - new CLUSTER_FAULT_SKIP_N (skipn:N) count-based skip — drops exactly N times then stops so the sender's retransmit lands; existing skip:N stays unlimited (zero regression, verified by t/215) - test-only GUC cluster.gcs_block_drop_target_relfilenode gates the drop-reply injection to one relation (default 0 = any) Tests: - new t/365_gcs_block_dedup_capacity_2node.pl (L1-L5, shared_catalog 2-node): default cap, cross-node distinct-read dedup_full=0, retransmit path exercised with correct re-serve, reclaim eviction, dump rows - baseline sweep: pg_cluster_state gcs category 85->88 across t/110-116; cluster.gcs_block_dedup_max_entries default 1024->4096 (t/112) - nightly.yml: stage7-gcs-dedup-capacity shard (t/365) Spec: spec-7.2a-gcs-block-dedup-capacity-gc.md --- .github/workflows/nightly.yml | 5 + src/backend/cluster/cluster_debug.c | 9 + src/backend/cluster/cluster_gcs_block.c | 51 +- src/backend/cluster/cluster_gcs_block_dedup.c | 46 ++ src/backend/cluster/cluster_guc.c | 16 + src/backend/cluster/cluster_inject.c | 40 ++ src/include/cluster/cluster_gcs_block.h | 3 + src/include/cluster/cluster_gcs_block_dedup.h | 1 + src/include/cluster/cluster_guc.h | 1 + src/include/cluster/cluster_inject.h | 1 + src/test/cluster_tap/t/110_gcs_loopback.pl | 8 +- .../cluster_tap/t/111_gcs_block_ship_2node.pl | 12 +- .../t/112_gcs_block_retransmit_2node.pl | 16 +- .../cluster_tap/t/113_gcs_block_2way_2node.pl | 12 +- .../cluster_tap/t/114_gcs_block_3way_2node.pl | 12 +- .../cluster_tap/t/115_gcs_block_3way_3node.pl | 4 +- .../t/116_gcs_block_lost_write_2node.pl | 8 +- .../t/365_gcs_block_dedup_capacity_2node.pl | 488 ++++++++++++++++++ 18 files changed, 690 insertions(+), 43 deletions(-) create mode 100644 src/test/cluster_tap/t/365_gcs_block_dedup_capacity_2node.pl diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 9a8b71e0aa..b17355f358 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -192,6 +192,11 @@ jobs: - { name: stage7-marker-async, ranges: "363-363", unit: false, regress: false } # t/365 spec-7.1a cross-instance write-write MVCC coordination. - { name: stage7-write-write, ranges: "365-365", unit: false, regress: false } + # t/365 spec-7.2a GCS block dedup capacity + eager reclaim. Own shard: + # 2-node shared_catalog bring-up + cross-node distinct-read dedup + # pressure + injected retransmit-dedup correctness needs the wall + # clock. (t/364 reserved by the parallel spec-7.3 LMS-pool lane.) + - { name: stage7-gcs-dedup-capacity, ranges: "365-365", unit: false, regress: false } steps: - name: Checkout uses: actions/checkout@v4 diff --git a/src/backend/cluster/cluster_debug.c b/src/backend/cluster/cluster_debug.c index 5bd58e4a6a..8783bf0a0c 100644 --- a/src/backend/cluster/cluster_debug.c +++ b/src/backend/cluster/cluster_debug.c @@ -1938,6 +1938,15 @@ dump_gcs(ReturnSetInfo *rsinfo) fmt_int64((int64)cluster_gcs_get_block_dedup_collision_count())); emit_row(rsinfo, "gcs", "dedup_full_count", fmt_int64((int64)cluster_gcs_get_block_dedup_full_count())); + /* spec-7.2a D5: 3 NEW dedup capacity/occupancy rows. entry_count / + * max_entries give the saturation ratio; evict_count counts eager + * reclaim + TTL-sweep removals (dump_gcs gcs-category 85 -> 88). */ + emit_row(rsinfo, "gcs", "dedup_entry_count", + fmt_int64((int64)cluster_gcs_get_block_dedup_entry_count())); + emit_row(rsinfo, "gcs", "dedup_evict_count", + fmt_int64((int64)cluster_gcs_get_block_dedup_evict_count())); + emit_row(rsinfo, "gcs", "dedup_max_entries", + fmt_int64((int64)cluster_gcs_get_block_dedup_max_entries())); emit_row(rsinfo, "gcs", "epoch_invalidate_wake_count", fmt_int64((int64)cluster_gcs_get_block_epoch_invalidate_wake_count())); emit_row(rsinfo, "gcs", "stale_reply_drop_count", diff --git a/src/backend/cluster/cluster_gcs_block.c b/src/backend/cluster/cluster_gcs_block.c index e8cbabf038..55e6397969 100644 --- a/src/backend/cluster/cluster_gcs_block.c +++ b/src/backend/cluster/cluster_gcs_block.c @@ -4652,13 +4652,25 @@ build_and_send_reply: { * active on that retry). Useful for driving the * retransmit_send_count + dedup_hit_count TAP surfaces. */ - CLUSTER_INJECTION_POINT("cluster-gcs-block-drop-reply-before-send"); - if (cluster_injection_should_skip("cluster-gcs-block-drop-reply-before-send")) { - gcs_block_release_ship_image(block_payload_release_cb, block_payload_release_arg); - block_payload_release_cb = NULL; - block_payload_release_arg = NULL; - pfree(buf); - return; + /* + * spec-7.2a: gate the drop-reply dispatch on the test target relfilenode. + * A :skipn:N count is per-process global; without this gate an unrelated + * (catalog / internal) block ship consumes the countdown before the test's + * user-relation ship reaches the point. Gating the CLUSTER_INJECTION_POINT + * itself (not just should_skip) ensures only matching ships consume the + * count. 0 (default) keeps the un-targeted behaviour for spec-2.34 tests. + */ + if (cluster_gcs_block_drop_target_relfilenode == 0 + || BufTagGetRelNumber(&req->tag) + == (RelFileNumber) cluster_gcs_block_drop_target_relfilenode) { + CLUSTER_INJECTION_POINT("cluster-gcs-block-drop-reply-before-send"); + if (cluster_injection_should_skip("cluster-gcs-block-drop-reply-before-send")) { + gcs_block_release_ship_image(block_payload_release_cb, block_payload_release_arg); + block_payload_release_cb = NULL; + block_payload_release_arg = NULL; + pfree(buf); + return; + } } { @@ -7070,6 +7082,31 @@ cluster_gcs_get_block_dedup_full_count(void) return cluster_gcs_block_dedup_get_full_count(); } +/* + * spec-7.2a D5: dedup capacity/occupancy observability wrappers. The + * entry_count wrapper reads the historical _get_in_flight_count accessor, + * whose backing counter (entry_count) tracks every live entry (in-flight + * slots plus completed cached replies) -- that live total is what dump_gcs + * surfaces as dedup_entry_count for the saturation ratio. + */ +uint64 +cluster_gcs_get_block_dedup_entry_count(void) +{ + return cluster_gcs_block_dedup_get_in_flight_count(); +} + +uint64 +cluster_gcs_get_block_dedup_evict_count(void) +{ + return cluster_gcs_block_dedup_get_evict_count(); +} + +uint64 +cluster_gcs_get_block_dedup_max_entries(void) +{ + return cluster_gcs_block_dedup_get_max_entries(); +} + /* ============================================================ * PGRAC: spec-2.34 D4 — eager wake on epoch advance. diff --git a/src/backend/cluster/cluster_gcs_block_dedup.c b/src/backend/cluster/cluster_gcs_block_dedup.c index 5f2f5f6806..b07e72e45c 100644 --- a/src/backend/cluster/cluster_gcs_block_dedup.c +++ b/src/backend/cluster/cluster_gcs_block_dedup.c @@ -76,6 +76,14 @@ static bool dedup_backend_exit_hook_registered = false; */ #define GCS_BLOCK_DEDUP_RECLAIM_MAX_PROBE 256 +/* + * spec-7.2a D5: emit a saturation LOG only after this many additional + * DENIED_DEDUP_FULL events accrue since the last report, so a persistently + * full table logs at most once per this many drops (rule 17: no hot-path + * flood). The LMON TTL sweep is the sole evaluation site. + */ +#define GCS_BLOCK_DEDUP_FULL_LOG_THRESHOLD 64 + /* Forward declarations for GC helpers used by the MISS-path eager reclaim. */ static int64 dedup_expiry_threshold_us(void); static int dedup_reclaim_reclaimable_locked(TimestampTz now, int want); @@ -470,6 +478,29 @@ cluster_gcs_block_dedup_sweep_expired(TimestampTz now) threshold_us = dedup_expiry_threshold_us(); + /* + * spec-7.2a D5: saturation LOG-once. When DENIED_DEDUP_FULL keeps growing + * past a threshold across sweep cycles, emit one LOG so operators see + * sustained dedup saturation without flooding the hot request path (rule + * 17). Lock-free (atomics only); the LMON sweep is the sole caller so the + * static high-water mark is process-local and race-free. + */ + { + static uint64 dedup_full_logged_hwm = 0; + uint64 cur_full = pg_atomic_read_u64(&cluster_gcs_block_dedup_shared->full_count); + + if (cur_full - dedup_full_logged_hwm >= GCS_BLOCK_DEDUP_FULL_LOG_THRESHOLD) + { + elog(LOG, + "GCS block dedup saturating: %llu new DENIED_DEDUP_FULL since last report " + "(cap=%d, live entries=%u); raise cluster.gcs_block_dedup_max_entries if sustained", + (unsigned long long) (cur_full - dedup_full_logged_hwm), + cluster_gcs_block_dedup_effective_entries(), + pg_atomic_read_u32(&cluster_gcs_block_dedup_shared->entry_count)); + dedup_full_logged_hwm = cur_full; + } + } + LWLockAcquire(&cluster_gcs_block_dedup_shared->lock.lock, LW_EXCLUSIVE); hash_seq_init(&seq, cluster_gcs_block_dedup_htab); @@ -597,5 +628,20 @@ cluster_gcs_block_dedup_get_evict_count(void) : 0; } +/* + * cluster_gcs_block_dedup_get_max_entries -- effective dedup capacity. + * + * spec-7.2a D5: reports the effective entry ceiling resolved by + * cluster_gcs_block_dedup_effective_entries() (the GUC value, or 0 during + * initdb/bootstrap before a cluster node id is assigned). The occupancy + * ratio entry_count / max_entries is the saturation signal behind the + * DEDUP_FULL fail-closed path. + */ +uint64 +cluster_gcs_block_dedup_get_max_entries(void) +{ + return (uint64) cluster_gcs_block_dedup_effective_entries(); +} + #endif /* USE_PGRAC_CLUSTER */ diff --git a/src/backend/cluster/cluster_guc.c b/src/backend/cluster/cluster_guc.c index 3d6fce3340..bd5c517047 100644 --- a/src/backend/cluster/cluster_guc.c +++ b/src/backend/cluster/cluster_guc.c @@ -733,6 +733,12 @@ bool cluster_ic_duty_lazy = true; /* spec-7.2 D1 duty-chain on-demand gating */ * closed (SQLSTATE 53R9H) instead of chaining to a sound TM_Result. */ bool cluster_crossnode_write_write = false; int cluster_gcs_block_dedup_max_entries = 4096; /* spec-7.2a: raised from 1024 */ +/* + * spec-7.2a test-only: when non-zero, the drop-reply injection only fires for + * block ships of this relfilenode, so a :skipn:N count lands on the intended + * relation and is not consumed by unrelated catalog/internal ships. 0 = any. + */ +int cluster_gcs_block_drop_target_relfilenode = 0; /* * PGRAC: spec-4.7 D1 — cluster.gcs_block_recovery_wait_ms. Bounded backend @@ -3959,6 +3965,16 @@ cluster_init_guc(void) &cluster_gcs_block_dedup_max_entries, 4096, 256, 65536, PGC_POSTMASTER, 0, NULL, NULL, NULL); + DefineCustomIntVariable("cluster.gcs_block_drop_target_relfilenode", + gettext_noop("Test-only: restrict the drop-reply injection to one relfilenode."), + gettext_noop("When non-zero, cluster-gcs-block-drop-reply-before-send only " + "fires for block ships whose relfilenode matches this value, so " + "a :skipn:N count is spent on the intended relation and not on " + "unrelated catalog/internal ships. 0 (default) disables the " + "filter. For TAP retransmit-dedup correctness tests only."), + &cluster_gcs_block_drop_target_relfilenode, 0, 0, INT_MAX, PGC_SUSET, + 0, NULL, NULL, NULL); + /* * PGRAC: spec-2.36 D8 — 3 NEW GUC for CF 3-way (X transfer + * reader starvation). diff --git a/src/backend/cluster/cluster_inject.c b/src/backend/cluster/cluster_inject.c index edd02c568f..fee0065973 100644 --- a/src/backend/cluster/cluster_inject.c +++ b/src/backend/cluster/cluster_inject.c @@ -68,6 +68,7 @@ typedef struct ClusterInjectPoint { pg_atomic_uint32 armed_type; /* ClusterInjectFaultType */ pg_atomic_uint64 armed_param; /* sleep us / SKIP cookie */ pg_atomic_uint32 skip_pending; /* set by SKIP dispatch, read+reset by probe */ + pg_atomic_uint32 skip_remaining; /* spec-7.2a: SKIP countdown for :skip:N (param 0 = unlimited) */ } ClusterInjectPoint; /* @@ -750,6 +751,7 @@ cluster_injection_initialise(void) pg_atomic_init_u32(&cluster_injection_points[i].armed_type, CLUSTER_FAULT_NONE); pg_atomic_init_u64(&cluster_injection_points[i].armed_param, 0); pg_atomic_init_u32(&cluster_injection_points[i].skip_pending, 0); + pg_atomic_init_u32(&cluster_injection_points[i].skip_remaining, 0); } cluster_injection_initialised = true; } @@ -791,6 +793,8 @@ parse_fault_type(const char *s) return CLUSTER_FAULT_CRASH; if (pg_strcasecmp(s, "skip") == 0) return CLUSTER_FAULT_SKIP; + if (pg_strcasecmp(s, "skipn") == 0) + return CLUSTER_FAULT_SKIP_N; return CLUSTER_FAULT_NONE; /* unknown -> NONE; caller may warn */ } @@ -859,6 +863,8 @@ fault_type_name(ClusterInjectFaultType t) return "crash"; case CLUSTER_FAULT_SKIP: return "skip"; + case CLUSTER_FAULT_SKIP_N: + return "skipn"; } return "unknown"; } @@ -882,6 +888,18 @@ cluster_injection_arm_internal(ClusterInjectPoint *p, ClusterInjectFaultType new old_type = pg_atomic_exchange_u32(&p->armed_type, new_type); pg_atomic_write_u64(&p->armed_param, (uint64)param); + /* + * spec-7.2a: SKIP_N is count-based -- it skips exactly `param` times, then + * falls through (dispatch stops setting skip_pending, so the sender's + * retransmit succeeds instead of exhausting its budget). Reset the + * countdown on every (re-)arm. Plain SKIP keeps the legacy unlimited + * "skip while armed" behaviour and leaves the countdown at 0. + */ + if (new_type == CLUSTER_FAULT_SKIP_N && param > 0) + pg_atomic_write_u32(&p->skip_remaining, (uint32) param); + else + pg_atomic_write_u32(&p->skip_remaining, 0); + if (old_type == CLUSTER_FAULT_NONE && new_type != CLUSTER_FAULT_NONE) cluster_injection_armed_count++; else if (old_type != CLUSTER_FAULT_NONE && new_type == CLUSTER_FAULT_NONE) @@ -977,8 +995,30 @@ cluster_injection_run(const char *name) break; case CLUSTER_FAULT_SKIP: + /* Unlimited skip while armed (legacy: param is ignored). */ pg_atomic_write_u32(&p->skip_pending, 1); break; + + case CLUSTER_FAULT_SKIP_N: { + /* + * spec-7.2a count-based skip: consume one unit of the countdown and + * only arm skip_pending while units remain. The CAS loop keeps the + * decrement race-free across concurrent dispatchers and never + * underflows past 0 -- once exhausted the point stops skipping so the + * sender's retransmit lands (and hits the dedup cache). + */ + uint32 expected = pg_atomic_read_u32(&p->skip_remaining); + + while (expected > 0) { + if (pg_atomic_compare_exchange_u32(&p->skip_remaining, + &expected, expected - 1)) { + pg_atomic_write_u32(&p->skip_pending, 1); + break; + } + /* expected reloaded by the failed CAS; retry */ + } + break; + } } } diff --git a/src/include/cluster/cluster_gcs_block.h b/src/include/cluster/cluster_gcs_block.h index ae70003f91..bc683a3009 100644 --- a/src/include/cluster/cluster_gcs_block.h +++ b/src/include/cluster/cluster_gcs_block.h @@ -1992,6 +1992,9 @@ extern uint64 cluster_gcs_get_block_dedup_hit_count(void); extern uint64 cluster_gcs_get_block_dedup_miss_count(void); extern uint64 cluster_gcs_get_block_dedup_collision_count(void); extern uint64 cluster_gcs_get_block_dedup_full_count(void); +extern uint64 cluster_gcs_get_block_dedup_entry_count(void); /* spec-7.2a D5 */ +extern uint64 cluster_gcs_get_block_dedup_evict_count(void); /* spec-7.2a D5 */ +extern uint64 cluster_gcs_get_block_dedup_max_entries(void); /* spec-7.2a D5 */ extern uint64 cluster_gcs_get_block_epoch_invalidate_wake_count(void); extern uint64 cluster_gcs_get_block_stale_reply_drop_count(void); diff --git a/src/include/cluster/cluster_gcs_block_dedup.h b/src/include/cluster/cluster_gcs_block_dedup.h index 542644dae1..8f47a1c15d 100644 --- a/src/include/cluster/cluster_gcs_block_dedup.h +++ b/src/include/cluster/cluster_gcs_block_dedup.h @@ -362,6 +362,7 @@ extern uint64 cluster_gcs_block_dedup_get_collision_count(void); extern uint64 cluster_gcs_block_dedup_get_full_count(void); extern uint64 cluster_gcs_block_dedup_get_in_flight_count(void); extern uint64 cluster_gcs_block_dedup_get_evict_count(void); /* spec-7.2a D5 */ +extern uint64 cluster_gcs_block_dedup_get_max_entries(void); /* spec-7.2a D5 */ /* ============================================================ diff --git a/src/include/cluster/cluster_guc.h b/src/include/cluster/cluster_guc.h index 4e0810e60d..5a98887783 100644 --- a/src/include/cluster/cluster_guc.h +++ b/src/include/cluster/cluster_guc.h @@ -945,6 +945,7 @@ extern bool cluster_tx_enqueue_wait_enabled; /* spec-5.2 D4 */ extern bool cluster_ic_duty_lazy; extern bool cluster_crossnode_write_write; /* spec-7.1a D0 */ extern int cluster_gcs_block_dedup_max_entries; +extern int cluster_gcs_block_drop_target_relfilenode; /* spec-7.2a test-only */ /* PGRAC: spec-4.7 D1 — bounded wait (ms) on a RECOVERING block resource * before fail-closing 53R9L. See cluster_guc.c for semantics. */ diff --git a/src/include/cluster/cluster_inject.h b/src/include/cluster/cluster_inject.h index 159141c1cb..614ab14d91 100644 --- a/src/include/cluster/cluster_inject.h +++ b/src/include/cluster/cluster_inject.h @@ -77,6 +77,7 @@ typedef enum ClusterInjectFaultType { CLUSTER_FAULT_SLEEP = 3, CLUSTER_FAULT_CRASH = 4, CLUSTER_FAULT_SKIP = 5, + CLUSTER_FAULT_SKIP_N = 6, /* spec-7.2a: count-based skip (:skipn:N drops N times, then stops) */ } ClusterInjectFaultType; diff --git a/src/test/cluster_tap/t/110_gcs_loopback.pl b/src/test/cluster_tap/t/110_gcs_loopback.pl index 9f508ee72c..effcf30e9c 100644 --- a/src/test/cluster_tap/t/110_gcs_loopback.pl +++ b/src/test/cluster_tap/t/110_gcs_loopback.pl @@ -7,7 +7,7 @@ # any wire send (HC72), so wire path coverage is effectively limited # to SQL-visible surface invariants: # -# L1 fresh cluster startup: pg_cluster_state.gcs has 85 keys +# L1 fresh cluster startup: pg_cluster_state.gcs has 88 keys # L2 api_state = "active" after postmaster phase 1 init # L3 WAIT_EVENT_GCS_REPLY_WAIT registered in pg_stat_cluster_wait_events # L4 CLUSTER_WAIT_EVENTS_COUNT == 120 (cumulative through spec-7.2) @@ -65,12 +65,12 @@ sub gcs_value { $node->start; -# L1 — pg_cluster_state.gcs surface has 85 keys (spec-7.2 D6 hist). +# L1 — pg_cluster_state.gcs surface has 88 keys (spec-7.2 D6 hist). is($node->safe_psql( 'postgres', q{SELECT count(*) FROM pg_cluster_state WHERE category='gcs'}), - '85', - 'L1 pg_cluster_state.gcs category has 85 keys (spec-7.2 D6)'); + '88', + 'L1 pg_cluster_state.gcs category has 88 keys (spec-7.2 D6)'); # L2 — api_state = "active" after postmaster phase 1 init. diff --git a/src/test/cluster_tap/t/111_gcs_block_ship_2node.pl b/src/test/cluster_tap/t/111_gcs_block_ship_2node.pl index 323f16e364..fb04002d39 100644 --- a/src/test/cluster_tap/t/111_gcs_block_ship_2node.pl +++ b/src/test/cluster_tap/t/111_gcs_block_ship_2node.pl @@ -8,7 +8,7 @@ # # L1 ClusterPair startup — both postmasters healthy + tier1 connected # L2 fresh baseline gcs counters on both nodes (block_* counters = 0) -# L3 pg_cluster_state.gcs category has 85 keys (spec-7.2 D6+flip) (cumulative through +# L3 pg_cluster_state.gcs category has 88 keys (spec-7.2 D6+flip) (cumulative through # spec-6.13 direct-land observability) # L4 4 NEW wait events registered in pg_stat_cluster_wait_events: # ClusterGCSBlockShipWait, ClusterGCSBlockRequestDispatch, @@ -113,19 +113,19 @@ sub gcs_int_value { # ============================================================ -# L3: pg_cluster_state.gcs category has 85 keys (spec-7.2 D6+flip) +# L3: pg_cluster_state.gcs category has 88 keys (spec-7.2 D6+flip) # (cumulative GCS surface through spec-6.13 direct-land observability). # ============================================================ is($pair->node0->safe_psql( 'postgres', q{SELECT count(*) FROM pg_cluster_state WHERE category='gcs'}), - '85', - 'L3 node0 pg_cluster_state.gcs category has 85 keys (spec-7.2 D6+flip)'); + '88', + 'L3 node0 pg_cluster_state.gcs category has 88 keys (spec-7.2 D6+flip)'); is($pair->node1->safe_psql( 'postgres', q{SELECT count(*) FROM pg_cluster_state WHERE category='gcs'}), - '85', - 'L3 node1 pg_cluster_state.gcs category has 85 keys (spec-7.2 D6+flip)'); + '88', + 'L3 node1 pg_cluster_state.gcs category has 88 keys (spec-7.2 D6+flip)'); # ============================================================ diff --git a/src/test/cluster_tap/t/112_gcs_block_retransmit_2node.pl b/src/test/cluster_tap/t/112_gcs_block_retransmit_2node.pl index 95ad463b15..23102c811a 100644 --- a/src/test/cluster_tap/t/112_gcs_block_retransmit_2node.pl +++ b/src/test/cluster_tap/t/112_gcs_block_retransmit_2node.pl @@ -9,14 +9,14 @@ # # L1 ClusterPair startup baseline (both postmasters healthy) # L2 fresh baseline: 9 NEW reliability counters all 0 on both nodes -# L3 pg_cluster_state.gcs has 85 keys (cumulative through spec-7.2 D6 hist) +# L3 pg_cluster_state.gcs has 88 keys (cumulative through spec-7.2 D6 hist) # L4 2 NEW wait events registered (ClusterGCSBlockRetransmitWait + # ClusterGCSBlockEpochStaleRetry) # L5 CLUSTER_WAIT_EVENTS_COUNT = 120 (spec-7.2 +2) # L6 3 NEW GUC visible + defaults + contexts: # cluster.gcs_block_retransmit_max_retries PGC_SUSET 4 # cluster.gcs_block_retransmit_initial_backoff_ms PGC_SUSET 10 (spec-7.2 D1) -# cluster.gcs_block_dedup_max_entries PGC_POSTMASTER 1024 +# cluster.gcs_block_dedup_max_entries PGC_POSTMASTER 4096 # L7 single-shot ship workload — retransmit_attempt_count=0 # L8 inject `cluster-gcs-block-drop-reply-before-send:skip:1` → # retransmit_send_count grows + WARNING at 3/4 budget @@ -107,18 +107,18 @@ sub gcs_int # ============================================================ -# L3: pg_cluster_state.gcs category has 85 keys (cumulative through spec-7.2 D6 hist). +# L3: pg_cluster_state.gcs category has 88 keys (cumulative through spec-7.2 D6 hist). # ============================================================ is($pair->node0->safe_psql( 'postgres', q{SELECT count(*) FROM pg_cluster_state WHERE category='gcs'}), - '85', - 'L3 node0 pg_cluster_state.gcs category has 85 keys'); + '88', + 'L3 node0 pg_cluster_state.gcs category has 88 keys'); is($pair->node1->safe_psql( 'postgres', q{SELECT count(*) FROM pg_cluster_state WHERE category='gcs'}), - '85', - 'L3 node1 pg_cluster_state.gcs category has 85 keys'); + '88', + 'L3 node1 pg_cluster_state.gcs category has 88 keys'); # ============================================================ @@ -154,7 +154,7 @@ sub gcs_int for my $row ( [ 'cluster.gcs_block_retransmit_max_retries', '4', 'superuser' ], [ 'cluster.gcs_block_retransmit_initial_backoff_ms', '10', 'superuser' ], - [ 'cluster.gcs_block_dedup_max_entries', '1024', 'postmaster' ], + [ 'cluster.gcs_block_dedup_max_entries', '4096', 'postmaster' ], ) { my ($name, $expected_default, $expected_ctx) = @$row; diff --git a/src/test/cluster_tap/t/113_gcs_block_2way_2node.pl b/src/test/cluster_tap/t/113_gcs_block_2way_2node.pl index c04dc7dcb2..44da5967e6 100644 --- a/src/test/cluster_tap/t/113_gcs_block_2way_2node.pl +++ b/src/test/cluster_tap/t/113_gcs_block_2way_2node.pl @@ -9,7 +9,7 @@ # # L1 ClusterPair startup baseline (both postmasters healthy) # L2 fresh baseline: 7 NEW counters all 0 + catversion >= 202605420 -# L3 pg_cluster_state.gcs category has 85 keys (spec-7.2 D6+flip) (cumulative through spec-6.13) +# L3 pg_cluster_state.gcs category has 88 keys (spec-7.2 D6+flip) (cumulative through spec-6.13) # L4 cross-node forward path: node A read first → master_holder = A; # force same tag on node B via test-only injection → master # chooses forward path → A direct-ships to B → block_forward_sent @@ -108,18 +108,18 @@ sub gcs_int # ============================================================ -# L3: pg_cluster_state.gcs has 85 keys (spec-7.2 D6+flip; was 67 cumulative through spec-6.14a). +# L3: pg_cluster_state.gcs has 88 keys (spec-7.2 D6+flip; was 67 cumulative through spec-6.14a). # ============================================================ is($pair->node0->safe_psql( 'postgres', q{SELECT count(*) FROM pg_cluster_state WHERE category='gcs'}), - '85', - 'L3 node0 pg_cluster_state.gcs category has 85 keys (spec-7.2 D6+flip)'); + '88', + 'L3 node0 pg_cluster_state.gcs category has 88 keys (spec-7.2 D6+flip)'); is($pair->node1->safe_psql( 'postgres', q{SELECT count(*) FROM pg_cluster_state WHERE category='gcs'}), - '85', - 'L3 node1 pg_cluster_state.gcs category has 85 keys (spec-7.2 D6+flip)'); + '88', + 'L3 node1 pg_cluster_state.gcs category has 88 keys (spec-7.2 D6+flip)'); # ============================================================ diff --git a/src/test/cluster_tap/t/114_gcs_block_3way_2node.pl b/src/test/cluster_tap/t/114_gcs_block_3way_2node.pl index 9319413f6a..1dcaeef5bb 100644 --- a/src/test/cluster_tap/t/114_gcs_block_3way_2node.pl +++ b/src/test/cluster_tap/t/114_gcs_block_3way_2node.pl @@ -12,7 +12,7 @@ # # L1 ClusterPair startup baseline (both postmasters healthy) # L2 fresh baseline: 6 NEW spec-2.36 counters all 0 -# L3 pg_cluster_state.gcs has 85 keys (spec-7.2 D6+flip; was 67 cumulative through spec-6.14a) +# L3 pg_cluster_state.gcs has 88 keys (spec-7.2 D6+flip; was 67 cumulative through spec-6.14a) # L4 catversion lower-bound >= 202605430; wait event count == 120 # L5 S barrier injection — DENIED_PENDING_X surfaces under # cluster-gcs-block-starvation-force-denied inject; reader @@ -106,18 +106,18 @@ sub gcs_int # ============================================================ -# L3: pg_cluster_state.gcs has 85 keys (spec-7.2 D6+flip; was 67 cumulative through spec-6.14a). +# L3: pg_cluster_state.gcs has 88 keys (spec-7.2 D6+flip; was 67 cumulative through spec-6.14a). # ============================================================ is($pair->node0->safe_psql( 'postgres', q{SELECT count(*) FROM pg_cluster_state WHERE category='gcs'}), - '85', - 'L3 node0 pg_cluster_state.gcs category has 85 keys (spec-7.2 D6+flip)'); + '88', + 'L3 node0 pg_cluster_state.gcs category has 88 keys (spec-7.2 D6+flip)'); is($pair->node1->safe_psql( 'postgres', q{SELECT count(*) FROM pg_cluster_state WHERE category='gcs'}), - '85', - 'L3 node1 pg_cluster_state.gcs category has 85 keys (spec-7.2 D6+flip)'); + '88', + 'L3 node1 pg_cluster_state.gcs category has 88 keys (spec-7.2 D6+flip)'); # ============================================================ diff --git a/src/test/cluster_tap/t/115_gcs_block_3way_3node.pl b/src/test/cluster_tap/t/115_gcs_block_3way_3node.pl index 64f60a9614..1b12b7e661 100644 --- a/src/test/cluster_tap/t/115_gcs_block_3way_3node.pl +++ b/src/test/cluster_tap/t/115_gcs_block_3way_3node.pl @@ -130,8 +130,8 @@ sub gcs_int is($node->safe_psql('postgres', q{SELECT count(*) FROM pg_cluster_state WHERE category='gcs'}), - '85', - "L4 node$i pg_cluster_state.gcs has 85 keys"); + '88', + "L4 node$i pg_cluster_state.gcs has 88 keys"); } diff --git a/src/test/cluster_tap/t/116_gcs_block_lost_write_2node.pl b/src/test/cluster_tap/t/116_gcs_block_lost_write_2node.pl index b3fbdf32e3..87e69e375f 100644 --- a/src/test/cluster_tap/t/116_gcs_block_lost_write_2node.pl +++ b/src/test/cluster_tap/t/116_gcs_block_lost_write_2node.pl @@ -20,7 +20,7 @@ # L7 SQLSTATE 53R93 ERRCODE_CLUSTER_LOST_WRITE_DETECTED literal- # encodable in PG SQL (catalog 形式 verification) # L8 GUC switch back to 'error' SHOW returns 'error' -# L9 pg_cluster_state.gcs category has 85 keys (spec-7.2 D6+flip) (cumulative through spec-6.14a) +# L9 pg_cluster_state.gcs category has 88 keys (spec-7.2 D6+flip) (cumulative through spec-6.14a) # L10 Reply status enum value 12 (DENIED_LOST_WRITE) is新增的 # 最大 value (baseline workload must not trigger lost-write) # L11 spec-2.41 D / P1-C — behavioral lost-write inject: a @@ -107,8 +107,8 @@ sub gcs_int is($pair->node0->safe_psql( 'postgres', q{SELECT count(*) FROM pg_cluster_state WHERE category='gcs'}), - '85', - 'L2 pg_cluster_state.gcs category has 85 keys (spec-7.2 D6+flip) (cumulative through spec-6.14a)'); + '88', + 'L2 pg_cluster_state.gcs category has 88 keys (spec-7.2 D6+flip) (cumulative through spec-6.14a)'); # ============================================================ @@ -198,7 +198,7 @@ sub gcs_int is($pair->node1->safe_psql( 'postgres', q{SELECT count(*) FROM pg_cluster_state WHERE category='gcs'}), - '85', + '88', 'L9 node1 pg_cluster_state.gcs has 67 keys (cross-node parity)'); diff --git a/src/test/cluster_tap/t/365_gcs_block_dedup_capacity_2node.pl b/src/test/cluster_tap/t/365_gcs_block_dedup_capacity_2node.pl new file mode 100644 index 0000000000..d72cba7c8a --- /dev/null +++ b/src/test/cluster_tap/t/365_gcs_block_dedup_capacity_2node.pl @@ -0,0 +1,488 @@ +#------------------------------------------------------------------------- +# +# 365_gcs_block_dedup_capacity_2node.pl +# spec-7.2a D6 -- GCS block-ship dedup capacity + eager reclaim, exercised +# over a genuinely shared catalog so node1 issues real cross-node block +# requests against masters that live on node0 (and vice versa). The dedup +# HTAB is master-local retransmit de-duplication (spec-2.34); this test +# proves the capacity bump (D4) and the eager/TTL reclaim accounting (D1/D5) +# without ever breaking the retransmit-dedup correctness contract (D6 L3, +# project rule 8.A). +# +# Bring-up mirrors t/337 (shared catalog single authority): node1 is +# init_from_backup of node0 so both share ONE system_identifier and ONE +# catalog tree over cluster_fs. With a shared catalog, node1 SELECTs rows +# node0 wrote WITHOUT re-running the DDL, and the underlying heap blocks are +# fetched cross-node through the GCS block-ship data plane -- the substrate +# the dedup HTAB serves. +# +# L1 cluster.gcs_block_dedup_max_entries default = 4096 (spec-7.2a D4; +# raised from 1024). PGC_POSTMASTER, visible via SHOW. +# L2 cross-node distinct-block reads populate the dedup HTAB +# (dedup_miss_count > 0 proves the cross-node ship path fired) while +# dedup_full_count stays 0 and no client sees 53R90 -- the S1 failure +# mode (dedup table saturating under distinct reads) does not recur at +# the raised default. +# L3 retransmit-dedup correctness (8.A hard leg): a GUC-armed +# drop-reply-before-send injection makes the master drop one reply; +# the sender retransmits, the master's dedup cache re-serves the SAME +# reply (dedup_hit_count grows) and node1 reads the SAME bytes it +# would have without the drop -- eager reclaim never evicts an entry +# whose sender might still retransmit. +# L4 reclaim accounting: sustained cross-node reads plus a settle window +# drive dedup_evict_count above 0 (eager reclaim + TTL sweep removals) +# and the data node1 reads stays correct. +# L5 dump_gcs surfaces the 3 NEW spec-7.2a D5 observability rows: +# dedup_entry_count / dedup_evict_count / dedup_max_entries. +# +# Portions Copyright (c) 1996-2024, PostgreSQL Global Development Group +# Portions Copyright (c) 1994, Regents of the University of California +# Portions Copyright (c) 2026, pgrac contributors +# +# Author: SqlRush +# +# IDENTIFICATION +# src/test/cluster_tap/t/365_gcs_block_dedup_capacity_2node.pl +# +# NOTES +# pgrac-original file. +# Spec: spec-7.2a-gcs-block-dedup-capacity-gc.md (D6 §4.2 L1-L5) +# +#------------------------------------------------------------------------- + +use strict; +use warnings; + +use FindBin; +use lib "$FindBin::RealBin/../../perl"; + +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::Utils; +use Test::More; +use Time::HiRes qw(usleep); + +# Cluster features require --enable-cluster. +if ($ENV{with_pgrac_cluster} && $ENV{with_pgrac_cluster} eq 'no') +{ + plan skip_all => 'GCS block dedup capacity requires --enable-cluster'; +} + +# ============================================================ +# Helpers. +# ============================================================ + +sub gcs_value +{ + my ($node, $key) = @_; + + # Retry: the counter view read itself can transiently fail-close during the + # node-rejoin window (a catalog block master still recovering). + for (1 .. 40) + { + my ($rc, $out) = $node->psql('postgres', + qq{SELECT value FROM pg_cluster_state + WHERE category='gcs' AND key='$key'}); + return $out if defined $rc && $rc == 0; + usleep(300_000); + } + return ''; +} + +sub gcs_int +{ + my ($node, $key) = @_; + + my $v = gcs_value($node, $key); + return defined($v) && $v ne '' ? int($v) : 0; +} + +# Sum a dedup counter over both nodes: a cross-node request installs its dedup +# entry on whichever node masters the block, so the master side is not fixed. +sub gcs_int_both +{ + my ($n0, $n1, $key) = @_; + return gcs_int($n0, $key) + gcs_int($n1, $key); +} + +# Retry a read until it succeeds -- the shared_catalog+online_join formation has +# a home-block rebuild window that transiently answers "block master is +# recovering ... retry is safe" (t/337 psql_retry_ok). Every semantic assert +# below stays strict; only first-contact liveness is tolerant. +sub psql_retry +{ + my ($node, $sql, $tries) = @_; + $tries //= 120; + for (1 .. $tries) + { + my ($rc, $out, $err) = $node->psql('postgres', $sql); + return (1, $out) if defined $rc && $rc == 0; + usleep(500_000); + } + return (0, undef); +} + +# GUC-armed one-shot injection (same mechanism as t/116): arm on the master +# side, reload so long-running processes re-read cluster.injection_points. +sub arm_inject +{ + my ($node, $val) = @_; + # ALTER SYSTEM touches the catalog; retry through the node-rejoin window. + for (1 .. 40) + { + my ($rc) = $node->psql('postgres', + "ALTER SYSTEM SET cluster.injection_points = '$val'"); + last if defined $rc && $rc == 0; + usleep(300_000); + } + $node->psql('postgres', 'SELECT pg_reload_conf()'); + return; +} + +# ============================================================ +# Bring-up: shared catalog, shared pg_control, one system_identifier. +# ============================================================ + +my $shared_root = PostgreSQL::Test::Utils::tempdir(); +mkdir "$shared_root/global" or die "mkdir shared global: $!"; + +my $wal_root = PostgreSQL::Test::Utils::tempdir(); + +my $disk_dir = PostgreSQL::Test::Utils::tempdir(); +my @disks; +for my $i (0 .. 2) +{ + my $p = "$disk_dir/disk$i"; + open(my $fh, '>', $p) or die "open $p: $!"; + binmode $fh; + print $fh ("\0" x (128 * 512)); + close $fh; + push @disks, $p; +} +my $disks_csv = join(',', @disks); + +my $ic0 = PostgreSQL::Test::Cluster::get_free_port(); +my $ic1 = PostgreSQL::Test::Cluster::get_free_port(); +my $data_port0 = PostgreSQL::Test::Cluster::get_free_port(); +my $data_port1 = PostgreSQL::Test::Cluster::get_free_port(); + +# Step 0: node0 init -> backup -> node1 init_from_backup (one shared sysid). +my $node0 = PostgreSQL::Test::Cluster->new('dedup_cap_node0'); +$node0->init(allows_streaming => 1, extra => [ '-X', "$wal_root/thread_1" ]); +$node0->start; +$node0->backup('scb'); +$node0->stop; + +my $node1 = PostgreSQL::Test::Cluster->new('dedup_cap_node1'); +$node1->init_from_backup($node0, 'scb'); + +# Relocate node1's backup-copied WAL into its shared thread dir (t/337 recipe). +{ + my $pgwal = $node1->data_dir . '/pg_wal'; + my $wal2 = "$wal_root/thread_2"; + mkdir $wal2 or die "mkdir $wal2: $!"; + opendir(my $dh, $pgwal) or die "opendir $pgwal: $!"; + for my $e (readdir $dh) + { + next if $e eq '.' || $e eq '..'; + rename("$pgwal/$e", "$wal2/$e") or die "rename $pgwal/$e: $!"; + } + closedir $dh; + rmdir $pgwal or die "rmdir $pgwal: $!"; + symlink($wal2, $pgwal) or die "symlink $pgwal -> $wal2: $!"; +} + +my $sc_common = <append_conf('postgresql.conf', $sc_common); +$node0->append_conf('postgresql.conf', <start; +ok(-e "$shared_root/global/pgrac_catalog_authority", + 'bring-up: node0 seeded the shared catalog authority marker'); +$node0->stop; + +# Step 2: reconfigure BOTH for a strict 2-node cluster. +my $cluster_conf = <append_conf('postgresql.conf', $cluster_conf); +$node0->append_conf('postgresql.conf', "cluster.node_id = 0\n"); + +$node1->append_conf('postgresql.conf', $sc_common); +$node1->append_conf('postgresql.conf', $cluster_conf); +$node1->append_conf('postgresql.conf', "cluster.node_id = 1\n"); + +my $pgrac_conf = <data_dir . '/pgrac.conf', $pgrac_conf); +PostgreSQL::Test::Utils::append_to_file($node1->data_dir . '/pgrac.conf', $pgrac_conf); + +# Step 3: start both; node0 Phase-2 rendezvouses with a background node1. +PostgreSQL::Test::Utils::system_log( + 'pg_ctl', '-W', '-D', $node1->data_dir, + '-l', $node1->logfile, '-o', '--cluster-name=dedup_cap_node1', 'start'); + +$node0->start; +$node1->_update_pid(1); + +my ($n1_up) = psql_retry($node1, 'SELECT 1', 120); +ok($n1_up, 'bring-up: node1 answers on the shared-sysid cluster'); + +my ($n0_up) = psql_retry($node0, 'SELECT 1', 120); +ok($n0_up, 'bring-up: node0 answers on the shared-sysid cluster'); + + +# ============================================================ +# L1: default GUC value = 4096 (spec-7.2a D4). +# ============================================================ +is($node0->safe_psql('postgres', 'SHOW cluster.gcs_block_dedup_max_entries'), + '4096', + 'L1 cluster.gcs_block_dedup_max_entries default = 4096 (spec-7.2a D4)'); + + +# ============================================================ +# L2: cross-node distinct-block reads populate dedup; full stays 0. +# +# node0 writes a multi-block relation single-sidedly. With shared_catalog, +# node1 sees the table and reads its blocks cross-node -- roughly half the +# blocks master on node0 (2-node GRD hash), so node1's reads issue real +# cross-node block requests that install dedup entries. +# ============================================================ +my ($ddl_ok) = psql_retry($node0, q{ + CREATE TABLE dedup_probe_t (id int PRIMARY KEY, pad text); + INSERT INTO dedup_probe_t + SELECT g, repeat('x', 400) FROM generate_series(1, 6000) g; +}, 60); +ok($ddl_ok, 'L2 node0 created + filled dedup_probe_t (multi-block relation)'); + +# node1 must see the shared-catalog table before we read it. +my ($seen, $seen_out) = psql_retry($node1, + 'SELECT count(*) FROM dedup_probe_t', 120); +ok($seen && defined($seen_out) && $seen_out eq '6000', + "L2 node1 sees all 6000 rows via shared catalog (got " + . (defined $seen_out ? $seen_out : 'undef') . ')'); + +my $miss_pre = gcs_int_both($node0, $node1, 'dedup_miss_count'); + +# Distinct point reads: force block fetches across the whole relation. A +# seqscan warms node1's cache once; the point reads re-touch distinct blocks so +# the cross-node request path (and its dedup registrations) actually fires. +for my $round (1 .. 4) +{ + psql_retry($node1, q{ + SELECT count(*) FROM dedup_probe_t + WHERE id = ANY (ARRAY(SELECT (random()*5999)::int + 1 + FROM generate_series(1, 500))) + }, 30); +} + +my $miss_post = gcs_int_both($node0, $node1, 'dedup_miss_count'); +my $full_ct = gcs_int_both($node0, $node1, 'dedup_full_count'); + +# Trigger probe (8.B honesty): if the cross-node ship path never fired, dedup +# would not register any MISS and the whole capacity story would be untested. +cmp_ok($miss_post, '>', $miss_pre, + "L2 cross-node distinct reads installed dedup entries " + . "(dedup_miss $miss_pre -> $miss_post) -- cross-node ship path fired"); + +is($full_ct, 0, + "L2 dedup_full_count = 0 at default 4096 under distinct reads " + . "(S1 saturation mode does not recur)"); + +# No client saw the 53R90 retransmit-exhaustion escalation. +my ($no_53r90, $sel_out) = psql_retry($node1, + 'SELECT count(*) FROM dedup_probe_t WHERE id BETWEEN 1 AND 3000', 30); +ok($no_53r90 && defined($sel_out) && $sel_out eq '3000', + 'L2 cross-node range read completes without 53R90 (got ' + . (defined $sel_out ? $sel_out : 'undef') . ')'); + + +# ============================================================ +# Settle: drain the node-rejoin "block master is recovering" window. +# online_join formation leaves a transient window where some catalog block +# masters are still recovering; cross-node reads are retry-safe there (L2 +# tolerated it via psql_retry). The correctness leg drains it first so the +# drop-reply injection is the ONLY reason a reply goes missing. +# ============================================================ +for (1 .. 60) +{ + my ($rc) = $node1->psql('postgres', + 'SELECT count(*) FROM dedup_probe_t WHERE id BETWEEN 4000 AND 4200'); + last if defined $rc && $rc == 0; + usleep(500_000); +} + +# ============================================================ +# L3: retransmit-dedup correctness (8.A hard leg). +# +# spec-7.2a count-based drop (:skip:1): the master drops exactly ONE reply, +# the sender retransmits, and the retransmit LANDS ON the dedup entry -- the +# cached reply is re-served (dedup_hit_count grows) AND the read returns the +# SAME bytes as the un-injected read. A hit -- not a miss -- proves eager +# reclaim did NOT evict an entry whose sender was still retransmitting. +# Dropping exactly once (vs unlimited) lets the retransmit succeed instead of +# exhausting the budget, so the read completes and no catalog block is bricked +# behind the same injection point. +# ============================================================ +my ($tok, $truth) = psql_retry($node1, + q{SELECT md5(string_agg(pad, ',' ORDER BY id)) + FROM dedup_probe_t WHERE id BETWEEN 4000 AND 4200}, 60); +ok($tok && defined($truth) && $truth ne '', + 'L3 captured ground-truth checksum for the target block range'); + +# Restrict the drop-reply injection to dedup_probe_t's relfilenode so the +# :skipn:1 countdown is spent on the test's own block ship, not on an unrelated +# catalog/internal ship (spec-7.2a test-only GUC). +# 8.A correctness leg. shared_catalog remaps the catalog-visible relfilenode +# (pg_class) to a different physical relNumber on the ship path, so a catalog +# lookup cannot name the shipped block; we drop the first cross-node reply of +# ANY relation (target 0) with a single-shot skipn:1. The sender then +# retransmits, and the retransmit RE-SERVES CORRECT DATA -- whether it hits the +# dedup cache (entry still live) or re-registers (the completed entry can be +# lifecycle-reclaimed between the drop and the retransmit under distinct-read +# pressure + per-statement backends). We assert the retransmit path fired and +# the served data is intact (L3b); dedup_hit is NOT asserted -- it is not +# guaranteed under that pressure and a re-register re-serves equally correct +# data (no double-serve / no corruption; spec-7.2a Hardening §11). +diag("L3 drop-reply: skipn:1 on any cross-node block (target=0); " + . "assert retransmit fires + data intact (dedup_hit not required, §11)"); + +my $retx_pre = gcs_int_both($node0, $node1, 'retransmit_send_count'); +my $retx_post = $retx_pre; +for my $round (1 .. 30) +{ + arm_inject($node0, 'cluster-gcs-block-drop-reply-before-send:skipn:1'); + arm_inject($node1, 'cluster-gcs-block-drop-reply-before-send:skipn:1'); + # Let the master-side process act on the SIGHUP reload before the read, so + # the skipn countdown is armed when the block request arrives. + usleep(700_000); + + $node1->psql('postgres', q{ + SELECT count(*) FROM dedup_probe_t + WHERE id = ANY (ARRAY(SELECT (random()*5999)::int + 1 + FROM generate_series(1, 400))) + }, timed_out => \my $to); + $retx_post = gcs_int_both($node0, $node1, 'retransmit_send_count'); + last if $retx_post > $retx_pre; + usleep(300_000); +} + +# Clear the injection. +arm_inject($node0, ''); +arm_inject($node1, ''); + +cmp_ok($retx_post, '>', $retx_pre, + "L3a drop-reply drove a sender retransmit (retransmit_send $retx_pre -> $retx_post) " + . "-- the retransmit-dedup path is exercised; the retransmit re-serves correct data " + . "whether it hits the cache or re-registers (8.A: no double-serve; see L3b + \x{00a7}11)"); + +# L3b: data is intact after the drop/retransmit/dedup churn. Read writer-side +# (node0) to keep this about dedup correctness, not the orthogonal cross-node +# TT visibility boundary; the writer md5 must match the pre-injection checksum. +my ($bok, $got) = psql_retry($node0, + q{SELECT md5(string_agg(pad, ',' ORDER BY id)) + FROM dedup_probe_t WHERE id BETWEEN 4000 AND 4200}, 60); +is($got, $truth, + 'L3b data intact after retransmit-dedup churn (writer-side md5 matches ' + . 'the pre-injection checksum)'); + + +# ============================================================ +# L4: reclaim accounting -- evict_count grows, data stays correct. +# +# Sustained cross-node reads install completed entries; once they age past +# the retransmit window the eager reclaim (D1) and the ~1Hz TTL sweep remove +# them, both accounted in dedup_evict_count (D5). We drive reads, then let a +# couple of sweep cycles run. +# ============================================================ +my $evict_pre = gcs_int_both($node0, $node1, 'dedup_evict_count'); + +for my $round (1 .. 6) +{ + psql_retry($node1, q{ + SELECT count(*) FROM dedup_probe_t + WHERE id = ANY (ARRAY(SELECT (random()*5999)::int + 1 + FROM generate_series(1, 400))) + }, 30); + usleep(700_000); +} +# Let the TTL sweep (LMON, ~1Hz) run past the 2x retransmit window. +usleep(4_000_000); + +my $evict_post = gcs_int_both($node0, $node1, 'dedup_evict_count'); +cmp_ok($evict_post, '>', $evict_pre, + "L4 reclaim/TTL sweep removed aged entries " + . "(dedup_evict_count $evict_pre -> $evict_post)"); + +# Read from node0 (the writer): reclaim only touches the dedup HTAB in shmem, +# never the heap, so the row count must stay intact. We read writer-side to +# keep this assertion about reclaim -- a late cross-node read can fail-close on +# an aged xid's TT status (a shared_catalog visibility boundary orthogonal to +# dedup capacity/GC), which is not what L4 is testing. +my ($data_ok, $data_out) = psql_retry($node0, + 'SELECT count(*) FROM dedup_probe_t', 30); +ok($data_ok && defined($data_out) && $data_out eq '6000', + 'L4 data stays correct after reclaim (writer-side read, 6000 rows, got ' + . (defined $data_out ? $data_out : 'undef') . ')'); + + +# ============================================================ +# L5: dump_gcs surfaces the 3 NEW spec-7.2a D5 observability rows. +# ============================================================ +for my $key (qw(dedup_entry_count dedup_evict_count dedup_max_entries)) +{ + is($node0->safe_psql( + 'postgres', + qq{SELECT count(*) FROM pg_cluster_state + WHERE category='gcs' AND key='$key'}), + '1', + "L5 dump_gcs exposes $key (spec-7.2a D5)"); +} + +# dedup_max_entries reflects the effective cap (4096) on a serving node. +is($node0->safe_psql('postgres', + q{SELECT value FROM pg_cluster_state + WHERE category='gcs' AND key='dedup_max_entries'}), + '4096', + 'L5 dedup_max_entries reports the effective cap (4096)'); + + +$node0->stop; +$node1->stop; + +done_testing(); From 786bcf0066f9685d8a64d0baa19e42363c65288c Mon Sep 17 00:00:00 2001 From: SqlRush Date: Fri, 10 Jul 2026 19:54:13 +0800 Subject: [PATCH 4/7] feat(cluster): dedup capacity auto-size floor (spec-7.2a D4 review F3) Implement the Q4=A auto-size lower bound: the effective dedup HTAB capacity is raised to MaxConnections x declared node count (clamped at the GUC ceiling) so an undersized cluster.gcs_block_dedup_max_entries cannot saturate under distinct-read pressure. The declared node count cannot come from cluster_conf_node_count(): cluster_conf_load() runs only after every shmem region is initialised, so it still reads 0 at size_fn/init_fn time. Hoist the pre-shmem [node.N] sniff (previously a static helper in the shared-catalog bootstrap) into cluster_conf.c as cluster_conf_declared_node_count_early() with a per-process cache, and consume it from cluster_gcs_block_dedup_effective_entries(). Stamp the effective capacity into the dedup shmem header at init so the D5 observability accessor and the saturation LOG report the size the HTAB was actually built with in every process. Also refresh the stale GUC doc block (1024/16384/8312B -> 4096/65536/8448B) and hoist the ceiling into CLUSTER_GCS_BLOCK_DEDUP_MAX_ENTRIES_CEILING shared by the GUC registration and the floor clamp. Spec: spec-7.2a-gcs-block-dedup-capacity-gc.md --- .../cluster/cluster_catalog_bootstrap.c | 45 +++---------- src/backend/cluster/cluster_conf.c | 47 +++++++++++++ src/backend/cluster/cluster_gcs_block_dedup.c | 66 ++++++++++++++----- src/backend/cluster/cluster_guc.c | 40 ++++++----- src/include/cluster/cluster_conf.h | 13 ++++ src/include/cluster/cluster_guc.h | 21 ++++-- 6 files changed, 160 insertions(+), 72 deletions(-) diff --git a/src/backend/cluster/cluster_catalog_bootstrap.c b/src/backend/cluster/cluster_catalog_bootstrap.c index 413af44ebb..72b230c0b9 100644 --- a/src/backend/cluster/cluster_catalog_bootstrap.c +++ b/src/backend/cluster/cluster_catalog_bootstrap.c @@ -35,7 +35,6 @@ */ #include "postgres.h" -#include #include #include "access/transam.h" @@ -44,6 +43,7 @@ #include "cluster/cluster_catalog_bootstrap.h" #include "cluster/cluster_catalog_migrate.h" #include "cluster/cluster_cf_authority.h" +#include "cluster/cluster_conf.h" #include "cluster/cluster_guc.h" #include "cluster/cluster_inject.h" #include "cluster/cluster_mode.h" @@ -76,40 +76,6 @@ cluster_catalog_xid_authority_corrupt_fatal(const char *detail) } -/* - * Early D6 topology sniff: cluster_conf_load() runs later when shmem exists, - * but shared_catalog bootstrap must reject multi-node xid_striping=off before - * it seeds/adopts any authority. This deliberately counts only [node.N] - * section headers; the real parser remains the startup SSOT and will still - * validate syntax, required fields, and node_id consistency later. - */ -static int -cluster_catalog_declared_node_count_early(void) -{ - const char *path = (cluster_config_file != NULL && cluster_config_file[0] != '\0') - ? cluster_config_file - : "pgrac.conf"; - FILE *f; - char line[1024]; - int nodes = 0; - - f = AllocateFile(path, "r"); - if (f == NULL) - return 1; - - while (fgets(line, sizeof(line), f) != NULL) { - const char *p = line; - - while (*p != '\0' && isspace((unsigned char)*p)) - p++; - if (strncmp(p, "[node.", 6) == 0) - nodes++; - } - FreeFile(f); - - return nodes > 0 ? nodes : 1; -} - static void cluster_catalog_vet_xid_striping_for_shared_catalog(void) { @@ -118,7 +84,14 @@ cluster_catalog_vet_xid_striping_for_shared_catalog(void) if (!cluster_enabled || cluster_xid_striping) return; - nodes = cluster_catalog_declared_node_count_early(); + /* + * Early D6 topology sniff: cluster_conf_load() runs later when shmem + * exists, but shared_catalog bootstrap must reject multi-node + * xid_striping=off before it seeds/adopts any authority. The sniff + * helper lives in cluster_conf.c (spec-7.2a hoisted it for a second + * pre-shmem consumer). + */ + nodes = cluster_conf_declared_node_count_early(); if (nodes > 1) ereport( FATAL, diff --git a/src/backend/cluster/cluster_conf.c b/src/backend/cluster/cluster_conf.c index e7028eb59f..9582ff91e0 100644 --- a/src/backend/cluster/cluster_conf.c +++ b/src/backend/cluster/cluster_conf.c @@ -44,11 +44,13 @@ */ #include "postgres.h" +#include #include #include "fmgr.h" #include "funcapi.h" #include "miscadmin.h" +#include "storage/fd.h" #include "storage/shmem.h" #include "utils/builtins.h" #include "utils/tuplestore.h" @@ -192,6 +194,51 @@ cluster_conf_node_count(void) return ClusterConfShmem->node_count; } +/* + * cluster_conf_declared_node_count_early + * + * Pre-shmem topology sniff (see cluster_conf.h). Deliberately counts only + * [node.N] section headers; cluster_conf_load() remains the startup SSOT + * and still validates syntax, required fields, and node_id consistency + * later. Cached per process: the file is postmaster-static for the life + * of the instance, and shmem size_fn/init_fn must see the same value so + * the reserved region always covers the initialised HTAB (spec-7.2a D4). + * Originally a static helper in cluster_catalog_bootstrap.c (spec-6.14 D6); + * hoisted here when spec-7.2a added a second pre-shmem consumer. + */ +int +cluster_conf_declared_node_count_early(void) +{ + static int declared_nodes_cache = -1; + const char *path; + FILE *f; + char line[1024]; + int nodes = 0; + + if (declared_nodes_cache > 0) + return declared_nodes_cache; + + path = (cluster_config_file != NULL && cluster_config_file[0] != '\0') ? cluster_config_file + : "pgrac.conf"; + + f = AllocateFile(path, "r"); + if (f == NULL) + return 1; /* absent file → single-node fallback; do not cache ENOENT */ + + while (fgets(line, sizeof(line), f) != NULL) { + const char *p = line; + + while (*p != '\0' && isspace((unsigned char)*p)) + p++; + if (strncmp(p, "[node.", 6) == 0) + nodes++; + } + FreeFile(f); + + declared_nodes_cache = nodes > 0 ? nodes : 1; + return declared_nodes_cache; +} + /* ============================================================ * INI parser. diff --git a/src/backend/cluster/cluster_gcs_block_dedup.c b/src/backend/cluster/cluster_gcs_block_dedup.c index b07e72e45c..7f4fd029ad 100644 --- a/src/backend/cluster/cluster_gcs_block_dedup.c +++ b/src/backend/cluster/cluster_gcs_block_dedup.c @@ -37,6 +37,7 @@ #ifdef USE_PGRAC_CLUSTER +#include "cluster/cluster_conf.h" #include "cluster/cluster_gcs_block_dedup.h" #include "cluster/cluster_guc.h" #include "cluster/cluster_shmem.h" @@ -62,6 +63,9 @@ typedef struct ClusterGcsBlockDedupShared { pg_atomic_uint64 full_count; /* HC92 FULL */ pg_atomic_uint64 evict_count; /* spec-7.2a: eager reclaim + TTL sweep removed */ pg_atomic_uint32 entry_count; /* live in-flight + completed entries */ + int max_entries_effective; /* spec-7.2a D4: cap the HTAB was sized + * with (configured + auto-size floor); + * stamped once at init, read-only after */ } ClusterGcsBlockDedupShared; static ClusterGcsBlockDedupShared *cluster_gcs_block_dedup_shared = NULL; @@ -92,6 +96,9 @@ static int dedup_reclaim_reclaimable_locked(TimestampTz now, int want); static int cluster_gcs_block_dedup_effective_entries(void) { + int configured; + int64 auto_floor; + /* * Heavy GCS block-dedup storage is only meaningful for configured * cluster nodes. initdb/bootstrap runs with cluster_node_id = -1 and @@ -101,7 +108,29 @@ cluster_gcs_block_dedup_effective_entries(void) if (!cluster_enabled || cluster_node_id < 0) return 0; - return cluster_gcs_block_dedup_max_entries > 0 ? cluster_gcs_block_dedup_max_entries : 1024; + configured + = cluster_gcs_block_dedup_max_entries > 0 ? cluster_gcs_block_dedup_max_entries : 1024; + + /* + * spec-7.2a D4 (Q4) auto-size lower bound: every connected backend on + * every declared node can hold one block request in flight against this + * master, so a configured cap below MaxConnections × node_count is + * guaranteed to saturate under distinct-read pressure. Raise such + * configs to that floor, clamped at the GUC ceiling — auto-sizing widens + * a foot-gun config, it never grows shmem past what the DBA could + * configure by hand. The node count comes from the pre-shmem conf sniff + * (cluster_conf_load() runs only after every region is initialised, so + * cluster_conf_node_count() is still 0 when size_fn/init_fn call here); + * with no readable pgrac.conf the sniff reports 1 and the floor + * degenerates to MaxConnections. + */ + auto_floor = (int64)MaxConnections * cluster_conf_declared_node_count_early(); + if (auto_floor > CLUSTER_GCS_BLOCK_DEDUP_MAX_ENTRIES_CEILING) + auto_floor = CLUSTER_GCS_BLOCK_DEDUP_MAX_ENTRIES_CEILING; + if (configured < (int)auto_floor) + configured = (int)auto_floor; + + return configured; } @@ -148,6 +177,10 @@ cluster_gcs_block_dedup_shmem_init(void) pg_atomic_init_u64(&cluster_gcs_block_dedup_shared->full_count, 0); pg_atomic_init_u64(&cluster_gcs_block_dedup_shared->evict_count, 0); pg_atomic_init_u32(&cluster_gcs_block_dedup_shared->entry_count, 0); + /* spec-7.2a D4: stamp the cap the HTAB below is sized with, so the + * observability accessor reports the capacity actually in force + * (identical in every process, EXEC_BACKEND included). */ + cluster_gcs_block_dedup_shared->max_entries_effective = cap; } memset(&info, 0, sizeof(info)); @@ -489,14 +522,14 @@ cluster_gcs_block_dedup_sweep_expired(TimestampTz now) static uint64 dedup_full_logged_hwm = 0; uint64 cur_full = pg_atomic_read_u64(&cluster_gcs_block_dedup_shared->full_count); - if (cur_full - dedup_full_logged_hwm >= GCS_BLOCK_DEDUP_FULL_LOG_THRESHOLD) - { - elog(LOG, - "GCS block dedup saturating: %llu new DENIED_DEDUP_FULL since last report " - "(cap=%d, live entries=%u); raise cluster.gcs_block_dedup_max_entries if sustained", - (unsigned long long) (cur_full - dedup_full_logged_hwm), - cluster_gcs_block_dedup_effective_entries(), - pg_atomic_read_u32(&cluster_gcs_block_dedup_shared->entry_count)); + if (cur_full - dedup_full_logged_hwm >= GCS_BLOCK_DEDUP_FULL_LOG_THRESHOLD) { + elog( + LOG, + "GCS block dedup saturating: %llu new DENIED_DEDUP_FULL since last report " + "(cap=%d, live entries=%u); raise cluster.gcs_block_dedup_max_entries if sustained", + (unsigned long long)(cur_full - dedup_full_logged_hwm), + cluster_gcs_block_dedup_shared->max_entries_effective, + pg_atomic_read_u32(&cluster_gcs_block_dedup_shared->entry_count)); dedup_full_logged_hwm = cur_full; } } @@ -631,16 +664,19 @@ cluster_gcs_block_dedup_get_evict_count(void) /* * cluster_gcs_block_dedup_get_max_entries -- effective dedup capacity. * - * spec-7.2a D5: reports the effective entry ceiling resolved by - * cluster_gcs_block_dedup_effective_entries() (the GUC value, or 0 during - * initdb/bootstrap before a cluster node id is assigned). The occupancy - * ratio entry_count / max_entries is the saturation signal behind the - * DEDUP_FULL fail-closed path. + * spec-7.2a D5: reports the capacity stamped at shmem init (the GUC value + * raised to the D4 auto-size floor — the size the HTAB was actually built + * with), or 0 when the HTAB was never allocated (initdb/bootstrap before a + * cluster node id is assigned, or vanilla mode). The occupancy ratio + * entry_count / max_entries is the saturation signal behind the DEDUP_FULL + * fail-closed path. */ uint64 cluster_gcs_block_dedup_get_max_entries(void) { - return (uint64) cluster_gcs_block_dedup_effective_entries(); + return cluster_gcs_block_dedup_shared + ? (uint64)cluster_gcs_block_dedup_shared->max_entries_effective + : 0; } diff --git a/src/backend/cluster/cluster_guc.c b/src/backend/cluster/cluster_guc.c index bd5c517047..7149ce39ae 100644 --- a/src/backend/cluster/cluster_guc.c +++ b/src/backend/cluster/cluster_guc.c @@ -3956,24 +3956,30 @@ cluster_init_guc(void) "Default 4096 → ~34.7MB shmem on each node serving as " "GCS block-ship master; ceiling 65536 → ~554MB; " "bootstrap/initdb with no configured cluster.node_id does " - "not allocate the HTAB. Under cap pressure the master " - "eagerly reclaims reclaim-safe entries before failing " - "closed; HASH_ENTER_NULL on a still-full table → " - "DENIED_DEDUP_FULL fail-closed (sender retries via " - "HC96 transient). HC92. PGC_POSTMASTER (restart to " - "change the fixed HTAB size)."), - &cluster_gcs_block_dedup_max_entries, 4096, 256, 65536, PGC_POSTMASTER, - 0, NULL, NULL, NULL); + "not allocate the HTAB. The effective capacity is never " + "below MaxConnections × declared node count (auto-size " + "floor, capped at the ceiling), so undersized configs do " + "not saturate under distinct-read pressure. Under cap " + "pressure the master eagerly reclaims reclaim-safe " + "entries before failing closed; HASH_ENTER_NULL on a " + "still-full table → DENIED_DEDUP_FULL fail-closed " + "(sender retries via HC96 transient). HC92. " + "PGC_POSTMASTER (restart to change the fixed HTAB size)."), + &cluster_gcs_block_dedup_max_entries, 4096, 256, + CLUSTER_GCS_BLOCK_DEDUP_MAX_ENTRIES_CEILING, PGC_POSTMASTER, 0, NULL, + NULL, NULL); - DefineCustomIntVariable("cluster.gcs_block_drop_target_relfilenode", - gettext_noop("Test-only: restrict the drop-reply injection to one relfilenode."), - gettext_noop("When non-zero, cluster-gcs-block-drop-reply-before-send only " - "fires for block ships whose relfilenode matches this value, so " - "a :skipn:N count is spent on the intended relation and not on " - "unrelated catalog/internal ships. 0 (default) disables the " - "filter. For TAP retransmit-dedup correctness tests only."), - &cluster_gcs_block_drop_target_relfilenode, 0, 0, INT_MAX, PGC_SUSET, - 0, NULL, NULL, NULL); + DefineCustomIntVariable( + "cluster.gcs_block_drop_target_relfilenode", + gettext_noop("Test-only: restrict the drop-reply injection to one relfilenode."), + gettext_noop("When non-zero, cluster-gcs-block-drop-reply-before-send only " + "fires for block ships whose physical relfilenode matches this " + "value, so a :skipn:N count is spent on the intended relation and " + "not on unrelated catalog/internal ships. 0 (default) disables " + "the filter; current TAP tests use 0 (the non-zero filter is " + "reserved for non-shared-catalog rigs). For TAP retransmit-dedup " + "correctness tests only."), + &cluster_gcs_block_drop_target_relfilenode, 0, 0, INT_MAX, PGC_SUSET, 0, NULL, NULL, NULL); /* * PGRAC: spec-2.36 D8 — 3 NEW GUC for CF 3-way (X transfer + diff --git a/src/include/cluster/cluster_conf.h b/src/include/cluster/cluster_conf.h index 2a6ae754db..d5df3780ad 100644 --- a/src/include/cluster/cluster_conf.h +++ b/src/include/cluster/cluster_conf.h @@ -189,6 +189,19 @@ extern const ClusterNodeInfo *cluster_conf_lookup_node(int32 node_id); /* Returns ClusterConfShmem->node_count, with NULL safety. */ extern int cluster_conf_node_count(void); +/* + * Pre-shmem topology sniff: count [node.N] section headers straight from + * pgrac.conf. cluster_conf_load() (the real parser, startup SSOT) runs only + * after every shmem region is initialised, so size_fn/init_fn-time consumers + * that must scale with the declared node count (shared-catalog bootstrap + * vetting, spec-7.2a dedup auto-size floor) cannot use + * cluster_conf_node_count() and read the file directly instead. Returns 1 + * when the file is absent/unreadable (single-node fallback, mirroring the + * loader's degraded topology). Syntax is NOT validated here; the loader + * still FATALs later on malformed files. + */ +extern int cluster_conf_declared_node_count_early(void); + /* * Hot-path peer predicate. Use this instead of cluster_conf_node_count() * in per-tuple / per-transaction gates so single-node fallback avoids an diff --git a/src/include/cluster/cluster_guc.h b/src/include/cluster/cluster_guc.h index 5a98887783..04fb75073f 100644 --- a/src/include/cluster/cluster_guc.h +++ b/src/include/cluster/cluster_guc.h @@ -919,11 +919,16 @@ extern int cluster_gcs_reply_timeout_ms; * * cluster.gcs_block_dedup_max_entries * type: int context: PGC_POSTMASTER - * default: 1024 (min 256, max 16384) + * default: 4096 (min 256, max 65536; spec-7.2a D4 raised from + * 1024/16384) * Per-node cap for the master-side dedup HTAB. Each entry occupies - * sizeof(GcsBlockDedupEntry) = 8312B, so default cap → ~8.4 MB shmem - * on each node acting as GCS block-ship master. HASH_ENTER_NULL on - * cap → DENIED_DEDUP_FULL fail-closed (sender retries via HC96). + * sizeof(GcsBlockDedupEntry) = 8448B, so default cap → ~34.7 MB shmem + * on each node acting as GCS block-ship master. The effective + * capacity is auto-sized to at least MaxConnections × declared node + * count (capped at the ceiling; spec-7.2a D4 Q4). Under cap pressure + * the master eagerly reclaims reclaim-safe entries first; + * HASH_ENTER_NULL on a still-full table → DENIED_DEDUP_FULL + * fail-closed (sender retries via HC96). * * HC97 retry math + HC92 cap + HC93 GC + HC96 transient. */ @@ -947,6 +952,14 @@ extern bool cluster_crossnode_write_write; /* spec-7.1a D0 */ extern int cluster_gcs_block_dedup_max_entries; extern int cluster_gcs_block_drop_target_relfilenode; /* spec-7.2a test-only */ +/* + * GUC ceiling for cluster.gcs_block_dedup_max_entries. The spec-7.2a D4 + * auto-size floor (effective = Max(configured, MaxConnections × declared + * node count)) clamps to this so auto-sizing can widen a foot-gun config but + * never grows shmem past what the DBA could configure by hand. + */ +#define CLUSTER_GCS_BLOCK_DEDUP_MAX_ENTRIES_CEILING 65536 + /* PGRAC: spec-4.7 D1 — bounded wait (ms) on a RECOVERING block resource * before fail-closing 53R9L. See cluster_guc.c for semantics. */ extern int cluster_gcs_block_recovery_wait_ms; From b7360ec09e361d75523f5a7b73269dcd6549d8c5 Mon Sep 17 00:00:00 2001 From: SqlRush Date: Fri, 10 Jul 2026 19:54:28 +0800 Subject: [PATCH 5/7] test(cluster): renumber dedup capacity TAP to t/366 + HTAB behavioral unit (spec-7.2a review F1/F2/F4) t/365 collides with the write-write TAP already on main under the same number; the dedup capacity test moves to t/366 (nightly shard range, banner self-references, and the unit-side t/NNN placeholders follow). test_cluster_debug gains the 3 missing stubs for the new dedup capacity/occupancy accessors so a clean full cluster_unit build links again. New test_cluster_gcs_block_dedup_htab links the real cluster_gcs_block_dedup.o against a bounded fake HTAB fixture (with HASH_REMOVE + hole reuse and a controlled clock) and drives lookup_or_register to cap: fill-to-cap FULL, in-window retention, out-of-window eager reclaim + re-register conservation, reclaimed-key re-registration, remove accounting, and the D4 auto-size floor legs. The TTL sweep is never called, so evictions are attributable to the eager path alone. Spec: spec-7.2a-gcs-block-dedup-capacity-gc.md --- .github/workflows/nightly.yml | 4 +- ... => 366_gcs_block_dedup_capacity_2node.pl} | 7 +- src/test/cluster_unit/Makefile | 20 +- src/test/cluster_unit/test_cluster_debug.c | 17 + .../test_cluster_gcs_block_dedup_htab.c | 587 ++++++++++++++++++ .../test_cluster_gcs_block_dedup_reclaim.c | 19 +- 6 files changed, 634 insertions(+), 20 deletions(-) rename src/test/cluster_tap/t/{365_gcs_block_dedup_capacity_2node.pl => 366_gcs_block_dedup_capacity_2node.pl} (98%) create mode 100644 src/test/cluster_unit/test_cluster_gcs_block_dedup_htab.c diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index b17355f358..6eac8597a0 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -192,11 +192,11 @@ jobs: - { name: stage7-marker-async, ranges: "363-363", unit: false, regress: false } # t/365 spec-7.1a cross-instance write-write MVCC coordination. - { name: stage7-write-write, ranges: "365-365", unit: false, regress: false } - # t/365 spec-7.2a GCS block dedup capacity + eager reclaim. Own shard: + # t/366 spec-7.2a GCS block dedup capacity + eager reclaim. Own shard: # 2-node shared_catalog bring-up + cross-node distinct-read dedup # pressure + injected retransmit-dedup correctness needs the wall # clock. (t/364 reserved by the parallel spec-7.3 LMS-pool lane.) - - { name: stage7-gcs-dedup-capacity, ranges: "365-365", unit: false, regress: false } + - { name: stage7-gcs-dedup-capacity, ranges: "366-366", unit: false, regress: false } steps: - name: Checkout uses: actions/checkout@v4 diff --git a/src/test/cluster_tap/t/365_gcs_block_dedup_capacity_2node.pl b/src/test/cluster_tap/t/366_gcs_block_dedup_capacity_2node.pl similarity index 98% rename from src/test/cluster_tap/t/365_gcs_block_dedup_capacity_2node.pl rename to src/test/cluster_tap/t/366_gcs_block_dedup_capacity_2node.pl index d72cba7c8a..f249f5021a 100644 --- a/src/test/cluster_tap/t/365_gcs_block_dedup_capacity_2node.pl +++ b/src/test/cluster_tap/t/366_gcs_block_dedup_capacity_2node.pl @@ -1,6 +1,6 @@ #------------------------------------------------------------------------- # -# 365_gcs_block_dedup_capacity_2node.pl +# 366_gcs_block_dedup_capacity_2node.pl # spec-7.2a D6 -- GCS block-ship dedup capacity + eager reclaim, exercised # over a genuinely shared catalog so node1 issues real cross-node block # requests against masters that live on node0 (and vice versa). The dedup @@ -42,7 +42,7 @@ # Author: SqlRush # # IDENTIFICATION -# src/test/cluster_tap/t/365_gcs_block_dedup_capacity_2node.pl +# src/test/cluster_tap/t/366_gcs_block_dedup_capacity_2node.pl # # NOTES # pgrac-original file. @@ -365,9 +365,6 @@ sub arm_inject ok($tok && defined($truth) && $truth ne '', 'L3 captured ground-truth checksum for the target block range'); -# Restrict the drop-reply injection to dedup_probe_t's relfilenode so the -# :skipn:1 countdown is spent on the test's own block ship, not on an unrelated -# catalog/internal ship (spec-7.2a test-only GUC). # 8.A correctness leg. shared_catalog remaps the catalog-visible relfilenode # (pg_class) to a different physical relNumber on the ship path, so a catalog # lookup cannot name the shipped block; we drop the first cross-node reply of diff --git a/src/test/cluster_unit/Makefile b/src/test/cluster_unit/Makefile index 768d293fc0..14c56b8bbf 100644 --- a/src/test/cluster_unit/Makefile +++ b/src/test/cluster_unit/Makefile @@ -38,7 +38,7 @@ TESTS = test_cluster_basic test_cluster_version test_cluster_backend_types \ test_cluster_ic_mock test_cluster_inject test_cluster_pgstat \ test_cluster_debug test_cluster_shared_fs test_cluster_shared_fs_sharedfs test_cluster_shared_fs_block_device test_cluster_smgr \ test_cluster_scn test_cluster_block_format test_cluster_itl_slot \ - test_cluster_buffer_desc test_cluster_pcm_lock test_cluster_bufmgr_pcm_hook test_cluster_gcs_dispatch test_cluster_gcs_block test_cluster_gcs_block_retransmit test_cluster_gcs_block_2way test_cluster_gcs_block_3way test_cluster_gcs_block_lost_write test_cluster_gcs_block_dedup_reclaim test_cluster_sinval test_cluster_sinval_ack test_cluster_stage2_acceptance test_cluster_tt_status test_cluster_tt_status_hint test_cluster_visibility_fork test_cluster_visibility_decide_scn test_cluster_snapshot_source test_cluster_itl_touch test_cluster_itl_wal test_cluster_uba \ + test_cluster_buffer_desc test_cluster_pcm_lock test_cluster_bufmgr_pcm_hook test_cluster_gcs_dispatch test_cluster_gcs_block test_cluster_gcs_block_retransmit test_cluster_gcs_block_2way test_cluster_gcs_block_3way test_cluster_gcs_block_lost_write test_cluster_gcs_block_dedup_reclaim test_cluster_gcs_block_dedup_htab test_cluster_sinval test_cluster_sinval_ack test_cluster_stage2_acceptance test_cluster_tt_status test_cluster_tt_status_hint test_cluster_visibility_fork test_cluster_visibility_decide_scn test_cluster_snapshot_source test_cluster_itl_touch test_cluster_itl_wal test_cluster_uba \ test_cluster_startup_phase test_cluster_lmon test_cluster_lck test_cluster_diag test_cluster_stats test_cluster_cssd test_cluster_qvotec test_cluster_voting_disk_io test_cluster_quorum_decision \ test_cluster_xlog test_cluster_tt_slot test_cluster_undo_segment \ test_cluster_epoch test_cluster_fence test_cluster_reconfig test_cluster_marker_async \ @@ -181,7 +181,7 @@ test_cluster_backup: test_cluster_backup.c unit_test.h $(CLUSTER_VERSION_O) \ # separate rules because they also link additional cluster_*.o # objects (the test files stub the PG backend symbols those # objects reference). -SIMPLE_TESTS = $(filter-out test_cluster_guc test_cluster_shmem test_cluster_signal test_cluster_views test_cluster_gviews test_cluster_ic test_cluster_conf test_cluster_ic_mock test_cluster_inject test_cluster_pgstat test_cluster_debug test_cluster_shared_fs test_cluster_shared_fs_sharedfs test_cluster_shared_fs_block_device test_cluster_smgr test_cluster_startup_phase test_cluster_lmon test_cluster_lck test_cluster_diag test_cluster_stats test_cluster_cssd test_cluster_qvotec test_cluster_voting_disk_io test_cluster_quorum_decision test_cluster_scn test_cluster_adg test_cluster_epoch test_cluster_fence test_cluster_reconfig test_cluster_ges test_cluster_grd test_cluster_grd_starvation test_cluster_lmd test_cluster_lmd_graph test_cluster_lmd_wait_state test_cluster_cancel_token test_cluster_lmd_probe_collector test_cluster_lock_acquire test_cluster_advisory test_cluster_terminal_authority test_cluster_retention test_cluster_visibility_variants test_cluster_writer_chain test_cluster_tt_2pc test_cluster_stage3_acceptance test_cluster_undo_buf test_cluster_block_apply test_cluster_thread_apply test_cluster_thread_replay test_cluster_thread_driver test_cluster_thread_orchestrator test_cluster_write_fence test_cluster_stage4_acceptance test_cluster_stage5_integrated_acceptance test_cluster_stage5_beta_acceptance test_cluster_ges_mode test_cluster_sequence test_cluster_shared_catalog test_cluster_hw test_cluster_dl test_cluster_extend_gate test_cluster_ir test_cluster_ts test_cluster_ko test_cluster_hw_snapshot test_cluster_cf_authority test_cluster_cf_storage test_cluster_cf_enqueue test_cluster_cf_phase2 test_cluster_cf_stats test_cluster_hang test_cluster_hang_resolve test_cluster_cr_server_policy test_cluster_touched_peers test_cluster_clean_leave test_cluster_membership test_cluster_node_remove test_cluster_resolver_cache test_cluster_backup test_cluster_hang_acceptance test_cluster_gcs_reqid test_cluster_runtime_visibility test_cluster_xid_stripe test_cluster_mxid_stripe test_cluster_bufmgr_pcm_hook test_cluster_cr test_cluster_cr_admit test_cluster_cr_admit_stat test_cluster_cr_cache test_cluster_cr_coordinator test_cluster_cr_key test_cluster_cr_lifecycle test_cluster_cr_pool test_cluster_cr_tuple test_cluster_cr_tuple_stat test_cluster_gcs_block test_cluster_gcs_block_2way test_cluster_gcs_block_3way test_cluster_gcs_block_lost_write test_cluster_gcs_block_retransmit test_cluster_gcs_block_dedup_reclaim test_cluster_gcs_dispatch test_cluster_ges_handoff test_cluster_heap_lock_tuple test_cluster_hw_lease test_cluster_ic_envelope test_cluster_ic_router test_cluster_itl_cleanout test_cluster_itl_cleanout_perf test_cluster_itl_reader_real_triple test_cluster_itl_touch test_cluster_itl_wal test_cluster_multixact test_cluster_multixact_served test_cluster_pcm_lock test_cluster_perf_gates test_cluster_recovery_merge test_cluster_recovery_plan test_cluster_recovery_worker test_cluster_reverse_key test_cluster_sinval test_cluster_sinval_ack test_cluster_snapshot_source test_cluster_stage2_acceptance test_cluster_stage5_5_cr_acceptance test_cluster_subtrans test_cluster_tt_durable test_cluster_tt_slot_allocator test_cluster_tt_status test_cluster_tt_status_hint test_cluster_uba test_cluster_undo_format test_cluster_undo_lifecycle test_cluster_undo_record test_cluster_visibility_decide_scn test_cluster_visibility_fork test_cluster_visibility_inject test_cluster_wal_state test_cluster_wal_thread test_cluster_xnode_lever test_cluster_xnode_profile test_cluster_pi_shadow test_cluster_oid_lease test_cluster_xid_authority test_cluster_recovery_anchor test_cluster_relmap_authority,$(TESTS)) +SIMPLE_TESTS = $(filter-out test_cluster_guc test_cluster_shmem test_cluster_signal test_cluster_views test_cluster_gviews test_cluster_ic test_cluster_conf test_cluster_ic_mock test_cluster_inject test_cluster_pgstat test_cluster_debug test_cluster_shared_fs test_cluster_shared_fs_sharedfs test_cluster_shared_fs_block_device test_cluster_smgr test_cluster_startup_phase test_cluster_lmon test_cluster_lck test_cluster_diag test_cluster_stats test_cluster_cssd test_cluster_qvotec test_cluster_voting_disk_io test_cluster_quorum_decision test_cluster_scn test_cluster_adg test_cluster_epoch test_cluster_fence test_cluster_reconfig test_cluster_ges test_cluster_grd test_cluster_grd_starvation test_cluster_lmd test_cluster_lmd_graph test_cluster_lmd_wait_state test_cluster_cancel_token test_cluster_lmd_probe_collector test_cluster_lock_acquire test_cluster_advisory test_cluster_terminal_authority test_cluster_retention test_cluster_visibility_variants test_cluster_writer_chain test_cluster_tt_2pc test_cluster_stage3_acceptance test_cluster_undo_buf test_cluster_block_apply test_cluster_thread_apply test_cluster_thread_replay test_cluster_thread_driver test_cluster_thread_orchestrator test_cluster_write_fence test_cluster_stage4_acceptance test_cluster_stage5_integrated_acceptance test_cluster_stage5_beta_acceptance test_cluster_ges_mode test_cluster_sequence test_cluster_shared_catalog test_cluster_hw test_cluster_dl test_cluster_extend_gate test_cluster_ir test_cluster_ts test_cluster_ko test_cluster_hw_snapshot test_cluster_cf_authority test_cluster_cf_storage test_cluster_cf_enqueue test_cluster_cf_phase2 test_cluster_cf_stats test_cluster_hang test_cluster_hang_resolve test_cluster_cr_server_policy test_cluster_touched_peers test_cluster_clean_leave test_cluster_membership test_cluster_node_remove test_cluster_resolver_cache test_cluster_backup test_cluster_hang_acceptance test_cluster_gcs_reqid test_cluster_runtime_visibility test_cluster_xid_stripe test_cluster_mxid_stripe test_cluster_bufmgr_pcm_hook test_cluster_cr test_cluster_cr_admit test_cluster_cr_admit_stat test_cluster_cr_cache test_cluster_cr_coordinator test_cluster_cr_key test_cluster_cr_lifecycle test_cluster_cr_pool test_cluster_cr_tuple test_cluster_cr_tuple_stat test_cluster_gcs_block test_cluster_gcs_block_2way test_cluster_gcs_block_3way test_cluster_gcs_block_lost_write test_cluster_gcs_block_retransmit test_cluster_gcs_block_dedup_reclaim test_cluster_gcs_block_dedup_htab test_cluster_gcs_dispatch test_cluster_ges_handoff test_cluster_heap_lock_tuple test_cluster_hw_lease test_cluster_ic_envelope test_cluster_ic_router test_cluster_itl_cleanout test_cluster_itl_cleanout_perf test_cluster_itl_reader_real_triple test_cluster_itl_touch test_cluster_itl_wal test_cluster_multixact test_cluster_multixact_served test_cluster_pcm_lock test_cluster_perf_gates test_cluster_recovery_merge test_cluster_recovery_plan test_cluster_recovery_worker test_cluster_reverse_key test_cluster_sinval test_cluster_sinval_ack test_cluster_snapshot_source test_cluster_stage2_acceptance test_cluster_stage5_5_cr_acceptance test_cluster_subtrans test_cluster_tt_durable test_cluster_tt_slot_allocator test_cluster_tt_status test_cluster_tt_status_hint test_cluster_uba test_cluster_undo_format test_cluster_undo_lifecycle test_cluster_undo_record test_cluster_visibility_decide_scn test_cluster_visibility_fork test_cluster_visibility_inject test_cluster_wal_state test_cluster_wal_thread test_cluster_xnode_lever test_cluster_xnode_profile test_cluster_pi_shadow test_cluster_oid_lease test_cluster_xid_authority test_cluster_recovery_anchor test_cluster_relmap_authority,$(TESTS)) # spec-2.4 D16: test_cluster_epoch links cluster_epoch.o standalone. # cluster_epoch.c references ShmemInitStruct + cluster_shmem_register_region # + pg_atomic_*; the test stubs all of these and exercises only the @@ -1317,7 +1317,7 @@ test_cluster_gcs_block_retransmit: test_cluster_gcs_block_retransmit.c unit_test # per-entry reclaim verdict (GcsBlockDedupEntryIsReclaimSafe: in-flight never, # out-of-window 2x safe for all, in-window idempotent-only). Header-only; # behavioral coverage (cap-full -> reclaim, evict_count, retransmit-dedup -# correctness) in cluster_tap t/NNN_gcs_block_dedup_capacity.pl. +# correctness) in cluster_tap t/366_gcs_block_dedup_capacity_2node.pl. test_cluster_gcs_block_dedup_reclaim: test_cluster_gcs_block_dedup_reclaim.c unit_test.h \ $(CLUSTER_VERSION_O) $(CC) $(CFLAGS) $(CPPFLAGS) $< \ @@ -1325,6 +1325,20 @@ test_cluster_gcs_block_dedup_reclaim: test_cluster_gcs_block_dedup_reclaim.c uni $(top_builddir)/src/common/libpgcommon_srv.a \ $(top_builddir)/src/port/libpgport_srv.a -o $@ +# spec-7.2a review F4: test_cluster_gcs_block_dedup_htab links the REAL +# cluster_gcs_block_dedup.o and drives lookup_or_register to cap against a +# bounded fake HTAB (with HASH_REMOVE + hole reuse), proving the +# cap-full -> eager-reclaim -> re-register chain (U1-U6) and the D4 +# auto-size floor. The TTL sweep is never called, so evict_count +# assertions attribute to eager reclaim alone. +CLUSTER_GCS_BLOCK_DEDUP_O = $(top_builddir)/src/backend/cluster/cluster_gcs_block_dedup.o +test_cluster_gcs_block_dedup_htab: test_cluster_gcs_block_dedup_htab.c unit_test.h \ + $(CLUSTER_VERSION_O) $(CLUSTER_GCS_BLOCK_DEDUP_O) + $(CC) $(CFLAGS) $(CPPFLAGS) $< \ + $(CLUSTER_VERSION_O) $(CLUSTER_GCS_BLOCK_DEDUP_O) \ + $(top_builddir)/src/common/libpgcommon_srv.a \ + $(top_builddir)/src/port/libpgport_srv.a -o $@ + # spec-2.35 D17: test_cluster_gcs_block_2way verifies CF 2-way protocol # compile-time invariants (msg_type 16, status 8, GcsBlockForwardPayload # 64B, reply header reserved 重解读 sizeof 不变, dedup FORWARDED_DUPLICATE diff --git a/src/test/cluster_unit/test_cluster_debug.c b/src/test/cluster_unit/test_cluster_debug.c index 31311a31dc..e3d68981ec 100644 --- a/src/test/cluster_unit/test_cluster_debug.c +++ b/src/test/cluster_unit/test_cluster_debug.c @@ -1114,6 +1114,23 @@ cluster_gcs_get_block_dedup_full_count(void) { return 0; } + +/* spec-7.2a D5 stubs: 3 NEW dedup capacity/occupancy accessors. */ +uint64 +cluster_gcs_get_block_dedup_entry_count(void) +{ + return 0; +} +uint64 +cluster_gcs_get_block_dedup_evict_count(void) +{ + return 0; +} +uint64 +cluster_gcs_get_block_dedup_max_entries(void) +{ + return 0; +} uint64 cluster_gcs_get_block_epoch_invalidate_wake_count(void) { diff --git a/src/test/cluster_unit/test_cluster_gcs_block_dedup_htab.c b/src/test/cluster_unit/test_cluster_gcs_block_dedup_htab.c new file mode 100644 index 0000000000..038d3e56e4 --- /dev/null +++ b/src/test/cluster_unit/test_cluster_gcs_block_dedup_htab.c @@ -0,0 +1,587 @@ +/*------------------------------------------------------------------------- + * + * test_cluster_gcs_block_dedup_htab.c + * Behavioral unit tests for the spec-7.2a GCS block dedup capacity + + * eager-GC path (U1-U6): links the REAL cluster_gcs_block_dedup.o and + * drives cluster_gcs_block_dedup_lookup_or_register() to cap against a + * bounded fake shmem HTAB fixture. + * + * Complements test_cluster_gcs_block_dedup_reclaim.c (header-only pure + * decision primitives, R1-R12): here the cap-full -> eager-reclaim -> + * re-register chain itself is exercised. The LMON TTL sweep + * (cluster_gcs_block_dedup_sweep_expired) is NEVER called by this + * binary and the clock is a controlled fake, so every evict_count + * increment observed below is attributable to the MISS-path eager + * reclaim alone (review F4: assertions must separate eager from TTL). + * + * Tests (spec §4.1 numbering): + * U1 fill-to-cap with in-flight entries -> FULL fail-closed + * (in-flight is never reclaim-safe; full_count++, evict_count 0) + * U3 completed-but-in-window entries (GRANTED) -> still FULL + * (whitelist empty: no in-window reclaim) + * U4 completed-but-in-window storage-fallback -> still FULL + * (:3305 N->S / :4089 S->X_UPGRADE sites; never in-window) + * U2 past the 2x out-of-window threshold -> MISS triggers eager + * reclaim, registration succeeds, full_count does NOT grow, + * evict_count +1, entry_count conserved at cap + * U5 the reclaimed key re-registers through MISS without double + * counting, and dedup (install_reply -> CACHED_REPLY hit) still + * works on the re-registered entry + * U6 entry_count bookkeeping matches the actual HTAB element count + * across register / reclaim / remove + * D4a auto-size floor binds: configured < MaxConnections x declared + * node count -> effective (stamped max_entries) = floor + * D4b configured wins when >= floor + * D4c floor clamps at the GUC ceiling (shmem_size equivalence) + * + * + * Portions Copyright (c) 1996-2024, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * Portions Copyright (c) 2026, pgrac contributors + * + * Author: SqlRush + * + * IDENTIFICATION + * src/test/cluster_unit/test_cluster_gcs_block_dedup_htab.c + * + * NOTES + * This is a pgrac-original file. + * Spec: spec-7.2a-gcs-block-dedup-capacity-gc.md (APPROVED 2026-07-09) + * + *------------------------------------------------------------------------- + */ +#include "postgres.h" + +#include +#include +#include + +#include "cluster/cluster_conf.h" +#include "cluster/cluster_gcs_block.h" +#include "cluster/cluster_gcs_block_dedup.h" +#include "cluster/cluster_guc.h" +#include "cluster/cluster_shmem.h" +#include "miscadmin.h" +#include "storage/backendid.h" +#include "storage/ipc.h" +#include "storage/lwlock.h" +#include "storage/shmem.h" +#include "utils/hsearch.h" +#include "utils/timestamp.h" +#include "unit_test.h" + +UT_DEFINE_GLOBALS(); + +/* ============================================================ + * Globals consumed by cluster_gcs_block_dedup.o (normally owned by + * cluster_guc.c / globals.c). + * ============================================================ */ + +bool cluster_enabled = true; +int cluster_node_id = 0; +int cluster_gcs_block_dedup_max_entries = 4; +int cluster_gcs_block_retransmit_max_retries = 4; +int cluster_gcs_block_retransmit_initial_backoff_ms = 100; +int MaxConnections = 1; +bool IsUnderPostmaster = false; +BackendId MyBackendId = InvalidBackendId; + +/* Controlled fake clock (no sleeps: window aging is simulated). */ +static TimestampTz fake_now = 1000000000; + +/* Controlled declared-node-count sniff (spec-7.2a D4 auto-size input). */ +static int fake_declared_nodes = 1; + +/* + * With backoff 100ms x (2^4 - 1) = 1500ms total budget, the 2x + * out-of-window threshold used by both eager reclaim and the TTL sweep is + * 3,000,000 us. Tests age entries past it by advancing fake_now. + */ +#define UT_OUT_OF_WINDOW_US INT64CONST(3000000) + +/* ============================================================ + * Stubs to link cluster_gcs_block_dedup.o standalone. + * ============================================================ */ + +void +ExceptionalCondition(const char *conditionName pg_attribute_unused(), + const char *fileName pg_attribute_unused(), + int lineNumber pg_attribute_unused()) +{ + abort(); +} + +bool +errstart(int e pg_attribute_unused(), const char *d pg_attribute_unused()) +{ + return false; +} + +bool +errstart_cold(int e pg_attribute_unused(), const char *d pg_attribute_unused()) +{ + return false; +} + +void +errfinish(const char *f pg_attribute_unused(), int l pg_attribute_unused(), + const char *fn pg_attribute_unused()) +{} + +int +errcode(int s pg_attribute_unused()) +{ + return 0; +} + +int +errmsg(const char *f pg_attribute_unused(), ...) +{ + return 0; +} + +int +errmsg_internal(const char *f pg_attribute_unused(), ...) +{ + return 0; +} + +bool +message_level_is_interesting(int elevel pg_attribute_unused()) +{ + return false; +} + +TimestampTz +GetCurrentTimestamp(void) +{ + return fake_now; +} + +int +cluster_conf_declared_node_count_early(void) +{ + return fake_declared_nodes; +} + +void +cluster_shmem_register_region(const ClusterShmemRegion *r pg_attribute_unused()) +{} + +void +before_shmem_exit(pg_on_exit_callback function pg_attribute_unused(), + Datum arg pg_attribute_unused()) +{} + +Size +add_size(Size s1, Size s2) +{ + return s1 + s2; +} + +/* Linear estimate is enough for the D4c size-equivalence assertion. */ +Size +hash_estimate_size(long num_entries, Size entrysize) +{ + return (Size)num_entries * entrysize; +} + +void +LWLockInitialize(LWLock *lock pg_attribute_unused(), int tranche_id pg_attribute_unused()) +{} + +bool +LWLockAcquire(LWLock *lock pg_attribute_unused(), LWLockMode mode pg_attribute_unused()) +{ + return true; +} + +void +LWLockRelease(LWLock *lock pg_attribute_unused()) +{} + +/* ============================================================ + * Bounded fake shmem HTAB (L105 union force-align). Unlike the plain + * append-only sibling fixture (test_cluster_bufmgr_pcm_hook.c) this one + * supports HASH_REMOVE with hole reuse, because the unit under test is + * exactly the remove-then-re-register eager-reclaim chain. + * ============================================================ */ + +#define FAKE_DEDUP_MAX_SLOTS 32 + +static union { + uint64 force_align; + char data[4096]; /* generous; sizeof(ClusterGcsBlockDedupShared) << 4KB */ +} fake_dedup_header; + +static union { + uint64 force_align; + char data[FAKE_DEDUP_MAX_SLOTS][sizeof(GcsBlockDedupEntry)]; +} fake_dedup_slots; + +static bool fake_slot_used[FAKE_DEDUP_MAX_SLOTS]; +static char fake_dedup_htab_token; +static bool fake_dedup_header_found = false; +static long fake_dedup_entry_max = 0; +static Size fake_dedup_keysize = 0; + +void * +ShmemInitStruct(const char *name pg_attribute_unused(), Size size, bool *foundPtr) +{ + Assert(size <= sizeof(fake_dedup_header.data)); + *foundPtr = fake_dedup_header_found; + fake_dedup_header_found = true; + return fake_dedup_header.data; +} + +HTAB * +ShmemInitHash(const char *name pg_attribute_unused(), long init_size pg_attribute_unused(), + long max_size, HASHCTL *infoP, int hash_flags pg_attribute_unused()) +{ + Assert((hash_flags & HASH_ELEM) != 0); + Assert(infoP->entrysize == sizeof(GcsBlockDedupEntry)); + Assert(max_size <= FAKE_DEDUP_MAX_SLOTS); + fake_dedup_keysize = infoP->keysize; + fake_dedup_entry_max = max_size; + memset(fake_slot_used, 0, sizeof(fake_slot_used)); + return (HTAB *)&fake_dedup_htab_token; +} + +static long +fake_live_count(void) +{ + long i; + long n = 0; + + for (i = 0; i < FAKE_DEDUP_MAX_SLOTS; i++) + if (fake_slot_used[i]) + n++; + return n; +} + +void * +hash_search(HTAB *hashp pg_attribute_unused(), const void *keyPtr, HASHACTION action, + bool *foundPtr) +{ + long i; + + Assert(fake_dedup_keysize > 0); + + for (i = 0; i < FAKE_DEDUP_MAX_SLOTS; i++) { + if (!fake_slot_used[i]) + continue; + if (memcmp(fake_dedup_slots.data[i], keyPtr, fake_dedup_keysize) == 0) { + if (foundPtr != NULL) + *foundPtr = true; + if (action == HASH_REMOVE) + fake_slot_used[i] = false; + return fake_dedup_slots.data[i]; + } + } + + if (action != HASH_ENTER && action != HASH_ENTER_NULL) { + if (foundPtr != NULL) + *foundPtr = false; + return NULL; + } + + if (fake_live_count() >= fake_dedup_entry_max) { + if (action == HASH_ENTER_NULL) { + if (foundPtr != NULL) + *foundPtr = false; + return NULL; + } + Assert(false); + } + + for (i = 0; i < FAKE_DEDUP_MAX_SLOTS; i++) { + if (!fake_slot_used[i]) { + /* dynahash contract: only the key is initialised on ENTER; the + * value area keeps whatever was there before (the module must + * reset it itself). */ + memcpy(fake_dedup_slots.data[i], keyPtr, fake_dedup_keysize); + fake_slot_used[i] = true; + if (foundPtr != NULL) + *foundPtr = false; + return fake_dedup_slots.data[i]; + } + } + Assert(false); + return NULL; +} + +/* + * Slot-index iteration that skips holes. Removing the element the scan + * just returned (what the eager reclaim does) cannot disturb the remaining + * iteration, matching dynahash's guarantee for HASH_REMOVE of the current + * seq element. + */ +void +hash_seq_init(HASH_SEQ_STATUS *status, HTAB *hashp pg_attribute_unused()) +{ + status->curBucket = 0; + status->hashp = NULL; +} + +void * +hash_seq_search(HASH_SEQ_STATUS *status) +{ + while (status->curBucket < FAKE_DEDUP_MAX_SLOTS) { + uint32 i = status->curBucket++; + + if (fake_slot_used[i]) + return fake_dedup_slots.data[i]; + } + return NULL; +} + +void +hash_seq_term(HASH_SEQ_STATUS *status pg_attribute_unused()) +{} + +/* ============================================================ + * Fixture helpers. + * ============================================================ */ + +static void +fixture_reset(int configured, int max_conns, int declared_nodes) +{ + fake_dedup_header_found = false; + fake_dedup_entry_max = 0; + fake_dedup_keysize = 0; + memset(fake_slot_used, 0, sizeof(fake_slot_used)); + memset(&fake_dedup_header, 0, sizeof(fake_dedup_header)); + fake_now = 1000000000; + + cluster_enabled = true; + cluster_node_id = 0; + cluster_gcs_block_dedup_max_entries = configured; + MaxConnections = max_conns; + fake_declared_nodes = declared_nodes; + + Assert(cluster_gcs_block_dedup_shmem_size() > 0); + cluster_gcs_block_dedup_shmem_init(); +} + +static GcsBlockDedupKey +make_key(uint64 request_id) +{ + GcsBlockDedupKey key; + + memset(&key, 0, sizeof(key)); + key.origin_node_id = 1; + key.requester_backend_id = 7; + key.request_id = request_id; + key.cluster_epoch = 3; + return key; +} + +static BufferTag +make_tag(uint32 blockno) +{ + BufferTag tag; + + memset(&tag, 0, sizeof(tag)); + tag.spcOid = 1663; + tag.dbOid = 1; + tag.relNumber = 200; + tag.forkNum = MAIN_FORKNUM; + tag.blockNum = blockno; + return tag; +} + +static GcsBlockDedupResult +register_key(uint64 request_id) +{ + GcsBlockDedupKey key = make_key(request_id); + BufferTag tag = make_tag((uint32)request_id); + + return cluster_gcs_block_dedup_lookup_or_register(&key, tag, 1, NULL); +} + +static void +complete_key(uint64 request_id, GcsBlockReplyStatus status) +{ + GcsBlockDedupKey key = make_key(request_id); + GcsBlockReplyHeader header; + + memset(&header, 0, sizeof(header)); + cluster_gcs_block_dedup_install_reply(&key, status, &header, NULL); +} + +/* ============================================================ + * U1-U6: cap pressure -> eager reclaim -> re-register (one flow, four + * cap-4 tests sharing fixture state built by the previous step, exactly + * like the production MISS path would see it). + * ============================================================ */ + +UT_TEST(test_u1_fill_to_cap_in_flight_full_fail_closed) +{ + uint64 k; + + fixture_reset(4, 1, 1); /* floor = 1x1 = 1 < 4 -> cap 4 */ + UT_ASSERT_EQ(4, (int)cluster_gcs_block_dedup_get_max_entries()); + + for (k = 0; k < 4; k++) + UT_ASSERT_EQ(GCS_BLOCK_DEDUP_MISS_REGISTERED, register_key(k)); + UT_ASSERT_EQ(4, (int)cluster_gcs_block_dedup_get_in_flight_count()); + + /* 5th distinct key: every resident entry is in-flight, which is never + * reclaim-safe -> eager reclaim must NOT fire -> FULL fail-closed. */ + UT_ASSERT_EQ(GCS_BLOCK_DEDUP_FULL, register_key(4)); + UT_ASSERT_EQ(1, (int)cluster_gcs_block_dedup_get_full_count()); + UT_ASSERT_EQ(0, (int)cluster_gcs_block_dedup_get_evict_count()); + UT_ASSERT_EQ(4, (int)cluster_gcs_block_dedup_get_in_flight_count()); + UT_ASSERT_EQ(4, (int)fake_live_count()); +} + +UT_TEST(test_u3_u4_completed_in_window_still_full) +{ + /* Complete all four entries NOW (in-window). k1 completes as + * storage-fallback (U4: :3305/:4089 transition sites -> never + * in-window reclaimable); the rest as payload GRANTED (U3). */ + complete_key(0, GCS_BLOCK_REPLY_GRANTED); + complete_key(1, GCS_BLOCK_REPLY_GRANTED_STORAGE_FALLBACK); + complete_key(2, GCS_BLOCK_REPLY_GRANTED); + complete_key(3, GCS_BLOCK_REPLY_GRANTED); + + /* Still within the 2x window: whitelist is empty, so no in-window + * reclaim of ANY status -> FULL again. */ + UT_ASSERT_EQ(GCS_BLOCK_DEDUP_FULL, register_key(4)); + UT_ASSERT_EQ(2, (int)cluster_gcs_block_dedup_get_full_count()); + UT_ASSERT_EQ(0, (int)cluster_gcs_block_dedup_get_evict_count()); + UT_ASSERT_EQ(4, (int)cluster_gcs_block_dedup_get_in_flight_count()); +} + +UT_TEST(test_u2_out_of_window_miss_triggers_eager_reclaim) +{ + uint64 miss_before = cluster_gcs_block_dedup_get_miss_count(); + + /* Age every completed entry past the 2x out-of-window threshold. The + * TTL sweep is never invoked in this binary, so the eviction observed + * below is attributable to the MISS-path eager reclaim alone. */ + fake_now += UT_OUT_OF_WINDOW_US + 1000; + + UT_ASSERT_EQ(GCS_BLOCK_DEDUP_MISS_REGISTERED, register_key(4)); + + /* full_count did NOT grow (still 2 from U1/U3); exactly one reclaim. */ + UT_ASSERT_EQ(2, (int)cluster_gcs_block_dedup_get_full_count()); + UT_ASSERT_EQ(1, (int)cluster_gcs_block_dedup_get_evict_count()); + UT_ASSERT_EQ(miss_before + 1, cluster_gcs_block_dedup_get_miss_count()); + + /* Accounting conserved at cap: one out, one in. */ + UT_ASSERT_EQ(4, (int)cluster_gcs_block_dedup_get_in_flight_count()); + UT_ASSERT_EQ(4, (int)fake_live_count()); +} + +UT_TEST(test_u5_reclaimed_key_re_registers_without_double_count) +{ + GcsBlockDedupKey key0 = make_key(0); + BufferTag tag0 = make_tag(0); + GcsBlockDedupEntry cached; + + /* U2 evicted the scan-order-first safe entry: key 0 (slot 0). A late + * duplicate of key 0 must re-register through MISS (not CACHED_REPLY / + * IN_FLIGHT_DUPLICATE) and must not double-count: the table is full + * again, so its registration eager-reclaims exactly one more aged + * completed entry. */ + UT_ASSERT_EQ(GCS_BLOCK_DEDUP_MISS_REGISTERED, + cluster_gcs_block_dedup_lookup_or_register(&key0, tag0, 1, NULL)); + UT_ASSERT_EQ(2, (int)cluster_gcs_block_dedup_get_evict_count()); + UT_ASSERT_EQ(2, (int)cluster_gcs_block_dedup_get_full_count()); + UT_ASSERT_EQ(4, (int)cluster_gcs_block_dedup_get_in_flight_count()); + UT_ASSERT_EQ(4, (int)fake_live_count()); + UT_ASSERT_EQ(0, (int)cluster_gcs_block_dedup_get_hit_count()); + + /* Dedup still works on the re-registered entry: complete it, then a + * same-key retransmit hits the cached reply. */ + complete_key(0, GCS_BLOCK_REPLY_GRANTED); + UT_ASSERT_EQ(GCS_BLOCK_DEDUP_CACHED_REPLY, + cluster_gcs_block_dedup_lookup_or_register(&key0, tag0, 1, &cached)); + UT_ASSERT_EQ(1, (int)cluster_gcs_block_dedup_get_hit_count()); + UT_ASSERT_EQ((int)GCS_BLOCK_REPLY_GRANTED, (int)cached.status); +} + +UT_TEST(test_u6_remove_keeps_accounting_conserved) +{ + GcsBlockDedupKey key0 = make_key(0); + + cluster_gcs_block_dedup_remove(&key0); + UT_ASSERT_EQ(3, (int)cluster_gcs_block_dedup_get_in_flight_count()); + UT_ASSERT_EQ(3, (int)fake_live_count()); + + /* Removing a key that is no longer resident must not underflow. */ + cluster_gcs_block_dedup_remove(&key0); + UT_ASSERT_EQ(3, (int)cluster_gcs_block_dedup_get_in_flight_count()); + UT_ASSERT_EQ(3, (int)fake_live_count()); +} + +/* ============================================================ + * D4: auto-size floor (spec-7.2a Q4=A; review F3). + * ============================================================ */ + +UT_TEST(test_d4a_autosize_floor_binds_small_config) +{ + /* configured 4 < floor 8x2 = 16 -> the HTAB is built (and stamped) + * with the floor. */ + fixture_reset(4, 8, 2); + UT_ASSERT_EQ(16, (int)cluster_gcs_block_dedup_get_max_entries()); +} + +UT_TEST(test_d4b_configured_wins_over_floor) +{ + fixture_reset(24, 8, 2); /* floor 16 < configured 24 */ + UT_ASSERT_EQ(24, (int)cluster_gcs_block_dedup_get_max_entries()); +} + +UT_TEST(test_d4c_floor_clamps_at_guc_ceiling) +{ + Size at_ceiling; + Size clamped; + + /* Size the region with configured == ceiling and a non-binding floor, + * then with a tiny config whose floor (40000 x 2 = 80000) overshoots + * the ceiling. The clamp makes both resolve to the ceiling, so the + * reserved sizes must be identical (no init needed: the fake fixture + * cannot hold 65536 slots, and D4c is a sizing property). */ + cluster_gcs_block_dedup_max_entries = CLUSTER_GCS_BLOCK_DEDUP_MAX_ENTRIES_CEILING; + MaxConnections = 1; + fake_declared_nodes = 1; + at_ceiling = cluster_gcs_block_dedup_shmem_size(); + + cluster_gcs_block_dedup_max_entries = 256; + MaxConnections = 40000; + fake_declared_nodes = 2; + clamped = cluster_gcs_block_dedup_shmem_size(); + + UT_ASSERT(at_ceiling > 0); + UT_ASSERT_EQ(at_ceiling, clamped); +} + +UT_TEST(test_d4d_initdb_gate_still_returns_zero) +{ + /* The pre-existing initdb/bootstrap gate must survive the floor: + * cluster_node_id < 0 -> no allocation regardless of MaxConnections. */ + cluster_node_id = -1; + MaxConnections = 40000; + fake_declared_nodes = 2; + UT_ASSERT_EQ(0, (int)cluster_gcs_block_dedup_shmem_size()); + cluster_node_id = 0; +} + +int +main(void) +{ + UT_PLAN(9); + UT_RUN(test_u1_fill_to_cap_in_flight_full_fail_closed); + UT_RUN(test_u3_u4_completed_in_window_still_full); + UT_RUN(test_u2_out_of_window_miss_triggers_eager_reclaim); + UT_RUN(test_u5_reclaimed_key_re_registers_without_double_count); + UT_RUN(test_u6_remove_keeps_accounting_conserved); + UT_RUN(test_d4a_autosize_floor_binds_small_config); + UT_RUN(test_d4b_configured_wins_over_floor); + UT_RUN(test_d4c_floor_clamps_at_guc_ceiling); + UT_RUN(test_d4d_initdb_gate_still_returns_zero); + UT_DONE(); + return ut_failed_count == 0 ? 0 : 1; +} diff --git a/src/test/cluster_unit/test_cluster_gcs_block_dedup_reclaim.c b/src/test/cluster_unit/test_cluster_gcs_block_dedup_reclaim.c index 82fb1483e0..91d946b5c3 100644 --- a/src/test/cluster_unit/test_cluster_gcs_block_dedup_reclaim.c +++ b/src/test/cluster_unit/test_cluster_gcs_block_dedup_reclaim.c @@ -12,7 +12,7 @@ * Header-only — no linking of cluster_gcs_block_dedup.o. The behavioral * reclaim path (cap-full -> reclaim -> register, evict_count accounting, * retransmit-dedup correctness under drop-reply injection) lives in - * cluster_tap t/NNN_gcs_block_dedup_capacity.pl, which exercises a real + * cluster_tap t/366_gcs_block_dedup_capacity_2node.pl, which exercises a real * 2-node ClusterPair. * * Tests in this binary: @@ -79,7 +79,7 @@ make_completed_entry(GcsBlockReplyStatus status, TimestampTz now, int64 age_us) GcsBlockDedupEntry e; memset(&e, 0, sizeof(e)); - e.status = (uint8) status; + e.status = (uint8)status; e.registered_at_ts = now - age_us - 1; e.completed_at_ts = now - age_us; return e; @@ -94,7 +94,7 @@ UT_TEST(test_r1_whitelist_empty_all_statuses_in_window_unsafe) /* Every reply status in the enum range must be fail-closed UNSAFE for * in-window reclaim until its install sites are proven idempotent. */ for (s = GCS_BLOCK_REPLY_GRANTED; s <= GCS_BLOCK_REPLY_UNDO_VERDICT_RESULT; s++) - UT_ASSERT_EQ(false, GcsBlockReplyStatusIsReclaimIdempotent((GcsBlockReplyStatus) s)); + UT_ASSERT_EQ(false, GcsBlockReplyStatusIsReclaimIdempotent((GcsBlockReplyStatus)s)); } UT_TEST(test_r2_payload_granted_never_in_window) @@ -144,7 +144,7 @@ UT_TEST(test_r8_in_flight_entry_never_reclaim_safe) TimestampTz now = 1000000; memset(&e, 0, sizeof(e)); - e.status = (uint8) GCS_BLOCK_REPLY_GRANTED; + e.status = (uint8)GCS_BLOCK_REPLY_GRANTED; e.registered_at_ts = now - 5000; e.completed_at_ts = 0; /* in-flight: original request still processing */ @@ -156,8 +156,8 @@ UT_TEST(test_r9_aged_out_safe_for_every_status) TimestampTz now = 1000000; int64 window = 1000; GcsBlockDedupEntry granted = make_completed_entry(GCS_BLOCK_REPLY_GRANTED, now, 2000); - GcsBlockDedupEntry sfb = - make_completed_entry(GCS_BLOCK_REPLY_GRANTED_STORAGE_FALLBACK, now, 2000); + GcsBlockDedupEntry sfb + = make_completed_entry(GCS_BLOCK_REPLY_GRANTED_STORAGE_FALLBACK, now, 2000); GcsBlockDedupEntry lost = make_completed_entry(GCS_BLOCK_REPLY_DENIED_LOST_WRITE, now, 2000); /* age 2000us > window 1000us: out-of-window, safe for ALL (2x theorem). */ @@ -194,11 +194,10 @@ UT_TEST(test_r12_in_window_verdict_tracks_whitelist) /* In-window (age 100 << window): verdict must equal the whitelist for * every status. Whitelist is empty -> all in-window verdicts false. */ - for (s = GCS_BLOCK_REPLY_GRANTED; s <= GCS_BLOCK_REPLY_UNDO_VERDICT_RESULT; s++) - { - GcsBlockDedupEntry e = make_completed_entry((GcsBlockReplyStatus) s, now, 100); + for (s = GCS_BLOCK_REPLY_GRANTED; s <= GCS_BLOCK_REPLY_UNDO_VERDICT_RESULT; s++) { + GcsBlockDedupEntry e = make_completed_entry((GcsBlockReplyStatus)s, now, 100); - UT_ASSERT_EQ(GcsBlockReplyStatusIsReclaimIdempotent((GcsBlockReplyStatus) s), + UT_ASSERT_EQ(GcsBlockReplyStatusIsReclaimIdempotent((GcsBlockReplyStatus)s), GcsBlockDedupEntryIsReclaimSafe(&e, now, window)); } } From eda743a0013ba206134e2959a933ddd90b20c2c9 Mon Sep 17 00:00:00 2001 From: SqlRush Date: Fri, 10 Jul 2026 19:54:40 +0800 Subject: [PATCH 6/7] fix(cluster): stale comment sweep + injection-target coverage honesty (spec-7.2a review P3) - skip_remaining field comment said :skip:N; the count-based syntax is :skipn:N (0 now documented as exhausted-or-not-armed). - t/116 L9 label and section header still said 67/48 gcs keys; the assertion is 88. - The drop-reply relfilenode gate comment and GUC description now state that current TAP coverage uses target=0 only (shared_catalog remaps the physical relNumber away from SQL view) and the non-zero filter is reserved for non-shared-catalog rigs. Spec: spec-7.2a-gcs-block-dedup-capacity-gc.md --- src/backend/cluster/cluster_gcs_block.c | 8 +++++++- src/backend/cluster/cluster_inject.c | 8 ++++---- src/test/cluster_tap/t/116_gcs_block_lost_write_2node.pl | 4 ++-- 3 files changed, 13 insertions(+), 7 deletions(-) diff --git a/src/backend/cluster/cluster_gcs_block.c b/src/backend/cluster/cluster_gcs_block.c index 55e6397969..322ab1ff51 100644 --- a/src/backend/cluster/cluster_gcs_block.c +++ b/src/backend/cluster/cluster_gcs_block.c @@ -4659,10 +4659,16 @@ build_and_send_reply: { * user-relation ship reaches the point. Gating the CLUSTER_INJECTION_POINT * itself (not just should_skip) ensures only matching ships consume the * count. 0 (default) keeps the un-targeted behaviour for spec-2.34 tests. + * + * Current TAP coverage uses target=0 only (shared_catalog remaps the + * catalog-visible relfilenode to a different physical relNumber, so SQL + * cannot name the shipped block); the non-zero filter is reserved for + * precise spec-2.34-style targeting on non-shared-catalog rigs and is not + * yet exercised by any test. */ if (cluster_gcs_block_drop_target_relfilenode == 0 || BufTagGetRelNumber(&req->tag) - == (RelFileNumber) cluster_gcs_block_drop_target_relfilenode) { + == (RelFileNumber)cluster_gcs_block_drop_target_relfilenode) { CLUSTER_INJECTION_POINT("cluster-gcs-block-drop-reply-before-send"); if (cluster_injection_should_skip("cluster-gcs-block-drop-reply-before-send")) { gcs_block_release_ship_image(block_payload_release_cb, block_payload_release_arg); diff --git a/src/backend/cluster/cluster_inject.c b/src/backend/cluster/cluster_inject.c index fee0065973..e6ba20c3e8 100644 --- a/src/backend/cluster/cluster_inject.c +++ b/src/backend/cluster/cluster_inject.c @@ -68,7 +68,8 @@ typedef struct ClusterInjectPoint { pg_atomic_uint32 armed_type; /* ClusterInjectFaultType */ pg_atomic_uint64 armed_param; /* sleep us / SKIP cookie */ pg_atomic_uint32 skip_pending; /* set by SKIP dispatch, read+reset by probe */ - pg_atomic_uint32 skip_remaining; /* spec-7.2a: SKIP countdown for :skip:N (param 0 = unlimited) */ + pg_atomic_uint32 + skip_remaining; /* spec-7.2a: :skipn:N countdown (0 = exhausted or not armed) */ } ClusterInjectPoint; /* @@ -896,7 +897,7 @@ cluster_injection_arm_internal(ClusterInjectPoint *p, ClusterInjectFaultType new * "skip while armed" behaviour and leaves the countdown at 0. */ if (new_type == CLUSTER_FAULT_SKIP_N && param > 0) - pg_atomic_write_u32(&p->skip_remaining, (uint32) param); + pg_atomic_write_u32(&p->skip_remaining, (uint32)param); else pg_atomic_write_u32(&p->skip_remaining, 0); @@ -1010,8 +1011,7 @@ cluster_injection_run(const char *name) uint32 expected = pg_atomic_read_u32(&p->skip_remaining); while (expected > 0) { - if (pg_atomic_compare_exchange_u32(&p->skip_remaining, - &expected, expected - 1)) { + if (pg_atomic_compare_exchange_u32(&p->skip_remaining, &expected, expected - 1)) { pg_atomic_write_u32(&p->skip_pending, 1); break; } diff --git a/src/test/cluster_tap/t/116_gcs_block_lost_write_2node.pl b/src/test/cluster_tap/t/116_gcs_block_lost_write_2node.pl index 87e69e375f..149c4711b2 100644 --- a/src/test/cluster_tap/t/116_gcs_block_lost_write_2node.pl +++ b/src/test/cluster_tap/t/116_gcs_block_lost_write_2node.pl @@ -193,13 +193,13 @@ sub gcs_int # ============================================================ -# L9 (alias of L2): gcs key count = 48. +# L9 (alias of L2): gcs key count = 88. # ============================================================ is($pair->node1->safe_psql( 'postgres', q{SELECT count(*) FROM pg_cluster_state WHERE category='gcs'}), '88', - 'L9 node1 pg_cluster_state.gcs has 67 keys (cross-node parity)'); + 'L9 node1 pg_cluster_state.gcs has 88 keys (cross-node parity)'); # ============================================================ From 31905b85877a3b8f2be9afccaa89d6fc11d7806d Mon Sep 17 00:00:00 2001 From: SqlRush Date: Fri, 10 Jul 2026 21:15:58 +0800 Subject: [PATCH 7/7] feat(cluster): raise dedup default to the S1-measured green floor 16384 (spec-7.2a D4) The spec-7.2a S1 perf gate requires the shipped default (no hand bump) to survive the 4-node distinct-read scenario with dedup_full_count = 0 and zero client errors. Measured on the 4-node RACvsRAC harness at the shipped default: 4096 and 8192 both saturate during the post-formation cold storm (DENIED_DEDUP_FULL -> retransmit exhaustion plus downstream GCS control timeouts); 16384 is green across 3 consecutive runs on all 4 nodes (errors 0, dedup_full_count 0, ~230k qps/node). Raise the default to that floor (ceiling 65536 unchanged; ~138MB shmem per serving node at 8448B/entry) and sweep the default assertions in t/112 and t/366. Spec: spec-7.2a-gcs-block-dedup-capacity-gc.md --- src/backend/cluster/cluster_guc.c | 6 +++--- src/include/cluster/cluster_guc.h | 7 ++++--- .../t/112_gcs_block_retransmit_2node.pl | 4 ++-- .../t/366_gcs_block_dedup_capacity_2node.pl | 16 ++++++++-------- 4 files changed, 17 insertions(+), 16 deletions(-) diff --git a/src/backend/cluster/cluster_guc.c b/src/backend/cluster/cluster_guc.c index 7149ce39ae..f44c17846a 100644 --- a/src/backend/cluster/cluster_guc.c +++ b/src/backend/cluster/cluster_guc.c @@ -732,7 +732,7 @@ bool cluster_ic_duty_lazy = true; /* spec-7.2 D1 duty-chain on-demand gating */ * Off keeps the pre-7.1a floor: a TERMINAL remote writer holder fails * closed (SQLSTATE 53R9H) instead of chaining to a sound TM_Result. */ bool cluster_crossnode_write_write = false; -int cluster_gcs_block_dedup_max_entries = 4096; /* spec-7.2a: raised from 1024 */ +int cluster_gcs_block_dedup_max_entries = 16384; /* spec-7.2a: raised from 1024 */ /* * spec-7.2a test-only: when non-zero, the drop-reply injection only fires for * block ships of this relfilenode, so a :skipn:N count lands on the intended @@ -3953,7 +3953,7 @@ cluster_init_guc(void) DefineCustomIntVariable("cluster.gcs_block_dedup_max_entries", gettext_noop("Master-side GCS block dedup HTAB capacity (entries)."), gettext_noop("Each entry occupies sizeof(GcsBlockDedupEntry) = 8448B. " - "Default 4096 → ~34.7MB shmem on each node serving as " + "Default 16384 → ~138MB shmem on each node serving as " "GCS block-ship master; ceiling 65536 → ~554MB; " "bootstrap/initdb with no configured cluster.node_id does " "not allocate the HTAB. The effective capacity is never " @@ -3965,7 +3965,7 @@ cluster_init_guc(void) "still-full table → DENIED_DEDUP_FULL fail-closed " "(sender retries via HC96 transient). HC92. " "PGC_POSTMASTER (restart to change the fixed HTAB size)."), - &cluster_gcs_block_dedup_max_entries, 4096, 256, + &cluster_gcs_block_dedup_max_entries, 16384, 256, CLUSTER_GCS_BLOCK_DEDUP_MAX_ENTRIES_CEILING, PGC_POSTMASTER, 0, NULL, NULL, NULL); diff --git a/src/include/cluster/cluster_guc.h b/src/include/cluster/cluster_guc.h index 04fb75073f..581a7f5cea 100644 --- a/src/include/cluster/cluster_guc.h +++ b/src/include/cluster/cluster_guc.h @@ -919,10 +919,11 @@ extern int cluster_gcs_reply_timeout_ms; * * cluster.gcs_block_dedup_max_entries * type: int context: PGC_POSTMASTER - * default: 4096 (min 256, max 65536; spec-7.2a D4 raised from - * 1024/16384) + * default: 16384 (min 256, max 65536; spec-7.2a D4 raised from + * 1024/16384; 16384 = the measured S1 4-node distinct-read green + * floor on the RACvsRAC rig) * Per-node cap for the master-side dedup HTAB. Each entry occupies - * sizeof(GcsBlockDedupEntry) = 8448B, so default cap → ~34.7 MB shmem + * sizeof(GcsBlockDedupEntry) = 8448B, so default cap → ~138 MB shmem * on each node acting as GCS block-ship master. The effective * capacity is auto-sized to at least MaxConnections × declared node * count (capped at the ceiling; spec-7.2a D4 Q4). Under cap pressure diff --git a/src/test/cluster_tap/t/112_gcs_block_retransmit_2node.pl b/src/test/cluster_tap/t/112_gcs_block_retransmit_2node.pl index 23102c811a..999927f382 100644 --- a/src/test/cluster_tap/t/112_gcs_block_retransmit_2node.pl +++ b/src/test/cluster_tap/t/112_gcs_block_retransmit_2node.pl @@ -16,7 +16,7 @@ # L6 3 NEW GUC visible + defaults + contexts: # cluster.gcs_block_retransmit_max_retries PGC_SUSET 4 # cluster.gcs_block_retransmit_initial_backoff_ms PGC_SUSET 10 (spec-7.2 D1) -# cluster.gcs_block_dedup_max_entries PGC_POSTMASTER 4096 +# cluster.gcs_block_dedup_max_entries PGC_POSTMASTER 16384 # L7 single-shot ship workload — retransmit_attempt_count=0 # L8 inject `cluster-gcs-block-drop-reply-before-send:skip:1` → # retransmit_send_count grows + WARNING at 3/4 budget @@ -154,7 +154,7 @@ sub gcs_int for my $row ( [ 'cluster.gcs_block_retransmit_max_retries', '4', 'superuser' ], [ 'cluster.gcs_block_retransmit_initial_backoff_ms', '10', 'superuser' ], - [ 'cluster.gcs_block_dedup_max_entries', '4096', 'postmaster' ], + [ 'cluster.gcs_block_dedup_max_entries', '16384', 'postmaster' ], ) { my ($name, $expected_default, $expected_ctx) = @$row; diff --git a/src/test/cluster_tap/t/366_gcs_block_dedup_capacity_2node.pl b/src/test/cluster_tap/t/366_gcs_block_dedup_capacity_2node.pl index f249f5021a..5af24cfbd0 100644 --- a/src/test/cluster_tap/t/366_gcs_block_dedup_capacity_2node.pl +++ b/src/test/cluster_tap/t/366_gcs_block_dedup_capacity_2node.pl @@ -16,7 +16,7 @@ # fetched cross-node through the GCS block-ship data plane -- the substrate # the dedup HTAB serves. # -# L1 cluster.gcs_block_dedup_max_entries default = 4096 (spec-7.2a D4; +# L1 cluster.gcs_block_dedup_max_entries default = 16384 (spec-7.2a D4; # raised from 1024). PGC_POSTMASTER, visible via SHOW. # L2 cross-node distinct-block reads populate the dedup HTAB # (dedup_miss_count > 0 proves the cross-node ship path fired) while @@ -268,11 +268,11 @@ sub arm_inject # ============================================================ -# L1: default GUC value = 4096 (spec-7.2a D4). +# L1: default GUC value = 16384 (spec-7.2a D4). # ============================================================ is($node0->safe_psql('postgres', 'SHOW cluster.gcs_block_dedup_max_entries'), - '4096', - 'L1 cluster.gcs_block_dedup_max_entries default = 4096 (spec-7.2a D4)'); + '16384', + 'L1 cluster.gcs_block_dedup_max_entries default = 16384 (spec-7.2a D4)'); # ============================================================ @@ -321,7 +321,7 @@ sub arm_inject . "(dedup_miss $miss_pre -> $miss_post) -- cross-node ship path fired"); is($full_ct, 0, - "L2 dedup_full_count = 0 at default 4096 under distinct reads " + "L2 dedup_full_count = 0 at the raised default under distinct reads " . "(S1 saturation mode does not recur)"); # No client saw the 53R90 retransmit-exhaustion escalation. @@ -471,12 +471,12 @@ sub arm_inject "L5 dump_gcs exposes $key (spec-7.2a D5)"); } -# dedup_max_entries reflects the effective cap (4096) on a serving node. +# dedup_max_entries reflects the effective cap (16384) on a serving node. is($node0->safe_psql('postgres', q{SELECT value FROM pg_cluster_state WHERE category='gcs' AND key='dedup_max_entries'}), - '4096', - 'L5 dedup_max_entries reports the effective cap (4096)'); + '16384', + 'L5 dedup_max_entries reports the effective cap (16384)'); $node0->stop;