From b2222fe9d6f36c6b431712e6c6250898438f8d2d Mon Sep 17 00:00:00 2001 From: Vadim Ponomarev Date: Fri, 3 Jul 2026 20:21:43 +0300 Subject: [PATCH 1/3] Limit concurrent backend WAL flushers --- doc/src/sgml/config.sgml | 15 ++ src/backend/access/transam/xlog.c | 240 ++++++++++++++++++ src/backend/storage/lmgr/proc.c | 1 + src/backend/utils/misc/guc_tables.c | 10 + src/backend/utils/misc/postgresql.conf.sample | 2 + src/include/access/xlog.h | 1 + src/include/storage/proc.h | 4 + src/test/recovery/meson.build | 1 + .../t/055_wal_flush_backend_flushers.pl | 120 +++++++++ src/test/regress/expected/guc.out | 7 + src/test/regress/sql/guc.sql | 3 + 11 files changed, 404 insertions(+) create mode 100644 src/test/recovery/t/055_wal_flush_backend_flushers.pl diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml index 607dafcb2ed16..6c0f00869974e 100644 --- a/doc/src/sgml/config.sgml +++ b/doc/src/sgml/config.sgml @@ -3588,6 +3588,21 @@ include_dir 'conf.d' This parameter can only be set in the postgresql.conf file or on the server command line. + + + + + wal_flush_backend_flushers (integer) + + wal_flush_backend_flushers configuration parameter + + + + + Sets the maximum number of backend processes that can flush WAL + concurrently. The value 0, which is the default, + disables this limit. This parameter can only be set at server start. + diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index e07cb9103515f..619cf122f0d98 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -90,6 +90,7 @@ #include "storage/predicate.h" #include "storage/proc.h" #include "storage/procarray.h" +#include "storage/proclist.h" #include "storage/reinit.h" #include "storage/spin.h" #include "storage/sync.h" @@ -132,6 +133,7 @@ int wal_sync_method = DEFAULT_WAL_SYNC_METHOD; int wal_level = WAL_LEVEL_REPLICA; int CommitDelay = 0; /* precommit delay in microseconds */ int CommitSiblings = 5; /* # concurrent xacts needed to sleep */ +int wal_flush_backend_flushers = 0; int wal_retrieve_retry_interval = 5000; int max_slot_wal_keep_size_mb = -1; int wal_decode_buffer_size = 512 * 1024; @@ -149,6 +151,14 @@ int wal_segment_size = DEFAULT_XLOG_SEG_SIZE; * which needs to iterate all the locks. */ #define NUM_XLOGINSERT_LOCKS 8 +#define WAL_BACKEND_FLUSH_SHARDS 16 + +struct WalBackendFlushShard +{ + pg_atomic_uint64 request; /* highest requested WAL flush LSN */ + proclist_head waiters; /* backends waiting for WAL flush */ + slock_t mutex; /* protects waiters */ +}; /* * Max distance from last checkpoint, before triggering a new xlog-based @@ -472,6 +482,8 @@ typedef struct XLogCtlData pg_atomic_uint64 logInsertResult; /* last byte + 1 inserted to buffers */ pg_atomic_uint64 logWriteResult; /* last byte + 1 written out */ pg_atomic_uint64 logFlushResult; /* last byte + 1 flushed */ + pg_atomic_uint32 backendFlushers; /* active backend WAL flushers */ + struct WalBackendFlushShard backendFlushShards[WAL_BACKEND_FLUSH_SHARDS]; /* * Latest initialized page in the cache (last byte position + 1). @@ -705,6 +717,13 @@ static void ReserveXLogInsertLocation(int size, XLogRecPtr *StartPos, static bool ReserveXLogSwitch(XLogRecPtr *StartPos, XLogRecPtr *EndPos, XLogRecPtr *PrevPtr); static XLogRecPtr WaitXLogInsertionsToFinish(XLogRecPtr upto); +static bool WalBackendFlushersTryAcquire(void); +static void WalBackendFlushersRelease(void); +static XLogRecPtr WalBackendFlushRequestMax(XLogRecPtr record); +static void WalBackendFlushRequestUpdate(struct WalBackendFlushShard *shard); +static void WalBackendFlushWait(struct WalBackendFlushShard *shard, XLogRecPtr record); +static void WalBackendFlushWakeWaiters(XLogRecPtr flushed_lsn); +static void WalBackendFlushWakeWaiterForRetry(void); static char *GetXLogBuffer(XLogRecPtr ptr, TimeLineID tli); static XLogRecPtr XLogBytePosToRecPtr(uint64 bytepos); static XLogRecPtr XLogBytePosToEndRecPtr(uint64 bytepos); @@ -2310,6 +2329,7 @@ XLogWrite(XLogwrtRqst WriteRqst, TimeLineID tli, bool flexible) int npages; int startidx; uint32 startoffset; + XLogRecPtr oldFlush; /* We should always be inside a critical section here */ Assert(CritSectionCount > 0); @@ -2318,6 +2338,7 @@ XLogWrite(XLogwrtRqst WriteRqst, TimeLineID tli, bool flexible) * Update local LogwrtResult (caller probably did this already, but...) */ RefreshXLogWriteResult(LogwrtResult); + oldFlush = LogwrtResult.Flush; /* * Since successive pages in the xlog cache are consecutively allocated, @@ -2578,6 +2599,8 @@ XLogWrite(XLogwrtRqst WriteRqst, TimeLineID tli, bool flexible) pg_atomic_write_u64(&XLogCtl->logWriteResult, LogwrtResult.Write); pg_write_barrier(); pg_atomic_write_u64(&XLogCtl->logFlushResult, LogwrtResult.Flush); + if (oldFlush < LogwrtResult.Flush) + WalBackendFlushWakeWaiters(LogwrtResult.Flush); #ifdef USE_ASSERT_CHECKING { @@ -2770,6 +2793,183 @@ UpdateMinRecoveryPoint(XLogRecPtr lsn, bool force) LWLockRelease(ControlFileLock); } +static bool +WalBackendFlushersTryAcquire(void) +{ + uint32 old_flushers; + + Assert(wal_flush_backend_flushers > 0); + + old_flushers = pg_atomic_read_u32(&XLogCtl->backendFlushers); + for (;;) + { + uint32 expected; + + if (old_flushers >= (uint32) wal_flush_backend_flushers) + return false; + + expected = old_flushers; + if (pg_atomic_compare_exchange_u32(&XLogCtl->backendFlushers, + &expected, old_flushers + 1)) + return true; + + old_flushers = expected; + } +} + +static void +WalBackendFlushersRelease(void) +{ + Assert(wal_flush_backend_flushers > 0); + pg_atomic_fetch_sub_u32(&XLogCtl->backendFlushers, 1); + WalBackendFlushWakeWaiterForRetry(); +} + +static void +WalBackendFlushRequestUpdate(struct WalBackendFlushShard *shard) +{ + XLogRecPtr requested = InvalidXLogRecPtr; + proclist_mutable_iter iter; + + proclist_foreach_modify(iter, &shard->waiters, + backendFlushWaitLink) + { + PGPROC *proc = GetPGProcByNumber(iter.cur); + + if (requested < proc->backendFlushWaitLSN) + requested = proc->backendFlushWaitLSN; + } + + pg_atomic_write_u64(&shard->request, requested); +} + +static XLogRecPtr +WalBackendFlushRequestMax(XLogRecPtr record) +{ + XLogRecPtr requested; + + for (int i = 0; i < WAL_BACKEND_FLUSH_SHARDS; i++) + { + requested = pg_atomic_read_u64(&XLogCtl->backendFlushShards[i].request); + if (record < requested) + record = requested; + } + return record; +} + +static void +WalBackendFlushWait(struct WalBackendFlushShard *shard, XLogRecPtr record) +{ + Assert(MyProc != NULL); + Assert(MyProcNumber != INVALID_PROC_NUMBER); + + ResetLatch(MyLatch); + + SpinLockAcquire(&shard->mutex); + MyProc->backendFlushWaitLSN = record; + proclist_push_tail(&shard->waiters, MyProcNumber, + backendFlushWaitLink); + if (pg_atomic_read_u64(&shard->request) < record) + pg_atomic_write_u64(&shard->request, record); + SpinLockRelease(&shard->mutex); + + INJECTION_POINT("wal-backend-flush-wait", NULL); + + RefreshXLogWriteResult(LogwrtResult); + if (record > LogwrtResult.Flush) + (void) WaitLatch(MyLatch, + WL_LATCH_SET | WL_TIMEOUT | WL_EXIT_ON_PM_DEATH, + 10, + WAIT_EVENT_WAL_SYNC); + ResetLatch(MyLatch); + + SpinLockAcquire(&shard->mutex); + if (proclist_contains(&shard->waiters, MyProcNumber, + backendFlushWaitLink)) + { + proclist_delete(&shard->waiters, MyProcNumber, + backendFlushWaitLink); + WalBackendFlushRequestUpdate(shard); + } + MyProc->backendFlushWaitLSN = InvalidXLogRecPtr; + SpinLockRelease(&shard->mutex); +} + +static void +WalBackendFlushWakeWaiters(XLogRecPtr flushed_lsn) +{ + if (wal_flush_backend_flushers == 0) + return; + + for (int shardno = 0; shardno < WAL_BACKEND_FLUSH_SHARDS; shardno++) + { + struct WalBackendFlushShard *shard = + &XLogCtl->backendFlushShards[shardno]; + + for (;;) + { + PGPROC *wakeups[64]; + int nwakeups = 0; + proclist_mutable_iter iter; + + SpinLockAcquire(&shard->mutex); + + proclist_foreach_modify(iter, &shard->waiters, + backendFlushWaitLink) + { + PGPROC *proc = GetPGProcByNumber(iter.cur); + + if (proc->backendFlushWaitLSN > flushed_lsn) + continue; + + wakeups[nwakeups++] = proc; + proclist_delete(&shard->waiters, iter.cur, + backendFlushWaitLink); + if (nwakeups == lengthof(wakeups)) + break; + } + + if (nwakeups > 0) + WalBackendFlushRequestUpdate(shard); + + SpinLockRelease(&shard->mutex); + + for (int i = 0; i < nwakeups; i++) + { + INJECTION_POINT("wal-backend-flush-wakeup", NULL); + SetLatch(&wakeups[i]->procLatch); + } + + if (nwakeups < lengthof(wakeups)) + break; + } + } +} + +static void +WalBackendFlushWakeWaiterForRetry(void) +{ + PGPROC *wakeups[WAL_BACKEND_FLUSH_SHARDS]; + int nwakeups = 0; + + if (wal_flush_backend_flushers == 0) + return; + + for (int shardno = 0; shardno < WAL_BACKEND_FLUSH_SHARDS; shardno++) + { + struct WalBackendFlushShard *shard = + &XLogCtl->backendFlushShards[shardno]; + + SpinLockAcquire(&shard->mutex); + if (!proclist_is_empty(&shard->waiters)) + wakeups[nwakeups++] = GetPGProcByNumber(shard->waiters.head); + SpinLockRelease(&shard->mutex); + } + + for (int i = 0; i < nwakeups; i++) + SetLatch(&wakeups[i]->procLatch); +} + /* * Ensure that all XLOG data through the given position is flushed to disk. * @@ -2782,6 +2982,8 @@ XLogFlush(XLogRecPtr record) XLogRecPtr WriteRqstPtr; XLogwrtRqst WriteRqst; TimeLineID insertTLI = XLogCtl->InsertTimeLineID; + bool backend_flush_limit_enabled; + struct WalBackendFlushShard *backend_flush_shard = NULL; /* * During REDO, we are reading not writing WAL. Therefore, instead of @@ -2810,6 +3012,15 @@ XLogFlush(XLogRecPtr record) START_CRIT_SECTION(); + backend_flush_limit_enabled = + wal_flush_backend_flushers > 0 && AmRegularBackendProcess(); + if (backend_flush_limit_enabled) + { + Assert(MyProcNumber != INVALID_PROC_NUMBER); + backend_flush_shard = + &XLogCtl->backendFlushShards[MyProcNumber % WAL_BACKEND_FLUSH_SHARDS]; + } + /* * Since fsync is usually a horribly expensive operation, we try to * piggyback as much data as we can on each fsync: if we see any more data @@ -2827,6 +3038,7 @@ XLogFlush(XLogRecPtr record) */ for (;;) { + bool backend_flusher_acquired = false; XLogRecPtr insertpos; /* done already? */ @@ -2842,6 +3054,18 @@ XLogFlush(XLogRecPtr record) if (WriteRqstPtr < XLogCtl->LogwrtRqst.Write) WriteRqstPtr = XLogCtl->LogwrtRqst.Write; SpinLockRelease(&XLogCtl->info_lck); + + if (backend_flush_limit_enabled && + !WalBackendFlushersTryAcquire()) + { + WalBackendFlushWait(backend_flush_shard, record); + continue; + } + backend_flusher_acquired = backend_flush_limit_enabled; + if (backend_flusher_acquired) + INJECTION_POINT("wal-backend-flush-after-acquire", NULL); + if (backend_flush_limit_enabled) + WriteRqstPtr = WalBackendFlushRequestMax(WriteRqstPtr); insertpos = WaitXLogInsertionsToFinish(WriteRqstPtr); /* @@ -2853,6 +3077,9 @@ XLogFlush(XLogRecPtr record) */ if (!LWLockAcquireOrWait(WALWriteLock, LW_EXCLUSIVE)) { + if (backend_flusher_acquired) + WalBackendFlushersRelease(); + /* * The lock is now free, but we didn't acquire it yet. Before we * do, loop back to check if someone else flushed the record for @@ -2866,6 +3093,8 @@ XLogFlush(XLogRecPtr record) if (record <= LogwrtResult.Flush) { LWLockRelease(WALWriteLock); + if (backend_flusher_acquired) + WalBackendFlushersRelease(); break; } @@ -2903,6 +3132,8 @@ XLogFlush(XLogRecPtr record) XLogWrite(WriteRqst, insertTLI, false); LWLockRelease(WALWriteLock); + if (backend_flusher_acquired) + WalBackendFlushersRelease(); /* done */ break; } @@ -5060,6 +5291,15 @@ XLOGShmemInit(void) pg_atomic_init_u64(&XLogCtl->logInsertResult, InvalidXLogRecPtr); pg_atomic_init_u64(&XLogCtl->logWriteResult, InvalidXLogRecPtr); pg_atomic_init_u64(&XLogCtl->logFlushResult, InvalidXLogRecPtr); + pg_atomic_init_u32(&XLogCtl->backendFlushers, 0); + for (i = 0; i < WAL_BACKEND_FLUSH_SHARDS; i++) + { + struct WalBackendFlushShard *shard = &XLogCtl->backendFlushShards[i]; + + pg_atomic_init_u64(&shard->request, InvalidXLogRecPtr); + proclist_init(&shard->waiters); + SpinLockInit(&shard->mutex); + } pg_atomic_init_u64(&XLogCtl->unloggedLSN, InvalidXLogRecPtr); } diff --git a/src/backend/storage/lmgr/proc.c b/src/backend/storage/lmgr/proc.c index b59e38adf87e1..8f96bcc18fa13 100644 --- a/src/backend/storage/lmgr/proc.c +++ b/src/backend/storage/lmgr/proc.c @@ -489,6 +489,7 @@ InitProcess(void) MyProc->statusFlags |= PROC_IS_AUTOVACUUM; MyProc->lwWaiting = LW_WS_NOT_WAITING; MyProc->lwWaitMode = 0; + MyProc->backendFlushWaitLSN = InvalidXLogRecPtr; MyProc->waitLock = NULL; MyProc->waitProcLock = NULL; pg_atomic_write_u64(&MyProc->waitStart, 0); diff --git a/src/backend/utils/misc/guc_tables.c b/src/backend/utils/misc/guc_tables.c index 6b82a23435efe..b132d1e5862c8 100644 --- a/src/backend/utils/misc/guc_tables.c +++ b/src/backend/utils/misc/guc_tables.c @@ -3048,6 +3048,16 @@ struct config_int ConfigureNamesInt[] = NULL, NULL, NULL }, + { + {"wal_flush_backend_flushers", PGC_POSTMASTER, WAL_SETTINGS, + gettext_noop("Sets the maximum number of backend processes that can flush WAL concurrently."), + gettext_noop("0 disables the limit.") + }, + &wal_flush_backend_flushers, + 0, 0, MAX_BACKENDS, + NULL, NULL, NULL + }, + { {"wal_skip_threshold", PGC_USERSET, WAL_SETTINGS, gettext_noop("Minimum size of new file to fsync instead of writing WAL."), diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample index d91133dbd7357..5ba646d11f7fe 100644 --- a/src/backend/utils/misc/postgresql.conf.sample +++ b/src/backend/utils/misc/postgresql.conf.sample @@ -257,6 +257,8 @@ # (change requires restart) #wal_writer_delay = 200ms # 1-10000 milliseconds #wal_writer_flush_after = 1MB # measured in pages, 0 disables +#wal_flush_backend_flushers = 0 # 0 disables the limit + # (change requires restart) #wal_skip_threshold = 2MB #commit_delay = 0 # range 0-100000, in microseconds diff --git a/src/include/access/xlog.h b/src/include/access/xlog.h index f20f5edb43876..47c9c80cac96e 100644 --- a/src/include/access/xlog.h +++ b/src/include/access/xlog.h @@ -54,6 +54,7 @@ extern PGDLLIMPORT char *wal_consistency_checking_string; extern PGDLLIMPORT bool log_checkpoints; extern PGDLLIMPORT int CommitDelay; extern PGDLLIMPORT int CommitSiblings; +extern PGDLLIMPORT int wal_flush_backend_flushers; extern PGDLLIMPORT bool track_wal_io_timing; extern PGDLLIMPORT int wal_decode_buffer_size; diff --git a/src/include/storage/proc.h b/src/include/storage/proc.h index a33c2f04eb437..3d05190c7016e 100644 --- a/src/include/storage/proc.h +++ b/src/include/storage/proc.h @@ -242,6 +242,10 @@ struct PGPROC /* Support for condition variables. */ proclist_node cvWaitLink; /* position in CV wait list */ + /* Support for limited backend WAL flush wait queue. */ + proclist_node backendFlushWaitLink; /* position in WAL flush wait list */ + XLogRecPtr backendFlushWaitLSN; /* WAL flush LSN this process needs */ + /* Info about lock the process is currently waiting for, if any. */ /* waitLock and waitProcLock are NULL if not currently waiting. */ LOCK *waitLock; /* Lock object we're sleeping on ... */ diff --git a/src/test/recovery/meson.build b/src/test/recovery/meson.build index 38e1e43e04154..df6d854f3a5ae 100644 --- a/src/test/recovery/meson.build +++ b/src/test/recovery/meson.build @@ -59,6 +59,7 @@ tests += { 't/048_vacuum_horizon_floor.pl', 't/053_standby_login_event_trigger.pl', 't/054_unlogged_sequence_promotion.pl', + 't/055_wal_flush_backend_flushers.pl', ], }, } diff --git a/src/test/recovery/t/055_wal_flush_backend_flushers.pl b/src/test/recovery/t/055_wal_flush_backend_flushers.pl new file mode 100644 index 0000000000000..b5d0631890e4e --- /dev/null +++ b/src/test/recovery/t/055_wal_flush_backend_flushers.pl @@ -0,0 +1,120 @@ +# Copyright (c) 2026, PostgreSQL Global Development Group + +# Test concurrent commits while limiting backend processes that may flush WAL. +use strict; +use warnings FATAL => 'all'; + +use PostgreSQL::Test::Cluster; +use Test::More; + +my $use_injection_points = ($ENV{enable_injection_points} // '') eq 'yes'; + +my $node = PostgreSQL::Test::Cluster->new('wal_flush_backend_flushers'); +$node->init; +$node->append_conf( + 'postgresql.conf', q{ +wal_flush_backend_flushers = 1 +}); +if ($use_injection_points) +{ + $node->append_conf( + 'postgresql.conf', q{ +shared_preload_libraries = 'injection_points' +}); +} +$node->start; + +is($node->safe_psql('postgres', 'SHOW wal_flush_backend_flushers'), + '1', 'wal_flush_backend_flushers can be set at server start'); + +$node->safe_psql('postgres', + 'CREATE TABLE wal_flush_backend_flushers_probe (id bigserial)'); +$node->safe_psql('postgres', + 'CREATE TABLE wal_flush_backend_flushers_test (id bigserial)'); + +if ($use_injection_points && $node->check_extension('injection_points')) +{ + $node->safe_psql('postgres', 'CREATE EXTENSION injection_points'); + $node->safe_psql( + 'postgres', q{ +SELECT injection_points_attach('wal-backend-flush-after-acquire', 'wait'); +SELECT injection_points_attach('wal-backend-flush-wait', 'wait'); +SELECT injection_points_attach('wal-backend-flush-wakeup', 'wait'); +}); + + my $flusher = $node->background_psql('postgres', on_error_stop => 1); + $flusher->query_until( + qr/flusher_started/, + q{ +\echo flusher_started +INSERT INTO wal_flush_backend_flushers_probe DEFAULT VALUES; +\echo flusher_done +}); + $node->wait_for_event('client backend', + 'wal-backend-flush-after-acquire'); + + my $waiter = $node->background_psql('postgres', on_error_stop => 1); + $waiter->query_until( + qr/waiter_started/, + q{ +\echo waiter_started +INSERT INTO wal_flush_backend_flushers_probe DEFAULT VALUES; +\echo waiter_done +}); + $node->wait_for_event('client backend', 'wal-backend-flush-wait'); + + pass('a backend waited in the limited WAL flush queue'); + + $node->safe_psql('postgres', + q{SELECT injection_points_wakeup('wal-backend-flush-after-acquire')}); + $node->wait_for_event('client backend', 'wal-backend-flush-wakeup'); + pass('WAL flush wakeup path reached a queued backend'); + + $node->safe_psql('postgres', + q{SELECT injection_points_wakeup('wal-backend-flush-wakeup')}); + $flusher->query_until(qr/flusher_done/, ''); + + $node->safe_psql('postgres', + q{SELECT injection_points_wakeup('wal-backend-flush-wait')}); + $waiter->query_until(qr/waiter_done/, ''); + + $flusher->quit; + $waiter->quit; + + $node->safe_psql( + 'postgres', q{ +SELECT injection_points_detach('wal-backend-flush-after-acquire'); +SELECT injection_points_detach('wal-backend-flush-wait'); +SELECT injection_points_detach('wal-backend-flush-wakeup'); +}); +} +elsif ($use_injection_points) +{ + note( + 'extension injection_points not installed; skipping wait/wakeup probe' + ); +} +else +{ + note( + 'injection points not supported by this build; skipping wait/wakeup probe' + ); +} + +$node->pgbench( + '--no-vacuum --client=8 --jobs=4 --transactions=200', + 0, + [qr/number of failed transactions: 0/], + [], + 'concurrent commits with one backend allowed to flush WAL', + { + 'wal_flush_backend_flushers.pgb' => + 'INSERT INTO wal_flush_backend_flushers_test DEFAULT VALUES' + }); + +is( $node->safe_psql( + 'postgres', 'SELECT count(*) FROM wal_flush_backend_flushers_test'), + '1600', + 'all concurrent WAL-writing transactions committed'); + +done_testing(); diff --git a/src/test/regress/expected/guc.out b/src/test/regress/expected/guc.out index 7f9e29c765cf1..00d21ba13ec99 100644 --- a/src/test/regress/expected/guc.out +++ b/src/test/regress/expected/guc.out @@ -6,6 +6,13 @@ SHOW datestyle; Postgres, MDY (1 row) +-- Check the default of the backend WAL flusher limit. +SHOW wal_flush_backend_flushers; + wal_flush_backend_flushers +---------------------------- + 0 +(1 row) + -- Check output style of CamelCase enum options SET intervalstyle to 'asd'; ERROR: invalid value for parameter "IntervalStyle": "asd" diff --git a/src/test/regress/sql/guc.sql b/src/test/regress/sql/guc.sql index f65f84a26320a..de0f9a17f863a 100644 --- a/src/test/regress/sql/guc.sql +++ b/src/test/regress/sql/guc.sql @@ -2,6 +2,9 @@ -- we can't rely on any specific default value of vacuum_cost_delay SHOW datestyle; +-- Check the default of the backend WAL flusher limit. +SHOW wal_flush_backend_flushers; + -- Check output style of CamelCase enum options SET intervalstyle to 'asd'; From 2ff7a08bc38fc659b7554532a4962be975edaeeb Mon Sep 17 00:00:00 2001 From: Vadim Ponomarev Date: Sat, 4 Jul 2026 10:02:41 +0300 Subject: [PATCH 2/3] Rework limited backend WAL flushing onto a ConditionVariable Address review findings against the initial implementation: - Replace the hand-rolled 16-shard wait queue (spinlock + proclist + 10ms latch polling) with a single ConditionVariable. Wakeups are now event-driven: a released slot is handed over with ConditionVariableSignal(), which dequeues the process it wakes, and a process that leaves without taking the slot passes the signal on, so no wakeup can be lost and no thundering herd occurs. This also removes the O(waiters) proclist scans under a spinlock and the extra PGPROC fields; CV wait links are cleaned up at proc exit by the existing machinery. - Track the maximum requested flush LSN in one monotonically-advancing atomic (pg_atomic_monotonic_advance_u64) instead of per-shard values recomputed under a spinlock. The published value is clamped to the end of inserted WAL so that a corrupted page LSN cannot make every slot holder repeatedly request a flush past end of WAL. The clamp uses the end-position conversion of the insert position (GetXLogInsertEndRecPtr), matching what WaitXLogInsertionsToFinish() compares against: the start-position variant points past the page header whenever the insert position sits exactly on a page boundary, and benchmarking showed such requests tripping that function's "request to flush past end of generated WAL" complaint. - Defer waiter wakeups out of XLogWrite(): they now run after WALWriteLock is released, mirroring how walsender wakeups are handled. A slot is kept for the whole XLogFlush() loop instead of being released and re-acquired around LWLockAcquireOrWait(). - Handle injection points safely around critical sections. Plain INJECTION_POINT() could allocate inside one; worse, XLogFlush() is often entered with the caller (e.g. RecordTransactionCommit) already holding a critical section, so even a load placed before START_CRIT_SECTION is not always safe. Load the points only when no critical section is active, have the TAP test pre-load them in each session with injection_points_load(), and preload the module so that the wait callback's lazily-initialized state cannot allocate either. Both injection points now live in code reachable only by client backends, so background processes can no longer consume them and hang the test. - Report the wait with a new IPC wait event WalFlushLimit instead of reusing IO/WALSync, which made throttled backends indistinguishable from real fsync waits in pg_stat_activity. - Make the GUC PGC_SIGHUP: no shared memory is sized by it, and slot accounting stays balanced across reloads (config reload happens only between statements, so each XLogFlush call sees one value). Document that only client backends are subject to the limit. - Drop the guc.sql default-value check, which would break installcheck against a cluster with a non-default setting; the TAP test covers the GUC. - Extend the TAP test: a second injection scenario where both the slot handoff signal and the flush broadcast arrive while the waiter is registered but not yet sleeping (it must complete via the recheck after preparing to sleep), commit traffic across limit reloads bouncing between off, high and one (an acquire/release imbalance would wedge the slots and hang), and a crash-recovery check that commits made through the limited flush path are durable. Also verify that the server log stays free of flush-past-end complaints. --- doc/src/sgml/config.sgml | 18 +- src/backend/access/transam/xlog.c | 396 ++++++++++-------- src/backend/storage/lmgr/proc.c | 1 - .../utils/activity/wait_event_names.txt | 2 + src/backend/utils/misc/guc_tables.c | 4 +- src/backend/utils/misc/postgresql.conf.sample | 1 - src/include/storage/proc.h | 4 - .../t/055_wal_flush_backend_flushers.pl | 157 ++++++- src/test/regress/expected/guc.out | 7 - src/test/regress/sql/guc.sql | 3 - 10 files changed, 371 insertions(+), 222 deletions(-) diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml index 6c0f00869974e..d7b4de4b6bc29 100644 --- a/doc/src/sgml/config.sgml +++ b/doc/src/sgml/config.sgml @@ -3588,8 +3588,8 @@ include_dir 'conf.d' This parameter can only be set in the postgresql.conf file or on the server command line. - - + + wal_flush_backend_flushers (integer) @@ -3599,9 +3599,17 @@ include_dir 'conf.d' - Sets the maximum number of backend processes that can flush WAL - concurrently. The value 0, which is the default, - disables this limit. This parameter can only be set at server start. + Sets the maximum number of client backends that can flush WAL + concurrently. When the limit is reached, further backends needing + their WAL flushed wait until one of the active flushers has flushed + it for them, or until they are allowed to flush themselves. This + can reduce contention on WAL flushing with very many concurrently + committing sessions. Background processes, such as autovacuum + workers and background workers, are not subject to this limit. + The value 0, which is the default, disables the + limit. This parameter can only be set in the + postgresql.conf file or on the server command + line. diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index 619cf122f0d98..620066ec89143 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -83,6 +83,7 @@ #include "replication/walreceiver.h" #include "replication/walsender.h" #include "storage/bufmgr.h" +#include "storage/condition_variable.h" #include "storage/fd.h" #include "storage/ipc.h" #include "storage/large_object.h" @@ -90,7 +91,6 @@ #include "storage/predicate.h" #include "storage/proc.h" #include "storage/procarray.h" -#include "storage/proclist.h" #include "storage/reinit.h" #include "storage/spin.h" #include "storage/sync.h" @@ -151,14 +151,6 @@ int wal_segment_size = DEFAULT_XLOG_SEG_SIZE; * which needs to iterate all the locks. */ #define NUM_XLOGINSERT_LOCKS 8 -#define WAL_BACKEND_FLUSH_SHARDS 16 - -struct WalBackendFlushShard -{ - pg_atomic_uint64 request; /* highest requested WAL flush LSN */ - proclist_head waiters; /* backends waiting for WAL flush */ - slock_t mutex; /* protects waiters */ -}; /* * Max distance from last checkpoint, before triggering a new xlog-based @@ -482,8 +474,6 @@ typedef struct XLogCtlData pg_atomic_uint64 logInsertResult; /* last byte + 1 inserted to buffers */ pg_atomic_uint64 logWriteResult; /* last byte + 1 written out */ pg_atomic_uint64 logFlushResult; /* last byte + 1 flushed */ - pg_atomic_uint32 backendFlushers; /* active backend WAL flushers */ - struct WalBackendFlushShard backendFlushShards[WAL_BACKEND_FLUSH_SHARDS]; /* * Latest initialized page in the cache (last byte position + 1). @@ -564,6 +554,30 @@ typedef struct XLogCtlData XLogRecPtr lastFpwDisableRecPtr; slock_t info_lck; /* locks shared variables shown above */ + + /* + * State for limiting the number of concurrent backend WAL flushers, see + * wal_flush_backend_flushers. + * + * backendFlushers counts the client backends currently allowed to perform + * the flush loop in XLogFlush(). backendFlushWaiters counts the backends + * waiting for a slot or for their WAL to be flushed by someone else; it + * gates the wakeup calls so that they cost nothing when the feature is + * idle. backendFlushRequest is a monotonically-advancing maximum of the + * flush LSNs the waiters need; slot holders extend their flush requests + * to cover it, so waiters piggyback on the holders' fsyncs. Waiters + * sleep on backendFlushCV; it is signaled when a slot is released and + * broadcast when the flushed position advances. + * + * These fields are deliberately kept away from logFlushResult above: + * backendFlushers takes compare-and-swap traffic from every throttled + * backend and must not share a cache line with the log*Result fields that + * RefreshXLogWriteResult() reads on every insert/write/flush. + */ + pg_atomic_uint32 backendFlushers; /* active backend WAL flushers */ + pg_atomic_uint32 backendFlushWaiters; /* backends waiting in XLogFlush */ + pg_atomic_uint64 backendFlushRequest; /* max flush LSN of waiters */ + ConditionVariable backendFlushCV; } XLogCtlData; /* @@ -624,6 +638,15 @@ static int UsableBytesInSegment; */ static XLogwrtResult LogwrtResult = {0, 0}; +/* + * True when an XLogWrite() performed by this process advanced the shared + * flushed position and backends waiting in WalBackendFlushAcquireOrWait() + * may need to be woken up. XLogWrite() runs with WALWriteLock held, so the + * wakeup is deferred until the caller has released the lock; see + * WalBackendFlushProcessWakeup(). + */ +static bool backendFlushWakeupPending = false; + /* * Update local copy of shared XLogCtl->log{Write,Flush}Result * @@ -717,13 +740,10 @@ static void ReserveXLogInsertLocation(int size, XLogRecPtr *StartPos, static bool ReserveXLogSwitch(XLogRecPtr *StartPos, XLogRecPtr *EndPos, XLogRecPtr *PrevPtr); static XLogRecPtr WaitXLogInsertionsToFinish(XLogRecPtr upto); -static bool WalBackendFlushersTryAcquire(void); -static void WalBackendFlushersRelease(void); -static XLogRecPtr WalBackendFlushRequestMax(XLogRecPtr record); -static void WalBackendFlushRequestUpdate(struct WalBackendFlushShard *shard); -static void WalBackendFlushWait(struct WalBackendFlushShard *shard, XLogRecPtr record); -static void WalBackendFlushWakeWaiters(XLogRecPtr flushed_lsn); -static void WalBackendFlushWakeWaiterForRetry(void); +static bool WalBackendFlushTryAcquire(void); +static bool WalBackendFlushAcquireOrWait(XLogRecPtr record); +static void WalBackendFlushRelease(void); +static void WalBackendFlushProcessWakeup(void); static char *GetXLogBuffer(XLogRecPtr ptr, TimeLineID tli); static XLogRecPtr XLogBytePosToRecPtr(uint64 bytepos); static XLogRecPtr XLogBytePosToEndRecPtr(uint64 bytepos); @@ -2079,6 +2099,13 @@ AdvanceXLInsertBuffer(XLogRecPtr upto, TimeLineID tli, bool opportunistic) WriteRqst.Flush = 0; XLogWrite(WriteRqst, tli, false); LWLockRelease(WALWriteLock); + + /* + * The flushed position advances here only when XLogWrite + * had to finish a segment, so waking the flush-limit + * waiters from this spot is rare. + */ + WalBackendFlushProcessWakeup(); pgWalUsage.wal_buffers_full++; TRACE_POSTGRESQL_WAL_BUFFER_WRITE_DIRTY_DONE(); @@ -2599,8 +2626,15 @@ XLogWrite(XLogwrtRqst WriteRqst, TimeLineID tli, bool flexible) pg_atomic_write_u64(&XLogCtl->logWriteResult, LogwrtResult.Write); pg_write_barrier(); pg_atomic_write_u64(&XLogCtl->logFlushResult, LogwrtResult.Flush); + + /* + * If we advanced the flushed position, remember to wake up backends + * waiting in WalBackendFlushAcquireOrWait() once our caller has released + * WALWriteLock; waking them here would lengthen the hold time of the most + * contended WAL lock. + */ if (oldFlush < LogwrtResult.Flush) - WalBackendFlushWakeWaiters(LogwrtResult.Flush); + backendFlushWakeupPending = true; #ifdef USE_ASSERT_CHECKING { @@ -2793,19 +2827,27 @@ UpdateMinRecoveryPoint(XLogRecPtr lsn, bool force) LWLockRelease(ControlFileLock); } +/* + * Try to acquire one of the wal_flush_backend_flushers slots. + * + * Returns true if the caller now holds a slot and must eventually call + * WalBackendFlushRelease(). Callers must have checked that the limit is + * enabled; it cannot change underneath us, see WalBackendFlushAcquireOrWait. + */ static bool -WalBackendFlushersTryAcquire(void) +WalBackendFlushTryAcquire(void) { + int limit = wal_flush_backend_flushers; uint32 old_flushers; - Assert(wal_flush_backend_flushers > 0); + Assert(limit > 0); old_flushers = pg_atomic_read_u32(&XLogCtl->backendFlushers); for (;;) { uint32 expected; - if (old_flushers >= (uint32) wal_flush_backend_flushers) + if (old_flushers >= (uint32) limit) return false; expected = old_flushers; @@ -2817,157 +2859,136 @@ WalBackendFlushersTryAcquire(void) } } +/* + * Release a slot acquired by WalBackendFlushTryAcquire() and hand it over + * to a waiter, if any. ConditionVariableSignal() removes the process it + * wakes from the wait queue, so the wakeup cannot be swallowed by a process + * that is no longer interested; a woken process that leaves without taking + * the slot passes the signal on (see WalBackendFlushAcquireOrWait). + */ static void -WalBackendFlushersRelease(void) +WalBackendFlushRelease(void) { - Assert(wal_flush_backend_flushers > 0); pg_atomic_fetch_sub_u32(&XLogCtl->backendFlushers, 1); - WalBackendFlushWakeWaiterForRetry(); -} - -static void -WalBackendFlushRequestUpdate(struct WalBackendFlushShard *shard) -{ - XLogRecPtr requested = InvalidXLogRecPtr; - proclist_mutable_iter iter; - proclist_foreach_modify(iter, &shard->waiters, - backendFlushWaitLink) - { - PGPROC *proc = GetPGProcByNumber(iter.cur); - - if (requested < proc->backendFlushWaitLSN) - requested = proc->backendFlushWaitLSN; - } - - pg_atomic_write_u64(&shard->request, requested); -} - -static XLogRecPtr -WalBackendFlushRequestMax(XLogRecPtr record) -{ - XLogRecPtr requested; - - for (int i = 0; i < WAL_BACKEND_FLUSH_SHARDS; i++) - { - requested = pg_atomic_read_u64(&XLogCtl->backendFlushShards[i].request); - if (record < requested) - record = requested; - } - return record; + /* + * The fetch_sub above is a full barrier, so if a concurrent waiter missed + * us here (read backendFlushWaiters as zero), it is guaranteed to see the + * freed slot when it retries the acquire after registering. + */ + if (pg_atomic_read_u32(&XLogCtl->backendFlushWaiters) > 0) + ConditionVariableSignal(&XLogCtl->backendFlushCV); } -static void -WalBackendFlushWait(struct WalBackendFlushShard *shard, XLogRecPtr record) +/* + * Wait until we acquire a WAL flusher slot, or until the wait becomes moot. + * + * Called by XLogFlush() when no slot was immediately available. Returns + * true if a slot was acquired, false if 'record' was flushed by someone + * else in the meantime. + * + * Note that wal_flush_backend_flushers cannot change while we are in here: + * config reload happens only between statements, so each XLogFlush() call + * works with a consistent value, keeping acquire and release balanced. + * + * This runs inside a critical section, which is fine: the condition + * variable sleep path performs no memory allocation, and + * CHECK_FOR_INTERRUPTS() is a no-op while in a critical section. + */ +static bool +WalBackendFlushAcquireOrWait(XLogRecPtr record) { - Assert(MyProc != NULL); - Assert(MyProcNumber != INVALID_PROC_NUMBER); - - ResetLatch(MyLatch); + bool acquired = false; - SpinLockAcquire(&shard->mutex); - MyProc->backendFlushWaitLSN = record; - proclist_push_tail(&shard->waiters, MyProcNumber, - backendFlushWaitLink); - if (pg_atomic_read_u64(&shard->request) < record) - pg_atomic_write_u64(&shard->request, record); - SpinLockRelease(&shard->mutex); + /* Fast path: slot free, no need to register as a waiter. */ + if (WalBackendFlushTryAcquire()) + return true; - INJECTION_POINT("wal-backend-flush-wait", NULL); + /* + * Publish the LSN we need flushed, so that the current slot holders + * extend their flush requests to cover it and we can piggyback on their + * fsyncs. Clamp the published value to the end of inserted WAL: an + * invalid 'record' (e.g. coming from a corrupted page LSN) must not cause + * every slot holder to request a flush past the end of generated WAL. The + * caller still detects and reports the bad LSN itself, just like in the + * unlimited case. + * + * The clamp must use the end-position conversion of the insert position, + * matching what WaitXLogInsertionsToFinish() compares its argument + * against. The start-position variant (GetXLogInsertRecPtr) points past + * the page header when the insert position sits exactly on a page + * boundary, and a request published from there would trip the + * flush-past-end-of-WAL complaint in the flushing backend. + * + * The maximum only ever advances; a stale-high value merely makes a + * holder flush WAL that has been inserted anyway, which is exactly the + * piggybacking XLogFlush() strives for. + */ + pg_atomic_monotonic_advance_u64(&XLogCtl->backendFlushRequest, + Min(record, GetXLogInsertEndRecPtr())); - RefreshXLogWriteResult(LogwrtResult); - if (record > LogwrtResult.Flush) - (void) WaitLatch(MyLatch, - WL_LATCH_SET | WL_TIMEOUT | WL_EXIT_ON_PM_DEATH, - 10, - WAIT_EVENT_WAL_SYNC); - ResetLatch(MyLatch); - - SpinLockAcquire(&shard->mutex); - if (proclist_contains(&shard->waiters, MyProcNumber, - backendFlushWaitLink)) - { - proclist_delete(&shard->waiters, MyProcNumber, - backendFlushWaitLink); - WalBackendFlushRequestUpdate(shard); - } - MyProc->backendFlushWaitLSN = InvalidXLogRecPtr; - SpinLockRelease(&shard->mutex); -} + /* + * Register as a waiter before the final condition checks below. The + * fetch_add is a full barrier, pairing with the one in + * WalBackendFlushRelease() and with the flushed-position update in + * XLogWrite(): whoever misses our registration is guaranteed to be + * visible to our rechecks, and vice versa, so no wakeup can be lost. + */ + pg_atomic_fetch_add_u32(&XLogCtl->backendFlushWaiters, 1); -static void -WalBackendFlushWakeWaiters(XLogRecPtr flushed_lsn) -{ - if (wal_flush_backend_flushers == 0) - return; + INJECTION_POINT_CACHED("wal-backend-flush-wait", NULL); - for (int shardno = 0; shardno < WAL_BACKEND_FLUSH_SHARDS; shardno++) + ConditionVariablePrepareToSleep(&XLogCtl->backendFlushCV); + for (;;) { - struct WalBackendFlushShard *shard = - &XLogCtl->backendFlushShards[shardno]; + /* Did someone else flush our record while we were waiting? */ + RefreshXLogWriteResult(LogwrtResult); + if (record <= LogwrtResult.Flush) + break; - for (;;) + if (WalBackendFlushTryAcquire()) { - PGPROC *wakeups[64]; - int nwakeups = 0; - proclist_mutable_iter iter; - - SpinLockAcquire(&shard->mutex); - - proclist_foreach_modify(iter, &shard->waiters, - backendFlushWaitLink) - { - PGPROC *proc = GetPGProcByNumber(iter.cur); - - if (proc->backendFlushWaitLSN > flushed_lsn) - continue; - - wakeups[nwakeups++] = proc; - proclist_delete(&shard->waiters, iter.cur, - backendFlushWaitLink); - if (nwakeups == lengthof(wakeups)) - break; - } + acquired = true; + break; + } - if (nwakeups > 0) - WalBackendFlushRequestUpdate(shard); + ConditionVariableSleep(&XLogCtl->backendFlushCV, + WAIT_EVENT_WAL_FLUSH_LIMIT); + } - SpinLockRelease(&shard->mutex); + /* + * If we consumed a slot-release signal but are not taking the slot, pass + * the wakeup on so that the freed slot is not lost on a sleeping waiter. + */ + if (ConditionVariableCancelSleep() && !acquired) + ConditionVariableSignal(&XLogCtl->backendFlushCV); - for (int i = 0; i < nwakeups; i++) - { - INJECTION_POINT("wal-backend-flush-wakeup", NULL); - SetLatch(&wakeups[i]->procLatch); - } + pg_atomic_fetch_sub_u32(&XLogCtl->backendFlushWaiters, 1); - if (nwakeups < lengthof(wakeups)) - break; - } - } + return acquired; } +/* + * Wake up backends waiting for their WAL to be flushed, if an XLogWrite() + * performed by this process advanced the flushed position. + * + * Call this after releasing WALWriteLock; the wakeups (one SetLatch per + * waiter) are too expensive to run under it. + */ static void -WalBackendFlushWakeWaiterForRetry(void) +WalBackendFlushProcessWakeup(void) { - PGPROC *wakeups[WAL_BACKEND_FLUSH_SHARDS]; - int nwakeups = 0; - - if (wal_flush_backend_flushers == 0) + if (!backendFlushWakeupPending) return; + backendFlushWakeupPending = false; - for (int shardno = 0; shardno < WAL_BACKEND_FLUSH_SHARDS; shardno++) - { - struct WalBackendFlushShard *shard = - &XLogCtl->backendFlushShards[shardno]; - - SpinLockAcquire(&shard->mutex); - if (!proclist_is_empty(&shard->waiters)) - wakeups[nwakeups++] = GetPGProcByNumber(shard->waiters.head); - SpinLockRelease(&shard->mutex); - } - - for (int i = 0; i < nwakeups; i++) - SetLatch(&wakeups[i]->procLatch); + /* + * Waiters whose flush LSN is still not reached will re-check their + * condition and go back to sleep; that is the same wake-all behavior the + * WALWriteLock wait queue has always had. + */ + if (pg_atomic_read_u32(&XLogCtl->backendFlushWaiters) > 0) + ConditionVariableBroadcast(&XLogCtl->backendFlushCV); } /* @@ -2982,8 +3003,7 @@ XLogFlush(XLogRecPtr record) XLogRecPtr WriteRqstPtr; XLogwrtRqst WriteRqst; TimeLineID insertTLI = XLogCtl->InsertTimeLineID; - bool backend_flush_limit_enabled; - struct WalBackendFlushShard *backend_flush_shard = NULL; + bool backend_flusher_acquired = false; /* * During REDO, we are reading not writing WAL. Therefore, instead of @@ -3010,17 +3030,24 @@ XLogFlush(XLogRecPtr record) LSN_FORMAT_ARGS(LogwrtResult.Flush)); #endif - START_CRIT_SECTION(); - - backend_flush_limit_enabled = - wal_flush_backend_flushers > 0 && AmRegularBackendProcess(); - if (backend_flush_limit_enabled) + /* + * The injection points below run inside the critical section, where the + * first use of an injection point is not allowed to allocate; load them + * into the local cache beforehand. Loading is itself only safe when no + * critical section is active yet, and some callers (such as + * RecordTransactionCommit()) already hold one here, so a test process + * that must reach these points from such a caller has to pre-load them + * with injection_points_load(). + */ + if (wal_flush_backend_flushers > 0 && AmRegularBackendProcess() && + CritSectionCount == 0) { - Assert(MyProcNumber != INVALID_PROC_NUMBER); - backend_flush_shard = - &XLogCtl->backendFlushShards[MyProcNumber % WAL_BACKEND_FLUSH_SHARDS]; + INJECTION_POINT_LOAD("wal-backend-flush-wait"); + INJECTION_POINT_LOAD("wal-backend-flush-after-acquire"); } + START_CRIT_SECTION(); + /* * Since fsync is usually a horribly expensive operation, we try to * piggyback as much data as we can on each fsync: if we see any more data @@ -3038,7 +3065,6 @@ XLogFlush(XLogRecPtr record) */ for (;;) { - bool backend_flusher_acquired = false; XLogRecPtr insertpos; /* done already? */ @@ -3055,17 +3081,26 @@ XLogFlush(XLogRecPtr record) WriteRqstPtr = XLogCtl->LogwrtRqst.Write; SpinLockRelease(&XLogCtl->info_lck); - if (backend_flush_limit_enabled && - !WalBackendFlushersTryAcquire()) + /* + * If the number of concurrent backend WAL flushers is limited, + * acquire a flusher slot before proceeding, and keep it until we are + * done: a slot holder that is merely waiting for WALWriteLock is + * still one of the configured number of active flushers. While + * holding a slot, extend our request to also cover the WAL the queued + * waiters need flushed, so they piggyback on our flush. + */ + if (!backend_flusher_acquired && + wal_flush_backend_flushers > 0 && AmRegularBackendProcess()) { - WalBackendFlushWait(backend_flush_shard, record); - continue; + if (!WalBackendFlushAcquireOrWait(record)) + continue; + backend_flusher_acquired = true; + INJECTION_POINT_CACHED("wal-backend-flush-after-acquire", NULL); } - backend_flusher_acquired = backend_flush_limit_enabled; if (backend_flusher_acquired) - INJECTION_POINT("wal-backend-flush-after-acquire", NULL); - if (backend_flush_limit_enabled) - WriteRqstPtr = WalBackendFlushRequestMax(WriteRqstPtr); + WriteRqstPtr = Max(WriteRqstPtr, + pg_atomic_read_u64(&XLogCtl->backendFlushRequest)); + insertpos = WaitXLogInsertionsToFinish(WriteRqstPtr); /* @@ -3077,9 +3112,6 @@ XLogFlush(XLogRecPtr record) */ if (!LWLockAcquireOrWait(WALWriteLock, LW_EXCLUSIVE)) { - if (backend_flusher_acquired) - WalBackendFlushersRelease(); - /* * The lock is now free, but we didn't acquire it yet. Before we * do, loop back to check if someone else flushed the record for @@ -3093,8 +3125,6 @@ XLogFlush(XLogRecPtr record) if (record <= LogwrtResult.Flush) { LWLockRelease(WALWriteLock); - if (backend_flusher_acquired) - WalBackendFlushersRelease(); break; } @@ -3132,14 +3162,18 @@ XLogFlush(XLogRecPtr record) XLogWrite(WriteRqst, insertTLI, false); LWLockRelease(WALWriteLock); - if (backend_flusher_acquired) - WalBackendFlushersRelease(); /* done */ break; } + if (backend_flusher_acquired) + WalBackendFlushRelease(); + END_CRIT_SECTION(); + /* wake up waiters now that we've released heavily contended locks */ + WalBackendFlushProcessWakeup(); + /* wake up walsenders now that we've released heavily contended locks */ WalSndWakeupProcessRequests(true, !RecoveryInProgress()); @@ -3315,6 +3349,9 @@ XLogBackgroundFlush(void) END_CRIT_SECTION(); + /* wake up waiters now that we've released heavily contended locks */ + WalBackendFlushProcessWakeup(); + /* wake up walsenders now that we've released heavily contended locks */ WalSndWakeupProcessRequests(true, !RecoveryInProgress()); @@ -5292,14 +5329,9 @@ XLOGShmemInit(void) pg_atomic_init_u64(&XLogCtl->logWriteResult, InvalidXLogRecPtr); pg_atomic_init_u64(&XLogCtl->logFlushResult, InvalidXLogRecPtr); pg_atomic_init_u32(&XLogCtl->backendFlushers, 0); - for (i = 0; i < WAL_BACKEND_FLUSH_SHARDS; i++) - { - struct WalBackendFlushShard *shard = &XLogCtl->backendFlushShards[i]; - - pg_atomic_init_u64(&shard->request, InvalidXLogRecPtr); - proclist_init(&shard->waiters); - SpinLockInit(&shard->mutex); - } + pg_atomic_init_u32(&XLogCtl->backendFlushWaiters, 0); + pg_atomic_init_u64(&XLogCtl->backendFlushRequest, InvalidXLogRecPtr); + ConditionVariableInit(&XLogCtl->backendFlushCV); pg_atomic_init_u64(&XLogCtl->unloggedLSN, InvalidXLogRecPtr); } diff --git a/src/backend/storage/lmgr/proc.c b/src/backend/storage/lmgr/proc.c index 8f96bcc18fa13..b59e38adf87e1 100644 --- a/src/backend/storage/lmgr/proc.c +++ b/src/backend/storage/lmgr/proc.c @@ -489,7 +489,6 @@ InitProcess(void) MyProc->statusFlags |= PROC_IS_AUTOVACUUM; MyProc->lwWaiting = LW_WS_NOT_WAITING; MyProc->lwWaitMode = 0; - MyProc->backendFlushWaitLSN = InvalidXLogRecPtr; MyProc->waitLock = NULL; MyProc->waitProcLock = NULL; pg_atomic_write_u64(&MyProc->waitStart, 0); diff --git a/src/backend/utils/activity/wait_event_names.txt b/src/backend/utils/activity/wait_event_names.txt index b9c1e6900ec1b..069135071714c 100644 --- a/src/backend/utils/activity/wait_event_names.txt +++ b/src/backend/utils/activity/wait_event_names.txt @@ -163,6 +163,8 @@ XACT_GROUP_UPDATE "Waiting for the group leader to update transaction status at ABI_compatibility: +WAL_FLUSH_LIMIT "Waiting to become one of the limited number of concurrent WAL flushers, or for another process to flush the needed WAL." + # # Wait Events - Timeout # diff --git a/src/backend/utils/misc/guc_tables.c b/src/backend/utils/misc/guc_tables.c index b132d1e5862c8..1319a02b861b1 100644 --- a/src/backend/utils/misc/guc_tables.c +++ b/src/backend/utils/misc/guc_tables.c @@ -3049,8 +3049,8 @@ struct config_int ConfigureNamesInt[] = }, { - {"wal_flush_backend_flushers", PGC_POSTMASTER, WAL_SETTINGS, - gettext_noop("Sets the maximum number of backend processes that can flush WAL concurrently."), + {"wal_flush_backend_flushers", PGC_SIGHUP, WAL_SETTINGS, + gettext_noop("Sets the maximum number of client backends that can flush WAL concurrently."), gettext_noop("0 disables the limit.") }, &wal_flush_backend_flushers, diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample index 5ba646d11f7fe..49bb342631ebb 100644 --- a/src/backend/utils/misc/postgresql.conf.sample +++ b/src/backend/utils/misc/postgresql.conf.sample @@ -258,7 +258,6 @@ #wal_writer_delay = 200ms # 1-10000 milliseconds #wal_writer_flush_after = 1MB # measured in pages, 0 disables #wal_flush_backend_flushers = 0 # 0 disables the limit - # (change requires restart) #wal_skip_threshold = 2MB #commit_delay = 0 # range 0-100000, in microseconds diff --git a/src/include/storage/proc.h b/src/include/storage/proc.h index 3d05190c7016e..a33c2f04eb437 100644 --- a/src/include/storage/proc.h +++ b/src/include/storage/proc.h @@ -242,10 +242,6 @@ struct PGPROC /* Support for condition variables. */ proclist_node cvWaitLink; /* position in CV wait list */ - /* Support for limited backend WAL flush wait queue. */ - proclist_node backendFlushWaitLink; /* position in WAL flush wait list */ - XLogRecPtr backendFlushWaitLSN; /* WAL flush LSN this process needs */ - /* Info about lock the process is currently waiting for, if any. */ /* waitLock and waitProcLock are NULL if not currently waiting. */ LOCK *waitLock; /* Lock object we're sleeping on ... */ diff --git a/src/test/recovery/t/055_wal_flush_backend_flushers.pl b/src/test/recovery/t/055_wal_flush_backend_flushers.pl index b5d0631890e4e..91b5a26e720d6 100644 --- a/src/test/recovery/t/055_wal_flush_backend_flushers.pl +++ b/src/test/recovery/t/055_wal_flush_backend_flushers.pl @@ -15,17 +15,10 @@ 'postgresql.conf', q{ wal_flush_backend_flushers = 1 }); -if ($use_injection_points) -{ - $node->append_conf( - 'postgresql.conf', q{ -shared_preload_libraries = 'injection_points' -}); -} $node->start; is($node->safe_psql('postgres', 'SHOW wal_flush_backend_flushers'), - '1', 'wal_flush_backend_flushers can be set at server start'); + '1', 'wal_flush_backend_flushers is set'); $node->safe_psql('postgres', 'CREATE TABLE wal_flush_backend_flushers_probe (id bigserial)'); @@ -34,15 +27,38 @@ if ($use_injection_points && $node->check_extension('injection_points')) { + # The wait-mode injection points used below fire inside critical + # sections, where the injection_points module must not allocate its + # lazily-initialized local state. Preloading the module sets that + # state up at backend start instead (compare test_slru/t/001_multixact.pl, + # which waits inside a critical section the same way). The availability + # check above ran without the preload, so a build without the module + # still skips this section gracefully instead of failing to start. + $node->append_conf( + 'postgresql.conf', q{ +shared_preload_libraries = 'injection_points' +}); + $node->restart; + $node->safe_psql('postgres', 'CREATE EXTENSION injection_points'); $node->safe_psql( 'postgres', q{ SELECT injection_points_attach('wal-backend-flush-after-acquire', 'wait'); SELECT injection_points_attach('wal-backend-flush-wait', 'wait'); -SELECT injection_points_attach('wal-backend-flush-wakeup', 'wait'); }); + # The injection points fire during the commit's XLogFlush, which runs + # inside the caller's critical section where the points cannot be loaded + # into the local cache anymore; pre-load them in each session first. + my $preload_points = q{ +SELECT injection_points_load('wal-backend-flush-after-acquire'); +SELECT injection_points_load('wal-backend-flush-wait'); +}; + + # The flusher acquires the single flusher slot and then blocks at the + # injection point, holding the slot. my $flusher = $node->background_psql('postgres', on_error_stop => 1); + $flusher->query_safe($preload_points); $flusher->query_until( qr/flusher_started/, q{ @@ -53,7 +69,11 @@ $node->wait_for_event('client backend', 'wal-backend-flush-after-acquire'); + # With the only slot taken, a second backend fails to acquire one and + # reaches the wait path. Both injection points sit in code reachable + # only by client backends, so no background process can consume them. my $waiter = $node->background_psql('postgres', on_error_stop => 1); + $waiter->query_safe($preload_points); $waiter->query_until( qr/waiter_started/, q{ @@ -65,27 +85,72 @@ pass('a backend waited in the limited WAL flush queue'); + # Let the waiter proceed into its condition-variable sleep, then let the + # flusher finish: it flushes the waiter's WAL too, releases the slot and + # wakes the waiter. Each backend passes its injection point exactly + # once, so a single wakeup per point suffices. + $node->safe_psql('postgres', + q{SELECT injection_points_wakeup('wal-backend-flush-wait')}); $node->safe_psql('postgres', q{SELECT injection_points_wakeup('wal-backend-flush-after-acquire')}); - $node->wait_for_event('client backend', 'wal-backend-flush-wakeup'); - pass('WAL flush wakeup path reached a queued backend'); - $node->safe_psql('postgres', - q{SELECT injection_points_wakeup('wal-backend-flush-wakeup')}); $flusher->query_until(qr/flusher_done/, ''); - - $node->safe_psql('postgres', - q{SELECT injection_points_wakeup('wal-backend-flush-wait')}); $waiter->query_until(qr/waiter_done/, ''); + pass('waiter completed after the flusher released the slot'); $flusher->quit; $waiter->quit; + # Second scenario: the slot is released and the flushed position + # advances while the waiter is registered but not yet sleeping on the + # condition variable (parked at the injection point placed before + # ConditionVariablePrepareToSleep), so both the handoff signal and the + # broadcast find an empty wait queue. The waiter must not rely on them: + # rechecking the flushed position after preparing to sleep has to let it + # complete. + my $flusher2 = $node->background_psql('postgres', on_error_stop => 1); + $flusher2->query_safe($preload_points); + $flusher2->query_until( + qr/flusher2_started/, + q{ +\echo flusher2_started +INSERT INTO wal_flush_backend_flushers_probe DEFAULT VALUES; +\echo flusher2_done +}); + $node->wait_for_event('client backend', + 'wal-backend-flush-after-acquire'); + + my $waiter2 = $node->background_psql('postgres', on_error_stop => 1); + $waiter2->query_safe($preload_points); + $waiter2->query_until( + qr/waiter2_started/, + q{ +\echo waiter2_started +INSERT INTO wal_flush_backend_flushers_probe DEFAULT VALUES; +\echo waiter2_done +}); + $node->wait_for_event('client backend', 'wal-backend-flush-wait'); + + # Let the flusher finish first: its group flush covers the waiter's + # already-published LSN, and its slot release signals an empty queue. + $node->safe_psql('postgres', + q{SELECT injection_points_wakeup('wal-backend-flush-after-acquire')}); + $flusher2->query_until(qr/flusher2_done/, ''); + + # Only now release the waiter; it must notice that its WAL has been + # flushed and complete without any wakeup arriving. + $node->safe_psql('postgres', + q{SELECT injection_points_wakeup('wal-backend-flush-wait')}); + $waiter2->query_until(qr/waiter2_done/, ''); + pass('waiter completed although wakeups preceded its sleep'); + + $flusher2->quit; + $waiter2->quit; + $node->safe_psql( 'postgres', q{ SELECT injection_points_detach('wal-backend-flush-after-acquire'); SELECT injection_points_detach('wal-backend-flush-wait'); -SELECT injection_points_detach('wal-backend-flush-wakeup'); }); } elsif ($use_injection_points) @@ -117,4 +182,62 @@ '1600', 'all concurrent WAL-writing transactions committed'); +# The limit can be changed with a reload; make sure commits still work after +# disabling it on the fly. +$node->adjust_conf('postgresql.conf', 'wal_flush_backend_flushers', '0'); +$node->reload; +$node->safe_psql('postgres', + 'INSERT INTO wal_flush_backend_flushers_test DEFAULT VALUES'); +is( $node->safe_psql( + 'postgres', 'SELECT count(*) FROM wal_flush_backend_flushers_test'), + '1601', + 'commits work after disabling the limit via reload'); + +# Stress the reload path: keep sessions committing while the limit bounces +# between off, high and one. Any acquire/release imbalance across the +# transitions would permanently wedge the flusher slots (with the limit back +# at one, a leaked slot blocks every commit), which shows up here as a hang. +my @churners = + map { $node->background_psql('postgres', on_error_stop => 1) } (1 .. 3); +my @limits = ('3', '1', '0', '1'); +my $churn_rows = 0; +foreach my $limit (@limits) +{ + $node->adjust_conf('postgresql.conf', 'wal_flush_backend_flushers', + $limit); + $node->reload; + for my $i (1 .. 5) + { + foreach my $session (@churners) + { + $session->query( + 'INSERT INTO wal_flush_backend_flushers_test DEFAULT VALUES'); + $churn_rows++; + } + } +} +$_->quit foreach @churners; +is( $node->safe_psql( + 'postgres', 'SELECT count(*) FROM wal_flush_backend_flushers_test'), + 1601 + $churn_rows, + 'commits survive limit changes between off, high and one'); + +# The published flush request is clamped to the end position of inserted +# WAL; a wrong clamp (e.g. the start-position conversion, which lands past +# the page header on page boundaries) surfaces as this complaint from the +# flushing backend. The trigger needs an unlucky insert position, so a +# clean pass is only probabilistic insurance, but the message must never +# appear with a correct clamp. +ok( !$node->log_contains('request to flush past end of generated WAL'), + 'no flush-past-end complaints from limited backend flushing'); + +# All the commits above went through the limited flush path with +# synchronous_commit on, so they must survive a crash. +$node->stop('immediate'); +$node->start; +is( $node->safe_psql( + 'postgres', 'SELECT count(*) FROM wal_flush_backend_flushers_test'), + 1601 + $churn_rows, + 'limited-path commits are durable across a crash'); + done_testing(); diff --git a/src/test/regress/expected/guc.out b/src/test/regress/expected/guc.out index 00d21ba13ec99..7f9e29c765cf1 100644 --- a/src/test/regress/expected/guc.out +++ b/src/test/regress/expected/guc.out @@ -6,13 +6,6 @@ SHOW datestyle; Postgres, MDY (1 row) --- Check the default of the backend WAL flusher limit. -SHOW wal_flush_backend_flushers; - wal_flush_backend_flushers ----------------------------- - 0 -(1 row) - -- Check output style of CamelCase enum options SET intervalstyle to 'asd'; ERROR: invalid value for parameter "IntervalStyle": "asd" diff --git a/src/test/regress/sql/guc.sql b/src/test/regress/sql/guc.sql index de0f9a17f863a..f65f84a26320a 100644 --- a/src/test/regress/sql/guc.sql +++ b/src/test/regress/sql/guc.sql @@ -2,9 +2,6 @@ -- we can't rely on any specific default value of vacuum_cost_delay SHOW datestyle; --- Check the default of the backend WAL flusher limit. -SHOW wal_flush_backend_flushers; - -- Check output style of CamelCase enum options SET intervalstyle to 'asd'; From 86b5dbebab2c1b342a04bd5d6df7cffd44024fe4 Mon Sep 17 00:00:00 2001 From: Vadim Ponomarev Date: Sat, 4 Jul 2026 17:07:01 +0300 Subject: [PATCH 3/3] Pad limited-flush shared state away from info_lck perf c2c profiling on a 4-socket stand (750 clients, limit 32) showed the limiter counters and the condition variable sharing one cache line with info_lck: the CV's internal spinlock alone accounted for 70% of that line's remote HITM traffic, competing with every XLogFlush entry, page-crossing insert, and hint-bit GetRedoRecPtr call. Give the counters and the condition variable cache lines of their own. --- src/backend/access/transam/xlog.c | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index 620066ec89143..dc96bf86d8a5d 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -569,14 +569,20 @@ typedef struct XLogCtlData * sleep on backendFlushCV; it is signaled when a slot is released and * broadcast when the flushed position advances. * - * These fields are deliberately kept away from logFlushResult above: - * backendFlushers takes compare-and-swap traffic from every throttled - * backend and must not share a cache line with the log*Result fields that - * RefreshXLogWriteResult() reads on every insert/write/flush. - */ + * All of these take atomic read-modify-write traffic from every + * committing backend, so they must not share cache lines with anything + * else that is hot: not with the log*Result fields that + * RefreshXLogWriteResult() reads on every insert/write/flush, and not + * with info_lck above, which every XLogFlush() entry and every + * GetRedoRecPtr() call takes. The condition variable's internal spinlock + * is pounded harder than the counters (two acquisitions per waiter plus + * every signal/broadcast), so it gets a cache line of its own as well. + */ + char backendFlushPad1[PG_CACHE_LINE_SIZE]; pg_atomic_uint32 backendFlushers; /* active backend WAL flushers */ pg_atomic_uint32 backendFlushWaiters; /* backends waiting in XLogFlush */ pg_atomic_uint64 backendFlushRequest; /* max flush LSN of waiters */ + char backendFlushPad2[PG_CACHE_LINE_SIZE]; ConditionVariable backendFlushCV; } XLogCtlData;